Merge branch 'nixos-unstable' of https://github.com/NixOS/nixpkgs into HEAD

Conflicts:
	nixpkgs/pkgs/servers/pr-tracker/default.nix

7 files changed, 319 insertions, 28 deletions

diff --git a/nixpkgs/lib/fileset/default.nix b/nixpkgs/lib/fileset/default.nix
index 0342be3e0371..4a97633b4a89 100644
--- a/nixpkgs/lib/fileset/default.nix
+++ b/nixpkgs/lib/fileset/default.nix
@@ -9,6 +9,7 @@ let
     _fileFilter
     _printFileset
     _intersection
+    _difference
     ;
 
   inherit (builtins)
@@ -369,6 +370,58 @@ If a directory does not recursively contain any file, it is omitted from the sto
       (elemAt filesets 1);
 
   /*
+    The file set containing all files from the first file set that are not in the second file set.
+    See also [Difference (set theory)](https://en.wikipedia.org/wiki/Complement_(set_theory)#Relative_complement).
+
+    The given file sets are evaluated as lazily as possible,
+    with the first argument being evaluated first if needed.
+
+    Type:
+      union :: FileSet -> FileSet -> FileSet
+
+    Example:
+      # Create a file set containing all files from the current directory,
+      # except ones under ./tests
+      difference ./. ./tests
+
+      let
+        # A set of Nix-related files
+        nixFiles = unions [ ./default.nix ./nix ./tests/default.nix ];
+      in
+      # Create a file set containing all files under ./tests, except ones in `nixFiles`,
+      # meaning only without ./tests/default.nix
+      difference ./tests nixFiles
+  */
+  difference =
+    # The positive file set.
+    # The result can only contain files that are also in this file set.
+    #
+    # This argument can also be a path,
+    # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion).
+    positive:
+    # The negative file set.
+    # The result will never contain files that are also in this file set.
+    #
+    # This argument can also be a path,
+    # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion).
+    negative:
+    let
+      filesets = _coerceMany "lib.fileset.difference" [
+        {
+          context = "first argument (positive set)";
+          value = positive;
+        }
+        {
+          context = "second argument (negative set)";
+          value = negative;
+        }
+      ];
+    in
+    _difference
+      (elemAt filesets 0)
+      (elemAt filesets 1);
+
+  /*
     Incrementally evaluate and trace a file set in a pretty way.
     This function is only intended for debugging purposes.
     The exact tracing format is unspecified and may change.
diff --git a/nixpkgs/lib/fileset/internal.nix b/nixpkgs/lib/fileset/internal.nix
index 2d52a8cb410b..b919a5de3eef 100644
--- a/nixpkgs/lib/fileset/internal.nix
+++ b/nixpkgs/lib/fileset/internal.nix
@@ -424,7 +424,7 @@ rec {
       # Filter suited when there's some files
       # This can't be used for when there's no files, because the base directory is always included
       nonEmpty =
-        path: _:
+        path: type:
         let
           # Add a slash to the path string, turning "/foo" to "/foo/",
           # making sure to not have any false prefix matches below.
@@ -433,25 +433,37 @@ rec {
           # meaning this function can never receive "/" as an argument
           pathSlash = path + "/";
         in
-        # Same as `hasPrefix pathSlash baseString`, but more efficient.
-        # With base /foo/bar we need to include /foo:
-        # hasPrefix "/foo/" "/foo/bar/"
-        if substring 0 (stringLength pathSlash) baseString == pathSlash then
-          true
-        # Same as `! hasPrefix baseString pathSlash`, but more efficient.
-        # With base /foo/bar we need to exclude /baz
-        # ! hasPrefix "/baz/" "/foo/bar/"
-        else if substring 0 baseLength pathSlash != baseString then
-          false
-        else
-          # Same as `removePrefix baseString path`, but more efficient.
-          # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe.
-          # We don't use pathSlash here because we only needed the trailing slash for the prefix matching.
-          # With base /foo and path /foo/bar/baz this gives
-          # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz"))
-          # == inTree (split "/" "bar/baz")
-          # == inTree [ "bar" "baz" ]
-          inTree (split "/" (substring baseLength (-1) path));
+        (
+          # Same as `hasPrefix pathSlash baseString`, but more efficient.
+          # With base /foo/bar we need to include /foo:
+          # hasPrefix "/foo/" "/foo/bar/"
+          if substring 0 (stringLength pathSlash) baseString == pathSlash then
+            true
+          # Same as `! hasPrefix baseString pathSlash`, but more efficient.
+          # With base /foo/bar we need to exclude /baz
+          # ! hasPrefix "/baz/" "/foo/bar/"
+          else if substring 0 baseLength pathSlash != baseString then
+            false
+          else
+            # Same as `removePrefix baseString path`, but more efficient.
+            # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe.
+            # We don't use pathSlash here because we only needed the trailing slash for the prefix matching.
+            # With base /foo and path /foo/bar/baz this gives
+            # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz"))
+            # == inTree (split "/" "bar/baz")
+            # == inTree [ "bar" "baz" ]
+            inTree (split "/" (substring baseLength (-1) path))
+        )
+        # This is a way have an additional check in case the above is true without any significant performance cost
+        && (
+          # This relies on the fact that Nix only distinguishes path types "directory", "regular", "symlink" and "unknown",
+          # so everything except "unknown" is allowed, seems reasonable to rely on that
+          type != "unknown"
+          || throw ''
+            lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: ${path}
+                This file is neither a regular file nor a symlink, the only file types supported by the Nix store.
+                Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.''
+        );
     in
     # Special case because the code below assumes that the _internalBase is always included in the result
     # which shouldn't be done when we have no files at all in the base
@@ -639,6 +651,86 @@ rec {
       # In all other cases it's the rhs
       rhs;
 
+  # Compute the set difference between two file sets.
+  # The filesets must already be coerced and validated to be in the same filesystem root.
+  # Type: Fileset -> Fileset -> Fileset
+  _difference = positive: negative:
+    let
+      # The common base components prefix, e.g.
+      # (/foo/bar, /foo/bar/baz) -> /foo/bar
+      # (/foo/bar, /foo/baz) -> /foo
+      commonBaseComponentsLength =
+        # TODO: Have a `lib.lists.commonPrefixLength` function such that we don't need the list allocation from commonPrefix here
+        length (
+          commonPrefix
+            positive._internalBaseComponents
+            negative._internalBaseComponents
+        );
+
+      # We need filesetTree's with the same base to be able to compute the difference between them
+      # This here is the filesetTree from the negative file set, but for a base path that matches the positive file set.
+      # Examples:
+      # For `difference /foo /foo/bar`, `negativeTreeWithPositiveBase = { bar = "directory"; }`
+      #   because under the base path of `/foo`, only `bar` from the negative file set is included
+      # For `difference /foo/bar /foo`, `negativeTreeWithPositiveBase = "directory"`
+      #   because under the base path of `/foo/bar`, everything from the negative file set is included
+      # For `difference /foo /bar`, `negativeTreeWithPositiveBase = null`
+      #   because under the base path of `/foo`, nothing from the negative file set is included
+      negativeTreeWithPositiveBase =
+        if commonBaseComponentsLength == length positive._internalBaseComponents then
+          # The common prefix is the same as the positive base path, so the second path is equal or longer.
+          # We need to _shorten_ the negative filesetTree to the same base path as the positive one
+          # E.g. for `difference /foo /foo/bar` the common prefix is /foo, equal to the positive file set's base
+          # So we need to shorten the base of the tree for the negative argument from /foo/bar to just /foo
+          _shortenTreeBase positive._internalBaseComponents negative
+        else if commonBaseComponentsLength == length negative._internalBaseComponents then
+          # The common prefix is the same as the negative base path, so the first path is longer.
+          # We need to lengthen the negative filesetTree to the same base path as the positive one.
+          # E.g. for `difference /foo/bar /foo` the common prefix is /foo, equal to the negative file set's base
+          # So we need to lengthen the base of the tree for the negative argument from /foo to /foo/bar
+          _lengthenTreeBase positive._internalBaseComponents negative
+        else
+          # The common prefix is neither the first nor the second path.
+          # This means there's no overlap between the two file sets,
+          # and nothing from the negative argument should get removed from the positive one
+          # E.g for `difference /foo /bar`, we remove nothing to get the same as `/foo`
+          null;
+
+      resultingTree =
+        _differenceTree
+        positive._internalBase
+        positive._internalTree
+        negativeTreeWithPositiveBase;
+    in
+    # If the first file set is empty, we can never have any files in the result
+    if positive._internalIsEmptyWithoutBase then
+      _emptyWithoutBase
+    # If the second file set is empty, nothing gets removed, so the result is just the first file set
+    else if negative._internalIsEmptyWithoutBase then
+      positive
+    else
+      # We use the positive file set base for the result,
+      # because only files from the positive side may be included,
+      # which is what base path is for
+      _create positive._internalBase resultingTree;
+
+  # Computes the set difference of two filesetTree's
+  # Type: Path -> filesetTree -> filesetTree
+  _differenceTree = path: lhs: rhs:
+    # If the lhs doesn't have any files, or the right hand side includes all files
+    if lhs == null || isString rhs then
+      # The result will always be empty
+      null
+    # If the right hand side has no files
+    else if rhs == null then
+      # The result is always the left hand side, because nothing gets removed
+      lhs
+    else
+      # Otherwise we always have two attribute sets to recurse into
+      mapAttrs (name: lhsValue:
+        _differenceTree (path + "/${name}") lhsValue (rhs.${name} or null)
+      ) (_directoryEntries path lhs);
+
   _fileFilter = predicate: fileset:
     let
       recurse = path: tree:
diff --git a/nixpkgs/lib/fileset/tests.sh b/nixpkgs/lib/fileset/tests.sh
index d8d8dd413189..2df0727bde38 100755
--- a/nixpkgs/lib/fileset/tests.sh
+++ b/nixpkgs/lib/fileset/tests.sh
@@ -332,7 +332,7 @@ expectFailure 'with ((import <nixpkgs/lib>).extend (import <nixpkgs/lib/fileset/
 \s*`root`: root "'"$work"'/foo/mock-root"
 \s*`fileset`: root "'"$work"'/bar/mock-root"
 \s*Different roots are not supported.'
-rm -rf *
+rm -rf -- *
 
 # `root` needs to exist
 expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: `root` \('"$work"'/a\) does not exist.'
@@ -342,7 +342,7 @@ touch a
 expectFailure 'toSource { root = ./a; fileset = ./a; }' 'lib.fileset.toSource: `root` \('"$work"'/a\) is a file, but it should be a directory instead. Potential solutions:
 \s*- If you want to import the file into the store _without_ a containing directory, use string interpolation or `builtins.path` instead of this function.
 \s*- If you want to import the file into the store _with_ a containing directory, set `root` to the containing directory, such as '"$work"', and set `fileset` to the file path.'
-rm -rf *
+rm -rf -- *
 
 # The fileset argument should be evaluated, even if the directory is empty
 expectFailure 'toSource { root = ./.; fileset = abort "This should be evaluated"; }' 'evaluation aborted with the following error message: '\''This should be evaluated'\'
@@ -352,7 +352,14 @@ mkdir a
 expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: `fileset` could contain files in '"$work"', which is not under the `root` \('"$work"'/a\). Potential solutions:
 \s*- Set `root` to '"$work"' or any directory higher up. This changes the layout of the resulting store path.
 \s*- Set `fileset` to a file set that cannot contain files outside the `root` \('"$work"'/a\). This could change the files included in the result.'
-rm -rf *
+rm -rf -- *
+
+# non-regular and non-symlink files cannot be added to the Nix store
+mkfifo a
+expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: '"$work"'/a
+\s*This file is neither a regular file nor a symlink, the only file types supported by the Nix store.
+\s*Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.'
+rm -rf -- *
 
 # Path coercion only works for paths
 expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.'
@@ -493,7 +500,7 @@ expectFailure 'with ((import <nixpkgs/lib>).extend (import <nixpkgs/lib/fileset/
 \s*element 0: root "'"$work"'/foo/mock-root"
 \s*element 1: root "'"$work"'/bar/mock-root"
 \s*Different roots are not supported.'
-rm -rf *
+rm -rf -- *
 
 # Coercion errors show the correct context
 expectFailure 'toSource { root = ./.; fileset = union ./a ./.; }' 'lib.fileset.union: first argument \('"$work"'/a\) does not exist.'
@@ -677,6 +684,104 @@ tree=(
 )
 checkFileset 'intersection (unions [ ./a/b ./c/d ./c/e ]) (unions [ ./a ./c/d/f ./c/e ])'
 
+## Difference
+
+# Subtracting something from itself results in nothing
+tree=(
+    [a]=0
+)
+checkFileset 'difference ./. ./.'
+
+# The tree of the second argument should not be evaluated if not needed
+checkFileset 'difference _emptyWithoutBase (_create ./. (abort "This should not be used!"))'
+checkFileset 'difference (_create ./. null) (_create ./. (abort "This should not be used!"))'
+
+# Subtracting nothing gives the same thing back
+tree=(
+    [a]=1
+)
+checkFileset 'difference ./. _emptyWithoutBase'
+checkFileset 'difference ./. (_create ./. null)'
+
+# Subtracting doesn't influence the base path
+mkdir a b
+touch {a,b}/x
+expectEqual 'toSource { root = ./a; fileset = difference ./a ./b; }' 'toSource { root = ./a; fileset = ./a; }'
+rm -rf -- *
+
+# Also not the other way around
+mkdir a
+expectFailure 'toSource { root = ./a; fileset = difference ./. ./a; }' 'lib.fileset.toSource: `fileset` could contain files in '"$work"', which is not under the `root` \('"$work"'/a\). Potential solutions:
+\s*- Set `root` to '"$work"' or any directory higher up. This changes the layout of the resulting store path.
+\s*- Set `fileset` to a file set that cannot contain files outside the `root` \('"$work"'/a\). This could change the files included in the result.'
+rm -rf -- *
+
+# Difference actually works
+# We test all combinations of ./., ./a, ./a/x and ./b
+tree=(
+    [a/x]=0
+    [a/y]=0
+    [b]=0
+    [c]=0
+)
+checkFileset 'difference ./. ./.'
+checkFileset 'difference ./a ./.'
+checkFileset 'difference ./a/x ./.'
+checkFileset 'difference ./b ./.'
+checkFileset 'difference ./a ./a'
+checkFileset 'difference ./a/x ./a'
+checkFileset 'difference ./a/x ./a/x'
+checkFileset 'difference ./b ./b'
+tree=(
+    [a/x]=0
+    [a/y]=0
+    [b]=1
+    [c]=1
+)
+checkFileset 'difference ./. ./a'
+tree=(
+    [a/x]=1
+    [a/y]=1
+    [b]=0
+    [c]=0
+)
+checkFileset 'difference ./a ./b'
+tree=(
+    [a/x]=1
+    [a/y]=0
+    [b]=0
+    [c]=0
+)
+checkFileset 'difference ./a/x ./b'
+tree=(
+    [a/x]=0
+    [a/y]=1
+    [b]=0
+    [c]=0
+)
+checkFileset 'difference ./a ./a/x'
+tree=(
+    [a/x]=0
+    [a/y]=0
+    [b]=1
+    [c]=0
+)
+checkFileset 'difference ./b ./a'
+checkFileset 'difference ./b ./a/x'
+tree=(
+    [a/x]=0
+    [a/y]=1
+    [b]=1
+    [c]=1
+)
+checkFileset 'difference ./. ./a/x'
+tree=(
+    [a/x]=1
+    [a/y]=1
+    [b]=0
+    [c]=1
+)
+checkFileset 'difference ./. ./b'
 
 ## File filter
 
diff --git a/nixpkgs/lib/fixed-points.nix b/nixpkgs/lib/fixed-points.nix
index 3444e95e15ad..3b5fdc9e8ea1 100644
--- a/nixpkgs/lib/fixed-points.nix
+++ b/nixpkgs/lib/fixed-points.nix
@@ -45,7 +45,7 @@ rec {
     }
     ```
 
-    This is where `fix` comes in, it contains the syntactic that's not in `f` anymore.
+    This is where `fix` comes in, it contains the syntactic recursion that's not in `f` anymore.
 
     ```nix
     nix-repl> fix = f:
diff --git a/nixpkgs/lib/meta.nix b/nixpkgs/lib/meta.nix
index 44730a71551e..2e817c42327d 100644
--- a/nixpkgs/lib/meta.nix
+++ b/nixpkgs/lib/meta.nix
@@ -162,5 +162,12 @@ rec {
        getExe' pkgs.imagemagick "convert"
        => "/nix/store/5rs48jamq7k6sal98ymj9l4k2bnwq515-imagemagick-7.1.1-15/bin/convert"
   */
-  getExe' = x: y: "${lib.getBin x}/bin/${y}";
+  getExe' = x: y:
+    assert lib.assertMsg (lib.isDerivation x)
+      "lib.meta.getExe': The first argument is of type ${builtins.typeOf x}, but it should be a derivation instead.";
+    assert lib.assertMsg (lib.isString y)
+     "lib.meta.getExe': The second argument is of type ${builtins.typeOf y}, but it should be a string instead.";
+    assert lib.assertMsg (builtins.length (lib.splitString "/" y) == 1)
+     "lib.meta.getExe': The second argument \"${y}\" is a nested path with a \"/\" character, but it should just be the name of the executable instead.";
+    "${lib.getBin x}/bin/${y}";
 }
diff --git a/nixpkgs/lib/tests/filesystem.sh b/nixpkgs/lib/tests/filesystem.sh
index cfd333d0001b..7e7e03bc667d 100755
--- a/nixpkgs/lib/tests/filesystem.sh
+++ b/nixpkgs/lib/tests/filesystem.sh
@@ -64,8 +64,14 @@ expectSuccess "pathType $PWD/directory" '"directory"'
 expectSuccess "pathType $PWD/regular" '"regular"'
 expectSuccess "pathType $PWD/symlink" '"symlink"'
 expectSuccess "pathType $PWD/fifo" '"unknown"'
-# Different errors depending on whether the builtins.readFilePath primop is available or not
-expectFailure "pathType $PWD/non-existent" "error: (evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'|getting status of '$PWD/non-existent': No such file or directory)"
+
+# Only check error message when a Nixpkgs-specified error is thrown,
+# which is only the case when `readFileType` is not available
+# and the fallback implementation needs to be used.
+if [[ "$(nix-instantiate --eval --expr 'builtins ? readFileType')" == false ]]; then
+    expectFailure "pathType $PWD/non-existent" \
+        "error: evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'"
+fi
 
 expectSuccess "pathIsDirectory /." "true"
 expectSuccess "pathIsDirectory $PWD/directory" "true"
diff --git a/nixpkgs/lib/tests/misc.nix b/nixpkgs/lib/tests/misc.nix
index 2e7fda2b1f8b..47853f47278a 100644
--- a/nixpkgs/lib/tests/misc.nix
+++ b/nixpkgs/lib/tests/misc.nix
@@ -1906,4 +1906,32 @@ runTests {
     expr = (with types; either int (listOf (either bool str))).description;
     expected = "signed integer or list of (boolean or string)";
   };
+
+# Meta
+  testGetExe'Output = {
+    expr = getExe' {
+      type = "derivation";
+      out = "somelonghash";
+      bin = "somelonghash";
+    } "executable";
+    expected = "somelonghash/bin/executable";
+  };
+
+  testGetExeOutput = {
+    expr = getExe {
+      type = "derivation";
+      out = "somelonghash";
+      bin = "somelonghash";
+      meta.mainProgram = "mainProgram";
+    };
+    expected = "somelonghash/bin/mainProgram";
+  };
+
+  testGetExe'FailureFirstArg = testingThrow (
+    getExe' "not a derivation" "executable"
+  );
+
+  testGetExe'FailureSecondArg = testingThrow (
+    getExe' { type = "derivation"; } "dir/executable"
+  );
 }