Merge branch 'nixos-unstable-small' of https://github.com/NixOS/nixpkgs

Conflicts:
	nixpkgs/pkgs/build-support/go/module.nix

24 files changed, 456 insertions, 79 deletions

diff --git a/nixpkgs/lib/README.md b/nixpkgs/lib/README.md
index a886cf5bfb55..1cf10670ecb2 100644
--- a/nixpkgs/lib/README.md
+++ b/nixpkgs/lib/README.md
@@ -36,6 +36,69 @@ The [module system](https://nixos.org/manual/nixpkgs/#module-system) spans multi
 - [`options.nix`](options.nix): `lib.options` for anything relating to option definitions
 - [`types.nix`](types.nix): `lib.types` for module system types
 
+## PR Guidelines
+
+Follow these guidelines for proposing a change to the interface of `lib`.
+
+### Provide a Motivation
+
+Clearly describe why the change is necessary and its use cases.
+
+Make sure that the change benefits the user more than the added mental effort of looking it up and keeping track of its definition.
+If the same can reasonably be done with the existing interface,
+consider just updating the documentation with more examples and links.
+This is also known as the [Fairbairn Threshold](https://wiki.haskell.org/Fairbairn_threshold).
+
+Through this principle we avoid the human cost of duplicated functionality in an overly large library.
+
+### Make one PR for each change
+
+Don't have multiple changes in one PR, instead split it up into multiple ones.
+
+This keeps the conversation focused and has a higher chance of getting merged.
+
+### Name the interface appropriately
+
+When introducing new names to the interface, such as new function, or new function attributes,
+make sure to name it appropriately.
+
+Names should be self-explanatory and consistent with the rest of `lib`.
+If there's no obvious best name, include the alternatives you considered.
+
+### Write documentation
+
+Update the [reference documentation](#reference-documentation) to reflect the change.
+
+Be generous with links to related functionality.
+
+### Write tests
+
+Add good test coverage for the change, including:
+
+- Tests for edge cases, such as empty values or lists.
+- Tests for tricky inputs, such as a string with string context or a path that doesn't exist.
+- Test all code paths, such as `if-then-else` branches and returned attributes.
+- If the tests for the sub-library are written in bash,
+  test messages of custom errors, such as `throw` or `abortMsg`,
+
+  At the time this is only not necessary for sub-libraries tested with [`tests/misc.nix`](./tests/misc.nix).
+
+See [running tests](#running-tests) for more details on the test suites.
+
+### Write tidy code
+
+Name variables well, even if they're internal.
+The code should be as self-explanatory as possible.
+Be generous with code comments when appropriate.
+
+As a baseline, follow the [Nixpkgs code conventions](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md#code-conventions).
+
+### Write efficient code
+
+Nix generally does not have free abstractions.
+Be aware that seemingly straightforward changes can cause more allocations and a decrease in performance.
+That said, don't optimise prematurely, especially in new code.
+
 ## Reference documentation
 
 Reference documentation for library functions is written above each function as a multi-line comment.
diff --git a/nixpkgs/lib/default.nix b/nixpkgs/lib/default.nix
index 35e31af364d8..f6c94ae91634 100644
--- a/nixpkgs/lib/default.nix
+++ b/nixpkgs/lib/default.nix
@@ -120,7 +120,8 @@ let
     inherit (self.meta) addMetaAttrs dontDistribute setName updateName
       appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio
       hiPrioSet getLicenseFromSpdxId getExe getExe';
-    inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile;
+    inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile
+      packagesFromDirectoryRecursive;
     inherit (self.sources) cleanSourceFilter
       cleanSource sourceByRegex sourceFilesBySuffices
       commitIdFromGitRepo cleanSourceWith pathHasContext
diff --git a/nixpkgs/lib/fileset/default.nix b/nixpkgs/lib/fileset/default.nix
index 18acb9a980ca..c007b60def0a 100644
--- a/nixpkgs/lib/fileset/default.nix
+++ b/nixpkgs/lib/fileset/default.nix
@@ -763,6 +763,12 @@ in {
     Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository.
     The first argument allows configuration with an attribute set,
     while the second argument is the path to the Git working tree.
+
+    `gitTrackedWith` does not perform any filtering when the path is a [Nix store path](https://nixos.org/manual/nix/stable/store/store-path.html#store-path) and not a repository.
+    In this way, it accommodates the use case where the expression that makes the `gitTracked` call does not reside in an actual git repository anymore,
+    and has presumably already been fetched in a way that excludes untracked files.
+    Fetchers with such equivalent behavior include `builtins.fetchGit`, `builtins.fetchTree` (experimental), and `pkgs.fetchgit` when used without `leaveDotGit`.
+
     If you don't need the configuration,
     you can use [`gitTracked`](#function-library-lib.fileset.gitTracked) instead.
 
diff --git a/nixpkgs/lib/fileset/internal.nix b/nixpkgs/lib/fileset/internal.nix
index 9de5590d3eff..4059d2e24426 100644
--- a/nixpkgs/lib/fileset/internal.nix
+++ b/nixpkgs/lib/fileset/internal.nix
@@ -41,6 +41,8 @@ let
   inherit (lib.path)
     append
     splitRoot
+    hasStorePathPrefix
+    splitStorePath
     ;
 
   inherit (lib.path.subpath)
@@ -861,28 +863,56 @@ rec {
   # Type: String -> String -> Path -> Attrs -> FileSet
   _fromFetchGit = function: argument: path: extraFetchGitAttrs:
     let
-      # This imports the files unnecessarily, which currently can't be avoided
-      # because `builtins.fetchGit` is the only function exposing which files are tracked by Git.
-      # With the [lazy trees PR](https://github.com/NixOS/nix/pull/6530),
-      # the unnecessarily import could be avoided.
-      # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944).
-      fetchResult = fetchGit ({
-        url = path;
-      } // extraFetchGitAttrs);
+      # The code path for when isStorePath is true
+      tryStorePath =
+        if pathExists (path + "/.git") then
+          # If there is a `.git` directory in the path,
+          # it means that the path was imported unfiltered into the Nix store.
+          # This function should throw in such a case, because
+          # - `fetchGit` doesn't generally work with `.git` directories in store paths
+          # - Importing the entire path could include Git-tracked files
+          throw ''
+            lib.fileset.${function}: The ${argument} (${toString path}) is a store path within a working tree of a Git repository.
+                This indicates that a source directory was imported into the store using a method such as `import "''${./.}"` or `path:.`.
+                This function currently does not support such a use case, since it currently relies on `builtins.fetchGit`.
+                You could make this work by using a fetcher such as `fetchGit` instead of copying the whole repository.
+                If you can't avoid copying the repo to the store, see https://github.com/NixOS/nix/issues/9292.''
+        else
+          # Otherwise we're going to assume that the path was a Git directory originally,
+          # but it was fetched using a method that already removed files not tracked by Git,
+          # such as `builtins.fetchGit`, `pkgs.fetchgit` or others.
+          # So we can just import the path in its entirety.
+          _singleton path;
+
+      # The code path for when isStorePath is false
+      tryFetchGit =
+        let
+          # This imports the files unnecessarily, which currently can't be avoided
+          # because `builtins.fetchGit` is the only function exposing which files are tracked by Git.
+          # With the [lazy trees PR](https://github.com/NixOS/nix/pull/6530),
+          # the unnecessarily import could be avoided.
+          # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944).
+          fetchResult = fetchGit ({
+            url = path;
+          } // extraFetchGitAttrs);
+        in
+        # We can identify local working directories by checking for .git,
+        # see https://git-scm.com/docs/gitrepository-layout#_description.
+        # Note that `builtins.fetchGit` _does_ work for bare repositories (where there's no `.git`),
+        # even though `git ls-files` wouldn't return any files in that case.
+        if ! pathExists (path + "/.git") then
+          throw "lib.fileset.${function}: Expected the ${argument} (${toString path}) to point to a local working tree of a Git repository, but it's not."
+        else
+          _mirrorStorePath path fetchResult.outPath;
+
     in
-    if inPureEvalMode then
-      throw "lib.fileset.${function}: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292."
-    else if ! isPath path then
+    if ! isPath path then
       throw "lib.fileset.${function}: Expected the ${argument} to be a path, but it's a ${typeOf path} instead."
     else if pathType path != "directory" then
       throw "lib.fileset.${function}: Expected the ${argument} (${toString path}) to be a directory, but it's a file instead."
-    # We can identify local working directories by checking for .git,
-    # see https://git-scm.com/docs/gitrepository-layout#_description.
-    # Note that `builtins.fetchGit` _does_ work for bare repositories (where there's no `.git`),
-    # even though `git ls-files` wouldn't return any files in that case.
-    else if ! pathExists (path + "/.git") then
-      throw "lib.fileset.${function}: Expected the ${argument} (${toString path}) to point to a local working tree of a Git repository, but it's not."
+    else if hasStorePathPrefix path then
+      tryStorePath
     else
-      _mirrorStorePath path fetchResult.outPath;
+      tryFetchGit;
 
 }
diff --git a/nixpkgs/lib/fileset/tests.sh b/nixpkgs/lib/fileset/tests.sh
index d55c4fbfdbeb..e809aef6935a 100755
--- a/nixpkgs/lib/fileset/tests.sh
+++ b/nixpkgs/lib/fileset/tests.sh
@@ -43,29 +43,17 @@ crudeUnquoteJSON() {
     cut -d \" -f2
 }
 
-prefixExpression() {
-    echo 'let
-      lib =
-        (import <nixpkgs/lib>)
-    '
-    if [[ "${1:-}" == "--simulate-pure-eval" ]]; then
-        echo '
-        .extend (final: prev: {
-          trivial = prev.trivial // {
-            inPureEvalMode = true;
-          };
-        })'
-    fi
-    echo '
-      ;
-      internal = import <nixpkgs/lib/fileset/internal.nix> {
-        inherit lib;
-      };
-    in
-    with lib;
-    with internal;
-    with lib.fileset;'
-}
+prefixExpression='
+  let
+    lib = import <nixpkgs/lib>;
+    internal = import <nixpkgs/lib/fileset/internal.nix> {
+      inherit lib;
+    };
+  in
+  with lib;
+  with internal;
+  with lib.fileset;
+'
 
 # Check that two nix expression successfully evaluate to the same value.
 # The expressions have `lib.fileset` in scope.
@@ -74,7 +62,7 @@ expectEqual() {
     local actualExpr=$1
     local expectedExpr=$2
     if actualResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/actualStderr \
-        --expr "$(prefixExpression) ($actualExpr)"); then
+        --expr "$prefixExpression ($actualExpr)"); then
         actualExitCode=$?
     else
         actualExitCode=$?
@@ -82,7 +70,7 @@ expectEqual() {
     actualStderr=$(< "$tmp"/actualStderr)
 
     if expectedResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/expectedStderr \
-        --expr "$(prefixExpression) ($expectedExpr)"); then
+        --expr "$prefixExpression ($expectedExpr)"); then
         expectedExitCode=$?
     else
         expectedExitCode=$?
@@ -110,7 +98,7 @@ expectEqual() {
 expectStorePath() {
     local expr=$1
     if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace 2>"$tmp"/stderr \
-        --expr "$(prefixExpression) ($expr)"); then
+        --expr "$prefixExpression ($expr)"); then
         cat "$tmp/stderr" >&2
         die "$expr failed to evaluate, but it was expected to succeed"
     fi
@@ -123,16 +111,10 @@ expectStorePath() {
 # The expression has `lib.fileset` in scope.
 # Usage: expectFailure NIX REGEX
 expectFailure() {
-    if [[ "$1" == "--simulate-pure-eval" ]]; then
-        maybePure="--simulate-pure-eval"
-        shift
-    else
-        maybePure=""
-    fi
     local expr=$1
     local expectedErrorRegex=$2
     if result=$(nix-instantiate --eval --strict --read-write-mode --show-trace 2>"$tmp/stderr" \
-        --expr "$(prefixExpression $maybePure) $expr"); then
+        --expr "$prefixExpression $expr"); then
         die "$expr evaluated successfully to $result, but it was expected to fail"
     fi
     stderr=$(<"$tmp/stderr")
@@ -149,12 +131,12 @@ expectTrace() {
     local expectedTrace=$2
 
     nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTrace \
-        --expr "$(prefixExpression) trace ($expr)" || true
+        --expr "$prefixExpression trace ($expr)" || true
 
     actualTrace=$(sed -n 's/^trace: //p' "$tmp/stderrTrace")
 
     nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTraceVal \
-        --expr "$(prefixExpression) traceVal ($expr)" || true
+        --expr "$prefixExpression traceVal ($expr)" || true
 
     actualTraceVal=$(sed -n 's/^trace: //p' "$tmp/stderrTraceVal")
 
@@ -1331,7 +1313,7 @@ expectFailure 'gitTrackedWith {} ./.' 'lib.fileset.gitTrackedWith: Expected the
 expectFailure 'gitTrackedWith { recurseSubmodules = null; } ./.' 'lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it'\''s a null instead.'
 
 # recurseSubmodules = true is not supported on all Nix versions
-if [[ "$(nix-instantiate --eval --expr "$(prefixExpression) (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then
+if [[ "$(nix-instantiate --eval --expr "$prefixExpression (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then
     fetchGitSupportsSubmodules=1
 else
     fetchGitSupportsSubmodules=
@@ -1401,10 +1383,60 @@ createGitRepo() {
     git -C "$1" commit -q --allow-empty -m "Empty commit"
 }
 
-# Check the error message for pure eval mode
+# Check that gitTracked[With] works as expected when evaluated out-of-tree
+
+## First we create a git repositories (and a subrepository) with `default.nix` files referring to their local paths
+## Simulating how it would be used in the wild
 createGitRepo .
-expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTracked ./.; }' 'lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.'
-expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTrackedWith {} ./.; }' 'lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.'
+echo '{ fs }: fs.toSource { root = ./.; fileset = fs.gitTracked ./.; }' > default.nix
+git add .
+
+## We can evaluate it locally just fine, `fetchGit` is used underneath to filter git-tracked files
+expectEqual '(import ./. { fs = lib.fileset; }).outPath' '(builtins.fetchGit ./.).outPath'
+
+## We can also evaluate when importing from fetched store paths
+storePath=$(expectStorePath 'builtins.fetchGit ./.')
+expectEqual '(import '"$storePath"' { fs = lib.fileset; }).outPath' \""$storePath"\"
+
+## But it fails if the path is imported with a fetcher that doesn't remove .git (like just using "${./.}")
+expectFailure 'import "${./.}" { fs = lib.fileset; }' 'lib.fileset.gitTracked: The argument \(.*\) is a store path within a working tree of a Git repository.
+\s*This indicates that a source directory was imported into the store using a method such as `import "\$\{./.\}"` or `path:.`.
+\s*This function currently does not support such a use case, since it currently relies on `builtins.fetchGit`.
+\s*You could make this work by using a fetcher such as `fetchGit` instead of copying the whole repository.
+\s*If you can'\''t avoid copying the repo to the store, see https://github.com/NixOS/nix/issues/9292.'
+
+## Even with submodules
+if [[ -n "$fetchGitSupportsSubmodules" ]]; then
+    ## Both the main repo with the submodule
+    echo '{ fs }: fs.toSource { root = ./.; fileset = fs.gitTrackedWith { recurseSubmodules = true; } ./.; }' > default.nix
+    createGitRepo sub
+    git submodule add ./sub sub >/dev/null
+    ## But also the submodule itself
+    echo '{ fs }: fs.toSource { root = ./.; fileset = fs.gitTracked ./.; }' > sub/default.nix
+    git -C sub add .
+
+    ## We can evaluate it locally just fine, `fetchGit` is used underneath to filter git-tracked files
+    expectEqual '(import ./. { fs = lib.fileset; }).outPath' '(builtins.fetchGit { url = ./.; submodules = true; }).outPath'
+    expectEqual '(import ./sub { fs = lib.fileset; }).outPath' '(builtins.fetchGit ./sub).outPath'
+
+    ## We can also evaluate when importing from fetched store paths
+    storePathWithSub=$(expectStorePath 'builtins.fetchGit { url = ./.; submodules = true; }')
+    expectEqual '(import '"$storePathWithSub"' { fs = lib.fileset; }).outPath' \""$storePathWithSub"\"
+    storePathSub=$(expectStorePath 'builtins.fetchGit ./sub')
+    expectEqual '(import '"$storePathSub"' { fs = lib.fileset; }).outPath' \""$storePathSub"\"
+
+    ## But it fails if the path is imported with a fetcher that doesn't remove .git (like just using "${./.}")
+    expectFailure 'import "${./.}" { fs = lib.fileset; }' 'lib.fileset.gitTrackedWith: The second argument \(.*\) is a store path within a working tree of a Git repository.
+    \s*This indicates that a source directory was imported into the store using a method such as `import "\$\{./.\}"` or `path:.`.
+    \s*This function currently does not support such a use case, since it currently relies on `builtins.fetchGit`.
+    \s*You could make this work by using a fetcher such as `fetchGit` instead of copying the whole repository.
+    \s*If you can'\''t avoid copying the repo to the store, see https://github.com/NixOS/nix/issues/9292.'
+    expectFailure 'import "${./.}/sub" { fs = lib.fileset; }' 'lib.fileset.gitTracked: The argument \(.*/sub\) is a store path within a working tree of a Git repository.
+    \s*This indicates that a source directory was imported into the store using a method such as `import "\$\{./.\}"` or `path:.`.
+    \s*This function currently does not support such a use case, since it currently relies on `builtins.fetchGit`.
+    \s*You could make this work by using a fetcher such as `fetchGit` instead of copying the whole repository.
+    \s*If you can'\''t avoid copying the repo to the store, see https://github.com/NixOS/nix/issues/9292.'
+fi
 rm -rf -- *
 
 # Go through all stages of Git files
diff --git a/nixpkgs/lib/filesystem.nix b/nixpkgs/lib/filesystem.nix
index 5569b8ac80fd..c416db02eb57 100644
--- a/nixpkgs/lib/filesystem.nix
+++ b/nixpkgs/lib/filesystem.nix
@@ -9,11 +9,22 @@ let
   inherit (builtins)
     readDir
     pathExists
+    toString
+    ;
+
+  inherit (lib.attrsets)
+    mapAttrs'
+    filterAttrs
     ;
 
   inherit (lib.filesystem)
     pathType
     ;
+
+  inherit (lib.strings)
+    hasSuffix
+    removeSuffix
+    ;
 in
 
 {
@@ -154,4 +165,147 @@ in
       dir + "/${name}"
   ) (builtins.readDir dir));
 
+  /*
+    Transform a directory tree containing package files suitable for
+    `callPackage` into a matching nested attribute set of derivations.
+
+    For a directory tree like this:
+
+    ```
+    my-packages
+    ├── a.nix
+    ├── b.nix
+    ├── c
+    │  ├── my-extra-feature.patch
+    │  ├── package.nix
+    │  └── support-definitions.nix
+    └── my-namespace
+       ├── d.nix
+       ├── e.nix
+       └── f
+          └── package.nix
+    ```
+
+    `packagesFromDirectoryRecursive` will produce an attribute set like this:
+
+    ```nix
+    # packagesFromDirectoryRecursive {
+    #   callPackage = pkgs.callPackage;
+    #   directory = ./my-packages;
+    # }
+    {
+      a = pkgs.callPackage ./my-packages/a.nix { };
+      b = pkgs.callPackage ./my-packages/b.nix { };
+      c = pkgs.callPackage ./my-packages/c/package.nix { };
+      my-namespace = {
+        d = pkgs.callPackage ./my-packages/my-namespace/d.nix { };
+        e = pkgs.callPackage ./my-packages/my-namespace/e.nix { };
+        f = pkgs.callPackage ./my-packages/my-namespace/f/package.nix { };
+      };
+    }
+    ```
+
+    In particular:
+    - If the input directory contains a `package.nix` file, then
+      `callPackage <directory>/package.nix { }` is returned.
+    - Otherwise, the input directory's contents are listed and transformed into
+      an attribute set.
+      - If a file name has the `.nix` extension, it is turned into attribute
+        where:
+        - The attribute name is the file name without the `.nix` extension
+        - The attribute value is `callPackage <file path> { }`
+      - Other files are ignored.
+      - Directories are turned into an attribute where:
+        - The attribute name is the name of the directory
+        - The attribute value is the result of calling
+          `packagesFromDirectoryRecursive { ... }` on the directory.
+
+        As a result, directories with no `.nix` files (including empty
+        directories) will be transformed into empty attribute sets.
+
+    Example:
+      packagesFromDirectoryRecursive {
+        inherit (pkgs) callPackage;
+        directory = ./my-packages;
+      }
+      => { ... }
+
+      lib.makeScope pkgs.newScope (
+        self: packagesFromDirectoryRecursive {
+          callPackage = self.callPackage;
+          directory = ./my-packages;
+        }
+      )
+      => { ... }
+
+    Type:
+      packagesFromDirectoryRecursive :: AttrSet -> AttrSet
+  */
+  packagesFromDirectoryRecursive =
+    # Options.
+    {
+      /*
+        `pkgs.callPackage`
+
+        Type:
+          Path -> AttrSet -> a
+      */
+      callPackage,
+      /*
+        The directory to read package files from
+
+        Type:
+          Path
+      */
+      directory,
+      ...
+    }:
+    let
+      # Determine if a directory entry from `readDir` indicates a package or
+      # directory of packages.
+      directoryEntryIsPackage = basename: type:
+        type == "directory" || hasSuffix ".nix" basename;
+
+      # List directory entries that indicate packages in the given `path`.
+      packageDirectoryEntries = path:
+        filterAttrs directoryEntryIsPackage (readDir path);
+
+      # Transform a directory entry (a `basename` and `type` pair) into a
+      # package.
+      directoryEntryToAttrPair = subdirectory: basename: type:
+        let
+          path = subdirectory + "/${basename}";
+        in
+        if type == "regular"
+        then
+        {
+          name = removeSuffix ".nix" basename;
+          value = callPackage path { };
+        }
+        else
+        if type == "directory"
+        then
+        {
+          name = basename;
+          value = packagesFromDirectory path;
+        }
+        else
+        throw
+          ''
+            lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString subdirectory}
+          '';
+
+      # Transform a directory into a package (if there's a `package.nix`) or
+      # set of packages (otherwise).
+      packagesFromDirectory = path:
+        let
+          defaultPackagePath = path + "/package.nix";
+        in
+        if pathExists defaultPackagePath
+        then callPackage defaultPackagePath { }
+        else mapAttrs'
+          (directoryEntryToAttrPair path)
+          (packageDirectoryEntries path);
+    in
+    packagesFromDirectory directory;
 }
diff --git a/nixpkgs/lib/licenses.nix b/nixpkgs/lib/licenses.nix
index baf92007123d..8dc01e560746 100644
--- a/nixpkgs/lib/licenses.nix
+++ b/nixpkgs/lib/licenses.nix
@@ -104,6 +104,7 @@ in mkLicense lset) ({
   };
 
   arphicpl = {
+    spdxId = "Arphic-1999";
     fullName = "Arphic Public License";
     url = "https://www.freedesktop.org/wiki/Arphic_Public_License/";
   };
@@ -236,6 +237,7 @@ in mkLicense lset) ({
   };
 
   cal10 = {
+    spdxId = "CAL-1.0";
     fullName = "Cryptographic Autonomy License version 1.0 (CAL-1.0)";
     url = "https://opensource.org/licenses/CAL-1.0";
   };
@@ -429,6 +431,7 @@ in mkLicense lset) ({
   };
 
   elastic20 = {
+    spdxId = "Elastic-2.0";
     fullName = "Elastic License 2.0";
     url = "https://github.com/elastic/elasticsearch/blob/main/licenses/ELASTIC-LICENSE-2.0.txt";
     free = false;
@@ -598,6 +601,7 @@ in mkLicense lset) ({
 
   # Intel's license, seems free
   iasl = {
+    spdxId = "Intel-ACPI";
     fullName = "iASL";
     url = "https://old.calculate-linux.org/packages/licenses/iASL";
   };
@@ -609,7 +613,7 @@ in mkLicense lset) ({
 
   imagemagick = {
     fullName = "ImageMagick License";
-    spdxId = "imagemagick";
+    spdxId = "ImageMagick";
   };
 
   imlib2 = {
@@ -803,6 +807,7 @@ in mkLicense lset) ({
   };
 
   miros = {
+    spdxId = "MirOS";
     fullName = "MirOS License";
     url = "https://opensource.org/licenses/MirOS";
   };
@@ -1138,6 +1143,7 @@ in mkLicense lset) ({
   };
 
   upl = {
+    spdxId = "UPL-1.0";
     fullName = "Universal Permissive License";
     url = "https://oss.oracle.com/licenses/upl/";
   };
@@ -1194,6 +1200,7 @@ in mkLicense lset) ({
   };
 
   xfig = {
+    spdxId = "Xfig";
     fullName = "xfig";
     url = "https://mcj.sourceforge.net/authors.html#xfig";
   };
diff --git a/nixpkgs/lib/tests/misc.nix b/nixpkgs/lib/tests/misc.nix
index 2884e880e13a..3059878ba069 100644
--- a/nixpkgs/lib/tests/misc.nix
+++ b/nixpkgs/lib/tests/misc.nix
@@ -1959,6 +1959,18 @@ runTests {
     expr = (with types; int).description;
     expected = "signed integer";
   };
+  testTypeDescriptionIntsPositive = {
+    expr = (with types; ints.positive).description;
+    expected = "positive integer, meaning >0";
+  };
+  testTypeDescriptionIntsPositiveOrEnumAuto = {
+    expr = (with types; either ints.positive (enum ["auto"])).description;
+    expected = ''positive integer, meaning >0, or value "auto" (singular enum)'';
+  };
+  testTypeDescriptionListOfPositive = {
+    expr = (with types; listOf ints.positive).description;
+    expected = "list of (positive integer, meaning >0)";
+  };
   testTypeDescriptionListOfInt = {
     expr = (with types; listOf int).description;
     expected = "list of signed integer";
@@ -2055,4 +2067,37 @@ runTests {
     expr = meta.platformMatch { } "x86_64-linux";
     expected = false;
   };
+
+  testPackagesFromDirectoryRecursive = {
+    expr = packagesFromDirectoryRecursive {
+      callPackage = path: overrides: import path overrides;
+      directory = ./packages-from-directory;
+    };
+    expected = {
+      a = "a";
+      b = "b";
+      # Note: Other files/directories in `./test-data/c/` are ignored and can be
+      # used by `package.nix`.
+      c = "c";
+      my-namespace = {
+        d = "d";
+        e = "e";
+        f = "f";
+        my-sub-namespace = {
+          g = "g";
+          h = "h";
+        };
+      };
+    };
+  };
+
+  # Check that `packagesFromDirectoryRecursive` can process a directory with a
+  # top-level `package.nix` file into a single package.
+  testPackagesFromDirectoryRecursiveTopLevelPackageNix = {
+    expr = packagesFromDirectoryRecursive {
+      callPackage = path: overrides: import path overrides;
+      directory = ./packages-from-directory/c;
+    };
+    expected = "c";
+  };
 }
diff --git a/nixpkgs/lib/tests/modules.sh b/nixpkgs/lib/tests/modules.sh
index 4cf9821a3063..a90ff4ad9a2f 100755
--- a/nixpkgs/lib/tests/modules.sh
+++ b/nixpkgs/lib/tests/modules.sh
@@ -24,14 +24,14 @@ evalConfig() {
     local attr=$1
     shift
     local script="import ./default.nix { modules = [ $* ];}"
-    nix-instantiate --timeout 1 -E "$script" -A "$attr" --eval-only --show-trace --read-write-mode
+    nix-instantiate --timeout 1 -E "$script" -A "$attr" --eval-only --show-trace --read-write-mode --json
 }
 
 reportFailure() {
     local attr=$1
     shift
     local script="import ./default.nix { modules = [ $* ];}"
-    echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only"
+    echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only --json"
     evalConfig "$attr" "$@" || true
     ((++fail))
 }
@@ -148,7 +148,7 @@ checkConfigOutput '^42$' config.value ./declare-either.nix ./define-value-int-po
 checkConfigOutput '^"24"$' config.value ./declare-either.nix ./define-value-string.nix
 # types.oneOf
 checkConfigOutput '^42$' config.value ./declare-oneOf.nix ./define-value-int-positive.nix
-checkConfigOutput '^\[ \]$' config.value ./declare-oneOf.nix ./define-value-list.nix
+checkConfigOutput '^\[\]$' config.value ./declare-oneOf.nix ./define-value-list.nix
 checkConfigOutput '^"24"$' config.value ./declare-oneOf.nix ./define-value-string.nix
 
 # Check mkForce without submodules.
@@ -328,7 +328,7 @@ checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-st
 # Shorthand modules interpret `meta` and `class` as config items
 checkConfigOutput '^true$' options._module.args.value.result ./freeform-attrsOf.nix ./define-freeform-keywords-shorthand.nix
 # No freeform assignments shouldn't make it error
-checkConfigOutput '^{ }$' config ./freeform-attrsOf.nix
+checkConfigOutput '^{}$' config ./freeform-attrsOf.nix
 # but only if the type matches
 checkConfigError 'A definition for option .* is not of type .*' config.value ./freeform-attrsOf.nix ./define-value-list.nix
 # and properties should be applied
@@ -366,19 +366,19 @@ checkConfigError 'The option .* has conflicting definitions' config.value ./type
 checkConfigOutput '^0$' config.value.int ./types-anything/equal-atoms.nix
 checkConfigOutput '^false$' config.value.bool ./types-anything/equal-atoms.nix
 checkConfigOutput '^""$' config.value.string ./types-anything/equal-atoms.nix
-checkConfigOutput '^/$' config.value.path ./types-anything/equal-atoms.nix
+checkConfigOutput '^"/[^"]+"$' config.value.path ./types-anything/equal-atoms.nix
 checkConfigOutput '^null$' config.value.null ./types-anything/equal-atoms.nix
 checkConfigOutput '^0.1$' config.value.float ./types-anything/equal-atoms.nix
 # Functions can't be merged together
 checkConfigError "The option .value.multiple-lambdas.<function body>. has conflicting option types" config.applied.multiple-lambdas ./types-anything/functions.nix
-checkConfigOutput '^<LAMBDA>$' config.value.single-lambda ./types-anything/functions.nix
+checkConfigOutput '^true$' config.valueIsFunction.single-lambda ./types-anything/functions.nix
 checkConfigOutput '^null$' config.applied.merging-lambdas.x ./types-anything/functions.nix
 checkConfigOutput '^null$' config.applied.merging-lambdas.y ./types-anything/functions.nix
 # Check that all mk* modifiers are applied
 checkConfigError 'attribute .* not found' config.value.mkiffalse ./types-anything/mk-mods.nix
-checkConfigOutput '^{ }$' config.value.mkiftrue ./types-anything/mk-mods.nix
+checkConfigOutput '^{}$' config.value.mkiftrue ./types-anything/mk-mods.nix
 checkConfigOutput '^1$' config.value.mkdefault ./types-anything/mk-mods.nix
-checkConfigOutput '^{ }$' config.value.mkmerge ./types-anything/mk-mods.nix
+checkConfigOutput '^{}$' config.value.mkmerge ./types-anything/mk-mods.nix
 checkConfigOutput '^true$' config.value.mkbefore ./types-anything/mk-mods.nix
 checkConfigOutput '^1$' config.value.nested.foo ./types-anything/mk-mods.nix
 checkConfigOutput '^"baz"$' config.value.nested.bar.baz ./types-anything/mk-mods.nix
@@ -398,16 +398,16 @@ checkConfigOutput '^"a b y z"$' config.resultFooBar ./declare-variants.nix ./def
 checkConfigOutput '^"a b c"$' config.resultFooFoo ./declare-variants.nix ./define-variant.nix
 
 ## emptyValue's
-checkConfigOutput "[ ]" config.list.a ./emptyValues.nix
-checkConfigOutput "{ }" config.attrs.a ./emptyValues.nix
+checkConfigOutput "\[\]" config.list.a ./emptyValues.nix
+checkConfigOutput "{}" config.attrs.a ./emptyValues.nix
 checkConfigOutput "null" config.null.a ./emptyValues.nix
-checkConfigOutput "{ }" config.submodule.a ./emptyValues.nix
+checkConfigOutput "{}" config.submodule.a ./emptyValues.nix
 # These types don't have empty values
 checkConfigError 'The option .int.a. is used but not defined' config.int.a ./emptyValues.nix
 checkConfigError 'The option .nonEmptyList.a. is used but not defined' config.nonEmptyList.a ./emptyValues.nix
 
 ## types.raw
-checkConfigOutput "{ foo = <CODE>; }" config.unprocessedNesting ./raw.nix
+checkConfigOutput '^true$' config.unprocessedNestingEvaluates.success ./raw.nix
 checkConfigOutput "10" config.processedToplevel ./raw.nix
 checkConfigError "The option .multiple. is defined multiple times" config.multiple ./raw.nix
 checkConfigOutput "bar" config.priorities ./raw.nix
@@ -441,13 +441,13 @@ checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survive
 checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix
 
 # Class checks, evalModules
-checkConfigOutput '^{ }$' config.ok.config ./class-check.nix
+checkConfigOutput '^{}$' config.ok.config ./class-check.nix
 checkConfigOutput '"nixos"' config.ok.class ./class-check.nix
 checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.fail.config ./class-check.nix
 checkConfigError 'The module foo.nix#darwinModules.default was imported into nixos instead of darwin.' config.fail-anon.config ./class-check.nix
 
 # Class checks, submoduleWith
-checkConfigOutput '^{ }$' config.sub.nixosOk ./class-check.nix
+checkConfigOutput '^{}$' config.sub.nixosOk ./class-check.nix
 checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.sub.nixosFail.config ./class-check.nix
 
 # submoduleWith type merge with different class
diff --git a/nixpkgs/lib/tests/modules/raw.nix b/nixpkgs/lib/tests/modules/raw.nix
index 418e671ed076..9eb7c5ce8f21 100644
--- a/nixpkgs/lib/tests/modules/raw.nix
+++ b/nixpkgs/lib/tests/modules/raw.nix
@@ -1,4 +1,4 @@
-{ lib, ... }: {
+{ lib, config, ... }: {
 
   options = {
     processedToplevel = lib.mkOption {
@@ -13,6 +13,9 @@
     priorities = lib.mkOption {
       type = lib.types.raw;
     };
+    unprocessedNestingEvaluates = lib.mkOption {
+      default = builtins.tryEval config.unprocessedNesting;
+    };
   };
 
   config = {
diff --git a/nixpkgs/lib/tests/modules/types-anything/equal-atoms.nix b/nixpkgs/lib/tests/modules/types-anything/equal-atoms.nix
index 972711201a09..9925cfd60892 100644
--- a/nixpkgs/lib/tests/modules/types-anything/equal-atoms.nix
+++ b/nixpkgs/lib/tests/modules/types-anything/equal-atoms.nix
@@ -9,7 +9,7 @@
       value.int = 0;
       value.bool = false;
       value.string = "";
-      value.path = /.;
+      value.path = ./.;
       value.null = null;
       value.float = 0.1;
     }
@@ -17,7 +17,7 @@
       value.int = 0;
       value.bool = false;
       value.string = "";
-      value.path = /.;
+      value.path = ./.;
       value.null = null;
       value.float = 0.1;
     }
diff --git a/nixpkgs/lib/tests/modules/types-anything/functions.nix b/nixpkgs/lib/tests/modules/types-anything/functions.nix
index 21edd4aff9c4..3288b64f9b7e 100644
--- a/nixpkgs/lib/tests/modules/types-anything/functions.nix
+++ b/nixpkgs/lib/tests/modules/types-anything/functions.nix
@@ -1,5 +1,9 @@
 { lib, config, ... }: {
 
+  options.valueIsFunction = lib.mkOption {
+    default = lib.mapAttrs (name: lib.isFunction) config.value;
+  };
+
   options.value = lib.mkOption {
     type = lib.types.anything;
   };
diff --git a/nixpkgs/lib/tests/packages-from-directory/a.nix b/nixpkgs/lib/tests/packages-from-directory/a.nix
new file mode 100644
index 000000000000..54f9eafd8e87
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/a.nix
@@ -0,0 +1,2 @@
+{ }:
+"a"
diff --git a/nixpkgs/lib/tests/packages-from-directory/b.nix b/nixpkgs/lib/tests/packages-from-directory/b.nix
new file mode 100644
index 000000000000..345b3c25fa2a
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/b.nix
@@ -0,0 +1,2 @@
+{ }:
+"b"
diff --git a/nixpkgs/lib/tests/packages-from-directory/c/my-extra-feature.patch b/nixpkgs/lib/tests/packages-from-directory/c/my-extra-feature.patch
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/c/my-extra-feature.patch
diff --git a/nixpkgs/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix b/nixpkgs/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix
diff --git a/nixpkgs/lib/tests/packages-from-directory/c/package.nix b/nixpkgs/lib/tests/packages-from-directory/c/package.nix
new file mode 100644
index 000000000000..c1203cdde960
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/c/package.nix
@@ -0,0 +1,2 @@
+{ }:
+"c"
diff --git a/nixpkgs/lib/tests/packages-from-directory/c/support-definitions.nix b/nixpkgs/lib/tests/packages-from-directory/c/support-definitions.nix
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/c/support-definitions.nix
diff --git a/nixpkgs/lib/tests/packages-from-directory/my-namespace/d.nix b/nixpkgs/lib/tests/packages-from-directory/my-namespace/d.nix
new file mode 100644
index 000000000000..6e5eaa09e995
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/my-namespace/d.nix
@@ -0,0 +1,2 @@
+{ }:
+"d"
diff --git a/nixpkgs/lib/tests/packages-from-directory/my-namespace/e.nix b/nixpkgs/lib/tests/packages-from-directory/my-namespace/e.nix
new file mode 100644
index 000000000000..50bd742f800c
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/my-namespace/e.nix
@@ -0,0 +1,2 @@
+{ }:
+"e"
diff --git a/nixpkgs/lib/tests/packages-from-directory/my-namespace/f/package.nix b/nixpkgs/lib/tests/packages-from-directory/my-namespace/f/package.nix
new file mode 100644
index 000000000000..c9a66c2eb120
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/my-namespace/f/package.nix
@@ -0,0 +1,2 @@
+{ }:
+"f"
diff --git a/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix b/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix
new file mode 100644
index 000000000000..4ecaffbf1dc7
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix
@@ -0,0 +1,2 @@
+{ }:
+"g"
diff --git a/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix b/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix
new file mode 100644
index 000000000000..3756275ba752
--- /dev/null
+++ b/nixpkgs/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix
@@ -0,0 +1,2 @@
+{ }:
+"h"
diff --git a/nixpkgs/lib/types.nix b/nixpkgs/lib/types.nix
index 4378568c141f..cea63c598321 100644
--- a/nixpkgs/lib/types.nix
+++ b/nixpkgs/lib/types.nix
@@ -113,9 +113,14 @@ rec {
     , # Description of the type, defined recursively by embedding the wrapped type if any.
       description ? null
       # A hint for whether or not this description needs parentheses. Possible values:
-      #  - "noun": a simple noun phrase such as "positive integer"
-      #  - "conjunction": a phrase with a potentially ambiguous "or" connective.
+      #  - "noun": a noun phrase
+      #    Example description: "positive integer",
+      #  - "conjunction": a phrase with a potentially ambiguous "or" connective
+      #    Example description: "int or string"
       #  - "composite": a phrase with an "of" connective
+      #    Example description: "list of string"
+      #  - "nonRestrictiveClause": a noun followed by a comma and a clause
+      #    Example description: "positive integer, meaning >0"
       # See the `optionDescriptionPhrase` function.
     , descriptionClass ? null
     , # DO NOT USE WITHOUT KNOWING WHAT YOU ARE DOING!
@@ -338,10 +343,12 @@ rec {
         unsigned = addCheck types.int (x: x >= 0) // {
           name = "unsignedInt";
           description = "unsigned integer, meaning >=0";
+          descriptionClass = "nonRestrictiveClause";
         };
         positive = addCheck types.int (x: x > 0) // {
           name = "positiveInt";
           description = "positive integer, meaning >0";
+          descriptionClass = "nonRestrictiveClause";
         };
         u8 = unsign 8 256;
         u16 = unsign 16 65536;
@@ -383,10 +390,12 @@ rec {
       nonnegative = addCheck number (x: x >= 0) // {
         name = "numberNonnegative";
         description = "nonnegative integer or floating point number, meaning >=0";
+        descriptionClass = "nonRestrictiveClause";
       };
       positive = addCheck number (x: x > 0) // {
         name = "numberPositive";
         description = "positive integer or floating point number, meaning >0";
+        descriptionClass = "nonRestrictiveClause";
       };
     };
 
@@ -463,6 +472,7 @@ rec {
     passwdEntry = entryType: addCheck entryType (str: !(hasInfix ":" str || hasInfix "\n" str)) // {
       name = "passwdEntry ${entryType.name}";
       description = "${optionDescriptionPhrase (class: class == "noun") entryType}, not containing newlines or colons";
+      descriptionClass = "nonRestrictiveClause";
     };
 
     attrs = mkOptionType {
@@ -870,7 +880,13 @@ rec {
     # Either value of type `t1` or `t2`.
     either = t1: t2: mkOptionType rec {
       name = "either";
-      description = "${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t1} or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction" || class == "composite") t2}";
+      description =
+        if t1.descriptionClass or null == "nonRestrictiveClause"
+        then
+          # Plain, but add comma
+          "${t1.description}, or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t2}"
+        else
+          "${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t1} or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction" || class == "composite") t2}";
       descriptionClass = "conjunction";
       check = x: t1.check x || t2.check x;
       merge = loc: defs: