Merge pull request #270537 from 9999years/packagesFromDirectory

lib.packagesFromDirectoryRecursive: init

14 files changed, 205 insertions, 1 deletions

diff --git a/lib/default.nix b/lib/default.nix
index 35e31af364d8..f6c94ae91634 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -120,7 +120,8 @@ let
     inherit (self.meta) addMetaAttrs dontDistribute setName updateName
       appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio
       hiPrioSet getLicenseFromSpdxId getExe getExe';
-    inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile;
+    inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile
+      packagesFromDirectoryRecursive;
     inherit (self.sources) cleanSourceFilter
       cleanSource sourceByRegex sourceFilesBySuffices
       commitIdFromGitRepo cleanSourceWith pathHasContext
diff --git a/lib/filesystem.nix b/lib/filesystem.nix
index 5569b8ac80fd..c416db02eb57 100644
--- a/lib/filesystem.nix
+++ b/lib/filesystem.nix
@@ -9,11 +9,22 @@ let
   inherit (builtins)
     readDir
     pathExists
+    toString
+    ;
+
+  inherit (lib.attrsets)
+    mapAttrs'
+    filterAttrs
     ;
 
   inherit (lib.filesystem)
     pathType
     ;
+
+  inherit (lib.strings)
+    hasSuffix
+    removeSuffix
+    ;
 in
 
 {
@@ -154,4 +165,147 @@ in
       dir + "/${name}"
   ) (builtins.readDir dir));
 
+  /*
+    Transform a directory tree containing package files suitable for
+    `callPackage` into a matching nested attribute set of derivations.
+
+    For a directory tree like this:
+
+    ```
+    my-packages
+    ├── a.nix
+    ├── b.nix
+    ├── c
+    │  ├── my-extra-feature.patch
+    │  ├── package.nix
+    │  └── support-definitions.nix
+    └── my-namespace
+       ├── d.nix
+       ├── e.nix
+       └── f
+          └── package.nix
+    ```
+
+    `packagesFromDirectoryRecursive` will produce an attribute set like this:
+
+    ```nix
+    # packagesFromDirectoryRecursive {
+    #   callPackage = pkgs.callPackage;
+    #   directory = ./my-packages;
+    # }
+    {
+      a = pkgs.callPackage ./my-packages/a.nix { };
+      b = pkgs.callPackage ./my-packages/b.nix { };
+      c = pkgs.callPackage ./my-packages/c/package.nix { };
+      my-namespace = {
+        d = pkgs.callPackage ./my-packages/my-namespace/d.nix { };
+        e = pkgs.callPackage ./my-packages/my-namespace/e.nix { };
+        f = pkgs.callPackage ./my-packages/my-namespace/f/package.nix { };
+      };
+    }
+    ```
+
+    In particular:
+    - If the input directory contains a `package.nix` file, then
+      `callPackage <directory>/package.nix { }` is returned.
+    - Otherwise, the input directory's contents are listed and transformed into
+      an attribute set.
+      - If a file name has the `.nix` extension, it is turned into attribute
+        where:
+        - The attribute name is the file name without the `.nix` extension
+        - The attribute value is `callPackage <file path> { }`
+      - Other files are ignored.
+      - Directories are turned into an attribute where:
+        - The attribute name is the name of the directory
+        - The attribute value is the result of calling
+          `packagesFromDirectoryRecursive { ... }` on the directory.
+
+        As a result, directories with no `.nix` files (including empty
+        directories) will be transformed into empty attribute sets.
+
+    Example:
+      packagesFromDirectoryRecursive {
+        inherit (pkgs) callPackage;
+        directory = ./my-packages;
+      }
+      => { ... }
+
+      lib.makeScope pkgs.newScope (
+        self: packagesFromDirectoryRecursive {
+          callPackage = self.callPackage;
+          directory = ./my-packages;
+        }
+      )
+      => { ... }
+
+    Type:
+      packagesFromDirectoryRecursive :: AttrSet -> AttrSet
+  */
+  packagesFromDirectoryRecursive =
+    # Options.
+    {
+      /*
+        `pkgs.callPackage`
+
+        Type:
+          Path -> AttrSet -> a
+      */
+      callPackage,
+      /*
+        The directory to read package files from
+
+        Type:
+          Path
+      */
+      directory,
+      ...
+    }:
+    let
+      # Determine if a directory entry from `readDir` indicates a package or
+      # directory of packages.
+      directoryEntryIsPackage = basename: type:
+        type == "directory" || hasSuffix ".nix" basename;
+
+      # List directory entries that indicate packages in the given `path`.
+      packageDirectoryEntries = path:
+        filterAttrs directoryEntryIsPackage (readDir path);
+
+      # Transform a directory entry (a `basename` and `type` pair) into a
+      # package.
+      directoryEntryToAttrPair = subdirectory: basename: type:
+        let
+          path = subdirectory + "/${basename}";
+        in
+        if type == "regular"
+        then
+        {
+          name = removeSuffix ".nix" basename;
+          value = callPackage path { };
+        }
+        else
+        if type == "directory"
+        then
+        {
+          name = basename;
+          value = packagesFromDirectory path;
+        }
+        else
+        throw
+          ''
+            lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString subdirectory}
+          '';
+
+      # Transform a directory into a package (if there's a `package.nix`) or
+      # set of packages (otherwise).
+      packagesFromDirectory = path:
+        let
+          defaultPackagePath = path + "/package.nix";
+        in
+        if pathExists defaultPackagePath
+        then callPackage defaultPackagePath { }
+        else mapAttrs'
+          (directoryEntryToAttrPair path)
+          (packageDirectoryEntries path);
+    in
+    packagesFromDirectory directory;
 }
diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix
index 2884e880e13a..cf7fa9f2e284 100644
--- a/lib/tests/misc.nix
+++ b/lib/tests/misc.nix
@@ -2055,4 +2055,37 @@ runTests {
     expr = meta.platformMatch { } "x86_64-linux";
     expected = false;
   };
+
+  testPackagesFromDirectoryRecursive = {
+    expr = packagesFromDirectoryRecursive {
+      callPackage = path: overrides: import path overrides;
+      directory = ./packages-from-directory;
+    };
+    expected = {
+      a = "a";
+      b = "b";
+      # Note: Other files/directories in `./test-data/c/` are ignored and can be
+      # used by `package.nix`.
+      c = "c";
+      my-namespace = {
+        d = "d";
+        e = "e";
+        f = "f";
+        my-sub-namespace = {
+          g = "g";
+          h = "h";
+        };
+      };
+    };
+  };
+
+  # Check that `packagesFromDirectoryRecursive` can process a directory with a
+  # top-level `package.nix` file into a single package.
+  testPackagesFromDirectoryRecursiveTopLevelPackageNix = {
+    expr = packagesFromDirectoryRecursive {
+      callPackage = path: overrides: import path overrides;
+      directory = ./packages-from-directory/c;
+    };
+    expected = "c";
+  };
 }
diff --git a/lib/tests/packages-from-directory/a.nix b/lib/tests/packages-from-directory/a.nix
new file mode 100644
index 000000000000..54f9eafd8e87
--- /dev/null
+++ b/lib/tests/packages-from-directory/a.nix
@@ -0,0 +1,2 @@
+{ }:
+"a"
diff --git a/lib/tests/packages-from-directory/b.nix b/lib/tests/packages-from-directory/b.nix
new file mode 100644
index 000000000000..345b3c25fa2a
--- /dev/null
+++ b/lib/tests/packages-from-directory/b.nix
@@ -0,0 +1,2 @@
+{ }:
+"b"
diff --git a/lib/tests/packages-from-directory/c/my-extra-feature.patch b/lib/tests/packages-from-directory/c/my-extra-feature.patch
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/lib/tests/packages-from-directory/c/my-extra-feature.patch
diff --git a/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix b/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/lib/tests/packages-from-directory/c/not-a-namespace/not-a-package.nix
diff --git a/lib/tests/packages-from-directory/c/package.nix b/lib/tests/packages-from-directory/c/package.nix
new file mode 100644
index 000000000000..c1203cdde960
--- /dev/null
+++ b/lib/tests/packages-from-directory/c/package.nix
@@ -0,0 +1,2 @@
+{ }:
+"c"
diff --git a/lib/tests/packages-from-directory/c/support-definitions.nix b/lib/tests/packages-from-directory/c/support-definitions.nix
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/lib/tests/packages-from-directory/c/support-definitions.nix
diff --git a/lib/tests/packages-from-directory/my-namespace/d.nix b/lib/tests/packages-from-directory/my-namespace/d.nix
new file mode 100644
index 000000000000..6e5eaa09e995
--- /dev/null
+++ b/lib/tests/packages-from-directory/my-namespace/d.nix
@@ -0,0 +1,2 @@
+{ }:
+"d"
diff --git a/lib/tests/packages-from-directory/my-namespace/e.nix b/lib/tests/packages-from-directory/my-namespace/e.nix
new file mode 100644
index 000000000000..50bd742f800c
--- /dev/null
+++ b/lib/tests/packages-from-directory/my-namespace/e.nix
@@ -0,0 +1,2 @@
+{ }:
+"e"
diff --git a/lib/tests/packages-from-directory/my-namespace/f/package.nix b/lib/tests/packages-from-directory/my-namespace/f/package.nix
new file mode 100644
index 000000000000..c9a66c2eb120
--- /dev/null
+++ b/lib/tests/packages-from-directory/my-namespace/f/package.nix
@@ -0,0 +1,2 @@
+{ }:
+"f"
diff --git a/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix b/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix
new file mode 100644
index 000000000000..4ecaffbf1dc7
--- /dev/null
+++ b/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/g.nix
@@ -0,0 +1,2 @@
+{ }:
+"g"
diff --git a/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix b/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix
new file mode 100644
index 000000000000..3756275ba752
--- /dev/null
+++ b/lib/tests/packages-from-directory/my-namespace/my-sub-namespace/h.nix
@@ -0,0 +1,2 @@
+{ }:
+"h"