Merge pull request #178717 from ShamrockLee/write-multiple-references

trivial-builders: replace writeReferencesToFile with writeClosure

10 files changed, 76 insertions, 39 deletions

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index ea464ebea18c..8e04944bc810 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -29,7 +29,7 @@
 , tarsum
 , util-linux
 , vmTools
-, writeReferencesToFile
+, writeClosure
 , writeScript
 , writeShellScriptBin
 , writeText
@@ -630,7 +630,7 @@ rec {
           imageName = lib.toLower name;
           imageTag = lib.optionalString (tag != null) tag;
           inherit fromImage baseJson;
-          layerClosure = writeReferencesToFile layer;
+          layerClosure = writeClosure [ layer ];
           passthru.buildArgs = args;
           passthru.layer = layer;
           passthru.imageTag =
diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix
index 67e081522d64..1f5507f2eb75 100644
--- a/pkgs/build-support/oci-tools/default.nix
+++ b/pkgs/build-support/oci-tools/default.nix
@@ -1,4 +1,4 @@
-{ lib, writeText, runCommand, writeReferencesToFile }:
+{ lib, writeText, runCommand, writeClosure }:
 
 {
   buildContainer =
@@ -72,7 +72,7 @@
       set -o pipefail
       mkdir -p $out/rootfs/{dev,proc,sys}
       cp ${config} $out/config.json
-      xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/
+      xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/
     '';
 }
 
diff --git a/pkgs/build-support/references-by-popularity/closure-graph.py b/pkgs/build-support/references-by-popularity/closure-graph.py
index 579f3b041fa8..4f8efd42ed81 100644
--- a/pkgs/build-support/references-by-popularity/closure-graph.py
+++ b/pkgs/build-support/references-by-popularity/closure-graph.py
@@ -8,8 +8,8 @@
 # and how deep in the tree they live. Equally-"popular" paths are then
 # sorted by name.
 #
-# The existing writeReferencesToFile prints the paths in a simple
-# ascii-based sorting of the paths.
+# The existing writeClosure prints the paths in a simple ascii-based
+# sorting of the paths.
 #
 # Sorting the paths by graph improves the chances that the difference
 # between two builds appear near the end of the list, instead of near
diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix
index 8d7ad9e742a1..c9e53a4cb706 100644
--- a/pkgs/build-support/singularity-tools/default.nix
+++ b/pkgs/build-support/singularity-tools/default.nix
@@ -4,7 +4,7 @@
 , storeDir ? builtins.storeDir
 , writeScript
 , singularity
-, writeReferencesToFile
+, writeClosure
 , bash
 , vmTools
 , gawk
@@ -50,18 +50,13 @@ rec {
     }:
     let
       projectName = singularity.projectName or "singularity";
-      layer = mkLayer {
-        inherit name;
-        contents = contents ++ [ bash runScriptFile ];
-        inherit projectName;
-      };
       runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
       runScriptFile = shellScript "run-script.sh" runScript;
       result = vmTools.runInLinuxVM (
         runCommand "${projectName}-image-${name}.img"
           {
             buildInputs = [ singularity e2fsprogs util-linux gawk ];
-            layerClosure = writeReferencesToFile layer;
+            layerClosure = writeClosure contents;
             preVM = vmTools.createEmptyImage {
               size = diskSize;
               fullName = "${projectName}-run-disk";
diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix
index df81d67d868d..d7438923a54b 100644
--- a/pkgs/build-support/trivial-builders/default.nix
+++ b/pkgs/build-support/trivial-builders/default.nix
@@ -1,4 +1,4 @@
-{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }:
+{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }:
 
 let
   inherit (lib)
@@ -625,18 +625,22 @@ rec {
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
   # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile
-  writeReferencesToFile = path: runCommand "runtime-deps"
+  # TODO: Convert to throw after Nixpkgs 24.05 branch-off.
+  writeReferencesToFile = (if config.allowAliases then lib.warn else throw)
+    "writeReferencesToFile is deprecated in favour of writeClosure"
+    (path: writeClosure [ path ]);
+
+  # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
+  # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure
+  writeClosure = paths: runCommand "runtime-deps"
     {
-      exportReferencesGraph = [ "graph" path ];
+      # Get the cleaner exportReferencesGraph interface
+      __structuredAttrs = true;
+      exportReferencesGraph.graph = paths;
+      nativeBuildInputs = [ jq ];
     }
     ''
-      touch $out
-      while read path; do
-        echo $path >> $out
-        read dummy
-        read nrRefs
-        for ((i = 0; i < nrRefs; i++)); do read ref; done
-      done < graph
+      jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out"
     '';
 
   # Docs in doc/build-helpers/trivial-build-helpers.chapter.md
diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix
index 59dbba3f1841..f41372d922bb 100644
--- a/pkgs/build-support/trivial-builders/test/default.nix
+++ b/pkgs/build-support/trivial-builders/test/default.nix
@@ -26,6 +26,9 @@ recurseIntoAttrs {
     then references
     else {};
   writeCBin = callPackage ./writeCBin.nix {};
+  writeClosure-union = callPackage ./writeClosure-union.nix {
+    inherit (references) samples;
+  };
   writeShellApplication = callPackage ./writeShellApplication.nix {};
   writeScriptBin = callPackage ./writeScriptBin.nix {};
   writeShellScript = callPackage ./write-shell-script.nix {};
diff --git a/pkgs/build-support/trivial-builders/test/references/default.nix b/pkgs/build-support/trivial-builders/test/references/default.nix
index 3e21c905321e..928cc1d9461f 100644
--- a/pkgs/build-support/trivial-builders/test/references/default.nix
+++ b/pkgs/build-support/trivial-builders/test/references/default.nix
@@ -12,7 +12,7 @@
 , cleanSamples ? lib.filterAttrs (n: lib.isStringLike)
   # Test targets
 , writeDirectReferencesToFile
-, writeReferencesToFile
+, writeClosure
 }:
 
 # -------------------------------------------------------------------------- #
@@ -46,8 +46,9 @@ let
   samplesToString = attrs:
     lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs);
 
-  references = lib.mapAttrs (n: v: writeReferencesToFile v) samples;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
   directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
 
   testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: {
     name = "references-test";
@@ -61,8 +62,9 @@ let
       mkdir -p "$out/bin"
       substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \
         --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \
-        --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \
-        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)}
+        --replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \
+        --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \
+        --replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure}
       runHook postInstall
       chmod +x "$out/bin/${finalAttrs.meta.mainProgram}"
     '';
@@ -79,8 +81,9 @@ let
 
     passthru = {
       inherit
+        collectiveClosure
         directReferences
-        references
+        closures
         samples
         ;
     };
@@ -109,8 +112,9 @@ testers.nixosTest {
     '';
   passthru = {
     inherit
+      collectiveClosure
       directReferences
-      references
+      closures
       samples
       testScriptBin
       ;
diff --git a/pkgs/build-support/trivial-builders/test/references/references-test.sh b/pkgs/build-support/trivial-builders/test/references/references-test.sh
index 1b8f8e1504ec..92e4467287ed 100755
--- a/pkgs/build-support/trivial-builders/test/references/references-test.sh
+++ b/pkgs/build-support/trivial-builders/test/references/references-test.sh
@@ -33,16 +33,17 @@ set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"  # nixpkgs root
 
-  # Injected by Nix (to avoid evaluating in a derivation)
-  # turn them into arrays
-  # shellcheck disable=SC2206 # deliberately unquoted
+  # Inject the path to compare from the Nix expression
+
+  # Associative Arrays
   declare -A samples=( @SAMPLES@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
   declare -A directRefs=( @DIRECT_REFS@ )
-  # shellcheck disable=SC2206 # deliberately unquoted
-  declare -A references=( @REFERENCES@ )
+  declare -A closures=( @CLOSURES@ )
+
+  # Path string
+  collectiveClosure=@COLLECTIVE_CLOSURE@
 
-echo >&2 Testing direct references...
+echo >&2 Testing direct closures...
 for i in "${!samples[@]}"; do
   echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}"
   diff -U3 \
@@ -52,10 +53,16 @@ done
 
 echo >&2 Testing closure...
 for i in "${!samples[@]}"; do
-  echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}"
+  echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}"
   diff -U3 \
-    <(sort <"${references[$i]}") \
+    <(sort <"${closures[$i]}") \
     <(nix-store -q --requisites "${samples[$i]}" | sort)
 done
 
+echo >&2 Testing mixed closures...
+echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure"
+diff -U3 \
+  <(sort <"$collectiveClosure") \
+  <(nix-store -q --requisites "${samples[@]}" | sort)
+
 echo 'OK!'
diff --git a/pkgs/build-support/trivial-builders/test/writeClosure-union.nix b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix
new file mode 100644
index 000000000000..92a2bf9f0988
--- /dev/null
+++ b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix
@@ -0,0 +1,23 @@
+{ lib
+, runCommandLocal
+  # Test targets
+, writeClosure
+, samples
+}:
+runCommandLocal "test-trivial-builders-writeClosure-union" {
+  __structuredAttrs = true;
+  closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
+  collectiveClosure = writeClosure (lib.attrValues samples);
+  inherit samples;
+  meta.maintainers = with lib.maintainers; [
+    ShamrockLee
+  ];
+} ''
+  set -eu -o pipefail
+  echo >&2 Testing mixed closures...
+  echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure"
+  diff -U3 \
+    <(sort <"$collectiveClosure") \
+    <(cat "''${closures[@]}" | sort | uniq)
+  touch "$out"
+''
diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix
index d8bf53af86aa..3b84026b4696 100644
--- a/pkgs/top-level/stage.nix
+++ b/pkgs/top-level/stage.nix
@@ -110,8 +110,9 @@ let
   trivialBuilders = self: super:
     import ../build-support/trivial-builders {
       inherit lib;
+      inherit (self) config;
       inherit (self) runtimeShell stdenv stdenvNoCC;
-      inherit (self.pkgsBuildHost) shellcheck-minimal;
+      inherit (self.pkgsBuildHost) jq shellcheck-minimal;
       inherit (self.pkgsBuildHost.xorg) lndir;
     };