minimal-bootstrap: make sources a non-tarballs.nixos.org FOD

This commit adjusts #232576 to break the
fetchurl<->minimal-bootstrap-sources dependency cycle without
needing an upload to tarballs.nixos.org.  It does this by appending
the low-level FOD attributes onto the `runCommand` derivation.

  https://nixos.org/manual/nix/unstable/language/advanced-attributes.html#adv-attr-outputHash

4 files changed, 74 insertions, 40 deletions

diff --git a/pkgs/os-specific/linux/minimal-bootstrap/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/default.nix
index 3e4e9e7f5cb8..3c1746716f06 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/default.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/default.nix
@@ -4,6 +4,7 @@
 , hostPlatform
 , fetchurl
 , checkMeta
+, make-minimal-bootstrap-sources
 }:
 
 lib.makeScope
@@ -60,6 +61,8 @@ lib.makeScope
 
     ln-boot = callPackage ./ln-boot { };
 
+    inherit make-minimal-bootstrap-sources;
+
     mes = lib.recurseIntoAttrs (callPackage ./mes { });
     mes-libc = callPackage ./mes/libc.nix { };
 
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix
index e7ddede2ffa4..4aa5d32f7498 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix
@@ -1,3 +1,6 @@
+{ make-minimal-bootstrap-sources
+}:
+
 rec {
   name = "stage0-posix-${version}-${rev}-source";
   # Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c
@@ -5,9 +8,6 @@ rec {
   rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c";
 
   # This 256 byte seed is the only pre-compiled binary in the bootstrap chain.
-  # While it is included in the stage0-posix source bundle and is synced with
-  # stage0-posix updates, we have split it out into its own derivation to highlight
-  # its unique status as a trusted binary seed.
   hex0-seed = import <nix/fetchurl.nix> {
     name = "hex0-seed-${version}";
     url = "https://github.com/oriansj/bootstrap-seeds/raw/b1263ff14a17835f4d12539226208c426ced4fba/POSIX/x86/hex0-seed";
@@ -15,34 +15,63 @@ rec {
     executable = true;
   };
 
-  # Packaged source files for the first bootstrapping stage.
-  #
-  # We don't have access to utilities such as fetchgit and fetchzip since this
-  # is this is part of the bootstrap process and would introduce a circular
-  # dependency. The only tool we have to fetch source trees is `import <nix/fetchurl.nix>`
-  # with the unpack option, taking a NAR file as input. This requires source
-  # tarballs to be repackaged.
-  #
-  # To build see `make-bootstrap-sources.nix`
-
-  #
-  # Files came from this Hydra build:
-  #
-  #   https://hydra.nixos.org/build/<placeholder>
-  #
-  # Which used nixpkgs revision <placeholder>
-  # to instantiate:
-  #
-  #   /nix/store/<placeholder>.drv
-  #
-  # and then built:
-  #
-  #   /nix/store/<placeholder>
-  #
-  src = import <nix/fetchurl.nix> {
-    inherit name;
-    url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-18/${name}.nar.xz";
-    hash = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U=";
-    unpack = true;
-  };
+  /*
+  Since `make-minimal-bootstrap-sources` requires nixpkgs and nix it
+  will create a circular dependency if it is used in place of the
+  binary bootstrap-files.  To break the circular dependency,
+  `minimal-bootstrap-sources` extends `make-minimal-bootstrap-sources`
+  by adding Fixed Output Derivation (FOD) attributes.  These cause
+  the builder to be skipped if the expected output is found (by
+  its hash) in the store or on a substituter.
+
+  # How do I update the hash?
+
+  Run the following command:
+  ```
+  nix hash file $(nix build --print-out-paths -f '<nixpkgs>' make-minimal-bootstrap-sources)
+  ```
+
+  # Why do we need this `.nar` archive?
+
+  This archive exists only because of a quirk/limitation of Nix: in
+  restricted mode the builtin fetchers can download only single
+  files; they have no way to unpack multi-file archives except for
+  NAR archives:
+
+  https://github.com/NixOS/nixpkgs/pull/232576#issuecomment-1592415619
+
+  # Why don't we have to upload this to tarballs.nixos.org like the binary bootstrap-files did?
+
+  Unlike this archive, the binary bootstrap-files contained binaries,
+  which meant that we had to:
+
+  1. Make sure they came from a trusted builder (Hydra)
+  2. Keep careful track of exactly what toolchain (i.e. nixpkgs
+     commit) that builder used to create them.
+  3. Keep copies of the built binaries, in case the toolchains that
+     produced them failed to be perfectly deterministic.
+
+  The curated archives at tarballs.nixos.org exist in order to
+  satisfy these requirements.
+
+  The second point created a significant burden: since the nixpkgs
+  toolchain used to build a given copy of the binary bootstrap-files
+  itself used a *previous* copy of the bootstrap-files, this meant
+  we had to track the provenance of all bootstrap-files tarballs
+  ever used, for all eternity.  There was no explanation of where
+  the "original" bootstrap-files came from: turtles all the way
+  down.  In spite of all this effort we still can't be sure of our
+  ability to reproduce the binary bootstrap-files, since the
+  compilers that built them don't always produce exactly bit-for-bit
+  deterministic results.
+
+  Since this archive contains no binaries and uses a format (NAR)
+  specifically designed for bit-exact reproducibility, none of the
+  requirements above apply to `minimal-bootstrap-sources`.
+  */
+  minimal-bootstrap-sources = make-minimal-bootstrap-sources.overrideAttrs(_: {
+    outputHashMode = "flat";
+    outputHashAlgo = "sha256";
+    outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg=";
+  });
 }
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix
index 2b41261add3a..8e0900100aed 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix
@@ -3,7 +3,10 @@
 }:
 
 lib.makeScope newScope (self: with self; {
-  inherit (import ./bootstrap-sources.nix) version hex0-seed src;
+  inherit (self.callPackage ./bootstrap-sources.nix)
+    version hex0-seed minimal-bootstrap-sources;
+
+  src = minimal-bootstrap-sources;
 
   m2libc = src + "/M2libc";
 
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
index be0974edf430..4fe9a9be7881 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
@@ -8,8 +8,7 @@
 #
 # To build:
 #
-#   nix-build . -A minimal-bootstrap-sources
-#   => ./result/stage0-posix-$version-$rev-source.nar.xz
+#   nix-build '<nixpkgs>' -o sources.nar.xz -A make-minimal-bootstrap-sources
 #
 
 { lib
@@ -19,7 +18,7 @@
 , xz
 }:
 let
-  inherit (import ./bootstrap-sources.nix) name rev;
+  inherit (import ./bootstrap-sources.nix { make-minimal-bootstrap-sources = null; }) name rev;
 
   src = fetchFromGitHub {
     owner = "oriansj";
@@ -41,8 +40,9 @@ let
         $out/mescc-tools-extra/M2libc
     '';
   };
+
 in
-runCommand name {
+runCommand "${name}.nar.xz" {
   nativeBuildInputs = [ nix xz ];
 
   passthru = { inherit src; };
@@ -55,6 +55,5 @@ runCommand name {
     platforms = platforms.all;
   };
 } ''
-  mkdir $out
-  nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz"
+  nix-store --dump ${src} | xz -c > $out
 ''