chromium: improve and move `recompressTarball`

Recap: We need that (arguably stupid) helper function/drv because the
chromium tarball is big -- and is likely to increase even more in the
future. So big, that we eventually exceeded hydra.nixos.org's
max-output-limit (3G).  Instead of raising global hydra's limit, it was
decided that we recompress the tarball after deleting unused vendored
files from it.

I spent a lot of time on a version/prototype that does everything
(downloading, decompression, tar extraction, deleting unused files,
reproducible tar recreation and finally recompression) via stdin but
eventually had to scratch that.

GNU tar does not allow to create a tarball just from stdin, nixpkgs'
stdenv isn't built with stdin/stdout/pipes in mind, and things a lot of
other things I probably already forgot.

Nonetheless, this version improves multiple things:
- No more `mv` (used to be multiple, not just ours, since fetchzip had
  some as well)
- No more `rm` to get rid of the extracted files before recompressing.
  Instead, we simply don't extract them in the first place (thanks to
  tar's --exlude).
- No more "no space left" that happened due to `downloadToTemp = true;`.
- Multithreaded xz decompression, since that commit is still in
  staging-next.

We cannot use stdenv's unpackFile() because that does not allow us to
specify the needed --exclude (and --strip-components=1 if we don't want
to rely on glob matching).

The hash changed because we now have a static base directory ("source")
in the tarball, instead of whatever upstream provided us with (e.g.
"chromium-120.0.6099.129").

5 files changed, 51 insertions, 40 deletions

diff --git a/pkgs/applications/networking/browsers/chromium/README.md b/pkgs/applications/networking/browsers/chromium/README.md
index c5a537147c48..410c47ee3a0c 100644
--- a/pkgs/applications/networking/browsers/chromium/README.md
+++ b/pkgs/applications/networking/browsers/chromium/README.md
@@ -39,16 +39,6 @@ update `upstream-info.nix`. After updates it is important to test at least
 `nixosTests.chromium` (or basic manual testing) and `google-chrome` (which
 reuses `upstream-info.nix`).
 
-Note: Due to the script downloading many large tarballs it might be
-necessary to adjust the available tmpfs size (it defaults to 10% of the
-systems memory)
-
-```nix
-services.logind.extraConfig = ''
-  RuntimeDirectorySize=4G
-'';
-```
-
 Note: The source tarball is often only available a few hours after the release
 was announced. The CI/CD status can be tracked here:
 - https://ci.chromium.org/p/infra/builders/cron/publish_tarball
diff --git a/pkgs/applications/networking/browsers/chromium/common.nix b/pkgs/applications/networking/browsers/chromium/common.nix
index ec355dccd193..0798be9372e9 100644
--- a/pkgs/applications/networking/browsers/chromium/common.nix
+++ b/pkgs/applications/networking/browsers/chromium/common.nix
@@ -1,5 +1,5 @@
 { stdenv, lib, fetchurl, fetchpatch
-, fetchzip, zstd
+, recompressTarball
 , buildPackages
 , pkgsBuildBuild
 , pkgsBuildTarget
@@ -148,33 +148,6 @@ let
       else throw "no chromium Rosetta Stone entry for os: ${platform.config}";
   };
 
-  recompressTarball = { version, hash ? "" }: fetchzip {
-    name = "chromium-${version}.tar.zstd";
-    url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz";
-    inherit hash;
-
-    nativeBuildInputs = [ zstd ];
-
-    postFetch = ''
-      echo removing unused code from tarball to stay under hydra limit
-      rm -r $out/third_party/{rust-src,llvm}
-
-      echo moving remains out of \$out
-      mv $out source
-
-      echo recompressing final contents into new tarball
-      # try to make a deterministic tarball
-      tar \
-        --use-compress-program "zstd -T$NIX_BUILD_CORES" \
-        --sort name \
-        --mtime 1970-01-01 \
-        --owner=root --group=root \
-        --numeric-owner --mode=go=rX,u+rw,a-s \
-        -cf $out source
-    '';
-  };
-
-
   base = rec {
     pname = "${lib.optionalString ungoogled "ungoogled-"}${packageName}-unwrapped";
     inherit (upstream-info) version;
diff --git a/pkgs/applications/networking/browsers/chromium/default.nix b/pkgs/applications/networking/browsers/chromium/default.nix
index f592344bdf65..9da0f725ed56 100644
--- a/pkgs/applications/networking/browsers/chromium/default.nix
+++ b/pkgs/applications/networking/browsers/chromium/default.nix
@@ -59,6 +59,7 @@ let
           inherit (upstream-info.deps.gn) url rev hash;
         };
       });
+      recompressTarball = callPackage ./recompress-tarball.nix { };
     });
 
     browser = callPackage ./browser.nix {
diff --git a/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix b/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix
new file mode 100644
index 000000000000..0e77dd230f65
--- /dev/null
+++ b/pkgs/applications/networking/browsers/chromium/recompress-tarball.nix
@@ -0,0 +1,47 @@
+{ zstd
+, fetchurl
+}:
+
+{ version
+, hash ? ""
+, ...
+} @ args:
+
+fetchurl ({
+  name = "chromium-${version}.tar.zstd";
+  url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz";
+  inherit hash;
+
+  # chromium xz tarballs are multiple gigabytes big and are sometimes downloaded multiples
+  # times for different versions as part of our update script.
+  # We originally inherited fetchzip's default for downloadToTemp (true).
+  # Given the size of the /run/user tmpfs used defaults to logind's RuntimeDirectorySize=,
+  # which in turn defaults to 10% of the total amount of physical RAM, this often lead to
+  # "no space left" errors, eventually resulting in its own section in our chromium
+  # README.md (for users wanting to run the update script).
+  # Nowadays, we use fetchurl instead of fetchzip, which defaults to false instead of true.
+  # We just want to be explicit and provide a place to document the history and reasoning
+  # behind this.
+  downloadToTemp = false;
+
+  nativeBuildInputs = [ zstd ];
+
+  postFetch = ''
+    cat "$downloadedFile" \
+    | xz -d --threads=$NIX_BUILD_CORES \
+    | tar xf - \
+      --warning=no-timestamp \
+      --one-top-level=source \
+      --exclude=third_party/llvm \
+      --exclude=third_party/rust-src \
+      --strip-components=1
+
+    tar \
+      --use-compress-program "zstd -T$NIX_BUILD_CORES" \
+      --sort name \
+      --mtime "1970-01-01" \
+      --owner=root --group=root \
+      --numeric-owner --mode=go=rX,u+rw,a-s \
+      -cf $out source
+  '';
+} // removeAttrs args [ "version" ])
diff --git a/pkgs/applications/networking/browsers/chromium/upstream-info.nix b/pkgs/applications/networking/browsers/chromium/upstream-info.nix
index eadcefe71bdc..4c35abaeb194 100644
--- a/pkgs/applications/networking/browsers/chromium/upstream-info.nix
+++ b/pkgs/applications/networking/browsers/chromium/upstream-info.nix
@@ -15,7 +15,7 @@
         version = "2023-10-23";
       };
     };
-    hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc=";
+    hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo=";
     hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc=";
     version = "120.0.6099.129";
   };
@@ -32,7 +32,7 @@
         rev = "120.0.6099.129-1";
       };
     };
-    hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc=";
+    hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo=";
     hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc=";
     version = "120.0.6099.129";
   };