Merge branch 'nixos-unstable-small' of https://github.com/NixOS/nixpkgs

7 files changed, 322 insertions, 18 deletions

diff --git a/nixpkgs/pkgs/build-support/docker/default.nix b/nixpkgs/pkgs/build-support/docker/default.nix
index 05a1a6fbbdaf..6c13a379f934 100644
--- a/nixpkgs/pkgs/build-support/docker/default.nix
+++ b/nixpkgs/pkgs/build-support/docker/default.nix
@@ -8,6 +8,7 @@
 , proot
 , fakeNss
 , fakeroot
+, file
 , go
 , jq
 , jshon
@@ -34,6 +35,7 @@
 , writeText
 , writeTextDir
 , writePython3
+, zstd
 }:
 
 let
@@ -76,6 +78,30 @@ let
   # mapping from the go package.
   defaultArchitecture = go.GOARCH;
 
+  compressors = {
+    none = {
+      ext = "";
+      nativeInputs = [ ];
+      compress = "cat";
+      decompress = "cat";
+    };
+    gz = {
+      ext = ".gz";
+      nativeInputs = [ pigz ];
+      compress = "pigz -p$NIX_BUILD_CORES -nTR";
+      decompress = "pigz -d -p$NIX_BUILD_CORES";
+    };
+    zstd = {
+      ext = ".zst";
+      nativeInputs = [ zstd ];
+      compress = "zstd -T$NIX_BUILD_CORES";
+      decompress = "zstd -d -T$NIX_BUILD_CORES";
+    };
+  };
+
+  compressorForImage = compressor: imageName: compressors.${compressor} or
+    (throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]");
+
 in
 rec {
   examples = callPackage ./examples.nix {
@@ -487,16 +513,17 @@ rec {
       '';
     };
 
-  buildLayeredImage = lib.makeOverridable ({ name, ... }@args:
+  buildLayeredImage = lib.makeOverridable ({ name, compressor ? "gz", ... }@args:
     let
       stream = streamLayeredImage args;
+      compress = compressorForImage compressor name;
     in
-    runCommand "${baseNameOf name}.tar.gz"
+    runCommand "${baseNameOf name}.tar${compress.ext}"
       {
         inherit (stream) imageName;
         passthru = { inherit (stream) imageTag; };
-        nativeBuildInputs = [ pigz ];
-      } "${stream} | pigz -nTR > $out"
+        nativeBuildInputs = compress.nativeInputs;
+      } "${stream} | ${compress.compress} > $out"
   );
 
   # 1. extract the base image
@@ -539,6 +566,8 @@ rec {
       buildVMMemorySize ? 512
     , # Time of creation of the image.
       created ? "1970-01-01T00:00:01Z"
+    , # Compressor to use. One of: none, gz, zstd.
+      compressor ? "gz"
     , # Deprecated.
       contents ? null
     ,
@@ -574,6 +603,8 @@ rec {
         in
         if created == "now" then impure else pure;
 
+      compress = compressorForImage compressor name;
+
       layer =
         if runAsRoot == null
         then
@@ -590,9 +621,9 @@ rec {
               extraCommands;
             copyToRoot = rootContents;
           };
-      result = runCommand "docker-image-${baseName}.tar.gz"
+      result = runCommand "docker-image-${baseName}.tar${compress.ext}"
         {
-          nativeBuildInputs = [ jshon pigz jq moreutils ];
+          nativeBuildInputs = [ jshon jq moreutils ] ++ compress.nativeInputs;
           # Image name must be lowercase
           imageName = lib.toLower name;
           imageTag = lib.optionalString (tag != null) tag;
@@ -746,7 +777,7 @@ rec {
         chmod -R a-w image
 
         echo "Cooking the image..."
-        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out
+        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | ${compress.compress} > $out
 
         echo "Finished."
       '';
@@ -761,16 +792,28 @@ rec {
   mergeImages = images: runCommand "merge-docker-images"
     {
       inherit images;
-      nativeBuildInputs = [ pigz jq ];
+      nativeBuildInputs = [ file jq ]
+        ++ compressors.none.nativeInputs
+        ++ compressors.gz.nativeInputs
+        ++ compressors.zstd.nativeInputs;
     } ''
     mkdir image inputs
     # Extract images
     repos=()
     manifests=()
+    last_image_mime="application/gzip"
     for item in $images; do
       name=$(basename $item)
       mkdir inputs/$name
-      tar -I pigz -xf $item -C inputs/$name
+
+      last_image_mime=$(file --mime-type -b $item)
+      case $last_image_mime in
+        "application/x-tar") ${compressors.none.decompress};;
+        "application/zstd") ${compressors.zstd.decompress};;
+        "application/gzip") ${compressors.gz.decompress};;
+        *) echo "error: unexpected layer type $last_image_mime" >&2; exit 1;;
+      esac < $item | tar -xC inputs/$name
+
       if [ -f inputs/$name/repositories ]; then
         repos+=(inputs/$name/repositories)
       fi
@@ -787,7 +830,14 @@ rec {
     mv repositories image/repositories
     mv manifest.json image/manifest.json
     # Create tarball and gzip
-    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out
+    tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | (
+      case $last_image_mime in
+        "application/x-tar") ${compressors.none.compress};;
+        "application/zstd") ${compressors.zstd.compress};;
+        "application/gzip") ${compressors.gz.compress};;
+        # `*)` not needed; already checked.
+      esac
+    ) > $out
   '';
 
 
@@ -923,6 +973,7 @@ rec {
                   --sort name \
                   --exclude=./proc \
                   --exclude=./sys \
+                  --exclude=.${builtins.storeDir} \
                   --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \
                   --hard-dereference \
                   -cf $out/layer.tar .
@@ -1238,14 +1289,15 @@ rec {
       };
 
   # Wrapper around streamNixShellImage to build an image from the result
-  buildNixShellImage = { drv, ... }@args:
+  buildNixShellImage = { drv, compressor ? "gz", ... }@args:
     let
       stream = streamNixShellImage args;
+      compress = compressorForImage compressor drv.name;
     in
-    runCommand "${drv.name}-env.tar.gz"
+    runCommand "${drv.name}-env.tar${compress.ext}"
       {
         inherit (stream) imageName;
         passthru = { inherit (stream) imageTag; };
-        nativeBuildInputs = [ pigz ];
-      } "${stream} | pigz -nTR > $out";
+        nativeBuildInputs = compress.nativeInputs;
+      } "${stream} | ${compress.compress} > $out";
 }
diff --git a/nixpkgs/pkgs/build-support/docker/examples.nix b/nixpkgs/pkgs/build-support/docker/examples.nix
index 5784e650dc2e..72c1cbe0d410 100644
--- a/nixpkgs/pkgs/build-support/docker/examples.nix
+++ b/nixpkgs/pkgs/build-support/docker/examples.nix
@@ -480,6 +480,22 @@ rec {
     layerC = layerOnTopOf layerB "c";
   in layerC;
 
+  bashUncompressed = pkgs.dockerTools.buildImage {
+    name = "bash-uncompressed";
+    tag = "latest";
+    compressor = "none";
+    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
+    copyToRoot = pkgs.bashInteractive;
+  };
+
+  bashZstdCompressed = pkgs.dockerTools.buildImage {
+    name = "bash-zstd";
+    tag = "latest";
+    compressor = "zstd";
+    # Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
+    copyToRoot = pkgs.bashInteractive;
+  };
+
   # buildImage without explicit tag
   bashNoTag = pkgs.dockerTools.buildImage {
     name = "bash-no-tag";
@@ -614,6 +630,12 @@ rec {
     layeredImageWithFakeRootCommands
   ];
 
+  mergeVaryingCompressor = pkgs.dockerTools.mergeImages [
+    redis
+    bashUncompressed
+    bashZstdCompressed
+  ];
+
   helloOnRoot = pkgs.dockerTools.streamLayeredImage {
     name = "hello";
     tag = "latest";
@@ -639,6 +661,20 @@ rec {
     includeStorePaths = false;
   };
 
+  helloOnRootNoStoreFakechroot = pkgs.dockerTools.streamLayeredImage {
+    name = "hello";
+    tag = "latest";
+    contents = [
+      (pkgs.buildEnv {
+        name = "hello-root";
+        paths = [ pkgs.hello ];
+      })
+    ];
+    config.Cmd = [ "hello" ];
+    includeStorePaths = false;
+    enableFakechroot = true;
+  };
+
   etc =
     let
       inherit (pkgs) lib;
diff --git a/nixpkgs/pkgs/build-support/fetchpypilegacy/default.nix b/nixpkgs/pkgs/build-support/fetchpypilegacy/default.nix
new file mode 100644
index 000000000000..bcd560449916
--- /dev/null
+++ b/nixpkgs/pkgs/build-support/fetchpypilegacy/default.nix
@@ -0,0 +1,45 @@
+# Fetch from PyPi legacy API as documented in https://warehouse.pypa.io/api-reference/legacy.html
+{ runCommand
+, lib
+, python3
+}:
+{
+  # package name
+  pname,
+  # Package index
+  url ? null,
+  # Multiple package indices to consider
+  urls ? [ ],
+  # filename including extension
+  file,
+  # SRI hash
+  hash,
+  # allow overriding the derivation name
+  name ? null,
+}:
+let
+  urls' = urls ++ lib.optional (url != null) url;
+
+  pathParts = lib.filter ({ prefix, path }: "NETRC" == prefix) builtins.nixPath;
+  netrc_file =
+    if (pathParts != [ ])
+    then (lib.head pathParts).path
+    else "";
+
+in
+# Assert that we have at least one URL
+assert urls' != [ ]; runCommand file
+  ({
+    nativeBuildInputs = [ python3 ];
+    impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+    outputHashMode = "flat";
+    # if hash is empty select a default algo to let nix propose the actual hash.
+    outputHashAlgo = if hash == "" then "sha256" else null;
+    outputHash = hash;
+    NETRC = netrc_file;
+  }
+  // (lib.optionalAttrs (name != null) {inherit name;}))
+  ''
+    python ${./fetch-legacy.py} ${lib.concatStringsSep " " (map (url: "--url ${lib.escapeShellArg url}") urls')} --pname ${pname} --filename ${file}
+    mv ${file} $out
+  ''
diff --git a/nixpkgs/pkgs/build-support/fetchpypilegacy/fetch-legacy.py b/nixpkgs/pkgs/build-support/fetchpypilegacy/fetch-legacy.py
new file mode 100644
index 000000000000..e031f244a771
--- /dev/null
+++ b/nixpkgs/pkgs/build-support/fetchpypilegacy/fetch-legacy.py
@@ -0,0 +1,162 @@
+# Some repositories (such as Devpi) expose the Pypi legacy API
+# (https://warehouse.pypa.io/api-reference/legacy.html).
+#
+# Note it is not possible to use pip
+# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12
+
+import base64
+import argparse
+import netrc
+import os
+import shutil
+import ssl
+import sys
+import urllib.request
+from html.parser import HTMLParser
+from os.path import normpath
+from typing import Optional
+from urllib.parse import urlparse, urlunparse
+
+
+# Parse the legacy index page to extract the href and package names
+class Pep503(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.sources: dict[str, str] = {}
+        self.url: Optional[str] = None
+        self.name: Optional[str] = None
+
+    def handle_data(self, data: str) -> None:
+        if self.url is not None:
+            self.name = data
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None:
+        if tag == "a":
+            for name, value in attrs:
+                if name == "href":
+                    self.url = value
+
+    def handle_endtag(self, tag: str) -> None:
+        if self.url is not None:
+            if not self.name:
+                raise ValueError("Name not set")
+
+            self.sources[self.name] = self.url
+        self.url = None
+
+
+def try_fetch(url: str, package_name: str, package_filename: str) -> None:
+    index_url = url + "/" + package_name + "/"
+
+    # Parse username and password for this host from the netrc file if given.
+    username: Optional[str] = None
+    password: Optional[str] = None
+    if os.environ.get("NETRC", "") != "":
+        netrc_obj = netrc.netrc(os.environ["NETRC"])
+        host = urlparse(index_url).netloc
+        # Strip port number if present
+        if ":" in host:
+            host = host.split(":")[0]
+        authenticators = netrc_obj.authenticators(host)
+        if authenticators:
+            username, _, password = authenticators
+
+    print("Reading index %s" % index_url)
+
+    context = ssl.create_default_context()
+
+    # Extract out username/password from index_url, if present.
+    parsed_url = urlparse(index_url)
+    username = parsed_url.username or username
+    password = parsed_url.password or password
+    index_url = parsed_url._replace(netloc=parsed_url.netloc.rpartition("@")[-1]).geturl()
+
+    req = urllib.request.Request(index_url)
+
+    if username and password:  # Add authentication
+        password_b64 = base64.b64encode(":".join((username, password)).encode()).decode("utf-8")
+        req.add_header("Authorization", "Basic {}".format(password_b64))
+    else:  # If we are not using authentication disable TLS verification for long term reproducibility
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+
+    response = urllib.request.urlopen(req, context=context)
+    index = response.read()
+
+    parser = Pep503()
+    parser.feed(str(index, "utf-8"))
+    if package_filename not in parser.sources:
+        print("The file %s has not be found in the index %s" % (package_filename, index_url))
+        exit(1)
+
+    package_file = open(package_filename, "wb")
+    # Sometimes the href is a relative or absolute path within the index's domain.
+    indicated_url = urlparse(parser.sources[package_filename])
+    if indicated_url.netloc == "":
+        parsed_url = urlparse(index_url)
+
+        if indicated_url.path.startswith("/"):
+            # An absolute path within the index's domain.
+            path = parser.sources[package_filename]
+        else:
+            # A relative path.
+            path = parsed_url.path + "/" + parser.sources[package_filename]
+
+        package_url = urlunparse(
+            (
+                parsed_url.scheme,
+                parsed_url.netloc,
+                path,
+                None,
+                None,
+                None,
+            )
+        )
+    else:
+        package_url = parser.sources[package_filename]
+
+    # Handle urls containing "../"
+    parsed_url = urlparse(package_url)
+    real_package_url = urlunparse(
+        (
+            parsed_url.scheme,
+            parsed_url.netloc,
+            normpath(parsed_url.path),
+            parsed_url.params,
+            parsed_url.query,
+            parsed_url.fragment,
+        )
+    )
+    print("Downloading %s" % real_package_url)
+
+    req = urllib.request.Request(real_package_url)
+    if username and password:
+        req.add_unredirected_header("Authorization", "Basic {}".format(password_b64))
+    response = urllib.request.urlopen(req, context=context)
+
+    with response as r:
+        shutil.copyfileobj(r, package_file)
+
+
+argparser = argparse.ArgumentParser(description="Fetch file from legacy pypi API")
+argparser.add_argument("--url", action="append", required=True)
+argparser.add_argument("--pname", action="store", required=True)
+argparser.add_argument("--filename", action="store", required=True)
+
+
+if __name__ == "__main__":
+    args = argparser.parse_args()
+    for url in args.url:
+        try:
+            try_fetch(url, args.pname, args.filename)
+        except urllib.error.HTTPError as e:
+            print("Got exception'", e, "', trying next package index", file=sys.stderr)
+            continue
+        else:
+            break
+    else:
+        print(
+            f"Could not fetch package '{args.pname}' file '{args.filename}' from any mirrors: {args.url}",
+            file=sys.stderr,
+        )
+        exit(1)
diff --git a/nixpkgs/pkgs/build-support/fetchpypilegacy/tests.nix b/nixpkgs/pkgs/build-support/fetchpypilegacy/tests.nix
new file mode 100644
index 000000000000..b16325b96b7e
--- /dev/null
+++ b/nixpkgs/pkgs/build-support/fetchpypilegacy/tests.nix
@@ -0,0 +1,9 @@
+{ testers, fetchPypiLegacy, ... }: {
+  # Tests that we can send custom headers with spaces in them
+  fetchSimple = testers.invalidateFetcherByDrvHash fetchPypiLegacy {
+    pname = "requests";
+    file = "requests-2.31.0.tar.gz";
+    url = "https://pypi.org/simple";
+    hash = "sha256-lCxadY+Y15Dq7Ropy27vx/+w0c968Fw9J5Flbb1q0eE=";
+  };
+}
diff --git a/nixpkgs/pkgs/build-support/php/build-composer-project.nix b/nixpkgs/pkgs/build-support/php/build-composer-project.nix
index 778aa35fa6a5..80c63bcde71b 100644
--- a/nixpkgs/pkgs/build-support/php/build-composer-project.nix
+++ b/nixpkgs/pkgs/build-support/php/build-composer-project.nix
@@ -57,9 +57,9 @@ let
 
       doInstallCheck = previousAttrs.doInstallCheck or false;
       installCheckPhase = previousAttrs.installCheckPhase or ''
-        runHook preCheckInstall
+        runHook preInstallCheck
 
-        runHook postCheckInstall
+        runHook postInstallCheck
       '';
 
       composerRepository = phpDrv.mkComposerRepository {
diff --git a/nixpkgs/pkgs/build-support/php/build-composer-repository.nix b/nixpkgs/pkgs/build-support/php/build-composer-repository.nix
index 5b31f86e61cf..e359c0829aaf 100644
--- a/nixpkgs/pkgs/build-support/php/build-composer-repository.nix
+++ b/nixpkgs/pkgs/build-support/php/build-composer-repository.nix
@@ -78,9 +78,9 @@ let
 
       doInstallCheck = previousAttrs.doInstallCheck or false;
       installCheckPhase = previousAttrs.installCheckPhase or ''
-        runHook preCheckInstall
+        runHook preInstallCheck
 
-        runHook postCheckInstall
+        runHook postInstallCheck
       '';
 
       COMPOSER_CACHE_DIR = "/dev/null";