Merge pull request #193033 from corngood/dotnet-deterministic-fetch-deps

dotnet: fix some problems with nuget-to-nix and make it deterministic

3 files changed, 93 insertions, 49 deletions

diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix
index b65aecd7e05c..5a465b48c1d7 100644
--- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix
+++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix
@@ -83,6 +83,11 @@
 } @ args:
 
 let
+  platforms =
+    if args ? meta.platforms
+    then lib.intersectLists args.meta.platforms dotnet-sdk.meta.platforms
+    else dotnet-sdk.meta.platforms;
+
   inherit (callPackage ./hooks {
     inherit dotnet-sdk dotnet-test-sdk disabledTests nuget-source dotnet-runtime runtimeDeps buildType;
   }) dotnetConfigureHook dotnetBuildHook dotnetCheckHook dotnetInstallHook dotnetFixupHook;
@@ -152,9 +157,6 @@ stdenvNoCC.mkDerivation (args // {
 
     fetch-deps =
       let
-        # Because this list is rather long its put in its own store path to maintain readability of the generated script
-        exclusions = writeText "nuget-package-exclusions" (lib.concatStringsSep "\n" (dotnet-sdk.passthru.packages { fetchNuGet = attrs: attrs.pname; }));
-
         # Derivations may set flags such as `--runtime <rid>` based on the host platform to avoid restoring/building nuget dependencies they dont have or dont need.
         # This introduces an issue; In this script we loop over all platforms from `meta` and add the RID flag for it, as to fetch all required dependencies.
         # The script would inherit the RID flag from the derivation based on the platform building the script, and set the flag for any iteration we do over the RIDs.
@@ -165,12 +167,12 @@ stdenvNoCC.mkDerivation (args // {
           in
           builtins.filter (flag: !(hasRid flag)) (dotnetFlags ++ dotnetRestoreFlags);
 
-        runtimeIds = map (system: dotnet-sdk.systemToDotnetRid system) (args.meta.platforms or dotnet-sdk.meta.platforms);
+        runtimeIds = map (system: dotnet-sdk.systemToDotnetRid system) platforms;
       in
       writeShellScript "fetch-${pname}-deps" ''
         set -euo pipefail
 
-        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}"
+        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}"
 
         for arg in "$@"; do
             case "$arg" in
@@ -179,7 +181,7 @@ stdenvNoCC.mkDerivation (args // {
                     shift
                     ;;
                 --help|-h)
-                    echo "usage: $0 <output path> [--keep-sources] [--help]"
+                    echo "usage: $0 [--keep-sources] [--help] <output path>"
                     echo "    <output path>   The path to write the lockfile to. A temporary file is used if this is not set"
                     echo "    --keep-sources  Dont remove temporary directories upon exit, useful for debugging"
                     echo "    --help          Show this help message"
@@ -188,14 +190,17 @@ stdenvNoCC.mkDerivation (args // {
             esac
         done
 
+        export tmp=$(mktemp -td "${pname}-tmp-XXXXXX")
+        HOME=$tmp/home
+
         exitTrap() {
             test -n "''${ranTrap-}" && return
             ranTrap=1
 
             if test -n "''${keepSources-}"; then
-                echo -e "Path to the source: $src\nPath to the fake home: $HOME"
+                echo -e "Path to the source: $tmp/src\nPath to the fake home: $tmp/home"
             else
-                rm -rf "$src" "$HOME"
+                rm -rf "$tmp"
             fi
 
             # Since mktemp is used this will be empty if the script didnt succesfully complete
@@ -211,8 +216,10 @@ stdenvNoCC.mkDerivation (args // {
             dotnet restore ''${project-} \
                 -p:ContinuousIntegrationBuild=true \
                 -p:Deterministic=true \
-                --packages "$HOME/nuget_pkgs" \
+                --packages "$tmp/nuget_pkgs" \
                 --runtime "$rid" \
+                --no-cache \
+                --force \
                 ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \
                 ${lib.optionalString (flags != []) (toString flags)}
         }
@@ -220,15 +227,14 @@ stdenvNoCC.mkDerivation (args // {
         declare -a projectFiles=( ${toString (lib.toList projectFile)} )
         declare -a testProjectFiles=( ${toString (lib.toList testProjectFile)} )
 
-        export HOME=$(mktemp -td "${pname}-home-XXXXXX")
         export DOTNET_NOLOGO=1
         export DOTNET_CLI_TELEMETRY_OPTOUT=1
 
-        depsFile="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")"
-        mkdir -p "$HOME/nuget_pkgs"
+        depsFile=$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")
+        mkdir -p "$tmp/nuget_pkgs"
 
         storeSrc="${srcOnly args}"
-        src="$(mktemp -td "${pname}-src-XXXXXX")"
+        src=$tmp/src
         cp -rT "$storeSrc" "$src"
         chmod -R +w "$src"
 
@@ -247,12 +253,10 @@ stdenvNoCC.mkDerivation (args // {
 
         echo "Writing lockfile..."
         echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n" > "$depsFile"
-        nuget-to-nix "$HOME/nuget_pkgs" "${exclusions}" >> "$depsFile"
+        nuget-to-nix "$tmp/nuget_pkgs" "${sdkDeps}" >> "$depsFile"
         echo "Succesfully wrote lockfile to $depsFile"
       '';
   } // args.passthru or { };
 
-  meta = {
-    platforms = dotnet-sdk.meta.platforms;
-  } // args.meta or { };
+  meta = (args.meta or { }) // { inherit platforms; };
 })
diff --git a/pkgs/build-support/dotnet/nuget-to-nix/default.nix b/pkgs/build-support/dotnet/nuget-to-nix/default.nix
index e3a3e45ac20c..3fdda4ac68d3 100644
--- a/pkgs/build-support/dotnet/nuget-to-nix/default.nix
+++ b/pkgs/build-support/dotnet/nuget-to-nix/default.nix
@@ -4,11 +4,12 @@
 , substituteAll
 , nix
 , coreutils
-, findutils
-, gnused
 , jq
+, yq
 , curl
 , gnugrep
+, gawk
+, dotnet-sdk
 }:
 
 runCommandLocal "nuget-to-nix" {
@@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" {
     binPath = lib.makeBinPath [
       nix
       coreutils
-      findutils
-      gnused
       jq
+      yq
       curl
       gnugrep
+      gawk
+      dotnet-sdk
     ];
   };
 
diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
index 5685b0f62bd9..760350ab0517 100755
--- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
+++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh
@@ -3,44 +3,82 @@
 set -euo pipefail
 
 export PATH="@binPath@"
+# used for glob ordering of package names
+export LC_ALL=C
 
 if [ $# -eq 0 ]; then
-  >&2 echo "Usage: $0 <packages directory> [path to file with a list of excluded packages] > deps.nix"
+  >&2 echo "Usage: $0 <packages directory> [path to excluded package source] > deps.nix"
   exit 1
 fi
 
 pkgs=$1
-exclusions="${2:-/dev/null}"
-tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX)
-trap "rm -f ${tmpfile}" EXIT
+tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)")
+trap 'rm -r "$tmp"' EXIT
+excluded_source=$(realpath "${2:-$tmp/empty}")
 
-declare -A nuget_sources_cache
+export DOTNET_NOLOGO=1
+export DOTNET_CLI_TELEMETRY_OPTOUT=1
 
-echo "{ fetchNuGet }: ["
+mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }')
+
+declare -A base_addresses
 
-while read pkg_spec; do
-  { read pkg_name; read pkg_version; } < <(
-    # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3`
-    sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkg_spec")
+for index in "${sources[@]}"; do
+  base_addresses[$index]=$(
+    curl --compressed --netrc -fsL "$index" | \
+      jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')
+done
 
-  if grep "$pkg_name" "$exclusions" > /dev/null; then
-    continue
-  fi
+echo "{ fetchNuGet }: ["
 
-  pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)"
+cd "$pkgs"
+for package in *; do
+  cd "$package"
+  for version in *; do
+    id=$(xq -r .package.metadata.id "$version/$package".nuspec)
 
-  pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")"
-  if [[ -d $pkg_src ]]; then
+    if [[ -e "$excluded_source/$id.$version".nupkg ]]; then
       continue
-  elif [[ $pkg_src != https://api.nuget.org/* ]]; then
-    pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}"
-    pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg"
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile}
-  else
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile}
-  fi
-done < <(find $1 -name '*.nuspec')
-
-LC_ALL=C sort --ignore-case ${tmpfile}
-
-echo "]"
+    fi
+
+    used_source="$(jq -r '.source' "$version"/.nupkg.metadata)"
+    for source in "${sources[@]}"; do
+      url="${base_addresses[$source]}$package/$version/$package.$version.nupkg"
+      if [[ "$source" == "$used_source" ]]; then
+        sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)"
+        found=true
+        break
+      else
+        if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then
+          # If multiple remote sources are enabled, nuget will try them all
+          # concurrently and use the one that responds first. We always use the
+          # first source that has the package.
+          echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2
+          found=true
+          break
+        else
+          if ! grep -q 'HTTP error 404' "$tmp/error"; then
+            cat "$tmp/error" 1>&2
+            exit 1
+          fi
+        fi
+      fi
+    done
+
+    if ! ${found-false}; then
+      echo "couldn't find $package $version" >&2
+      exit 1
+    fi
+
+    if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })"
+    else
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })"
+    fi
+  done
+  cd ..
+done
+
+cat << EOL
+]
+EOL