diff options
author | github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> | 2022-10-03 00:04:07 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-03 00:04:07 +0000 |
commit | 74f2e360359aaf3d42706c1f70911ae939d966e8 (patch) | |
tree | 28d303ba9d9a6c98855dbbf1d31437c71ee87be2 /pkgs/build-support | |
parent | 0ae3aa5345b2840a7490e9f17a7d1c5cf1b2f6ad (diff) | |
parent | a62844b302507c7531ad68a86cb7aa54704c9cb4 (diff) | |
download | nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar.gz nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar.bz2 nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar.lz nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar.xz nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.tar.zst nixlib-74f2e360359aaf3d42706c1f70911ae939d966e8.zip |
Merge master into staging-next
Diffstat (limited to 'pkgs/build-support')
4 files changed, 105 insertions, 56 deletions
diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index b65aecd7e05c..5a465b48c1d7 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -83,6 +83,11 @@ } @ args: let + platforms = + if args ? meta.platforms + then lib.intersectLists args.meta.platforms dotnet-sdk.meta.platforms + else dotnet-sdk.meta.platforms; + inherit (callPackage ./hooks { inherit dotnet-sdk dotnet-test-sdk disabledTests nuget-source dotnet-runtime runtimeDeps buildType; }) dotnetConfigureHook dotnetBuildHook dotnetCheckHook dotnetInstallHook dotnetFixupHook; @@ -152,9 +157,6 @@ stdenvNoCC.mkDerivation (args // { fetch-deps = let - # Because this list is rather long its put in its own store path to maintain readability of the generated script - exclusions = writeText "nuget-package-exclusions" (lib.concatStringsSep "\n" (dotnet-sdk.passthru.packages { fetchNuGet = attrs: attrs.pname; })); - # Derivations may set flags such as `--runtime <rid>` based on the host platform to avoid restoring/building nuget dependencies they dont have or dont need. # This introduces an issue; In this script we loop over all platforms from `meta` and add the RID flag for it, as to fetch all required dependencies. # The script would inherit the RID flag from the derivation based on the platform building the script, and set the flag for any iteration we do over the RIDs. @@ -165,12 +167,12 @@ stdenvNoCC.mkDerivation (args // { in builtins.filter (flag: !(hasRid flag)) (dotnetFlags ++ dotnetRestoreFlags); - runtimeIds = map (system: dotnet-sdk.systemToDotnetRid system) (args.meta.platforms or dotnet-sdk.meta.platforms); + runtimeIds = map (system: dotnet-sdk.systemToDotnetRid system) platforms; in writeShellScript "fetch-${pname}-deps" '' set -euo pipefail - export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}" + export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}" for arg in "$@"; do case "$arg" in @@ -179,7 +181,7 @@ stdenvNoCC.mkDerivation (args // { shift ;; --help|-h) - echo "usage: $0 <output path> [--keep-sources] [--help]" + echo "usage: $0 [--keep-sources] [--help] <output path>" echo " <output path> The path to write the lockfile to. A temporary file is used if this is not set" echo " --keep-sources Dont remove temporary directories upon exit, useful for debugging" echo " --help Show this help message" @@ -188,14 +190,17 @@ stdenvNoCC.mkDerivation (args // { esac done + export tmp=$(mktemp -td "${pname}-tmp-XXXXXX") + HOME=$tmp/home + exitTrap() { test -n "''${ranTrap-}" && return ranTrap=1 if test -n "''${keepSources-}"; then - echo -e "Path to the source: $src\nPath to the fake home: $HOME" + echo -e "Path to the source: $tmp/src\nPath to the fake home: $tmp/home" else - rm -rf "$src" "$HOME" + rm -rf "$tmp" fi # Since mktemp is used this will be empty if the script didnt succesfully complete @@ -211,8 +216,10 @@ stdenvNoCC.mkDerivation (args // { dotnet restore ''${project-} \ -p:ContinuousIntegrationBuild=true \ -p:Deterministic=true \ - --packages "$HOME/nuget_pkgs" \ + --packages "$tmp/nuget_pkgs" \ --runtime "$rid" \ + --no-cache \ + --force \ ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ ${lib.optionalString (flags != []) (toString flags)} } @@ -220,15 +227,14 @@ stdenvNoCC.mkDerivation (args // { declare -a projectFiles=( ${toString (lib.toList projectFile)} ) declare -a testProjectFiles=( ${toString (lib.toList testProjectFile)} ) - export HOME=$(mktemp -td "${pname}-home-XXXXXX") export DOTNET_NOLOGO=1 export DOTNET_CLI_TELEMETRY_OPTOUT=1 - depsFile="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")" - mkdir -p "$HOME/nuget_pkgs" + depsFile=$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}") + mkdir -p "$tmp/nuget_pkgs" storeSrc="${srcOnly args}" - src="$(mktemp -td "${pname}-src-XXXXXX")" + src=$tmp/src cp -rT "$storeSrc" "$src" chmod -R +w "$src" @@ -247,12 +253,10 @@ stdenvNoCC.mkDerivation (args // { echo "Writing lockfile..." echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n" > "$depsFile" - nuget-to-nix "$HOME/nuget_pkgs" "${exclusions}" >> "$depsFile" + nuget-to-nix "$tmp/nuget_pkgs" "${sdkDeps}" >> "$depsFile" echo "Succesfully wrote lockfile to $depsFile" ''; } // args.passthru or { }; - meta = { - platforms = dotnet-sdk.meta.platforms; - } // args.meta or { }; + meta = (args.meta or { }) // { inherit platforms; }; }) diff --git a/pkgs/build-support/dotnet/nuget-to-nix/default.nix b/pkgs/build-support/dotnet/nuget-to-nix/default.nix index e3a3e45ac20c..3fdda4ac68d3 100644 --- a/pkgs/build-support/dotnet/nuget-to-nix/default.nix +++ b/pkgs/build-support/dotnet/nuget-to-nix/default.nix @@ -4,11 +4,12 @@ , substituteAll , nix , coreutils -, findutils -, gnused , jq +, yq , curl , gnugrep +, gawk +, dotnet-sdk }: runCommandLocal "nuget-to-nix" { @@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" { binPath = lib.makeBinPath [ nix coreutils - findutils - gnused jq + yq curl gnugrep + gawk + dotnet-sdk ]; }; diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh index 5685b0f62bd9..760350ab0517 100755 --- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh +++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh @@ -3,44 +3,82 @@ set -euo pipefail export PATH="@binPath@" +# used for glob ordering of package names +export LC_ALL=C if [ $# -eq 0 ]; then - >&2 echo "Usage: $0 <packages directory> [path to file with a list of excluded packages] > deps.nix" + >&2 echo "Usage: $0 <packages directory> [path to excluded package source] > deps.nix" exit 1 fi pkgs=$1 -exclusions="${2:-/dev/null}" -tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX) -trap "rm -f ${tmpfile}" EXIT +tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)") +trap 'rm -r "$tmp"' EXIT +excluded_source=$(realpath "${2:-$tmp/empty}") -declare -A nuget_sources_cache +export DOTNET_NOLOGO=1 +export DOTNET_CLI_TELEMETRY_OPTOUT=1 -echo "{ fetchNuGet }: [" +mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }') + +declare -A base_addresses -while read pkg_spec; do - { read pkg_name; read pkg_version; } < <( - # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3` - sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkg_spec") +for index in "${sources[@]}"; do + base_addresses[$index]=$( + curl --compressed --netrc -fsL "$index" | \ + jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"') +done - if grep "$pkg_name" "$exclusions" > /dev/null; then - continue - fi +echo "{ fetchNuGet }: [" - pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)" +cd "$pkgs" +for package in *; do + cd "$package" + for version in *; do + id=$(xq -r .package.metadata.id "$version/$package".nuspec) - pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")" - if [[ -d $pkg_src ]]; then + if [[ -e "$excluded_source/$id.$version".nupkg ]]; then continue - elif [[ $pkg_src != https://api.nuget.org/* ]]; then - pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}" - pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg" - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile} - else - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile} - fi -done < <(find $1 -name '*.nuspec') - -LC_ALL=C sort --ignore-case ${tmpfile} - -echo "]" + fi + + used_source="$(jq -r '.source' "$version"/.nupkg.metadata)" + for source in "${sources[@]}"; do + url="${base_addresses[$source]}$package/$version/$package.$version.nupkg" + if [[ "$source" == "$used_source" ]]; then + sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)" + found=true + break + else + if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then + # If multiple remote sources are enabled, nuget will try them all + # concurrently and use the one that responds first. We always use the + # first source that has the package. + echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2 + found=true + break + else + if ! grep -q 'HTTP error 404' "$tmp/error"; then + cat "$tmp/error" 1>&2 + exit 1 + fi + fi + fi + done + + if ! ${found-false}; then + echo "couldn't find $package $version" >&2 + exit 1 + fi + + if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })" + else + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })" + fi + done + cd .. +done + +cat << EOL +] +EOL diff --git a/pkgs/build-support/node/fetch-yarn-deps/default.nix b/pkgs/build-support/node/fetch-yarn-deps/default.nix index 6d67b87915ac..4cf2507706ae 100644 --- a/pkgs/build-support/node/fetch-yarn-deps/default.nix +++ b/pkgs/build-support/node/fetch-yarn-deps/default.nix @@ -42,31 +42,36 @@ in { fetchYarnDeps = let f = { name ? "offline", - yarnLock, + src ? null, hash ? "", sha256 ? "", - }: let + ... + }@args: let hash_ = if hash != "" then { outputHashAlgo = null; outputHash = hash; } else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; } else { outputHashAlgo = "sha256"; outputHash = lib.fakeSha256; }; - in stdenv.mkDerivation { + in stdenv.mkDerivation ({ inherit name; - dontUnpack = true; + dontUnpack = src == null; dontInstall = true; nativeBuildInputs = [ prefetch-yarn-deps ]; GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; buildPhase = '' + runHook preBuild + + yarnLock=''${yarnLock:=$PWD/yarn.lock} mkdir -p $out - (cd $out; prefetch-yarn-deps --verbose --builder ${yarnLock}) + (cd $out; prefetch-yarn-deps --verbose --builder $yarnLock) + + runHook postBuild ''; outputHashMode = "recursive"; - inherit (hash_) outputHashAlgo outputHash; - }; + } // hash_ // (removeAttrs args ["src" "name" "hash" "sha256"])); in lib.setFunctionArgs f (lib.functionArgs f) // { tests = callPackage ./tests {}; |