fetch-cargo-deps: factor into fetchCargoDeps function

2 files changed, 172 insertions, 175 deletions

diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps
index 5074e26848ed..4fbc53d1039e 100755
--- a/pkgs/build-support/rust/fetch-cargo-deps
+++ b/pkgs/build-support/rust/fetch-cargo-deps
@@ -1,183 +1,177 @@
-source $stdenv/setup
+fetchCargoDeps() {
+    src=$(realpath $1)
+    out=$(realpath $2)
 
-set -euo pipefail
+    echo "Fetching $src to $out"
 
-src=$(realpath $1)
-out=$(realpath $2)
+    mkdir $out
 
-echo "Fetching $src to $out"
+    # Configure cargo to fetch from a local copy of the crates.io registry
 
-mkdir $out
+    echo "Using rust registry from $rustRegistry"
 
-# Configure cargo to fetch from a local copy of the crates.io registry
-
-echo "Using rust registry from $rustRegistry"
-
-cat <<EOF > $out/config
+    cat <<EOF > $out/config
 [registry]
 index = "file://$rustRegistry"
 EOF
 
-export CARGO_HOME=$out
-cd $src
-
-if [[ ! -f Cargo.lock ]]; then
-    echo
-    echo "ERROR: The Cargo.lock file doesn't exist"
-    echo
-    echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
-    echo "when the registry is updated."
-    echo
-
-    exit 1
-fi
-
-# We need to do the following string replacement so that 'cargo fetch'
-# doesn't ignore the versions specified in Cargo.lock
-set +u
-substituteInPlace Cargo.lock \
-    --replace "registry+https://github.com/rust-lang/crates.io-index" \
-              "registry+file://$rustRegistry"
-set -u
-
-# Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
-eval "$cargoUpdateHook"
-
-# Do the fetch
-cargo fetch --verbose
-
-# Now that we have fetched everything, let's make the output deterministic
-
-# Cargo uses the following directory structure for fetched data, where
-# $indexHash is a hash of the registry index URL:
-#
-#
-# /config:
-#
-#     Cargo config file. We'll delete this because it's not deterministic,
-#     and instead recreate it just before running 'cargo build'.
-#
-# /registry/cache/$indexHash/:
-#
-#     This is where tarballs of registry package dependencies are kept
-#     We'll need to keep this, but make sure $indexHash is a fixed name.
-#
-# /registry/index/$indexHash/:
-#
-#     A copy of the registry index is kept here. We can delete this, and
-#     instead, just before running 'cargo build', we'll symlink this
-#     directory to our static copy of the registry in the Nix store.
-#
-# /registry/src/$indexHash/{pkgName-pkgVersion}/:
-#
-#     Here cargo keeps extracted sources of the cached tarballs.
-#     We'll just delete this because cargo will re-populate them from the
-#     tarballs.
-#
-# /git/db/{domain-hash}/:
-#
-#     Here cargo keeps the `.git` directories of git dependencies.
-#     We'll need to keep these, but make them deterministic.
-#
-# /git/checkouts/{domain-hash}/{branchName}/:
-#
-#     Here cargo keeps checked-out sources of the git dependencies.
-#     We can delete this, because cargo will re-populate them from the above
-#     `.git` directories.
-#
-# Let's start
-
-# Remove cargo config file, which points to the ever-changing registry
-rm $out/config
-
-# Save the Cargo.lock file into the output, so that we don't have to do another
-# 'cargo update' during the build (which would try to access the network) for
-# any ad-hoc package updates (through $cargoUpdateHook).
-#
-# We need to replace the rustRegistry URL with something deterministic.
-# Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
-
-set +u
-substituteInPlace Cargo.lock \
-    --replace "registry+file://$rustRegistry" \
-              "registry+file:///dev/null"
-set -u
-mv Cargo.lock $out/
-
-
-# Let's replace $indexHash with something more deterministic
-mv $out/registry/cache/* $out/registry/cache/HASH
-
-# The registry index changes all the time, so it's not deterministic
-# We'll symlink it before running 'cargo build'
-rm -rf $out/registry/index/*
-
-# Make git DBs deterministic
-# TODO: test with git submodules
-[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
-    revs=""
-    cd "$out/git/checkouts/$name"
-    while read dir; do
-        # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
-        branch="${dir:2:$((${#dir}-7))}"
-
-        cd "$out/git/checkouts/$name/$branch"
-        rev="$(git rev-parse HEAD)"
-        revs="$revs $rev"
-    done < <(find . -type d -name .git -print)
-
-    echo "List of revs to keep for git db $name: $revs"
-
-    (
-        # The following code was adapted from nix-prefetch-git
-
-        cd "$out/git/db/$name"
-
-        export GIT_DIR=.
-
-        # Remove all remote branches
-        git branch -r | while read branch; do
-            git branch -rD "$branch" >&2
-        done
-
-        # Remove all tags
-        git tag | while read tag; do
-            git tag -d "$tag" >&2
-        done
-
-        # Remove all local branches
-        branchrefs=()
-        eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
-
-        for branchref in "${branchrefs[@]}"; do
-            git update-ref -d "$branchref" >&2
-        done
-
-        # Create ad-hoc branches for the revs we need
-        echo "$revs" | while read rev; do
-            echo "Creating git branch b_$rev $rev"
-            git branch b_$rev $rev
-        done
-
-        # Remove files that have timestamps or otherwise have non-deterministic
-        # properties.
-        rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
-
-        # Do a full repack. Must run single-threaded, or else we lose determinism.
-        git config pack.threads 1
-        git repack -A -d -f
-        rm -f config
-
-        # Garbage collect unreferenced objects.
-        git gc --prune=all
-    )
-done)
-
-# Remove unneeded outputs
-[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
-[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
-
-# XXX: provide some debugging output to see find out why we are seeing
-# sporadic hash mismatches
-find $out ! -type f
-find $out -type f -exec sha256sum {} +
+    export CARGO_HOME=$out
+    cd $src
+
+    if [[ ! -f Cargo.lock ]]; then
+        echo
+        echo "ERROR: The Cargo.lock file doesn't exist"
+        echo
+        echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
+        echo "when the registry is updated."
+        echo
+
+        exit 1
+    fi
+
+    # We need to do the following string replacement so that 'cargo fetch'
+    # doesn't ignore the versions specified in Cargo.lock
+    substituteInPlace Cargo.lock \
+        --replace "registry+https://github.com/rust-lang/crates.io-index" \
+                  "registry+file://$rustRegistry"
+
+    # Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
+    eval "$cargoUpdateHook"
+
+    # Do the fetch
+    cargo fetch --verbose
+
+    # Now that we have fetched everything, let's make the output deterministic
+
+    # Cargo uses the following directory structure for fetched data, where
+    # $indexHash is a hash of the registry index URL:
+    #
+    #
+    # /config:
+    #
+    #     Cargo config file. We'll delete this because it's not deterministic,
+    #     and instead recreate it just before running 'cargo build'.
+    #
+    # /registry/cache/$indexHash/:
+    #
+    #     This is where tarballs of registry package dependencies are kept
+    #     We'll need to keep this, but make sure $indexHash is a fixed name.
+    #
+    # /registry/index/$indexHash/:
+    #
+    #     A copy of the registry index is kept here. We can delete this, and
+    #     instead, just before running 'cargo build', we'll symlink this
+    #     directory to our static copy of the registry in the Nix store.
+    #
+    # /registry/src/$indexHash/{pkgName-pkgVersion}/:
+    #
+    #     Here cargo keeps extracted sources of the cached tarballs.
+    #     We'll just delete this because cargo will re-populate them from the
+    #     tarballs.
+    #
+    # /git/db/{domain-hash}/:
+    #
+    #     Here cargo keeps the `.git` directories of git dependencies.
+    #     We'll need to keep these, but make them deterministic.
+    #
+    # /git/checkouts/{domain-hash}/{branchName}/:
+    #
+    #     Here cargo keeps checked-out sources of the git dependencies.
+    #     We can delete this, because cargo will re-populate them from the above
+    #     `.git` directories.
+    #
+    # Let's start
+
+    # Remove cargo config file, which points to the ever-changing registry
+    rm $out/config
+
+    # Save the Cargo.lock file into the output, so that we don't have to do another
+    # 'cargo update' during the build (which would try to access the network) for
+    # any ad-hoc package updates (through $cargoUpdateHook).
+    #
+    # We need to replace the rustRegistry URL with something deterministic.
+    # Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
+
+    substituteInPlace Cargo.lock \
+        --replace "registry+file://$rustRegistry" \
+                  "registry+file:///dev/null"
+    mv Cargo.lock $out/
+
+
+    # Let's replace $indexHash with something more deterministic
+    mv $out/registry/cache/* $out/registry/cache/HASH
+
+    # The registry index changes all the time, so it's not deterministic
+    # We'll symlink it before running 'cargo build'
+    rm -rf $out/registry/index/*
+
+    # Make git DBs deterministic
+    # TODO: test with git submodules
+    [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
+        revs=""
+        cd "$out/git/checkouts/$name"
+        while read dir; do
+            # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
+            branch="${dir:2:$((${#dir}-7))}"
+
+            cd "$out/git/checkouts/$name/$branch"
+            rev="$(git rev-parse HEAD)"
+            revs="$revs $rev"
+        done < <(find . -type d -name .git -print)
+
+        echo "List of revs to keep for git db $name: $revs"
+
+        (
+            # The following code was adapted from nix-prefetch-git
+
+            cd "$out/git/db/$name"
+
+            export GIT_DIR=.
+
+            # Remove all remote branches
+            git branch -r | while read branch; do
+                git branch -rD "$branch" >&2
+            done
+
+            # Remove all tags
+            git tag | while read tag; do
+                git tag -d "$tag" >&2
+            done
+
+            # Remove all local branches
+            branchrefs=()
+            eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
+
+            for branchref in "${branchrefs[@]}"; do
+                git update-ref -d "$branchref" >&2
+            done
+
+            # Create ad-hoc branches for the revs we need
+            echo "$revs" | while read rev; do
+                echo "Creating git branch b_$rev $rev"
+                git branch b_$rev $rev
+            done
+
+            # Remove files that have timestamps or otherwise have non-deterministic
+            # properties.
+            rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
+
+            # Do a full repack. Must run single-threaded, or else we lose determinism.
+            git config pack.threads 1
+            git repack -A -d -f
+            rm -f config
+
+            # Garbage collect unreferenced objects.
+            git gc --prune=all
+        )
+    done)
+
+    # Remove unneeded outputs
+    [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
+    [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
+
+    # XXX: provide some debugging output to see find out why we are seeing
+    # sporadic hash mismatches
+    find $out ! -type f
+    find $out -type f -exec sha256sum {} +
+}
diff --git a/pkgs/build-support/rust/fetchcargo.nix b/pkgs/build-support/rust/fetchcargo.nix
index a77540c396f2..518420002622 100644
--- a/pkgs/build-support/rust/fetchcargo.nix
+++ b/pkgs/build-support/rust/fetchcargo.nix
@@ -9,8 +9,11 @@ stdenv.mkDerivation {
   phases = "unpackPhase installPhase";
 
   installPhase = ''
+    source ${./fetch-cargo-deps}
+
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
-    bash ${./fetch-cargo-deps} . "$out"
+
+    fetchCargoDeps . "$out"
   '';
 
   outputHashAlgo = "sha256";