Merge pull request #12324 from rickynils/nixos-rebuild-remote-try2

Fix NixOS installer tests failures introduced by nixos-rebuild changes

1 files changed, 160 insertions, 27 deletions

diff --git a/nixos/modules/installer/tools/nixos-rebuild.sh b/nixos/modules/installer/tools/nixos-rebuild.sh
index 6792690b4c3b..105d1cd16252 100644
--- a/nixos/modules/installer/tools/nixos-rebuild.sh
+++ b/nixos/modules/installer/tools/nixos-rebuild.sh
@@ -19,6 +19,8 @@ rollback=
 upgrade=
 repair=
 profile=/nix/var/nix/profiles/system
+buildHost=
+targetHost=
 
 while [ "$#" -gt 0 ]; do
     i="$1"; shift 1
@@ -73,6 +75,14 @@ while [ "$#" -gt 0 ]; do
         fi
         shift 1
         ;;
+      --build-host|h)
+        buildHost="$1"
+        shift 1
+        ;;
+      --target-host|t)
+        targetHost="$1"
+        shift 1
+        ;;
       *)
         echo "$0: unknown option \`$i'"
         exit 1
@@ -80,6 +90,90 @@ while [ "$#" -gt 0 ]; do
     esac
 done
 
+
+if [ -z "$buildHost" -a -n "$targetHost" ]; then
+    buildHost="$targetHost"
+fi
+if [ "$targetHost" = localhost ]; then
+    targetHost=
+fi
+if [ "$buildHost" = localhost ]; then
+    buildHost=
+fi
+
+buildHostCmd() {
+    if [ -z "$buildHost" ]; then
+        "$@"
+    elif [ -n "$remoteNix" ]; then
+        ssh $SSHOPTS "$buildHost" PATH="$remoteNix:$PATH" "$@"
+    else
+        ssh $SSHOPTS "$buildHost" "$@"
+    fi
+}
+
+targetHostCmd() {
+    if [ -z "$targetHost" ]; then
+        "$@"
+    else
+        ssh $SSHOPTS "$targetHost" "$@"
+    fi
+}
+
+copyToTarget() {
+    if ! [ "$targetHost" = "$buildHost" ]; then
+        if [ -z "$targetHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --from "$buildHost" "$1"
+        elif [ -z "$buildHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$targetHost" "$1"
+        else
+            buildHostCmd nix-copy-closure --to "$targetHost" "$1"
+        fi
+    fi
+}
+
+nixBuild() {
+    if [ -z "$buildHost" ]; then
+        nix-build "$@"
+    else
+        local instArgs=()
+        local buildArgs=()
+
+        while [ "$#" -gt 0 ]; do
+            local i="$1"; shift 1
+            case "$i" in
+              -o)
+                local out="$1"; shift 1
+                buildArgs+=("--add-root" "$out" "--indirect")
+                ;;
+              -A)
+                local j="$1"; shift 1
+                instArgs+=("$i" "$j")
+                ;;
+              -I)
+                # We don't want this in buildArgs
+                shift 1
+                ;;
+              "<"*) # nix paths
+                instArgs+=("$i")
+                ;;
+              *)
+                buildArgs+=("$i")
+                ;;
+            esac
+        done
+
+        local drv="$(nix-instantiate "${instArgs[@]}" "${extraBuildFlags[@]}")"
+        if [ -a "$drv" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$buildHost" "$drv"
+            buildHostCmd nix-store -r "$drv" "${buildArgs[@]}"
+        else
+            echo "nix-instantiate failed"
+            exit 1
+        fi
+  fi
+}
+
+
 if [ -z "$action" ]; then showSyntax; fi
 
 # Only run shell scripts from the Nixpkgs tree if the action is
@@ -128,7 +222,16 @@ fi
 
 
 tmpDir=$(mktemp -t -d nixos-rebuild.XXXXXX)
-trap 'rm -rf "$tmpDir"' EXIT
+SSHOPTS="$NIX_SSHOPTS -o ControlMaster=auto -o ControlPath=$tmpDir/ssh-%n -o ControlPersist=60"
+
+cleanup() {
+    for ctrl in "$tmpDir"/ssh-*; do
+        ssh -o ControlPath="$ctrl" -O exit dummyhost 2>/dev/null || true
+    done
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT
+
 
 
 # If the Nix daemon is running, then use it.  This allows us to use
@@ -150,30 +253,56 @@ if [ -n "$rollback" -o "$action" = dry-build ]; then
     buildNix=
 fi
 
+prebuiltNix() {
+    machine="$1"
+    if [ "$machine" = x86_64 ]; then
+        return /nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
+    elif [[ "$machine" =~ i.86 ]]; then
+        return /nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
+    else
+        echo "$0: unsupported platform"
+        exit 1
+    fi
+}
+
+remotePATH=
+
 if [ -n "$buildNix" ]; then
     echo "building Nix..." >&2
-    if ! nix-build '<nixpkgs/nixos>' -A config.nix.package -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-        if ! nix-build '<nixpkgs/nixos>' -A nixFallback -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-            if ! nix-build '<nixpkgs>' -A nix -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-                machine="$(uname -m)"
-                if [ "$machine" = x86_64 ]; then
-                    nixStorePath=/nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
-                elif [[ "$machine" =~ i.86 ]]; then
-                    nixStorePath=/nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
-                else
-                    echo "$0: unsupported platform"
-                    exit 1
-                fi
+    nixDrv=
+    if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nix.drv --indirect -A config.nix.package "${extraBuildFlags[@]}")"; then
+        if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nix.drv --indirect -A nixFallback "${extraBuildFlags[@]}")"; then
+            if ! nixDrv="$(nix-instantiate '<nixpkgs>' --add-root $tmpDir/nix.drv --indirect -A nix "${extraBuildFlags[@]}")"; then
+                nixStorePath="$(prebuiltNix "$(uname -m)")"
                 if ! nix-store -r $nixStorePath --add-root $tmpDir/nix --indirect \
                     --option extra-binary-caches https://cache.nixos.org/; then
                     echo "warning: don't know how to get latest Nix" >&2
                 fi
                 # Older version of nix-store -r don't support --add-root.
                 [ -e $tmpDir/nix ] || ln -sf $nixStorePath $tmpDir/nix
+                if [ -n "$buildHost" ]; then
+                    remoteNixStorePath="$(prebuiltNix "$(buildHostCmd uname -m)")"
+                    remoteNix="$remoteNixStorePath/bin"
+                    if ! buildHostCmd nix-store -r $remoteNixStorePath \
+                      --option extra-binary-caches https://cache.nixos.org/ >/dev/null; then
+                        remoteNix=
+                        echo "warning: don't know how to get latest Nix" >&2
+                    fi
+                fi
             fi
         fi
     fi
-    PATH=$tmpDir/nix/bin:$PATH
+    if [ -a "$nixDrv" ]; then
+        nix-store -r "$nixDrv"'!'"out" --add-root $tmpDir/nix --indirect >/dev/null
+        if [ -n "$buildHost" ]; then
+            nix-copy-closure --to "$buildHost" "$nixDrv"
+            # The nix build produces multiple outputs, we add them all to the remote path
+            for p in $(buildHostCmd nix-store -r "$(readlink "$nixDrv")" "${buildArgs[@]}"); do
+                remoteNix="$remoteNix${remoteNix:+:}$p/bin"
+            done
+        fi
+    fi
+    PATH="$tmpDir/nix/bin:$PATH"
 fi
 
 
@@ -200,31 +329,35 @@ fi
 if [ -z "$rollback" ]; then
     echo "building the system configuration..." >&2
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env "${extraBuildFlags[@]}" -p "$profile" -f '<nixpkgs/nixos>' --set -A system
-        pathToConfig="$profile"
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system "${extraBuildFlags[@]}")"
+        copyToTarget "$pathToConfig"
+        targetHostCmd nix-env -p "$profile" --set "$pathToConfig"
     elif [ "$action" = test -o "$action" = build -o "$action" = dry-build -o "$action" = dry-activate ]; then
-        nix-build '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}")"
     elif [ "$action" = build-vm ]; then
-        nix-build '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}")"
     elif [ "$action" = build-vm-with-bootloader ]; then
-        nix-build '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}")"
     else
         showSyntax
     fi
+    # Copy build to target host if we haven't already done it
+    if ! [ "$action" = switch -o "$action" = boot ]; then
+        copyToTarget "$pathToConfig"
+    fi
 else # [ -n "$rollback" ]
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env --rollback -p "$profile"
+        targetHostCmd nix-env --rollback -p "$profile"
         pathToConfig="$profile"
     elif [ "$action" = test -o "$action" = build ]; then
         systemNumber=$(
-            nix-env -p "$profile" --list-generations |
+            targetHostCmd nix-env -p "$profile" --list-generations |
             sed -n '/current/ {g; p;}; s/ *\([0-9]*\).*/\1/; h'
         )
-        ln -sT "$profile"-${systemNumber}-link ./result
-        pathToConfig=./result
+        pathToConfig="$profile"-${systemNumber}-link
+        if [ -z "$targetHost" ]; then
+            ln -sT "$pathToConfig" ./result
+        fi
     else
         showSyntax
     fi
@@ -234,7 +367,7 @@ fi
 # If we're not just building, then make the new configuration the boot
 # default and/or activate it now.
 if [ "$action" = switch -o "$action" = boot -o "$action" = test -o "$action" = dry-activate ]; then
-    if ! $pathToConfig/bin/switch-to-configuration "$action"; then
+    if ! targetHostCmd $pathToConfig/bin/switch-to-configuration "$action"; then
         echo "warning: error(s) occurred while switching to the new configuration" >&2
         exit 1
     fi