Merge pull request #11279 from rickynils/nixos-rebuild-remote

nixos-rebuild: Add option for building and/or deploying on a remote host

nixos/doc/manual/man-nixos-rebuild.xml

@@ -281,6 +281,51 @@ $ nixos-rebuild switch -p test -I nixos-config=./test.nix
     </listitem>
   </varlistentry>
 
+  <varlistentry>
+    <term><option>--build-host</option></term>
+    <listitem>
+      <para>Instead of building the new configuration locally, use the
+      specified host to perform the build. The host needs to be accessible
+      with ssh, and must be able to perform Nix builds. If the option
+      <option>--target-host</option> is not set, the build will be copied back
+      to the local machine when done.</para>
+
+      <para>Note that, if <option>--no-build-nix</option> is not specified,
+      Nix will be built both locally and remotely. This is because the
+      configuration will always be evaluated locally even though the building
+      might be performed remotely.</para>
+
+      <para>You can include a remote user name in
+      the host name (<replaceable>user@host</replaceable>). You can also set
+      ssh options by defining the <envar>NIX_SSHOPTS</envar> environment
+      variable.</para>
+    </listitem>
+  </varlistentry>
+
+  <varlistentry>
+    <term><option>--target-host</option></term>
+    <listitem>
+      <para>Specifies the NixOS target host. By setting this to something other
+      than <replaceable>localhost</replaceable>, the system activation will
+      happen on the remote host instead of the local machine. The remote host
+      needs to be accessible over ssh, and for the commands
+      <option>switch</option>, <option>boot</option> and <option>test</option>
+      you need root access.</para>
+
+      <para>If <option>--build-host</option> is not explicitly
+      specified, <option>--build-host</option> will implicitly be set to the
+      same value as <option>--target-host</option>. So, if you only specify
+      <option>--target-host</option> both building and activation will take
+      place remotely (and no build artifacts will be copied to the local
+      machine).</para>
+
+      <para>You can include a remote user name in
+      the host name (<replaceable>user@host</replaceable>). You can also set
+      ssh options by defining the <envar>NIX_SSHOPTS</envar> environment
+      variable.</para>
+    </listitem>
+  </varlistentry>
+
 </variablelist>
 
 <para>In addition, <command>nixos-rebuild</command> accepts various
@@ -305,6 +350,13 @@ the Nix manual for details.</para>
     </listitem>
   </varlistentry>
 
+  <varlistentry><term><envar>NIX_SSHOPTS</envar></term>
+
+    <listitem><para>Additional options to be passed to
+    <command>ssh</command> on the command line.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </refsection>

nixos/modules/installer/tools/nixos-rebuild.sh

@@ -19,6 +19,8 @@ rollback=
 upgrade=
 repair=
 profile=/nix/var/nix/profiles/system
+buildHost=
+targetHost=
 
 while [ "$#" -gt 0 ]; do
     i="$1"; shift 1
@@ -73,6 +75,14 @@ while [ "$#" -gt 0 ]; do
         fi
         shift 1
         ;;
+      --build-host|h)
+        buildHost="$1"
+        shift 1
+        ;;
+      --target-host|t)
+        targetHost="$1"
+        shift 1
+        ;;
       *)
         echo "$0: unknown option \`$i'"
         exit 1
@@ -80,6 +90,90 @@ while [ "$#" -gt 0 ]; do
     esac
 done
 
+
+if [ -z "$buildHost" -a -n "$targetHost" ]; then
+    buildHost="$targetHost"
+fi
+if [ "$targetHost" = localhost ]; then
+    targetHost=
+fi
+if [ "$buildHost" = localhost ]; then
+    buildHost=
+fi
+
+buildHostCmd() {
+    if [ -z "$buildHost" ]; then
+        "$@"
+    elif [ -n "$remoteNix" ]; then
+        ssh $SSHOPTS "$buildHost" PATH="$remoteNix:$PATH" "$@"
+    else
+        ssh $SSHOPTS "$buildHost" "$@"
+    fi
+}
+
+targetHostCmd() {
+    if [ -z "$targetHost" ]; then
+        "$@"
+    else
+        ssh $SSHOPTS "$targetHost" "$@"
+    fi
+}
+
+copyToTarget() {
+    if ! [ "$targetHost" = "$buildHost" ]; then
+        if [ -z "$targetHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --from "$buildHost" "$1"
+        elif [ -z "$buildHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$targetHost" "$1"
+        else
+            buildHostCmd nix-copy-closure --to "$targetHost" "$1"
+        fi
+    fi
+}
+
+nixBuild() {
+    if [ -z "$buildHost" ]; then
+        nix-build "$@"
+    else
+        local instArgs=()
+        local buildArgs=()
+
+        while [ "$#" -gt 0 ]; do
+            local i="$1"; shift 1
+            case "$i" in
+              -o)
+                local out="$1"; shift 1
+                buildArgs+=("--add-root" "$out" "--indirect")
+                ;;
+              -A)
+                local j="$1"; shift 1
+                instArgs+=("$i" "$j")
+                ;;
+              -I)
+                # We don't want this in buildArgs
+                shift 1
+                ;;
+              "<"*) # nix paths
+                instArgs+=("$i")
+                ;;
+              *)
+                buildArgs+=("$i")
+                ;;
+            esac
+        done
+
+        local drv="$(nix-instantiate "${instArgs[@]}" "${extraBuildFlags[@]}")"
+        if [ -a "$drv" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$buildHost" "$drv"
+            buildHostCmd nix-store -r "$drv" "${buildArgs[@]}"
+        else
+            echo "nix-instantiate failed"
+            exit 1
+        fi
+  fi
+}
+
+
 if [ -z "$action" ]; then showSyntax; fi
 
 # Only run shell scripts from the Nixpkgs tree if the action is
@@ -128,7 +222,16 @@ fi
 
 
 tmpDir=$(mktemp -t -d nixos-rebuild.XXXXXX)
-trap 'rm -rf "$tmpDir"' EXIT
+SSHOPTS="$NIX_SSHOPTS -o ControlMaster=auto -o ControlPath=$tmpDir/ssh-%n -o ControlPersist=60"
+
+cleanup() {
+    for ctrl in "$tmpDir"/ssh-*; do
+        ssh -o ControlPath="$ctrl" -O exit dummyhost 2>/dev/null || true
+    done
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT
+
 
 
 # If the Nix daemon is running, then use it.  This allows us to use
@@ -150,30 +253,56 @@ if [ -n "$rollback" -o "$action" = dry-build ]; then
     buildNix=
 fi
 
-if [ -n "$buildNix" ]; then
+prebuiltNix() {
-    echo "building Nix..." >&2
+    machine="$1"
-    if ! nix-build '<nixpkgs/nixos>' -A config.nix.package -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-        if ! nix-build '<nixpkgs/nixos>' -A nixFallback -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-            if ! nix-build '<nixpkgs>' -A nix -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-                machine="$(uname -m)"
     if [ "$machine" = x86_64 ]; then
-                    nixStorePath=/nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
+        return /nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
     elif [[ "$machine" =~ i.86 ]]; then
-                    nixStorePath=/nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
+        return /nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
     else
         echo "$0: unsupported platform"
         exit 1
     fi
+}
+
+remotePATH=
+
+if [ -n "$buildNix" ]; then
+    echo "building Nix..." >&2
+    nixDrv=
+    if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nixdrv --indirect -A config.nix.package "${extraBuildFlags[@]}")"; then
+        if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nixdrv --indirect -A nixFallback "${extraBuildFlags[@]}")"; then
+            if ! nixDrv="$(nix-instantiate '<nixpkgs>' --add-root $tmpDir/nixdrv --indirect -A nix "${extraBuildFlags[@]}")"; then
+                nixStorePath="$(prebuiltNix "$(uname -m)")"
                 if ! nix-store -r $nixStorePath --add-root $tmpDir/nix --indirect \
                     --option extra-binary-caches https://cache.nixos.org/; then
                     echo "warning: don't know how to get latest Nix" >&2
                 fi
                 # Older version of nix-store -r don't support --add-root.
                 [ -e $tmpDir/nix ] || ln -sf $nixStorePath $tmpDir/nix
+                if [ -n "$buildHost" ]; then
+                    remoteNixStorePath="$(prebuiltNix "$(buildHostCmd uname -m)")"
+                    remoteNix="$remoteNixStorePath/bin"
+                    if ! buildHostCmd nix-store -r $remoteNixStorePath \
+                      --option extra-binary-caches https://cache.nixos.org/ >/dev/null; then
+                        remoteNix=
+                        echo "warning: don't know how to get latest Nix" >&2
                     fi
                 fi
             fi
-    PATH=$tmpDir/nix/bin:$PATH
+        fi
+    fi
+    if [ -a "$nixDrv" ]; then
+        nix-store -r "$nixDrv" --add-root $tmpDir/nix --indirect >/dev/null
+        if [ -n "$buildHost" ]; then
+            nix-copy-closure --to "$buildHost" "$nixDrv"
+            # The nix build produces multiple outputs, we add them all to the remote path
+            for p in $(buildHostCmd nix-store -r "$(readlink "$nixDrv")" "${buildArgs[@]}"); do
+                remoteNix="$remoteNix${remoteNix:+:}$p/bin"
+            done
+        fi
+    fi
+    PATH="$tmpDir/nix/bin:$PATH"
 fi
 
 
@@ -200,31 +329,35 @@ fi
 if [ -z "$rollback" ]; then
     echo "building the system configuration..." >&2
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env "${extraBuildFlags[@]}" -p "$profile" -f '<nixpkgs/nixos>' --set -A system
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system "${extraBuildFlags[@]}")"
-        pathToConfig="$profile"
+        copyToTarget "$pathToConfig"
+        targetHostCmd nix-env -p "$profile" --set "$pathToConfig"
     elif [ "$action" = test -o "$action" = build -o "$action" = dry-build -o "$action" = dry-activate ]; then
-        nix-build '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}" > /dev/null
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}")"
-        pathToConfig=./result
     elif [ "$action" = build-vm ]; then
-        nix-build '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}" > /dev/null
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}")"
-        pathToConfig=./result
     elif [ "$action" = build-vm-with-bootloader ]; then
-        nix-build '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}" > /dev/null
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}")"
-        pathToConfig=./result
     else
         showSyntax
     fi
+    # Copy build to target host if we haven't already done it
+    if ! [ "$action" = switch -o "$action" = boot ]; then
+        copyToTarget "$pathToConfig"
+    fi
 else # [ -n "$rollback" ]
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env --rollback -p "$profile"
+        targetHostCmd nix-env --rollback -p "$profile"
         pathToConfig="$profile"
     elif [ "$action" = test -o "$action" = build ]; then
         systemNumber=$(
-            nix-env -p "$profile" --list-generations |
+            targetHostCmd nix-env -p "$profile" --list-generations |
             sed -n '/current/ {g; p;}; s/ *\([0-9]*\).*/\1/; h'
         )
-        ln -sT "$profile"-${systemNumber}-link ./result
+        pathToConfig="$profile"-${systemNumber}-link
-        pathToConfig=./result
+        if [ -z "$targetHost" ]; then
+            ln -sT "$pathToConfig" ./result
+        fi
     else
         showSyntax
     fi
@@ -234,7 +367,7 @@ fi
 # If we're not just building, then make the new configuration the boot
 # default and/or activate it now.
 if [ "$action" = switch -o "$action" = boot -o "$action" = test -o "$action" = dry-activate ]; then
-    if ! $pathToConfig/bin/switch-to-configuration "$action"; then
+    if ! targetHostCmd $pathToConfig/bin/switch-to-configuration "$action"; then
         echo "warning: error(s) occurred while switching to the new configuration" >&2
         exit 1
     fi