Merge commit 'a2e06fc3423c4be53181b15c28dfbe0bcf67dd73'

8 files changed, 76 insertions, 37 deletions

diff --git a/nixpkgs/nixos/lib/eval-config.nix b/nixpkgs/nixos/lib/eval-config.nix
index 77490ca3762a..9892d6f160f7 100644
--- a/nixpkgs/nixos/lib/eval-config.nix
+++ b/nixpkgs/nixos/lib/eval-config.nix
@@ -41,6 +41,12 @@ let
       # default to the argument. That way this new default could propagate all
       # they way through, but has the last priority behind everything else.
       nixpkgs.system = lib.mkDefault system;
+
+      # Stash the value of the `system` argument. When using `nesting.children`
+      # we want to have the same default value behavior (immediately above)
+      # without any interference from the user's configuration.
+      nixpkgs.initialSystem = system;
+
       _module.args.pkgs = lib.mkIf (pkgs_ != null) (lib.mkForce pkgs_);
     };
   };
diff --git a/nixpkgs/nixos/lib/make-ext4-fs.nix b/nixpkgs/nixos/lib/make-ext4-fs.nix
index f46d3990c06b..627ac324cf57 100644
--- a/nixpkgs/nixos/lib/make-ext4-fs.nix
+++ b/nixpkgs/nixos/lib/make-ext4-fs.nix
@@ -64,7 +64,7 @@ pkgs.stdenv.mkDerivation {
       echo "copying files to image..."
       cptofs -t ext4 -i $img ./files/* /
 
-
+      export EXT2FS_NO_MTAB_OK=yes
       # I have ended up with corrupted images sometimes, I suspect that happens when the build machine's disk gets full during the build.
       if ! fsck.ext4 -n -f $img; then
         echo "--- Fsck failed for EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks) ---"
@@ -72,21 +72,8 @@ pkgs.stdenv.mkDerivation {
         return 1
       fi
 
-      (
-        # Resizes **snugly** to its actual limits (or closer to)
-        free=$(dumpe2fs $img | grep '^Free blocks:')
-        blocksize=$(dumpe2fs $img | grep '^Block size:')
-        blocks=$(dumpe2fs $img | grep '^Block count:')
-        blocks=$((''${blocks##*:})) # format the number.
-        blocksize=$((''${blocksize##*:})) # format the number.
-        # System can't boot with 0 blocks free.
-        # Add 16MiB of free space
-        fudge=$(( 16 * 1024 * 1024 / blocksize ))
-        size=$(( blocks - ''${free##*:} + fudge ))
-
-        echo "Resizing from $blocks blocks to $size blocks. (~ $((size*blocksize/1024/1024))MiB)"
-        EXT2FS_NO_MTAB_OK=yes resize2fs $img -f $size
-      )
+      echo "Resizing to minimum allowed size"
+      resize2fs -M $img
 
       # And a final fsck, because of the previous truncating.
       fsck.ext4 -n -f $img
diff --git a/nixpkgs/nixos/lib/qemu-flags.nix b/nixpkgs/nixos/lib/qemu-flags.nix
index 774f66b4804e..859d9e975fec 100644
--- a/nixpkgs/nixos/lib/qemu-flags.nix
+++ b/nixpkgs/nixos/lib/qemu-flags.nix
@@ -17,9 +17,9 @@ in
         else throw "Unknown QEMU serial device for system '${pkgs.stdenv.hostPlatform.system}'";
 
   qemuBinary = qemuPkg: {
-    x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu kvm64";
+    x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu host";
     armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -enable-kvm -machine virt -cpu host";
     aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -enable-kvm -machine virt,gic-version=host -cpu host";
-    x86_64-darwin = "${qemuPkg}/bin/qemu-kvm -cpu kvm64";
+    x86_64-darwin = "${qemuPkg}/bin/qemu-kvm -cpu host";
   }.${pkgs.stdenv.hostPlatform.system} or "${qemuPkg}/bin/qemu-kvm";
 }
diff --git a/nixpkgs/nixos/lib/test-driver/test-driver.py b/nixpkgs/nixos/lib/test-driver/test-driver.py
index cf204a2619f5..c27947bc610d 100644
--- a/nixpkgs/nixos/lib/test-driver/test-driver.py
+++ b/nixpkgs/nixos/lib/test-driver/test-driver.py
@@ -1,13 +1,17 @@
 #! /somewhere/python3
 from contextlib import contextmanager, _GeneratorContextManager
+from queue import Queue, Empty
+from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List
 from xml.sax.saxutils import XMLGenerator
 import _thread
 import atexit
+import base64
 import os
+import pathlib
 import ptpython.repl
 import pty
-from queue import Queue, Empty
 import re
+import shlex
 import shutil
 import socket
 import subprocess
@@ -15,9 +19,6 @@ import sys
 import tempfile
 import time
 import unicodedata
-from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List
-import shlex
-import pathlib
 
 CHAR_TO_KEY = {
     "A": "shift-a",
@@ -395,7 +396,7 @@ class Machine:
         status_code_pattern = re.compile(r"(.*)\|\!EOF\s+(\d+)")
 
         while True:
-            chunk = self.shell.recv(4096).decode()
+            chunk = self.shell.recv(4096).decode(errors="ignore")
             match = status_code_pattern.match(chunk)
             if match:
                 output += match[1]
@@ -566,6 +567,41 @@ class Machine:
             if ret.returncode != 0:
                 raise Exception("Cannot convert screenshot")
 
+    def copy_from_host_via_shell(self, source: str, target: str) -> None:
+        """Copy a file from the host into the guest by piping it over the
+        shell into the destination file. Works without host-guest shared folder.
+        Prefer copy_from_host for whenever possible.
+        """
+        with open(source, "rb") as fh:
+            content_b64 = base64.b64encode(fh.read()).decode()
+            self.succeed(
+                f"mkdir -p $(dirname {target})",
+                f"echo -n {content_b64} | base64 -d > {target}",
+            )
+
+    def copy_from_host(self, source: str, target: str) -> None:
+        """Copy a file from the host into the guest via the `shared_dir` shared
+        among all the VMs (using a temporary directory).
+        """
+        host_src = pathlib.Path(source)
+        vm_target = pathlib.Path(target)
+        with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td:
+            shared_temp = pathlib.Path(shared_td)
+            host_intermediate = shared_temp / host_src.name
+            vm_shared_temp = pathlib.Path("/tmp/shared") / shared_temp.name
+            vm_intermediate = vm_shared_temp / host_src.name
+
+            self.succeed(make_command(["mkdir", "-p", vm_shared_temp]))
+            if host_src.is_dir():
+                shutil.copytree(host_src, host_intermediate)
+            else:
+                shutil.copy(host_src, host_intermediate)
+            self.succeed("sync")
+            self.succeed(make_command(["mkdir", "-p", vm_target.parent]))
+            self.succeed(make_command(["cp", "-r", vm_intermediate, vm_target]))
+        # Make sure the cleanup is synced into VM
+        self.succeed("sync")
+
     def copy_from_vm(self, source: str, target_dir: str = "") -> None:
         """Copy a file from the VM (specified by an in-VM source path) to a path
         relative to `$out`. The file is copied via the `shared_dir` shared among
@@ -875,7 +911,7 @@ def subtest(name: str) -> Iterator[None]:
 if __name__ == "__main__":
     log = Logger()
 
-    vlan_nrs = list(dict.fromkeys(os.environ["VLANS"].split()))
+    vlan_nrs = list(dict.fromkeys(os.environ.get("VLANS", "").split()))
     vde_sockets = [create_vlan(v) for v in vlan_nrs]
     for nr, vde_socket, _, _ in vde_sockets:
         os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
diff --git a/nixpkgs/nixos/lib/testing-python.nix b/nixpkgs/nixos/lib/testing-python.nix
index a7f6d7926514..3891adc10435 100644
--- a/nixpkgs/nixos/lib/testing-python.nix
+++ b/nixpkgs/nixos/lib/testing-python.nix
@@ -175,13 +175,13 @@ in rec {
 
       nodeNames = builtins.attrNames nodes;
       invalidNodeNames = lib.filter
-        (node: builtins.match "^[A-z_][A-z0-9_]+$" node == null) nodeNames;
+        (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) nodeNames;
 
     in
       if lib.length invalidNodeNames > 0 then
         throw ''
           Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
-          All machines are referenced as perl variables in the testing framework which will break the
+          All machines are referenced as python variables in the testing framework which will break the
           script when special characters are used.
 
           Please stick to alphanumeric chars and underscores as separation.
@@ -218,12 +218,12 @@ in rec {
       '';
 
       testScript = ''
-        startAll;
-        $client->waitForUnit("multi-user.target");
+        start_all()
+        client.wait_for_unit("multi-user.target")
         ${preBuild}
-        $client->succeed("env -i ${bash}/bin/bash ${buildrunner} /tmp/xchg/saved-env >&2");
+        client.succeed("env -i ${bash}/bin/bash ${buildrunner} /tmp/xchg/saved-env >&2")
         ${postBuild}
-        $client->succeed("sync"); # flush all data before pulling the plug
+        client.succeed("sync") # flush all data before pulling the plug
       '';
 
       vmRunCommand = writeText "vm-run" ''
@@ -263,9 +263,12 @@ in rec {
         { ... }:
         {
           inherit require;
+          imports = [
+            ../tests/common/auto.nix
+          ];
           virtualisation.memorySize = 1024;
           services.xserver.enable = true;
-          services.xserver.displayManager.auto.enable = true;
+          test-support.displayManager.auto.enable = true;
           services.xserver.displayManager.defaultSession = "none+icewm";
           services.xserver.windowManager.icewm.enable = true;
         };
@@ -274,7 +277,7 @@ in rec {
         machine = client;
         preBuild =
           ''
-            $client->waitForX;
+            client.wait_for_x()
           '';
       } // args);
 
diff --git a/nixpkgs/nixos/lib/testing.nix b/nixpkgs/nixos/lib/testing.nix
index ae8ecd6270ce..7d6a5c0a2900 100644
--- a/nixpkgs/nixos/lib/testing.nix
+++ b/nixpkgs/nixos/lib/testing.nix
@@ -19,7 +19,11 @@ in rec {
   inherit pkgs;
 
 
-  testDriver = stdenv.mkDerivation {
+  testDriver = lib.warn ''
+    Perl VM tests are deprecated and will be removed for 20.09.
+    Please update your tests to use the python test driver.
+    See https://github.com/NixOS/nixpkgs/pull/71684 for details.
+  '' stdenv.mkDerivation {
     name = "nixos-test-driver";
 
     buildInputs = [ makeWrapper perl ];
@@ -246,9 +250,12 @@ in rec {
         { ... }:
         {
           inherit require;
+          imports = [
+            ../tests/common/auto.nix
+          ];
           virtualisation.memorySize = 1024;
           services.xserver.enable = true;
-          services.xserver.displayManager.auto.enable = true;
+          test-support.displayManager.auto.enable = true;
           services.xserver.displayManager.defaultSession = "none+icewm";
           services.xserver.windowManager.icewm.enable = true;
         };
diff --git a/nixpkgs/nixos/lib/testing/jquery-ui.nix b/nixpkgs/nixos/lib/testing/jquery-ui.nix
index e65107a3c2fb..abd59da2d285 100644
--- a/nixpkgs/nixos/lib/testing/jquery-ui.nix
+++ b/nixpkgs/nixos/lib/testing/jquery-ui.nix
@@ -4,7 +4,7 @@ stdenv.mkDerivation rec {
   name = "jquery-ui-1.11.4";
 
   src = fetchurl {
-    url = "http://jqueryui.com/resources/download/${name}.zip";
+    url = "https://jqueryui.com/resources/download/${name}.zip";
     sha256 = "0ciyaj1acg08g8hpzqx6whayq206fvf4whksz2pjgxlv207lqgjh";
   };
 
@@ -17,7 +17,7 @@ stdenv.mkDerivation rec {
     '';
 
   meta = {
-    homepage = http://jqueryui.com/;
+    homepage = https://jqueryui.com/;
     description = "A library of JavaScript widgets and effects";
     platforms = stdenv.lib.platforms.all;
   };
diff --git a/nixpkgs/nixos/lib/utils.nix b/nixpkgs/nixos/lib/utils.nix
index a522834e4294..21f4c7c6988f 100644
--- a/nixpkgs/nixos/lib/utils.nix
+++ b/nixpkgs/nixos/lib/utils.nix
@@ -14,7 +14,7 @@ rec {
   # becomes dev-xyzzy.  FIXME: slow.
   escapeSystemdPath = s:
    replaceChars ["/" "-" " "] ["-" "\\x2d" "\\x20"]
-    (if hasPrefix "/" s then substring 1 (stringLength s) s else s);
+   (removePrefix "/" s);
 
   # Returns a system path for a given shell package
   toShellPath = shell: