Merge commit '432fc2d9a67f92e05438dff5fdc2b39d33f77997'

# Conflicts:
#	nixpkgs/pkgs/applications/editors/emacs/elisp-packages/elpa-generated.nix
#	nixpkgs/pkgs/applications/networking/mailreaders/thunderbird/default.nix
#	nixpkgs/pkgs/applications/window-managers/sway/default.nix
#	nixpkgs/pkgs/build-support/rust/default.nix
#	nixpkgs/pkgs/development/go-modules/generic/default.nix

4 files changed, 101 insertions, 31 deletions

diff --git a/nixpkgs/nixos/lib/build-vms.nix b/nixpkgs/nixos/lib/build-vms.nix
index ebbb0296bef6..064e44f643b2 100644
--- a/nixpkgs/nixos/lib/build-vms.nix
+++ b/nixpkgs/nixos/lib/build-vms.nix
@@ -36,6 +36,13 @@ rec {
         [ ../modules/virtualisation/qemu-vm.nix
           ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
           { key = "no-manual"; documentation.nixos.enable = false; }
+          { key = "no-revision";
+            # Make the revision metadata constant, in order to avoid needless retesting.
+            # The human version (e.g. 21.05-pre) is left as is, because it is useful
+            # for external modules that test with e.g. nixosTest and rely on that
+            # version number.
+            config.system.nixos.revision = "constant-nixos-revision";
+          }
           { key = "nodes"; _module.args.nodes = nodes; }
         ] ++ optional minimal ../modules/testing/minimal-kernel.nix;
     };
diff --git a/nixpkgs/nixos/lib/make-ext4-fs.nix b/nixpkgs/nixos/lib/make-ext4-fs.nix
index 33dbc8f5ec46..416beeb32f2f 100644
--- a/nixpkgs/nixos/lib/make-ext4-fs.nix
+++ b/nixpkgs/nixos/lib/make-ext4-fs.nix
@@ -74,11 +74,9 @@ pkgs.stdenv.mkDerivation {
         return 1
       fi
 
-      echo "Resizing to minimum allowed size"
-      resize2fs -M $img
-
-      # And a final fsck, because of the previous truncating.
-      fsck.ext4 -n -f $img
+      # We may want to shrink the file system and resize the image to
+      # get rid of the unnecessary slack here--but see
+      # https://github.com/NixOS/nixpkgs/issues/125121 for caveats.
 
       if [ ${builtins.toString compressImage} ]; then
         echo "Compressing image"
diff --git a/nixpkgs/nixos/lib/test-driver/test-driver.py b/nixpkgs/nixos/lib/test-driver/test-driver.py
index 7800a49e4107..9c97ce383437 100644
--- a/nixpkgs/nixos/lib/test-driver/test-driver.py
+++ b/nixpkgs/nixos/lib/test-driver/test-driver.py
@@ -3,6 +3,7 @@ from contextlib import contextmanager, _GeneratorContextManager
 from queue import Queue, Empty
 from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List, Iterable
 from xml.sax.saxutils import XMLGenerator
+from colorama import Style
 import queue
 import io
 import _thread
@@ -20,6 +21,7 @@ import shutil
 import socket
 import subprocess
 import sys
+import telnetlib
 import tempfile
 import time
 import traceback
@@ -127,18 +129,18 @@ def create_vlan(vlan_nr: str) -> Tuple[str, str, "subprocess.Popen[bytes]", Any]
     return (vlan_nr, vde_socket, vde_process, fd)
 
 
-def retry(fn: Callable) -> None:
+def retry(fn: Callable, timeout: int = 900) -> None:
     """Call the given function repeatedly, with 1 second intervals,
     until it returns True or a timeout is reached.
     """
 
-    for _ in range(900):
+    for _ in range(timeout):
         if fn(False):
             return
         time.sleep(1)
 
     if not fn(True):
-        raise Exception("action timed out")
+        raise Exception(f"action timed out after {timeout} seconds")
 
 
 class Logger:
@@ -151,6 +153,8 @@ class Logger:
         self.xml.startDocument()
         self.xml.startElement("logfile", attrs={})
 
+        self._print_serial_logs = True
+
     def close(self) -> None:
         self.xml.endElement("logfile")
         self.xml.endDocument()
@@ -174,15 +178,21 @@ class Logger:
         self.drain_log_queue()
         self.log_line(message, attributes)
 
-    def enqueue(self, message: Dict[str, str]) -> None:
-        self.queue.put(message)
+    def log_serial(self, message: str, machine: str) -> None:
+        self.enqueue({"msg": message, "machine": machine, "type": "serial"})
+        if self._print_serial_logs:
+            eprint(Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL)
+
+    def enqueue(self, item: Dict[str, str]) -> None:
+        self.queue.put(item)
 
     def drain_log_queue(self) -> None:
         try:
             while True:
                 item = self.queue.get_nowait()
-                attributes = {"machine": item["machine"], "type": "serial"}
-                self.log_line(self.sanitise(item["msg"]), attributes)
+                msg = self.sanitise(item["msg"])
+                del item["msg"]
+                self.log_line(msg, item)
         except Empty:
             pass
 
@@ -307,8 +317,9 @@ class Machine:
             start_command += "-cdrom " + args["cdrom"] + " "
 
         if "usb" in args:
+            # https://github.com/qemu/qemu/blob/master/docs/usb2.txt
             start_command += (
-                "-device piix3-usb-uhci -drive "
+                "-device usb-ehci -drive "
                 + "id=usbdisk,file="
                 + args["usb"]
                 + ",if=none,readonly "
@@ -327,6 +338,9 @@ class Machine:
     def log(self, msg: str) -> None:
         self.logger.log(msg, {"machine": self.name})
 
+    def log_serial(self, msg: str) -> None:
+        self.logger.log_serial(msg, self.name)
+
     def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager:
         my_attrs = {"machine": self.name}
         my_attrs.update(attrs)
@@ -427,7 +441,7 @@ class Machine:
     def execute(self, command: str) -> Tuple[int, str]:
         self.connect()
 
-        out_command = "( {} ); echo '|!=EOF' $?\n".format(command)
+        out_command = "( set -euo pipefail; {} ); echo '|!=EOF' $?\n".format(command)
         self.shell.send(out_command.encode())
 
         output = ""
@@ -442,6 +456,16 @@ class Machine:
                 return (status_code, output)
             output += chunk
 
+    def shell_interact(self) -> None:
+        """Allows you to interact with the guest shell
+
+        Should only be used during test development, not in the production test."""
+        self.connect()
+        self.log("Terminal is ready (there is no prompt):")
+        telnet = telnetlib.Telnet()
+        telnet.sock = self.shell  # type: ignore
+        telnet.interact()
+
     def succeed(self, *commands: str) -> str:
         """Execute each command and check that it succeeds."""
         output = ""
@@ -735,6 +759,7 @@ class Machine:
         shell_path = os.path.join(self.state_dir, "shell")
         self.shell_socket = create_socket(shell_path)
 
+        display_available = any(x in os.environ for x in ["DISPLAY", "WAYLAND_DISPLAY"])
         qemu_options = (
             " ".join(
                 [
@@ -744,7 +769,7 @@ class Machine:
                     "-device virtio-serial",
                     "-device virtconsole,chardev=shell",
                     "-device virtio-rng-pci",
-                    "-serial stdio" if "DISPLAY" in os.environ else "-nographic",
+                    "-serial stdio" if display_available else "-nographic",
                 ]
             )
             + " "
@@ -783,8 +808,7 @@ class Machine:
                 # Ignore undecodable bytes that may occur in boot menus
                 line = _line.decode(errors="ignore").replace("\r", "").rstrip()
                 self.last_lines.put(line)
-                eprint("{} # {}".format(self.name, line))
-                self.logger.enqueue({"msg": line, "machine": self.name})
+                self.log_serial(line)
 
         _thread.start_new_thread(process_serial_output, ())
 
@@ -883,7 +907,6 @@ class Machine:
 def create_machine(args: Dict[str, Any]) -> Machine:
     global log
     args["log"] = log
-    args["redirectSerial"] = os.environ.get("USE_SERIAL", "0") == "1"
     return Machine(args)
 
 
@@ -926,6 +949,16 @@ def run_tests() -> None:
             machine.execute("sync")
 
 
+def serial_stdout_on() -> None:
+    global log
+    log._print_serial_logs = True
+
+
+def serial_stdout_off() -> None:
+    global log
+    log._print_serial_logs = False
+
+
 @contextmanager
 def subtest(name: str) -> Iterator[None]:
     with log.nested(name):
diff --git a/nixpkgs/nixos/lib/testing-python.nix b/nixpkgs/nixos/lib/testing-python.nix
index 6192be1cd053..715482e87304 100644
--- a/nixpkgs/nixos/lib/testing-python.nix
+++ b/nixpkgs/nixos/lib/testing-python.nix
@@ -25,13 +25,21 @@ rec {
       name = "nixos-test-driver";
 
       nativeBuildInputs = [ makeWrapper ];
-      buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
+      buildInputs = [ (python3.withPackages (p: [ p.ptpython p.colorama ])) ];
       checkInputs = with python3Packages; [ pylint black mypy ];
 
       dontUnpack = true;
 
       preferLocalBuild = true;
 
+      buildPhase = ''
+        python <<EOF
+        from pydoc import importfile
+        with open('driver-exports', 'w') as fp:
+          fp.write(','.join(dir(importfile('${testDriverScript}'))))
+        EOF
+      '';
+
       doCheck = true;
       checkPhase = ''
         mypy --disallow-untyped-defs \
@@ -50,12 +58,19 @@ rec {
 
           wrapProgram $out/bin/nixos-test-driver \
             --prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \
+
+          install -m 0644 -vD driver-exports $out/nix-support/driver-exports
         '';
     };
 
   # Run an automated test suite in the given virtual network.
-  # `driver' is the script that runs the network.
-  runTests = driver:
+  runTests = {
+    # the script that runs the network
+    driver,
+    # a source position in the format of builtins.unsafeGetAttrPos
+    # for meta.position
+    pos,
+  }:
     stdenv.mkDerivation {
       name = "vm-test-run-${driver.testName}";
 
@@ -68,7 +83,11 @@ rec {
           LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
         '';
 
-      passthru = driver.passthru;
+      passthru = driver.passthru // {
+        inherit driver;
+      };
+
+      inherit pos;
     };
 
 
@@ -79,6 +98,11 @@ rec {
       # Skip linting (mainly intended for faster dev cycles)
     , skipLint ? false
     , passthru ? {}
+    , # For meta.position
+      pos ? # position used in error messages and for meta.position
+        (if t.meta.description or null != null
+          then builtins.unsafeGetAttrPos "description" t.meta
+          else builtins.unsafeGetAttrPos "testScript" t)
     , ...
     } @ t:
     let
@@ -106,7 +130,13 @@ rec {
               {
                 virtualisation.qemu.package = qemu_pkg;
               }
-            );
+            ) ++ [(
+              {
+                # Ensure we do not use aliases. Ideally this is only set
+                # when the test framework is used by Nixpkgs NixOS tests.
+                nixpkgs.config.allowAliases = false;
+              }
+            )];
           };
 
           # FIXME: get this pkg from the module system
@@ -131,12 +161,10 @@ rec {
                   "it's currently ${toString testNameLen} characters long.")
             else
               "nixos-test-driver-${name}";
-
-          warn = if skipLint then lib.warn "Linting is disabled!" else lib.id;
         in
-        warn (runCommand testDriverName
+        lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName
           {
-            buildInputs = [ makeWrapper ];
+            nativeBuildInputs = [ makeWrapper ];
             testScript = testScript';
             preferLocalBuild = true;
             testName = name;
@@ -149,7 +177,10 @@ rec {
 
             echo -n "$testScript" > $out/test-script
             ${lib.optionalString (!skipLint) ''
-              ${python3Packages.black}/bin/black --check --diff $out/test-script
+              PYFLAKES_BUILTINS="$(
+                echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
+                < ${lib.escapeShellArg "${testDriver}/nix-support/driver-exports"}
+              )" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
             ''}
 
             ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
@@ -165,8 +196,7 @@ rec {
               --add-flags "''${vms[*]}" \
               ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \
               --set tests 'start_all(); join_all();' \
-              --set VLANS '${toString vlans}' \
-              ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"}
+              --set VLANS '${toString vlans}'
           ''); # "
 
       passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
@@ -176,13 +206,15 @@ rec {
       driver = mkDriver null;
       driverInteractive = mkDriver pkgs.qemu;
 
-      test = passMeta (runTests driver);
+      test = passMeta (runTests { inherit driver pos; });
 
       nodeNames = builtins.attrNames driver.nodes;
       invalidNodeNames = lib.filter
         (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
         nodeNames;
 
+      nodeHostNames = map (c: c.config.system.name) (lib.attrValues driver.nodes);
+
     in
     if lib.length invalidNodeNames > 0 then
       throw ''