Merge master into staging-next

14 files changed, 131 insertions, 56 deletions

diff --git a/nixos/doc/manual/configuration/modularity.xml b/nixos/doc/manual/configuration/modularity.xml
index cda36eba25c1..5ff5bc22c85e 100644
--- a/nixos/doc/manual/configuration/modularity.xml
+++ b/nixos/doc/manual/configuration/modularity.xml
@@ -113,12 +113,10 @@ $ nixos-option <xref linkend="opt-boot.kernelModules"/>
 [ "tun" "ipv6" "loop" <replaceable>...</replaceable> ]
 </screen>
   Interactive exploration of the configuration is possible using
-  <command
-xlink:href="https://github.com/edolstra/nix-repl">nix-repl</command>,
-  a read-eval-print loop for Nix expressions. It’s not installed by default;
-  run <literal>nix-env -i nix-repl</literal> to get it. A typical use:
+  <command>nix repl</command>, a read-eval-print loop for Nix expressions.
+  A typical use:
 <screen>
-$ nix-repl '&lt;nixpkgs/nixos>'
+$ nix repl '&lt;nixpkgs/nixos>'
 
 nix-repl> config.<xref linkend="opt-networking.hostName"/>
 "mandark"
diff --git a/nixos/lib/make-ext4-fs.nix b/nixos/lib/make-ext4-fs.nix
index 88be8b73ab37..694142a5123a 100644
--- a/nixos/lib/make-ext4-fs.nix
+++ b/nixos/lib/make-ext4-fs.nix
@@ -94,5 +94,24 @@ pkgs.stdenv.mkDerivation {
         cat errorlog
         return 1
       fi
+
+      (
+        # Resizes **snugly** to its actual limits (or closer to)
+        free=$(dumpe2fs $out | grep '^Free blocks:')
+        blocksize=$(dumpe2fs $out | grep '^Block size:')
+        blocks=$(dumpe2fs $out | grep '^Block count:')
+        blocks=$((''${blocks##*:})) # format the number.
+        blocksize=$((''${blocksize##*:})) # format the number.
+        # System can't boot with 0 blocks free.
+        # Add 16MiB of free space
+        fudge=$(( 16 * 1024 * 1024 / blocksize ))
+        size=$(( blocks - ''${free##*:} + fudge ))
+
+        echo "Resizing from $blocks blocks to $size blocks. (~ $((size*blocksize/1024/1024))MiB)"
+        EXT2FS_NO_MTAB_OK=yes resize2fs $out -f $size
+      )
+
+      # And a final fsck, because of the previous truncating.
+      fsck.ext4 -n -f $out
     '';
 }
diff --git a/nixos/modules/services/continuous-integration/jenkins/job-builder.nix b/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
index 861b46a2d642..5d1bfe4ec407 100644
--- a/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
+++ b/nixos/modules/services/continuous-integration/jenkins/job-builder.nix
@@ -42,6 +42,18 @@ in {
         type = types.str;
         description = ''
           User token in Jenkins used to reload config.
+          WARNING: This token will be world readable in the Nix store. To keep
+          it secret, use the <option>accessTokenFile</option> option instead.
+        '';
+      };
+
+      accessTokenFile = mkOption {
+        default = "";
+        type = types.str;
+        example = "/run/keys/jenkins-job-builder-access-token";
+        description = ''
+          File containing the API token for the <option>accessUser</option>
+          user.
         '';
       };
 
@@ -103,6 +115,21 @@ in {
   };
 
   config = mkIf (jenkinsCfg.enable && cfg.enable) {
+    assertions = [
+      { assertion =
+          if cfg.accessUser != ""
+          then (cfg.accessToken != "" && cfg.accessTokenFile == "") ||
+               (cfg.accessToken == "" && cfg.accessTokenFile != "")
+          else true;
+        message = ''
+          One of accessToken and accessTokenFile options must be non-empty
+          strings, but not both. Current values:
+            services.jenkins.jobBuilder.accessToken = "${cfg.accessToken}"
+            services.jenkins.jobBuilder.accessTokenFile = "${cfg.accessTokenFile}"
+        '';
+      }
+    ];
+
     systemd.services.jenkins-job-builder = {
       description = "Jenkins Job Builder Service";
       # JJB can run either before or after jenkins. We chose after, so we can
@@ -128,8 +155,13 @@ in {
           ownerStamp = ".config-xml-managed-by-nixos-jenkins-job-builder";
           reloadScript = ''
             echo "Asking Jenkins to reload config"
-            CRUMB=$(curl -s 'http://${cfg.accessUser}:${cfg.accessToken}@${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-            curl --silent -X POST -H "$CRUMB" http://${cfg.accessUser}:${cfg.accessToken}@${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}/reload
+            curl_opts="--silent --fail --show-error"
+            access_token=${if cfg.accessTokenFile != ""
+                           then "$(cat '${cfg.accessTokenFile}')"
+                           else cfg.accessToken}
+            jenkins_url="http://${cfg.accessUser}:$access_token@${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}"
+            crumb=$(curl $curl_opts "$jenkins_url"'/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+            curl $curl_opts -X POST -H "$crumb" "$jenkins_url"/reload
           '';
         in
           ''
diff --git a/nixos/modules/services/databases/cockroachdb.nix b/nixos/modules/services/databases/cockroachdb.nix
index 8de1e78633d5..e977751b21ef 100644
--- a/nixos/modules/services/databases/cockroachdb.nix
+++ b/nixos/modules/services/databases/cockroachdb.nix
@@ -13,7 +13,7 @@ let
     [ # Basic startup
       "${crdb}/bin/cockroach start"
       "--logtostderr"
-      "--store=${cfg.dataDir}"
+      "--store=/var/lib/cockroachdb"
       (ifNotNull cfg.locality "--locality='${cfg.locality}'")
 
       # WebUI settings
@@ -41,7 +41,7 @@ let
       };
 
       port = mkOption {
-        type = types.int;
+        type = types.port;
         default = defaultPort;
         description = "Port to bind to for ${descr}";
       };
@@ -70,10 +70,12 @@ in
           like datacenter.  The tiers and order must be the same on all nodes.
           Including more tiers is better than including fewer. For example:
 
+          <literal>
               country=us,region=us-west,datacenter=us-west-1b,rack=12
               country=ca,region=ca-east,datacenter=ca-east-2,rack=4
 
               planet=earth,province=manitoba,colo=secondary,power=3
+          </literal>
         '';
       };
 
@@ -83,12 +85,6 @@ in
         description = "The addresses for connecting the node to a cluster.";
       };
 
-      dataDir = mkOption {
-        type = types.path;
-        default = "/var/lib/cockroachdb";
-        description = "Location where CockroachDB stores its table files";
-      };
-
       insecure = mkOption {
         type = types.bool;
         default = false;
@@ -126,9 +122,12 @@ in
           The total size for caches.
 
           This can be a percentage, expressed with a fraction sign or as a
-          decimal-point number, or any bytes-based unit. For example, "25%",
-          "0.25" both represent 25% of the available system memory. The values
-          "1000000000" and "1GB" both represent 1 gigabyte of memory.
+          decimal-point number, or any bytes-based unit. For example,
+          <literal>"25%"</literal>, <literal>"0.25"</literal> both represent
+          25% of the available system memory. The values
+          <literal>"1000000000"</literal> and <literal>"1GB"</literal> both
+          represent 1 gigabyte of memory.
+
         '';
       };
 
@@ -140,9 +139,11 @@ in
           data for SQL queries.
 
           This can be a percentage, expressed with a fraction sign or as a
-          decimal-point number, or any bytes-based unit. For example, "25%",
-          "0.25" both represent 25% of the available system memory. The values
-          "1000000000" and "1GB" both represent 1 gigabyte of memory.
+          decimal-point number, or any bytes-based unit. For example,
+          <literal>"25%"</literal>, <literal>"0.25"</literal> both represent
+          25% of the available system memory. The values
+          <literal>"1000000000"</literal> and <literal>"1GB"</literal> both
+          represent 1 gigabyte of memory.
         '';
       };
 
@@ -193,27 +194,21 @@ in
         requires = [ "time-sync.target" ];
         wantedBy = [ "multi-user.target" ];
 
-        unitConfig.RequiresMountsFor = "${cfg.dataDir}";
-
-        preStart = ''
-          if ! test -e ${cfg.dataDir}; then
-            mkdir -m 0700 -p ${cfg.dataDir}
-            chown -R ${cfg.user} ${cfg.dataDir}
-          fi
-        '';
+        unitConfig.RequiresMountsFor = "/var/lib/cockroachdb";
 
         serviceConfig =
           { ExecStart = startupCommand;
             Type = "notify";
             User = cfg.user;
-            PermissionsStartOnly = true;
+            StateDirectory = "cockroachdb";
+            StateDirectoryMode = "0700";
 
             Restart = "always";
-            TimeoutStopSec="60";
-            RestartSec="10";
-            StandardOutput="syslog";
-            StandardError="syslog";
-            SyslogIdentifier="cockroach";
+
+            # A conservative-ish timeout is alright here, because for Type=notify
+            # cockroach will send systemd pings during startup to keep it alive
+            TimeoutStopSec = 60;
+            RestartSec = 10;
           };
       };
   };
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index 950af848c0f6..5308c9c4ee08 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -127,7 +127,7 @@ let
         serviceConfig.Restart = mkDefault "always";
         serviceConfig.PrivateTmp = mkDefault true;
         serviceConfig.WorkingDirectory = mkDefault /tmp;
-      } serviceOpts ] ++ optional (serviceOpts.serviceConfig.DynamicUser or false) {
+      } serviceOpts ] ++ optional (!(serviceOpts.serviceConfig.DynamicUser or false)) {
         serviceConfig.User = conf.user;
         serviceConfig.Group = conf.group;
       });
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/tor.nix b/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
index 0e2a13c44ab7..e0ae83802425 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/tor.nix
@@ -36,5 +36,10 @@ in
           ${concatStringsSep " \\\n  " cfg.extraFlags}
       '';
     };
+
+    # CPython requires a process to either have $HOME defined or run as a UID
+    # defined in /etc/passwd. The latter is false with DynamicUser, so define a
+    # dummy $HOME. https://bugs.python.org/issue10496
+    environment = { HOME = "/var/empty"; };
   };
 }
diff --git a/nixos/modules/services/networking/wireguard.nix b/nixos/modules/services/networking/wireguard.nix
index 564632a85ae5..41aff1480a05 100644
--- a/nixos/modules/services/networking/wireguard.nix
+++ b/nixos/modules/services/networking/wireguard.nix
@@ -202,7 +202,7 @@ let
         };
 
         script = ''
-          modprobe wireguard
+          ${optionalString (!config.boot.isContainer) "modprobe wireguard"}
 
           ${values.preSetup}
 
diff --git a/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix b/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix
index 4269f6cfb088..e871ae6ff15a 100644
--- a/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix
+++ b/nixos/modules/services/web-servers/apache-httpd/mediawiki.nix
@@ -83,11 +83,11 @@ let
 
   # Unpack Mediawiki and put the config file in its root directory.
   mediawikiRoot = pkgs.stdenv.mkDerivation rec {
-    name= "mediawiki-1.29.1";
+    name= "mediawiki-1.31.1";
 
     src = pkgs.fetchurl {
-      url = "https://releases.wikimedia.org/mediawiki/1.29/${name}.tar.gz";
-      sha256 = "03mpazbxvb011s2nmlw5p6dc43yjgl5yrsilmj1imyykm57bwb3m";
+      url = "https://releases.wikimedia.org/mediawiki/1.31/${name}.tar.gz";
+      sha256 = "13x48clij21cmysjkpnx68vggchrdasqp7b290j87xlfgjhdhnnf";
     };
 
     skins = config.skins;
@@ -111,7 +111,7 @@ let
         sed -i \
         -e 's|/bin/bash|${pkgs.bash}/bin/bash|g' \
         -e 's|/usr/bin/timeout|${pkgs.coreutils}/bin/timeout|g' \
-          $out/includes/limit.sh \
+          $out/includes/shell/limit.sh \
           $out/includes/GlobalFunctions.php
       '';
   };
diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix
index 2fcc0f254256..3dd36f9b12e1 100644
--- a/nixos/modules/virtualisation/containers.nix
+++ b/nixos/modules/virtualisation/containers.nix
@@ -36,7 +36,7 @@ let
         #! ${pkgs.runtimeShell} -e
 
         # Initialise the container side of the veth pair.
-        if [ "$PRIVATE_NETWORK" = 1 ]; then
+        if [ -n "$HOST_ADDRESS" ] || [ -n "$LOCAL_ADDRESS" ]; then
 
           ip link set host0 name eth0
           ip link set dev eth0 up
@@ -85,6 +85,10 @@ let
       cp --remove-destination /etc/resolv.conf "$root/etc/resolv.conf"
 
       if [ "$PRIVATE_NETWORK" = 1 ]; then
+        extraFlags+=" --private-network"
+      fi
+
+      if [ -n "$HOST_ADDRESS" ] || [ -n "$LOCAL_ADDRESS" ]; then
         extraFlags+=" --network-veth"
         if [ -n "$HOST_BRIDGE" ]; then
           extraFlags+=" --network-bridge=$HOST_BRIDGE"
@@ -153,7 +157,7 @@ let
       # Clean up existing machined registration and interfaces.
       machinectl terminate "$INSTANCE" 2> /dev/null || true
 
-      if [ "$PRIVATE_NETWORK" = 1 ]; then
+      if [ -n "$HOST_ADDRESS" ] || [ -n "$LOCAL_ADDRESS" ]; then
         ip link del dev "ve-$INSTANCE" 2> /dev/null || true
         ip link del dev "vb-$INSTANCE" 2> /dev/null || true
       fi
@@ -200,7 +204,7 @@ let
           '';
     in
       ''
-        if [ "$PRIVATE_NETWORK" = 1 ]; then
+        if [ -n "$HOST_ADDRESS" ] || [ -n "$LOCAL_ADDRESS" ]; then
           if [ -z "$HOST_BRIDGE" ]; then
             ifaceHost=ve-$INSTANCE
             ip link set dev $ifaceHost up
@@ -352,7 +356,7 @@ let
         List of forwarded ports from host to container. Each forwarded port
         is specified by protocol, hostPort and containerPort. By default,
         protocol is tcp and hostPort and containerPort are assumed to be
-        the same if containerPort is not explicitly given. 
+        the same if containerPort is not explicitly given.
       '';
     };
 
@@ -457,6 +461,16 @@ in
                       { boot.isContainer = true;
                         networking.hostName = mkDefault name;
                         networking.useDHCP = false;
+                        assertions = [
+                          {
+                            assertion =  config.privateNetwork -> stringLength name < 12;
+                            message = ''
+                              Container name `${name}` is too long: When `privateNetwork` is enabled, container names can
+                              not be longer than 11 characters, because the container's interface name is derived from it.
+                              This might be fixed in the future. See https://github.com/NixOS/nixpkgs/issues/38509
+                            '';
+                          }
+                        ];
                       };
                     in [ extraConfig ] ++ (map (x: x.value) defs);
                   prefix = [ "containers" name ];
@@ -699,7 +713,7 @@ in
     # container so that container@.target can get the container
     # configuration.
     environment.etc =
-      let mkPortStr = p: p.protocol + ":" + (toString p.hostPort) + ":" + (if p.containerPort == null then toString p.hostPort else toString p.containerPort); 
+      let mkPortStr = p: p.protocol + ":" + (toString p.hostPort) + ":" + (if p.containerPort == null then toString p.hostPort else toString p.containerPort);
       in mapAttrs' (name: cfg: nameValuePair "containers/${name}.conf"
       { text =
           ''
diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix
index 360b32faae72..ecd14b274eb3 100644
--- a/nixos/tests/docker-tools.nix
+++ b/nixos/tests/docker-tools.nix
@@ -62,5 +62,9 @@ import ./make-test.nix ({ pkgs, ... }: {
       # Ensure Layered Docker images work
       $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-image}'");
       $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-image.imageName}");
+
+      # Ensure building an image on top of a layered Docker images work
+      $docker->succeed("docker load --input='${pkgs.dockerTools.examples.layered-on-top}'");
+      $docker->succeed("docker run --rm ${pkgs.dockerTools.examples.layered-on-top.imageName}");
     '';
 })
diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab.nix
index 269da8aa215f..f401fe098dcc 100644
--- a/nixos/tests/gitlab.nix
+++ b/nixos/tests/gitlab.nix
@@ -8,7 +8,7 @@ import ./make-test.nix ({ pkgs, lib, ...} : with lib; {
 
   nodes = {
     gitlab = { ... }: {
-      virtualisation.memorySize = 4096;
+      virtualisation.memorySize = 2047;
       systemd.services.gitlab.serviceConfig.Restart = mkForce "no";
       systemd.services.gitlab-workhorse.serviceConfig.Restart = mkForce "no";
       systemd.services.gitaly.serviceConfig.Restart = mkForce "no";
diff --git a/nixos/tests/home-assistant.nix b/nixos/tests/home-assistant.nix
index 0b3da0d59c68..7627bb07901d 100644
--- a/nixos/tests/home-assistant.nix
+++ b/nixos/tests/home-assistant.nix
@@ -31,6 +31,9 @@ in {
               latitude = "0.0";
               longitude = "0.0";
               elevation = 0;
+              auth_providers = [
+                { type = "legacy_api_password"; }
+              ];
             };
             frontend = { };
             http.api_password = apiPassword;
diff --git a/nixos/tests/kafka.nix b/nixos/tests/kafka.nix
index a833e01f9f5e..72f91f6428a5 100644
--- a/nixos/tests/kafka.nix
+++ b/nixos/tests/kafka.nix
@@ -40,7 +40,7 @@ let
 
         networking.firewall.allowedTCPPorts = [ 9092 ];
         # i686 tests: qemu-system-i386 can simulate max 2047MB RAM (not 2048)
-        virtualisation.memorySize = 2047; 
+        virtualisation.memorySize = 2047;
       };
     };
 
@@ -70,4 +70,6 @@ in with pkgs; {
   kafka_0_11 = makeKafkaTest "kafka_0_11" apacheKafka_0_11;
   kafka_1_0  = makeKafkaTest "kafka_1_0"  apacheKafka_1_0;
   kafka_1_1  = makeKafkaTest "kafka_1_1"  apacheKafka_1_1;
+  kafka_2_0  = makeKafkaTest "kafka_2_0"  apacheKafka_2_0;
+  kafka_2_1  = makeKafkaTest "kafka_2_1"  apacheKafka_2_1;
 }
diff --git a/nixos/tests/run-in-machine.nix b/nixos/tests/run-in-machine.nix
index 116f5dc28a62..339a4b9a7404 100644
--- a/nixos/tests/run-in-machine.nix
+++ b/nixos/tests/run-in-machine.nix
@@ -10,11 +10,14 @@ let
     drv = pkgs.hello;
     machine = { ... }: { /* services.sshd.enable = true; */ };
   };
-in pkgs.runCommand "verify-output" { inherit output; } ''
-  if [ ! -e "$output/bin/hello" ]; then
-    echo "Derivation built using runInMachine produced incorrect output:" >&2
-    ls -laR "$output" >&2
-    exit 1
-  fi
-  "$output/bin/hello" > "$out"
-''
+
+  test = pkgs.runCommand "verify-output" { inherit output; } ''
+    if [ ! -e "$output/bin/hello" ]; then
+      echo "Derivation built using runInMachine produced incorrect output:" >&2
+      ls -laR "$output" >&2
+      exit 1
+    fi
+    "$output/bin/hello" > "$out"
+  '';
+
+in test // { inherit test; } # To emulate behaviour of makeTest