Merge branch 'master' into staging

This apparently fixes some broken src fetches (gnuradio, twisted).

22 files changed, 418 insertions, 30 deletions

diff --git a/nixos/doc/manual/administration/imperative-containers.xml b/nixos/doc/manual/administration/imperative-containers.xml
index 9cc7ca3e672a..9bb62bc2ece9 100644
--- a/nixos/doc/manual/administration/imperative-containers.xml
+++ b/nixos/doc/manual/administration/imperative-containers.xml
@@ -27,7 +27,7 @@
 <screen>
 # nixos-container create foo --config '
   <xref linkend="opt-services.openssh.enable"/> = true;
-  <link linkend="opt-users.users._name__.openssh.authorizedKeys.keys">users.extraUsers.root.openssh.authorizedKeys.keys</link> = ["ssh-dss AAAAB3N…"];
+  <link linkend="opt-users.users._name__.openssh.authorizedKeys.keys">users.users.root.openssh.authorizedKeys.keys</link> = ["ssh-dss AAAAB3N…"];
 '
 </screen>
  </para>
diff --git a/nixos/doc/manual/configuration/network-manager.xml b/nixos/doc/manual/configuration/network-manager.xml
index e217a99148b9..d103ee249783 100644
--- a/nixos/doc/manual/configuration/network-manager.xml
+++ b/nixos/doc/manual/configuration/network-manager.xml
@@ -19,7 +19,7 @@
   All users that should have permission to change network settings must belong
   to the <code>networkmanager</code> group:
 <programlisting>
-<link linkend="opt-users.users._name__.extraGroups">users.extraUsers.youruser.extraGroups</link> = [ "networkmanager" ];
+<link linkend="opt-users.users._name__.extraGroups">users.users.alice.extraGroups</link> = [ "networkmanager" ];
 </programlisting>
  </para>
 
diff --git a/nixos/doc/manual/configuration/ssh.xml b/nixos/doc/manual/configuration/ssh.xml
index 6e883e3fbbc1..a4af1b96583d 100644
--- a/nixos/doc/manual/configuration/ssh.xml
+++ b/nixos/doc/manual/configuration/ssh.xml
@@ -20,7 +20,7 @@
   follows:
 <!-- FIXME: this might not work if the user is unmanaged. -->
 <programlisting>
-<link linkend="opt-users.users._name__.openssh.authorizedKeys.keys">users.extraUsers.alice.openssh.authorizedKeys.keys</link> =
+<link linkend="opt-users.users._name__.openssh.authorizedKeys.keys">users.users.alice.openssh.authorizedKeys.keys</link> =
   [ "ssh-dss AAAAB3NzaC1kc3MAAACBAPIkGWVEt4..." ];
 </programlisting>
  </para>
diff --git a/nixos/doc/manual/installation/changing-config.xml b/nixos/doc/manual/installation/changing-config.xml
index 680160a3cb7e..1a116ec0b655 100644
--- a/nixos/doc/manual/installation/changing-config.xml
+++ b/nixos/doc/manual/installation/changing-config.xml
@@ -66,7 +66,7 @@ $ ./result/bin/run-*-vm
   <literal>mutableUsers = false</literal>. Another way is to temporarily add
   the following to your configuration:
 <screen>
-<link linkend="opt-users.users._name__.initialHashedPassword">users.extraUsers.your-user.initialHashedPassword</link> = "test";
+<link linkend="opt-users.users._name__.initialHashedPassword">users.users.your-user.initialHashedPassword</link> = "test";
 </screen>
   <emphasis>Important:</emphasis> delete the $hostname.qcow2 file if you have
   started the virtual machine at least once without the right users, otherwise
diff --git a/nixos/doc/manual/installation/installing-from-other-distro.xml b/nixos/doc/manual/installation/installing-from-other-distro.xml
index c55aa90267fb..d1e49a2a1597 100644
--- a/nixos/doc/manual/installation/installing-from-other-distro.xml
+++ b/nixos/doc/manual/installation/installing-from-other-distro.xml
@@ -211,7 +211,7 @@ $ sudo groupdel nixbld</screen>
     use <literal>sudo</literal>)
    </para>
 <programlisting>
-<link linkend="opt-users.users._name__.initialHashedPassword">users.extraUsers.root.initialHashedPassword</link> = "";
+<link linkend="opt-users.users._name__.initialHashedPassword">users.users.root.initialHashedPassword</link> = "";
               </programlisting>
   </listitem>
   <listitem>
diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml
index 667437a24135..51e77d24a74e 100644
--- a/nixos/doc/manual/release-notes/rl-1809.xml
+++ b/nixos/doc/manual/release-notes/rl-1809.xml
@@ -330,6 +330,11 @@ inherit (pkgs.nixos {
      will be added to <literal>environment.systemPackages</literal>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The module <option>services.networking.hostapd</option> now uses WPA2 by default.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>
diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix
index 73231edf077b..aac86087f9ec 100644
--- a/nixos/modules/misc/ids.nix
+++ b/nixos/modules/misc/ids.nix
@@ -317,6 +317,10 @@
       restic = 291;
       openvpn = 292;
       meguca = 293;
+      yarn = 294;
+      hdfs = 295;
+      mapred = 296;
+      hadoop = 297;
 
       # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
 
@@ -594,6 +598,10 @@
       restic = 291;
       openvpn = 292;
       meguca = 293;
+      yarn = 294;
+      hdfs = 295;
+      mapred = 296;
+      hadoop = 297;
 
       # When adding a gid, make sure it doesn't match an existing
       # uid. Users and groups with the same name should have equal
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index ea996acebb20..c70d52c94cdb 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -178,6 +178,7 @@
   ./services/backup/rsnapshot.nix
   ./services/backup/tarsnap.nix
   ./services/backup/znapzend.nix
+  ./services/cluster/hadoop/default.nix
   ./services/cluster/kubernetes/default.nix
   ./services/cluster/kubernetes/dns.nix
   ./services/cluster/kubernetes/dashboard.nix
diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
new file mode 100644
index 000000000000..38db10406b9a
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -0,0 +1,31 @@
+{ hadoop, pkgs }:
+let
+  propertyXml = name: value: ''
+    <property>
+      <name>${name}</name>
+      <value>${builtins.toString value}</value>
+    </property>
+  '';
+  siteXml = fileName: properties: pkgs.writeTextDir fileName ''
+    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+    <!-- generated by NixOS -->
+    <configuration>
+      ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}
+    </configuration>
+  '';
+  userFunctions = ''
+    hadoop_verify_logdir() {
+      echo Skipping verification of log directory
+    }
+  '';
+in
+pkgs.buildEnv {
+  name = "hadoop-conf";
+  paths = [
+    (siteXml "core-site.xml" hadoop.coreSite)
+    (siteXml "hdfs-site.xml" hadoop.hdfsSite)
+    (siteXml "mapred-site.xml" hadoop.mapredSite)
+    (siteXml "yarn-site.xml" hadoop.yarnSite)
+    (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions)
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
new file mode 100644
index 000000000000..53c13fd0603d
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -0,0 +1,63 @@
+{ config, lib, pkgs, ...}:
+let 
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  imports = [ ./yarn.nix ./hdfs.nix ];
+
+  options.services.hadoop = {
+    coreSite = mkOption {
+      default = {};
+      example = {
+        "fs.defaultFS" = "hdfs://localhost";
+      };
+      description = "Hadoop core-site.xml definition";
+    };
+
+    hdfsSite = mkOption {
+      default = {};
+      example = {
+        "dfs.nameservices" = "namenode1";
+      };
+      description = "Hadoop hdfs-site.xml definition";
+    };
+
+    mapredSite = mkOption {
+      default = {};
+      example = {
+        "mapreduce.map.cpu.vcores" = "1";
+      };
+      description = "Hadoop mapred-site.xml definition";
+    };
+
+    yarnSite = mkOption {
+      default = {};
+      example = {
+        "yarn.resourcemanager.ha.id" = "resourcemanager1";
+      };
+      description = "Hadoop yarn-site.xml definition";
+    };
+
+    package = mkOption {
+      type = types.package;
+      default = pkgs.hadoop;
+      defaultText = "pkgs.hadoop";
+      example = literalExample "pkgs.hadoop";
+      description = ''
+      '';
+    };
+  };
+
+
+  config = mkMerge [
+    (mkIf (builtins.hasAttr "yarn" config.users.extraUsers ||
+           builtins.hasAttr "hdfs" config.users.extraUsers ) {
+      users.extraGroups.hadoop = {
+        gid = config.ids.gids.hadoop;
+      };
+    })
+
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix
new file mode 100644
index 000000000000..48020e6139cb
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/hdfs.nix
@@ -0,0 +1,73 @@
+{ config, lib, pkgs, ...}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  options.services.hadoop.hdfs = {
+    namenode.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN NameNode
+      '';
+    };
+    datanode.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN DataNode
+      '';
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.hdfs.namenode.enabled {
+      systemd.services."hdfs-namenode" = {
+        description = "Hadoop HDFS NameNode";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        preStart = ''
+          ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true
+        '';
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-namenode";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode";
+        };
+      };
+    })
+    (mkIf cfg.hdfs.datanode.enabled {
+      systemd.services."hdfs-datanode" = {
+        description = "Hadoop HDFS DataNode";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "hdfs";
+          SyslogIdentifier = "hdfs-datanode";
+          ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode";
+        };
+      };
+    })
+    (mkIf (
+        cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled
+    ) {
+      users.extraUsers.hdfs = {
+        description = "Hadoop HDFS user";
+        group = "hadoop";
+        uid = config.ids.uids.hdfs;
+      };
+    })
+
+  ];
+}
diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix
new file mode 100644
index 000000000000..ce5b04a331c5
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@@ -0,0 +1,74 @@
+{ config, lib, pkgs, ...}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; };
+in
+with lib;
+{
+  options.services.hadoop.yarn = {
+    resourcemanager.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN ResourceManager
+      '';
+    };
+    nodemanager.enabled = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to run the Hadoop YARN NodeManager
+      '';
+    };
+  };
+
+  config = mkMerge [
+    (mkIf (
+        cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled
+    ) {
+
+      users.extraUsers.yarn = {
+        description = "Hadoop YARN user";
+        group = "hadoop";
+        uid = config.ids.uids.yarn;
+      };
+    })
+
+    (mkIf cfg.yarn.resourcemanager.enabled {
+      systemd.services."yarn-resourcemanager" = {
+        description = "Hadoop YARN ResourceManager";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-resourcemanager";
+          ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
+                      " resourcemanager";
+        };
+      };
+    })
+
+    (mkIf cfg.yarn.nodemanager.enabled {
+      systemd.services."yarn-nodemanager" = {
+        description = "Hadoop YARN NodeManager";
+        wantedBy = [ "multi-user.target" ];
+
+        environment = {
+          HADOOP_HOME = "${cfg.package}";
+        };
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-nodemanager";
+          ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
+                      " nodemanager";
+        };
+      };
+    })
+
+  ];
+}
diff --git a/nixos/modules/services/hardware/fwupd.nix b/nixos/modules/services/hardware/fwupd.nix
index d8abde2a600a..d97d690920a6 100644
--- a/nixos/modules/services/hardware/fwupd.nix
+++ b/nixos/modules/services/hardware/fwupd.nix
@@ -85,6 +85,6 @@ in {
   };
 
   meta = {
-    maintainers = pkgs.fwupd.maintainers;
+    maintainers = pkgs.fwupd.meta.maintainers;
   };
 }
diff --git a/nixos/modules/services/network-filesystems/openafs/client.nix b/nixos/modules/services/network-filesystems/openafs/client.nix
index 3826fe3edfd0..52c0966e05bc 100644
--- a/nixos/modules/services/network-filesystems/openafs/client.nix
+++ b/nixos/modules/services/network-filesystems/openafs/client.nix
@@ -1,6 +1,7 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, pkgs, ... }:
 
-with import ./lib.nix { inherit lib; };
+# openafsMod, openafsBin, mkCellServDB
+with import ./lib.nix { inherit config lib pkgs; };
 
 let
   inherit (lib) getBin mkOption mkIf optionalString singleton types;
@@ -8,8 +9,8 @@ let
   cfg = config.services.openafsClient;
 
   cellServDB = pkgs.fetchurl {
-    url = http://dl.central.org/dl/cellservdb/CellServDB.2017-03-14;
-    sha256 = "1197z6c5xrijgf66rhaymnm5cvyg2yiy1i20y4ah4mrzmjx0m7sc";
+    url = http://dl.central.org/dl/cellservdb/CellServDB.2018-05-14;
+    sha256 = "1wmjn6mmyy2r8p10nlbdzs4nrqxy8a9pjyrdciy5nmppg4053rk2";
   };
 
   clientServDB = pkgs.writeText "client-cellServDB-${cfg.cellName}" (mkCellServDB cfg.cellName cfg.cellServDB);
@@ -21,8 +22,6 @@ let
     echo "${cfg.mountPoint}:${cfg.cache.directory}:${toString cfg.cache.blocks}" > $out/cacheinfo
   '';
 
-  openafsMod = config.boot.kernelPackages.openafs;
-  openafsBin = lib.getBin pkgs.openafs;
 in
 {
   ###### interface
@@ -147,6 +146,19 @@ in
         '';
       };
 
+      packages = {
+        module = mkOption {
+          default = config.boot.kernelPackages.openafs;
+          type = types.package;
+          description = "OpenAFS kernel module package. MUST match the userland package!";
+        };
+        programs = mkOption {
+          default = getBin pkgs.openafs;
+          type = types.package;
+          description = "OpenAFS programs package. MUST match the kernel module package!";
+        };
+      };
+
       sparse = mkOption {
         default = true;
         type = types.bool;
@@ -180,7 +192,7 @@ in
       }
     ];
 
-    environment.systemPackages = [ pkgs.openafs ];
+    environment.systemPackages = [ openafsBin ];
 
     environment.etc = {
       clientCellServDB = {
diff --git a/nixos/modules/services/network-filesystems/openafs/lib.nix b/nixos/modules/services/network-filesystems/openafs/lib.nix
index ecfc72d2eaf9..255740ac65ef 100644
--- a/nixos/modules/services/network-filesystems/openafs/lib.nix
+++ b/nixos/modules/services/network-filesystems/openafs/lib.nix
@@ -1,14 +1,15 @@
-{ lib, ...}:
+{ config, lib, pkgs, ...}:
 
 let
-  inherit (lib) concatStringsSep mkOption types;
+  inherit (lib) concatStringsSep getBin mkOption types;
 
 in rec {
 
   mkCellServDB = cellName: db: ''
     >${cellName}
   '' + (concatStringsSep "\n" (map (dbm: if (dbm.ip != "" && dbm.dnsname != "") then dbm.ip + " #" + dbm.dnsname else "")
-                                   db));
+                                   db))
+     + "\n";
 
   # CellServDB configuration type
   cellServDBConfig = {
@@ -25,4 +26,8 @@ in rec {
       description = "DNS full-qualified domain name of a database server";
     };
   };
+
+  openafsMod = config.services.openafsClient.packages.module;
+  openafsBin = config.services.openafsClient.packages.programs;
+  openafsSrv = config.services.openafsServer.package;
 }
diff --git a/nixos/modules/services/network-filesystems/openafs/server.nix b/nixos/modules/services/network-filesystems/openafs/server.nix
index 429eb945ac9e..aa8640fd240e 100644
--- a/nixos/modules/services/network-filesystems/openafs/server.nix
+++ b/nixos/modules/services/network-filesystems/openafs/server.nix
@@ -1,6 +1,7 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, pkgs, ... }:
 
-with import ./lib.nix { inherit lib; };
+# openafsBin, openafsSrv, mkCellServDB
+with import ./lib.nix { inherit config lib pkgs; };
 
 let
   inherit (lib) concatStringsSep intersperse mapAttrsToList mkForce mkIf mkMerge mkOption optionalString types;
@@ -11,21 +12,21 @@ let
     checkbintime 3 0 5 0 0
   '' + (optionalString cfg.roles.database.enable ''
     bnode simple vlserver 1
-    parm ${openafsBin}/libexec/openafs/vlserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.vlserverArgs}
+    parm ${openafsSrv}/libexec/openafs/vlserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.vlserverArgs}
     end
     bnode simple ptserver 1
-    parm ${openafsBin}/libexec/openafs/ptserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.ptserverArgs}
+    parm ${openafsSrv}/libexec/openafs/ptserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.ptserverArgs}
     end
   '') + (optionalString cfg.roles.fileserver.enable ''
     bnode dafs dafs 1
-    parm ${openafsBin}/libexec/openafs/dafileserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.fileserverArgs}
-    parm ${openafsBin}/libexec/openafs/davolserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.volserverArgs}
-    parm ${openafsBin}/libexec/openafs/salvageserver ${cfg.roles.fileserver.salvageserverArgs}
-    parm ${openafsBin}/libexec/openafs/dasalvager ${cfg.roles.fileserver.salvagerArgs}
+    parm ${openafsSrv}/libexec/openafs/dafileserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.fileserverArgs}
+    parm ${openafsSrv}/libexec/openafs/davolserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.volserverArgs}
+    parm ${openafsSrv}/libexec/openafs/salvageserver ${cfg.roles.fileserver.salvageserverArgs}
+    parm ${openafsSrv}/libexec/openafs/dasalvager ${cfg.roles.fileserver.salvagerArgs}
     end
   '') + (optionalString (cfg.roles.database.enable && cfg.roles.backup.enable) ''
     bnode simple buserver 1
-    parm ${openafsBin}/libexec/openafs/buserver ${cfg.roles.backup.buserverArgs} ${optionalString (cfg.roles.backup.cellServDB != []) "-cellservdb /etc/openafs/backup/"}
+    parm ${openafsSrv}/libexec/openafs/buserver ${cfg.roles.backup.buserverArgs} ${optionalString (cfg.roles.backup.cellServDB != []) "-cellservdb /etc/openafs/backup/"}
     end
   ''));
 
@@ -39,8 +40,6 @@ let
 
   udpSizeStr = toString cfg.udpPacketSize;
 
-  openafsBin = lib.getBin pkgs.openafs;
-
 in {
 
   options = {
@@ -79,6 +78,12 @@ in {
         description = "Definition of all cell-local database server machines.";
       };
 
+      package = mkOption {
+        default = pkgs.openafs.server or pkgs.openafs;
+        type = types.package;
+        description = "OpenAFS package for the server binaries";
+      };
+
       roles = {
         fileserver = {
           enable = mkOption {
@@ -213,7 +218,7 @@ in {
       }
     ];
 
-    environment.systemPackages = [ pkgs.openafs ];
+    environment.systemPackages = [ openafsBin ];
 
     environment.etc = {
       bosConfig = {
@@ -244,7 +249,10 @@ in {
         after = [ "syslog.target" "network.target" ];
         wantedBy = [ "multi-user.target" ];
         restartIfChanged = false;
-        unitConfig.ConditionPathExists = [ "/etc/openafs/server/rxkad.keytab" ];
+        unitConfig.ConditionPathExists = [
+          "|/etc/openafs/server/rxkad.keytab"
+          "|/etc/openafs/server/KeyFileExt"
+        ];
         preStart = ''
           mkdir -m 0755 -p /var/openafs
           ${optionalString (netInfo != null) "cp ${netInfo} /var/openafs/netInfo"}
diff --git a/nixos/modules/services/networking/hostapd.nix b/nixos/modules/services/networking/hostapd.nix
index 63f56437d1c8..3af0441a89d8 100644
--- a/nixos/modules/services/networking/hostapd.nix
+++ b/nixos/modules/services/networking/hostapd.nix
@@ -29,7 +29,7 @@ let
     ctrl_interface_group=${cfg.group}
 
     ${if cfg.wpa then ''
-      wpa=1
+      wpa=2
       wpa_passphrase=${cfg.wpaPassphrase}
       '' else ""}
 
diff --git a/nixos/modules/services/networking/iwd.nix b/nixos/modules/services/networking/iwd.nix
index 344212ad8329..cfc536fc5b5f 100644
--- a/nixos/modules/services/networking/iwd.nix
+++ b/nixos/modules/services/networking/iwd.nix
@@ -28,6 +28,10 @@ in {
 
       serviceConfig.ExecStart = "${pkgs.iwd}/libexec/iwd";
     };
+
+    systemd.tmpfiles.rules = [
+      "d /var/lib/iwd 0700 root root -"
+    ];
   };
 
   meta.maintainers = with lib.maintainers; [ mic92 ];
diff --git a/nixos/release.nix b/nixos/release.nix
index 0d3ae997469f..67820149437d 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -299,6 +299,8 @@ in rec {
   tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
   tests.grafana = callTest tests/grafana.nix {};
   tests.graphite = callTest tests/graphite.nix {};
+  tests.hadoop.hdfs = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/hdfs.nix {};
+  tests.hadoop.yarn = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/yarn.nix {};
   tests.hardened = callTest tests/hardened.nix { };
   tests.haproxy = callTest tests/haproxy.nix {};
   tests.hibernate = callTest tests/hibernate.nix {};
diff --git a/nixos/tests/common/letsencrypt.nix b/nixos/tests/common/letsencrypt.nix
index 7c6b3b29e36d..2c86fe8d68bc 100644
--- a/nixos/tests/common/letsencrypt.nix
+++ b/nixos/tests/common/letsencrypt.nix
@@ -193,6 +193,7 @@ let
 
   snakeOilCa = pkgs.runCommand "snakeoil-ca" {
     buildInputs = [ pkgs.openssl ];
+    allowSubstitutes = false;
   } ''
     mkdir "$out"
     openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 \
@@ -215,6 +216,7 @@ let
     '';
   in pkgs.runCommand "snakeoil-certs-${fqdn}" {
     buildInputs = [ pkgs.openssl ];
+    allowSubstitutes = false;
   } ''
     mkdir "$out"
     openssl genrsa -out "$out/snakeoil.key" 4096
diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix
new file mode 100644
index 000000000000..4206c940c1af
--- /dev/null
+++ b/nixos/tests/hadoop/hdfs.nix
@@ -0,0 +1,54 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    namenode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.namenode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+        hdfsSite = {
+          "dfs.replication" = 1;
+          "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+          "dfs.namenode.http-bind-host" = "0.0.0.0";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+      ];
+    };
+    datanode = {pkgs, config, ...}: {
+      services.hadoop = {
+        package = pkgs.hadoop_3_1;
+        hdfs.datanode.enabled = true;
+        coreSite = {
+          "fs.defaultFS" = "hdfs://namenode:8020";
+        };
+      };
+      networking.firewall.allowedTCPPorts = [
+        9864 # datanode.http.address
+        9866 # datanode.address
+        9867 # datanode.ipc.address
+      ];
+    };
+  };
+
+  testScript = ''
+    startAll
+
+    $namenode->waitForUnit("hdfs-namenode");
+    $namenode->waitForUnit("network.target");
+    $namenode->waitForOpenPort(8020);
+    $namenode->waitForOpenPort(9870);
+
+    $datanode->waitForUnit("hdfs-datanode");
+    $datanode->waitForUnit("network.target");
+    $datanode->waitForOpenPort(9864);
+    $datanode->waitForOpenPort(9866);
+    $datanode->waitForOpenPort(9867);
+
+    $namenode->succeed("curl http://namenode:9870");
+    $datanode->succeed("curl http://datanode:9864");
+  '';
+})
diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix
new file mode 100644
index 000000000000..e97cc1acc902
--- /dev/null
+++ b/nixos/tests/hadoop/yarn.nix
@@ -0,0 +1,46 @@
+import ../make-test.nix ({pkgs, ...}: {
+  nodes = {
+    resourcemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.resourcemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8088 # resourcemanager.webapp.address
+        8031 # resourcemanager.resource-tracker.address
+      ];
+    };
+    nodemanager = {pkgs, config, ...}: {
+      services.hadoop.package = pkgs.hadoop_3_1;
+      services.hadoop.yarn.nodemanager.enabled = true;
+      services.hadoop.yarnSite = {
+        "yarn.resourcemanager.hostname" = "resourcemanager";
+        "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
+        "yarn.nodemanager.address" = "0.0.0.0:8041";
+      };
+      networking.firewall.allowedTCPPorts = [
+        8042 # nodemanager.webapp.address
+        8041 # nodemanager.address
+      ];
+    };
+
+  };
+
+  testScript = ''
+    startAll;
+
+    $resourcemanager->waitForUnit("yarn-resourcemanager");
+    $resourcemanager->waitForUnit("network.target");
+    $resourcemanager->waitForOpenPort(8031);
+    $resourcemanager->waitForOpenPort(8088);
+
+    $nodemanager->waitForUnit("yarn-nodemanager");
+    $nodemanager->waitForUnit("network.target");
+    $nodemanager->waitForOpenPort(8042);
+    $nodemanager->waitForOpenPort(8041);
+
+    $resourcemanager->succeed("curl http://localhost:8088");
+    $nodemanager->succeed("curl http://localhost:8042");
+  '';
+})