Merge remote-tracking branch 'upstream/master' into staging

8 files changed, 266 insertions, 42 deletions

diff --git a/nixos/modules/services/databases/foundationdb.nix b/nixos/modules/services/databases/foundationdb.nix
index 22acddc8ca91..693d2fde9916 100644
--- a/nixos/modules/services/databases/foundationdb.nix
+++ b/nixos/modules/services/databases/foundationdb.nix
@@ -4,6 +4,7 @@ with lib;
 
 let
   cfg = config.services.foundationdb;
+  pkg = cfg.package;
 
   # used for initial cluster configuration
   initialIpAddr = if (cfg.publicAddress != "auto") then cfg.publicAddress else "127.0.0.1";
@@ -24,7 +25,7 @@ let
     group         = ${cfg.group}
 
     [fdbserver]
-    command        = ${pkgs.foundationdb}/bin/fdbserver
+    command        = ${pkg}/bin/fdbserver
     public_address = ${cfg.publicAddress}:$ID
     listen_address = ${cfg.listenAddress}
     datadir        = ${cfg.dataDir}/$ID
@@ -35,6 +36,13 @@ let
     memory         = ${cfg.memory}
     storage_memory = ${cfg.storageMemory}
 
+    ${optionalString (cfg.tls != null) ''
+      tls_plugin           = ${pkg}/libexec/plugins/FDBLibTLS.so
+      tls_certificate_file = ${cfg.tls.certificate}
+      tls_key_file         = ${cfg.tls.key}
+      tls_verify_peers     = ${cfg.tls.allowedPeers}
+    ''}
+
     ${optionalString (cfg.locality.machineId    != null) "locality_machineid=${cfg.locality.machineId}"}
     ${optionalString (cfg.locality.zoneId       != null) "locality_zoneid=${cfg.locality.zoneId}"}
     ${optionalString (cfg.locality.datacenterId != null) "locality_dcid=${cfg.locality.datacenterId}"}
@@ -43,7 +51,7 @@ let
     ${fdbServers cfg.serverProcesses}
 
     [backup_agent]
-    command = ${pkgs.foundationdb}/libexec/backup_agent
+    command = ${pkg}/libexec/backup_agent
     ${backupAgents cfg.backupProcesses}
   '';
 in
@@ -52,6 +60,14 @@ in
 
     enable = mkEnableOption "FoundationDB Server";
 
+    package = mkOption {
+      type        = types.package;
+      description = ''
+        The FoundationDB package to use for this server. This must be specified by the user
+        in order to ensure migrations and upgrades are controlled appropriately.
+      '';
+    };
+
     publicAddress = mkOption {
       type        = types.str;
       default     = "auto";
@@ -188,6 +204,43 @@ in
       '';
     };
 
+    tls = mkOption {
+      default = null;
+      description = ''
+        FoundationDB Transport Security Layer (TLS) settings.
+      '';
+
+      type = types.nullOr (types.submodule ({
+        options = {
+          certificate = mkOption {
+            type = types.str;
+            description = ''
+              Path to the TLS certificate file. This certificate will
+              be offered to, and may be verified by, clients.
+            '';
+          };
+
+          key = mkOption {
+            type = types.str;
+            description = "Private key file for the certificate.";
+          };
+
+          allowedPeers = mkOption {
+            type = types.str;
+            default = "Check.Valid=1,Check.Unexpired=1";
+            description = ''
+	      "Peer verification string". This may be used to adjust which TLS
+              client certificates a server will accept, as a form of user
+              authorization; for example, it may only accept TLS clients who
+              offer a certificate abiding by some locality or organization name.
+
+              For more information, please see the FoundationDB documentation.
+            '';
+          };
+        };
+      }));
+    };
+
     locality = mkOption {
       default = {
         machineId    = null;
@@ -270,7 +323,7 @@ in
     meta.doc         = ./foundationdb.xml;
     meta.maintainers = with lib.maintainers; [ thoughtpolice ];
 
-    environment.systemPackages = [ pkgs.foundationdb ];
+    environment.systemPackages = [ pkg ];
 
     users.extraUsers = optionalAttrs (cfg.user == "foundationdb") (singleton
       { name        = "foundationdb";
@@ -324,34 +377,37 @@ in
           ReadWritePaths        = lib.concatStringsSep " " (map (x: "-" + x) rwpaths);
         };
 
-      path = [ pkgs.foundationdb pkgs.coreutils ];
+      path = [ pkg pkgs.coreutils ];
 
       preStart = ''
         rm -f ${cfg.pidfile}   && \
           touch ${cfg.pidfile} && \
           chown -R ${cfg.user}:${cfg.group} ${cfg.pidfile}
 
-        for x in "${cfg.logDir}" "${cfg.dataDir}" /etc/foundationdb; do
-          [ ! -d "$x" ] && mkdir -m 0700 -vp "$x" && chown -R ${cfg.user}:${cfg.group} "$x";
+        for x in "${cfg.logDir}" "${cfg.dataDir}"; do
+          [ ! -d "$x" ] && mkdir -m 0700 -vp "$x";
+          chown -R ${cfg.user}:${cfg.group} "$x";
         done
 
+        [ ! -d /etc/foundationdb ] && \
+          mkdir -m 0775 -vp /etc/foundationdb && \
+          chown -R ${cfg.user}:${cfg.group} "/etc/foundationdb"
+
         if [ ! -f /etc/foundationdb/fdb.cluster ]; then
             cf=/etc/foundationdb/fdb.cluster
             desc=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
             rand=$(tr -dc A-Za-z0-9 </dev/urandom 2>/dev/null | head -c8)
             echo ''${desc}:''${rand}@${initialIpAddr}:${builtins.toString cfg.listenPortStart} > $cf
-            chmod 0660 $cf && chown -R ${cfg.user}:${cfg.group} $cf
+            chmod 0664 $cf && chown -R ${cfg.user}:${cfg.group} $cf
             touch "${cfg.dataDir}/.first_startup"
         fi
       '';
 
-      script = ''
-        exec fdbmonitor --lockfile ${cfg.pidfile} --conffile ${configFile};
-      '';
+      script = "exec fdbmonitor --lockfile ${cfg.pidfile} --conffile ${configFile}";
 
       postStart = ''
         if [ -e "${cfg.dataDir}/.first_startup" ]; then
-          fdbcli --exec "configure new single ssd"
+          fdbcli --exec "configure new single memory"
           rm -f "${cfg.dataDir}/.first_startup";
         fi
       '';
diff --git a/nixos/modules/services/databases/foundationdb.xml b/nixos/modules/services/databases/foundationdb.xml
index 2a0e3c76c9d8..def9cc436691 100644
--- a/nixos/modules/services/databases/foundationdb.xml
+++ b/nixos/modules/services/databases/foundationdb.xml
@@ -12,7 +12,7 @@
 
 <para><emphasis>Maintainer:</emphasis> Austin Seipp</para>
 
-<para><emphasis>Default version:</emphasis> 5.1.x</para>
+<para><emphasis>Available version(s):</emphasis> 5.1.x</para>
 
 <para>FoundationDB (or "FDB") is a distributed, open source, high performance,
 transactional key-value store. It can store petabytes of data and deliver
@@ -26,9 +26,17 @@ exceptional performance while maintaining consistency and ACID semantics
 
 <programlisting>
 services.foundationdb.enable = true;
+services.foundationdb.package = pkgs.foundationdb51; # FoundationDB 5.1.x
 </programlisting>
 </para>
 
+<para>The <option>services.foundationdb.package</option> option is required,
+and must always be specified. Because FoundationDB network protocols and
+on-disk storage formats may change between (major) versions, and upgrades must
+be explicitly handled by the user, you must always manually specify this
+yourself so that the NixOS module will use the proper version. Note that minor,
+bugfix releases are always compatible.</para>
+
 <para>After running <command>nixos-rebuild</command>, you can verify whether
 FoundationDB is running by executing <command>fdbcli</command> (which is added
 to <option>environment.systemPackages</option>):
@@ -192,6 +200,44 @@ to a new node in order to connect, if it is not part of the cluster.</para>
 
 </section>
 
+<section><title>Client authorization and TLS</title>
+
+<para>By default, any user who can connect to a FoundationDB process with the
+correct cluster configuration can access anything. FoundationDB uses a
+pluggable design to transport security, and out of the box it supports a
+LibreSSL-based plugin for TLS support. This plugin not only does in-flight
+encryption, but also performs client authorization based on the given
+endpoint's certificate chain. For example, a FoundationDB server may be
+configured to only accept client connections over TLS, where the client TLS
+certificate is from organization <emphasis>Acme Co</emphasis> in the
+<emphasis>Research and Development</emphasis> unit.</para>
+
+<para>Configuring TLS with FoundationDB is done using the
+<option>services.foundationdb.tls</option> options in order to control the peer
+verification string, as well as the certificate and its private key.</para>
+
+<para>Note that the certificate and its private key must be accessible to the
+FoundationDB user account that the server runs under. These files are also NOT
+managed by NixOS, as putting them into the store may reveal private
+information.</para>
+
+<para>After you have a key and certificate file in place, it is not enough to
+simply set the NixOS module options -- you must also configure the
+<command>fdb.cluster</command> file to specify that a given set of coordinators
+use TLS. This is as simple as adding the suffix <command>:tls</command> to your
+cluster coordinator configuration, after the port number. For example, assuming
+you have a coordinator on localhost with the default configuration, simply
+specifying:</para>
+
+<programlisting>
+XXXXXX:XXXXXX@127.0.0.1:4500:tls
+</programlisting>
+
+<para>will configure all clients and server processes to use TLS from now
+on.</para>
+
+</section>
+
 <section><title>Backups and Disaster Recovery</title>
 
 <para>The usual rules for doing FoundationDB backups apply on NixOS as written
@@ -245,16 +291,12 @@ FoundationDB is not new software, but the NixOS compilation and integration has
 only undergone fairly basic testing of all the available functionality.</para>
 
 <itemizedlist>
-  <listitem><para>TLS plugin support is compiled in, but it's currently not
-      possible to specify the set of TLS certificate options in
-      <command>services.foundationdb</command></para></listitem>
   <listitem><para>There is no way to specify individual parameters for
       individual <command>fdbserver</command> processes. Currently, all server
       processes inherit all the global <command>fdbmonitor</command> settings.
       </para></listitem>
   <listitem><para>Python bindings are not currently installed.</para></listitem>
   <listitem><para>Ruby bindings are not currently installed.</para></listitem>
-  <listitem><para>Java bindings are not currently installed.</para></listitem>
   <listitem><para>Go bindings are not currently installed.</para></listitem>
 </itemizedlist>
 
diff --git a/nixos/modules/services/misc/docker-registry.nix b/nixos/modules/services/misc/docker-registry.nix
index 96ac2a1cf2c9..45931cb42b54 100644
--- a/nixos/modules/services/misc/docker-registry.nix
+++ b/nixos/modules/services/misc/docker-registry.nix
@@ -5,6 +5,45 @@ with lib;
 let
   cfg = config.services.dockerRegistry;
 
+  blobCache = if cfg.enableRedisCache
+    then "redis"
+    else "inmemory";
+
+  registryConfig = {
+    version =  "0.1";
+    log.fields.service = "registry";
+    storage = {
+      cache.blobdescriptor = blobCache;
+      filesystem.rootdirectory = cfg.storagePath;
+      delete.enabled = cfg.enableDelete;
+    };
+    http = {
+      addr = ":${builtins.toString cfg.port}";
+      headers.X-Content-Type-Options = ["nosniff"];
+    };
+    health.storagedriver = {
+      enabled = true;
+      interval = "10s";
+      threshold = 3;
+    };
+  };
+
+  registryConfig.redis = mkIf cfg.enableRedisCache {
+    addr = "${cfg.redisUrl}";
+    password = "${cfg.redisPassword}";
+    db = 0;
+    dialtimeout = "10ms";
+    readtimeout = "10ms";
+    writetimeout = "10ms";
+    pool = {
+      maxidle = 16;
+      maxactive = 64;
+      idletimeout = "300s";
+    };
+  };
+
+  configFile = pkgs.writeText "docker-registry-config.yml" (builtins.toJSON (registryConfig // cfg.extraConfig));
+
 in {
   options.services.dockerRegistry = {
     enable = mkEnableOption "Docker Registry";
@@ -27,6 +66,26 @@ in {
       description = "Docker registry storage path.";
     };
 
+    enableDelete = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Enable delete for manifests and blobs.";
+    };
+
+    enableRedisCache = mkEnableOption "redis as blob cache";
+
+    redisUrl = mkOption {
+      type = types.str;
+      default = "localhost:6379";
+      description = "Set redis host and port.";
+    };
+
+    redisPassword = mkOption {
+      type = types.str;
+      default = "";
+      description = "Set redis password.";
+    };
+
     extraConfig = mkOption {
       description = ''
         Docker extra registry configuration via environment variables.
@@ -34,6 +93,19 @@ in {
       default = {};
       type = types.attrsOf types.str;
     };
+
+    enableGarbageCollect = mkEnableOption "garbage collect";
+
+    garbageCollectDates = mkOption {
+      default = "daily";
+      type = types.str;
+      description = ''
+        Specification (in the format described by
+        <citerefentry><refentrytitle>systemd.time</refentrytitle>
+        <manvolnum>7</manvolnum></citerefentry>) of the time at
+        which the garbage collect will occur.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
@@ -41,15 +113,8 @@ in {
       description = "Docker Container Registry";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
-
-      environment = {
-        REGISTRY_HTTP_ADDR = "${cfg.listenAddress}:${toString cfg.port}";
-        REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY = cfg.storagePath;
-      } // cfg.extraConfig;
-
       script = ''
-        ${pkgs.docker-distribution}/bin/registry serve \
-          ${pkgs.docker-distribution.out}/share/go/src/github.com/docker/distribution/cmd/registry/config-example.yml
+        ${pkgs.docker-distribution}/bin/registry serve ${configFile}
       '';
 
       serviceConfig = {
@@ -58,6 +123,22 @@ in {
       };
     };
 
+    systemd.services.docker-registry-garbage-collect = {
+      description = "Run Garbage Collection for docker registry";
+
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+
+      serviceConfig.Type = "oneshot";
+
+      script = ''
+        ${pkgs.docker-distribution}/bin/registry garbage-collect ${configFile}
+        ${pkgs.systemd}/bin/systemctl restart docker-registry.service
+      '';
+
+      startAt = optional cfg.enableGarbageCollect cfg.garbageCollectDates;
+    };
+
     users.extraUsers.docker-registry = {
       createHome = true;
       home = cfg.storagePath;
diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix
index 180b273d177e..780448d8bad8 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -18,18 +18,19 @@ let
   #  systemd service must be provided by specifying either
   #  `serviceOpts.script` or `serviceOpts.serviceConfig.ExecStart`
   exporterOpts = {
-    blackbox = import ./exporters/blackbox.nix { inherit config lib pkgs; };
-    collectd = import ./exporters/collectd.nix { inherit config lib pkgs; };
-    dovecot  = import ./exporters/dovecot.nix  { inherit config lib pkgs; };
-    fritzbox = import ./exporters/fritzbox.nix { inherit config lib pkgs; };
-    json     = import ./exporters/json.nix     { inherit config lib pkgs; };
-    minio    = import ./exporters/minio.nix    { inherit config lib pkgs; };
-    nginx    = import ./exporters/nginx.nix    { inherit config lib pkgs; };
-    node     = import ./exporters/node.nix     { inherit config lib pkgs; };
-    postfix  = import ./exporters/postfix.nix  { inherit config lib pkgs; };
-    snmp     = import ./exporters/snmp.nix     { inherit config lib pkgs; };
-    unifi    = import ./exporters/unifi.nix    { inherit config lib pkgs; };
-    varnish  = import ./exporters/varnish.nix  { inherit config lib pkgs; };
+    blackbox  = import ./exporters/blackbox.nix  { inherit config lib pkgs; };
+    collectd  = import ./exporters/collectd.nix  { inherit config lib pkgs; };
+    dovecot   = import ./exporters/dovecot.nix   { inherit config lib pkgs; };
+    fritzbox  = import ./exporters/fritzbox.nix  { inherit config lib pkgs; };
+    json      = import ./exporters/json.nix      { inherit config lib pkgs; };
+    minio     = import ./exporters/minio.nix     { inherit config lib pkgs; };
+    nginx     = import ./exporters/nginx.nix     { inherit config lib pkgs; };
+    node      = import ./exporters/node.nix      { inherit config lib pkgs; };
+    postfix   = import ./exporters/postfix.nix   { inherit config lib pkgs; };
+    snmp      = import ./exporters/snmp.nix      { inherit config lib pkgs; };
+    surfboard = import ./exporters/surfboard.nix { inherit config lib pkgs; };
+    unifi     = import ./exporters/unifi.nix     { inherit config lib pkgs; };
+    varnish   = import ./exporters/varnish.nix   { inherit config lib pkgs; };
   };
 
   mkExporterOpts = ({ name, port }: {
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix b/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
index 39054c2b73dd..431dd8b4ead7 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters/nginx.nix
@@ -39,7 +39,7 @@ in
           --nginx.scrape_uri '${cfg.scrapeUri}' \
           --telemetry.address ${cfg.listenAddress}:${toString cfg.port} \
           --telemetry.endpoint ${cfg.telemetryEndpoint} \
-          --insecure ${cfg.insecure} \
+          --insecure ${toString cfg.insecure} \
           ${concatStringsSep " \\\n  " cfg.extraFlags}
       '';
     };
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/surfboard.nix b/nixos/modules/services/monitoring/prometheus/exporters/surfboard.nix
new file mode 100644
index 000000000000..715dba06a3dc
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/surfboard.nix
@@ -0,0 +1,32 @@
+{ config, lib, pkgs }:
+
+with lib;
+
+let
+  cfg = config.services.prometheus.exporters.surfboard;
+in
+{
+  port = 9239;
+  extraOpts = {
+    modemAddress = mkOption {
+      type = types.str;
+      default = "192.168.100.1";
+      description = ''
+        The hostname or IP of the cable modem.
+      '';
+    };
+  };
+  serviceOpts = {
+    description = "Prometheus exporter for surfboard cable modem";
+    unitConfig.Documentation = "https://github.com/ipstatic/surfboard_exporter";
+    serviceConfig = {
+      DynamicUser = true;
+      ExecStart = ''
+        ${pkgs.prometheus-surfboard-exporter}/bin/surfboard_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --modem-address ${cfg.modemAddress} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}
diff --git a/nixos/modules/virtualisation/google-compute-image.nix b/nixos/modules/virtualisation/google-compute-image.nix
index 0b6bec786da4..374a84332357 100644
--- a/nixos/modules/virtualisation/google-compute-image.nix
+++ b/nixos/modules/virtualisation/google-compute-image.nix
@@ -221,7 +221,7 @@ in
           echo "Obtaining SSH keys..."
           mkdir -m 0700 -p /root/.ssh
           AUTH_KEYS=$(${mktemp})
-          ${wget} -O $AUTH_KEYS --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/sshKeys
+          ${wget} -O $AUTH_KEYS http://metadata.google.internal/computeMetadata/v1/instance/attributes/sshKeys
           if [ -s $AUTH_KEYS ]; then
 
             # Read in key one by one, split in case Google decided
@@ -246,6 +246,18 @@ in
             false
           fi
           rm -f $AUTH_KEYS
+          SSH_HOST_KEYS_DIR=$(${mktemp} -d)
+          ${wget} -O $SSH_HOST_KEYS_DIR/ssh_host_ed25519_key http://metadata.google.internal/computeMetadata/v1/instance/attributes/ssh_host_ed25519_key
+          ${wget} -O $SSH_HOST_KEYS_DIR/ssh_host_ed25519_key.pub http://metadata.google.internal/computeMetadata/v1/instance/attributes/ssh_host_ed25519_key_pub
+          if [ -s $SSH_HOST_KEYS_DIR/ssh_host_ed25519_key -a -s $SSH_HOST_KEYS_DIR/ssh_host_ed25519_key.pub ]; then
+              mv -f $SSH_HOST_KEYS_DIR/ssh_host_ed25519_key* /etc/ssh/
+              chmod 600 /etc/ssh/ssh_host_ed25519_key
+              chmod 644 /etc/ssh/ssh_host_ed25519_key.pub
+          else
+              echo "Setup of ssh host keys from http://metadata.google.internal/computeMetadata/v1/instance/attributes/ failed."
+              false
+          fi
+          rm -f $SSH_HOST_KEYS_DIR
         '';
       serviceConfig.Type = "oneshot";
       serviceConfig.RemainAfterExit = true;
diff --git a/nixos/modules/virtualisation/virtualbox-host.nix b/nixos/modules/virtualisation/virtualbox-host.nix
index 7413e12c8f3d..885d752577d5 100644
--- a/nixos/modules/virtualisation/virtualbox-host.nix
+++ b/nixos/modules/virtualisation/virtualbox-host.nix
@@ -6,7 +6,7 @@ let
   cfg = config.virtualisation.virtualbox.host;
 
   virtualbox = pkgs.virtualbox.override {
-    inherit (cfg) enableHardening headless;
+    inherit (cfg) enableExtensionPack enableHardening headless;
   };
 
   kernelModules = config.boot.kernelPackages.virtualbox.override {
@@ -17,9 +17,7 @@ in
 
 {
   options.virtualisation.virtualbox.host = {
-    enable = mkOption {
-      type = types.bool;
-      default = false;
+    enable = mkEnableOption "VirtualBox" // {
       description = ''
         Whether to enable VirtualBox.
 
@@ -30,6 +28,8 @@ in
       '';
     };
 
+    enableExtensionPack = mkEnableOption "VirtualBox extension pack";
+
     addNetworkInterface = mkOption {
       type = types.bool;
       default = true;