Merge pull request #17969 from offlinehacker/pkgs/etcd/update-3.0.6

Update etcd, improve nixos module, fix nixos tests

3 files changed, 229 insertions, 101 deletions

diff --git a/nixos/modules/services/misc/etcd.nix b/nixos/modules/services/misc/etcd.nix
index 0d6ed8eb9043..d30cc5fd7e89 100644
--- a/nixos/modules/services/misc/etcd.nix
+++ b/nixos/modules/services/misc/etcd.nix
@@ -28,13 +28,13 @@ in {
 
     listenClientUrls = mkOption {
       description = "Etcd list of URLs to listen on for client traffic.";
-      default = ["http://localhost:4001"];
+      default = ["http://127.0.0.1:2379"];
       type = types.listOf types.str;
     };
 
     listenPeerUrls = mkOption {
       description = "Etcd list of URLs to listen on for peer traffic.";
-      default = ["http://localhost:7001"];
+      default = ["http://127.0.0.1:2380"];
       type = types.listOf types.str;
     };
 
@@ -46,7 +46,7 @@ in {
 
     initialCluster = mkOption {
       description = "Etcd initial cluster configuration for bootstrapping.";
-      default = ["${cfg.name}=http://localhost:7001"];
+      default = ["${cfg.name}=http://127.0.0.1:2380"];
       type = types.listOf types.str;
     };
 
@@ -68,6 +68,54 @@ in {
       type = types.str;
     };
 
+    clientCertAuth = mkOption {
+      description = "Whether to use certs for client authentication";
+      default = false;
+      type = types.bool;
+    };
+
+    trustedCaFile = mkOption {
+      description = "Certificate authority file to use for clients";
+      default = null;
+      type = types.nullOr types.path;
+    };
+
+    certFile = mkOption {
+      description = "Cert file to use for clients";
+      default = null;
+      type = types.nullOr types.path;
+    };
+
+    keyFile = mkOption {
+      description = "Key file to use for clients";
+      default = null;
+      type = types.nullOr types.path;
+    };
+
+    peerCertFile = mkOption {
+      description = "Cert file to use for peer to peer communication";
+      default = cfg.certFile;
+      type = types.nullOr types.path;
+    };
+
+    peerKeyFile = mkOption {
+      description = "Key file to use for peer to peer communication";
+      default = cfg.keyFile;
+      type = types.nullOr types.path;
+    };
+
+    peerTrustedCaFile = mkOption {
+      description = "Certificate authority file to use for peer to peer communication";
+      default = cfg.trustedCaFile;
+      type = types.nullOr types.path;
+    };
+
+    peerClientCertAuth = mkOption {
+      description = "Whether to check all incoming peer requests from the cluster for valid client certificates signed by the supplied CA";
+      default = false;
+      type = types.bool;
+    };
+
     extraConf = mkOption {
       description = ''
         Etcd extra configuration. See
@@ -99,7 +147,7 @@ in {
       wantedBy = [ "multi-user.target" ];
       after = [ "network-interfaces.target" ];
 
-      environment = {
+      environment = (filterAttrs (n: v: v != null) {
         ETCD_NAME = cfg.name;
         ETCD_DISCOVERY = cfg.discovery;
         ETCD_DATA_DIR = cfg.dataDir;
@@ -107,7 +155,14 @@ in {
         ETCD_LISTEN_CLIENT_URLS = concatStringsSep "," cfg.listenClientUrls;
         ETCD_LISTEN_PEER_URLS = concatStringsSep "," cfg.listenPeerUrls;
         ETCD_INITIAL_ADVERTISE_PEER_URLS = concatStringsSep "," cfg.initialAdvertisePeerUrls;
-      } // (optionalAttrs (cfg.discovery == ""){
+        ETCD_PEER_TRUSTED_CA_FILE = cfg.peerTrustedCaFile;
+        ETCD_PEER_CERT_FILE = cfg.peerCertFile;
+        ETCD_PEER_KEY_FILE = cfg.peerKeyFile;
+        ETCD_CLIENT_CERT_AUTH = toString cfg.peerClientCertAuth;
+        ETCD_TRUSTED_CA_FILE = cfg.trustedCaFile;
+        ETCD_CERT_FILE = cfg.certFile;
+        ETCD_KEY_FILE = cfg.keyFile;
+      }) // (optionalAttrs (cfg.discovery == ""){
         ETCD_INITIAL_CLUSTER = concatStringsSep "," cfg.initialCluster;
         ETCD_INITIAL_CLUSTER_STATE = cfg.initialClusterState;
         ETCD_INITIAL_CLUSTER_TOKEN = cfg.initialClusterToken;
diff --git a/nixos/tests/etcd-cluster.nix b/nixos/tests/etcd-cluster.nix
new file mode 100644
index 000000000000..3971997a9bf7
--- /dev/null
+++ b/nixos/tests/etcd-cluster.nix
@@ -0,0 +1,157 @@
+# This test runs simple etcd cluster
+
+import ./make-test.nix ({ pkgs, ... } : let
+
+  runWithOpenSSL = file: cmd: pkgs.runCommand file {
+    buildInputs = [ pkgs.openssl ];
+  } cmd;
+
+  ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
+  ca_pem = runWithOpenSSL "ca.pem" ''
+    openssl req \
+      -x509 -new -nodes -key ${ca_key} \
+      -days 10000 -out $out -subj "/CN=etcd-ca"
+  '';
+  etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
+  etcd_csr = runWithOpenSSL "etcd.csr" ''
+    openssl req \
+       -new -key ${etcd_key} \
+       -out $out -subj "/CN=etcd" \
+       -config ${openssl_cnf}
+  '';
+  etcd_cert = runWithOpenSSL "etcd.pem" ''
+    openssl x509 \
+      -req -in ${etcd_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} \
+      -CAcreateserial -out $out \
+      -days 365 -extensions v3_req \
+      -extfile ${openssl_cnf}
+  '';
+
+  etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
+    "openssl genrsa -out $out 2048";
+
+  etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
+    openssl req \
+      -new -key ${etcd_client_key} \
+      -out $out -subj "/CN=etcd-client" \
+      -config ${client_openssl_cnf}
+  '';
+
+  etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
+    openssl x509 \
+      -req -in ${etcd_client_csr} \
+      -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+      -out $out -days 365 -extensions v3_req \
+      -extfile ${client_openssl_cnf}
+  '';
+
+  openssl_cnf = pkgs.writeText "openssl.cnf" ''
+    ions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = serverAuth
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = node1
+    DNS.2 = node2
+    DNS.3 = node3
+    IP.1 = 127.0.0.1
+  '';
+
+  client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
+    ions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = digitalSignature, keyEncipherment
+    extendedKeyUsage = clientAuth
+  '';
+
+  nodeConfig = {
+    services = {
+      etcd = {
+        enable = true;
+        keyFile = etcd_key;
+        certFile = etcd_cert;
+        trustedCaFile = ca_pem;
+        peerClientCertAuth = true;
+        listenClientUrls = ["https://127.0.0.1:2379"];
+        listenPeerUrls = ["https://0.0.0.0:2380"];
+      };
+    };
+
+    environment.variables = {
+      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
+      ETCDCTL_KEY_FILE = "${etcd_client_key}";
+      ETCDCTL_CA_FILE = "${ca_pem}";
+      ETCDCTL_PEERS = "https://127.0.0.1:2379";
+    };
+
+    networking.firewall.allowedTCPPorts = [ 2380 ];
+  };
+in {
+  name = "etcd";
+
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ offline ];
+  };
+
+  nodes = {
+    node1 = { config, pkgs, nodes, ... }: {
+      require = [nodeConfig];
+      services.etcd = {
+        initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380"];
+        initialAdvertisePeerUrls = ["https://node1:2380"];
+      };
+    };
+
+    node2 = { config, pkgs, ... }: {
+      require = [nodeConfig];
+      services.etcd = {
+        initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380"];
+        initialAdvertisePeerUrls = ["https://node2:2380"];
+      };
+    };
+
+    node3 = { config, pkgs, ... }: {
+      require = [nodeConfig];
+      services.etcd = {
+        initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380" "node3=https://node3:2380"];
+        initialAdvertisePeerUrls = ["https://node3:2380"];
+        initialClusterState = "existing";
+      };
+    };
+  };
+
+  testScript = ''
+    subtest "should start etcd cluster", sub {
+      $node1->start();
+      $node2->start();
+      $node1->waitForUnit("etcd.service");
+      $node2->waitForUnit("etcd.service");
+      $node2->waitUntilSucceeds("etcdctl cluster-health");
+      $node1->succeed("etcdctl set /foo/bar 'Hello world'");
+      $node2->succeed("etcdctl get /foo/bar | grep 'Hello world'");
+    };
+
+    subtest "should add another member", sub {
+      $node1->succeed("etcdctl member add node3 https://node3:2380");
+      $node3->start();
+      $node3->waitForUnit("etcd.service");
+      $node3->waitUntilSucceeds("etcdctl member list | grep 'node3'");
+      $node3->succeed("etcdctl cluster-health");
+    };
+
+    subtest "should survive member crash", sub {
+      $node3->crash;
+      $node1->succeed("etcdctl cluster-health");
+      $node1->succeed("etcdctl set /foo/bar 'Hello degraded world'");
+      $node1->succeed("etcdctl get /foo/bar | grep 'Hello degraded world'");
+    };
+  '';
+})
diff --git a/nixos/tests/etcd.nix b/nixos/tests/etcd.nix
index bac4ec6a918b..f8a6791a834f 100644
--- a/nixos/tests/etcd.nix
+++ b/nixos/tests/etcd.nix
@@ -1,111 +1,27 @@
-# This test runs etcd as single node, multy node and using discovery
+# This test runs simple etcd node
 
 import ./make-test.nix ({ pkgs, ... } : {
   name = "etcd";
+
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ offline ];
   };
 
   nodes = {
-    simple =
-      { config, pkgs, nodes, ... }:
-        {
-          services.etcd.enable = true;
-          services.etcd.listenClientUrls = ["http://0.0.0.0:4001"];
-          environment.systemPackages = [ pkgs.curl ];
-          networking.firewall.allowedTCPPorts = [ 4001 ];
-        };
-
-
-    node1 =
-      { config, pkgs, nodes, ... }:
-        {
-          services = {
-            etcd = {
-              enable = true;
-              listenPeerUrls = ["http://0.0.0.0:7001"];
-              initialAdvertisePeerUrls = ["http://node1:7001"];
-              initialCluster = ["node1=http://node1:7001" "node2=http://node2:7001"];
-            };
-          };
-
-          networking.firewall.allowedTCPPorts = [ 7001 ];
-        };
-
-    node2 =
-      { config, pkgs, ... }:
-        {
-          services = {
-            etcd = {
-              enable = true;
-              listenPeerUrls = ["http://0.0.0.0:7001"];
-              initialAdvertisePeerUrls = ["http://node2:7001"];
-              initialCluster = ["node1=http://node1:7001" "node2=http://node2:7001"];
-            };
-          };
-
-          networking.firewall.allowedTCPPorts = [ 7001 ];
-        };
-
-    discovery1 =
-      { config, pkgs, nodes, ... }:
-        {
-          services = {
-            etcd = {
-              enable = true;
-              listenPeerUrls = ["http://0.0.0.0:7001"];
-              initialAdvertisePeerUrls = ["http://discovery1:7001"];
-              discovery = "http://simple:4001/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/";
-            };
-          };
-
-          networking.firewall.allowedTCPPorts = [ 7001 ];
-        };
-
-    discovery2 =
-      { config, pkgs, ... }:
-        {
-          services = {
-            etcd = {
-              enable = true;
-              listenPeerUrls = ["http://0.0.0.0:7001"];
-              initialAdvertisePeerUrls = ["http://discovery2:7001"];
-              discovery = "http://simple:4001/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/";
-            };
-          };
-
-          networking.firewall.allowedTCPPorts = [ 7001 ];
-        };
+    node = { config, pkgs, nodes, ... }: {
+      services.etcd.enable = true;
     };
+  };
 
   testScript = ''
-    subtest "single node", sub {
-      $simple->start();
-      $simple->waitForUnit("etcd.service");
-      $simple->waitUntilSucceeds("etcdctl set /foo/bar 'Hello world'");
-      $simple->waitUntilSucceeds("etcdctl get /foo/bar | grep 'Hello world'");
+    subtest "should start etcd node", sub {
+      $node->start();
+      $node->waitForUnit("etcd.service");
     };
 
-    subtest "multy node", sub {
-      $node1->start();
-      $node2->start();
-      $node1->waitForUnit("etcd.service");
-      $node2->waitForUnit("etcd.service");
-      $node1->waitUntilSucceeds("etcdctl set /foo/bar 'Hello world'");
-      $node2->waitUntilSucceeds("etcdctl get /foo/bar | grep 'Hello world'");
-      $node1->shutdown();
-      $node2->shutdown();
-    };
-
-    subtest "discovery", sub {
-      $simple->succeed("curl -X PUT http://localhost:4001/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/_config/size -d value=2");
-
-      $discovery1->start();
-      $discovery2->start();
-      $discovery1->waitForUnit("etcd.service");
-      $discovery2->waitForUnit("etcd.service");
-      $discovery1->waitUntilSucceeds("etcdctl set /foo/bar 'Hello world'");
-      $discovery2->waitUntilSucceeds("etcdctl get /foo/bar | grep 'Hello world'");
-    };
+    subtest "should write and read some values to etcd", sub {
+      $node->succeed("etcdctl set /foo/bar 'Hello world'");
+      $node->succeed("etcdctl get /foo/bar | grep 'Hello world'");
+    }
   '';
 })