Merge pull request #19023 from offlinehacker/kube-update

WIP: kubernetes update package and module

1 files changed, 388 insertions, 162 deletions

diff --git a/nixos/tests/kubernetes.nix b/nixos/tests/kubernetes.nix
index b19ea67b0baf..273bd3c80c19 100644
--- a/nixos/tests/kubernetes.nix
+++ b/nixos/tests/kubernetes.nix
@@ -1,182 +1,408 @@
-# This test runs two node kubernetes cluster and checks if simple redis pod works
+{ system ? builtins.currentSystem }:
 
-import ./make-test.nix ({ pkgs, ...} : rec {
-  name = "kubernetes";
-  meta = with pkgs.stdenv.lib.maintainers; {
-    maintainers = [ offline ];
+with import ../lib/testing.nix { inherit system; };
+with import ../lib/qemu-flags.nix;
+with pkgs.lib;
+
+let
+  redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
+    kind = "Pod";
+    apiVersion = "v1";
+    metadata.name = "redis";
+    metadata.labels.name = "redis";
+    spec.containers = [{
+      name = "redis";
+      image = "redis";
+      args = ["--bind" "0.0.0.0"];
+      imagePullPolicy = "Never";
+      ports = [{
+        name = "redis-server";
+        containerPort = 6379;
+      }];
+    }];
+  });
+
+  redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
+    kind = "Service";
+    apiVersion = "v1";
+    metadata.name = "redis";
+    spec = {
+      ports = [{port = 6379; targetPort = 6379;}];
+      selector = {name = "redis";};
+    };
+  });
+
+  redisImage = pkgs.dockerTools.buildImage {
+    name = "redis";
+    tag = "latest";
+    contents = pkgs.redis;
+    config.Entrypoint = "/bin/redis-server";
   };
 
-  redisMaster = builtins.toFile "redis-master-pod.yaml" ''
-      id: redis-master-pod
-      kind: Pod
-      apiVersion: v1beta1
-      desiredState:
-        manifest:
-          version: v1beta1
-          id: redis-master-pod
-          containers:
-            - name: master
-              image: master:5000/nix
-              cpu: 100
-              ports:
-                - name: redis-server
-                  containerPort: 6379
-                  hostPort: 6379
-              volumeMounts:
-                - name: nix-store
-                  mountPath: /nix/store
-                  readOnly: true
-              volumeMounts:
-                - name: system-profile
-                  mountPath: /bin
-                  readOnly: true
-              command:
-                - /bin/redis-server
-          volumes:
-            - name: nix-store
-              source:
-                hostDir:
-                  path: /nix/store
-            - name: system-profile
-              source:
-                hostDir:
-                  path: /run/current-system/sw/bin
-      labels:
-        name: redis
-        role: master
+  testSimplePod = ''
+    $kubernetes->execute("docker load < ${redisImage}");
+    $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");
+    $kubernetes->succeed("kubectl create -f ${redisService}");
+    $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
+    $kubernetes->succeed("nc -z \$\(dig \@10.10.0.1 redis.default.svc.cluster.local +short\) 6379");
   '';
+in {
+  # This test runs kubernetes on a single node
+  trivial = makeTest {
+    name = "kubernetes-trivial";
 
-  nodes = {
-    master =
-      { config, pkgs, lib, nodes, ... }:
-        {
-          virtualisation.memorySize = 768;
-          services.kubernetes = {
-            roles = ["master" "node"];
-            dockerCfg = ''{"master:5000":{}}'';
-            controllerManager.machines = ["master" "node"];
-            apiserver.address = "0.0.0.0";
-            verbose = true;
-          };
-          virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
+    nodes = {
+      kubernetes =
+        { config, pkgs, lib, nodes, ... }:
+          {
+            virtualisation.memorySize = 768;
+            virtualisation.diskSize = 2048;
 
-          services.etcd = {
-            listenPeerUrls = ["http://0.0.0.0:7001"];
-            initialAdvertisePeerUrls = ["http://master:7001"];
-            initialCluster = ["master=http://master:7001" "node=http://node:7001"];
-          };
-          services.dockerRegistry.enable = true;
-          services.dockerRegistry.host = "0.0.0.0";
-          services.dockerRegistry.port = 5000;
+            programs.bash.enableCompletion = true;
 
-          virtualisation.vlans = [ 1 2 ];
-          networking.bridges = {
-            cbr0.interfaces = [ "eth2" ];
-          };
-          networking.interfaces = {
-            cbr0 = {
-              ipAddress = "10.10.0.1";
-              prefixLength = 24;
-            };
-            eth2.ip4 = lib.mkOverride 0 [ ];
+            services.kubernetes.roles = ["master" "node"];
+            virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";
+
+            networking.bridges.cbr0.interfaces = [];
+            networking.interfaces.cbr0 = {};
           };
-          networking.localCommands = ''
-            ip route add 10.10.0.0/16 dev cbr0
-            ip route flush cache
-          '';
-          networking.extraHosts = "127.0.0.1 master";
+    };
+
+    testScript = ''
+      startAll;
+
+      $kubernetes->waitUntilSucceeds("kubectl get nodes | grep kubernetes | grep Ready");
+
+      ${testSimplePod}
+    '';
+  };
+
+  cluster = let
+    runWithOpenSSL = file: cmd: pkgs.runCommand file {
+      buildInputs = [ pkgs.openssl ];
+    } cmd;
+
+    ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
+    ca_pem = runWithOpenSSL "ca.pem" ''
+      openssl req \
+        -x509 -new -nodes -key ${ca_key} \
+        -days 10000 -out $out -subj "/CN=etcd-ca"
+    '';
+    etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
+    etcd_csr = runWithOpenSSL "etcd.csr" ''
+      openssl req \
+        -new -key ${etcd_key} \
+        -out $out -subj "/CN=etcd" \
+        -config ${openssl_cnf}
+    '';
+    etcd_cert = runWithOpenSSL "etcd.pem" ''
+      openssl x509 \
+        -req -in ${etcd_csr} \
+        -CA ${ca_pem} -CAkey ${ca_key} \
+        -CAcreateserial -out $out \
+        -days 365 -extensions v3_req \
+        -extfile ${openssl_cnf}
+    '';
+
+    etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
+      "openssl genrsa -out $out 2048";
+
+    etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
+      openssl req \
+        -new -key ${etcd_client_key} \
+        -out $out -subj "/CN=etcd-client" \
+        -config ${client_openssl_cnf}
+    '';
+
+    etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
+      openssl x509 \
+        -req -in ${etcd_client_csr} \
+        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+        -out $out -days 365 -extensions v3_req \
+        -extfile ${client_openssl_cnf}
+    '';
 
-          networking.firewall.enable = false;
-          #networking.firewall.allowedTCPPorts = [ 4001 7001 ];
+    apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";
 
-          environment.systemPackages = [ pkgs.redis ];
+    apiserver_csr = runWithOpenSSL "apiserver.csr" ''
+      openssl req \
+        -new -key ${apiserver_key} \
+        -out $out -subj "/CN=kube-apiserver" \
+        -config ${apiserver_cnf}
+    '';
+
+    apiserver_cert = runWithOpenSSL "apiserver.pem" ''
+      openssl x509 \
+        -req -in ${apiserver_csr} \
+        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+        -out $out -days 365 -extensions v3_req \
+        -extfile ${apiserver_cnf}
+    '';
+
+    worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
+
+    worker_csr = runWithOpenSSL "worker.csr" ''
+      openssl req \
+        -new -key ${worker_key} \
+        -out $out -subj "/CN=kube-worker" \
+        -config ${worker_cnf}
+    '';
+
+    worker_cert = runWithOpenSSL "worker.pem" ''
+      openssl x509 \
+        -req -in ${worker_csr} \
+        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
+        -out $out -days 365 -extensions v3_req \
+        -extfile ${worker_cnf}
+    '';
+
+    openssl_cnf = pkgs.writeText "openssl.cnf" ''
+      [req]
+      req_extensions = v3_req
+      distinguished_name = req_distinguished_name
+      [req_distinguished_name]
+      [ v3_req ]
+      basicConstraints = CA:FALSE
+      keyUsage = digitalSignature, keyEncipherment
+      extendedKeyUsage = serverAuth
+      subjectAltName = @alt_names
+      [alt_names]
+      DNS.1 = etcd1
+      DNS.2 = etcd2
+      DNS.3 = etcd3
+      IP.1 = 127.0.0.1
+    '';
+
+    client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
+      [req]
+      req_extensions = v3_req
+      distinguished_name = req_distinguished_name
+      [req_distinguished_name]
+      [ v3_req ]
+      basicConstraints = CA:FALSE
+      keyUsage = digitalSignature, keyEncipherment
+      extendedKeyUsage = clientAuth
+    '';
+
+    apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
+      [req]
+      req_extensions = v3_req
+      distinguished_name = req_distinguished_name
+      [req_distinguished_name]
+      [ v3_req ]
+      basicConstraints = CA:FALSE
+      keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+      subjectAltName = @alt_names
+      [alt_names]
+      DNS.1 = kubernetes
+      DNS.2 = kubernetes.default
+      DNS.3 = kubernetes.default.svc
+      DNS.4 = kubernetes.default.svc.cluster.local
+      IP.1 = 10.10.10.1
+    '';
+
+    worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
+      [req]
+      req_extensions = v3_req
+      distinguished_name = req_distinguished_name
+      [req_distinguished_name]
+      [ v3_req ]
+      basicConstraints = CA:FALSE
+      keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+      subjectAltName = @alt_names
+      [alt_names]
+      DNS.1 = kubeWorker1
+      DNS.2 = kubeWorker2
+    '';
+
+    etcdNodeConfig = {
+      virtualisation.memorySize = 128;
+
+      services = {
+        etcd = {
+          enable = true;
+          keyFile = etcd_key;
+          certFile = etcd_cert;
+          trustedCaFile = ca_pem;
+          peerClientCertAuth = true;
+          listenClientUrls = ["https://0.0.0.0:2379"];
+          listenPeerUrls = ["https://0.0.0.0:2380"];
         };
+      };
 
-    node =
-      { config, pkgs, lib, nodes, ... }:
-        {
-          services.kubernetes = {
-            roles = ["node"];
-            dockerCfg = ''{"master:5000":{}}'';
-            kubelet.apiServers = ["master:8080"];
-            verbose = true;
-          };
-          virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
-          services.etcd = {
-            listenPeerUrls = ["http://0.0.0.0:7001"];
-            initialAdvertisePeerUrls = ["http://node:7001"];
-            initialCluster = ["master=http://master:7001" "node=http://node:7001"];
-          };
+      environment.variables = {
+        ETCDCTL_CERT_FILE = "${etcd_client_cert}";
+        ETCDCTL_KEY_FILE = "${etcd_client_key}";
+        ETCDCTL_CA_FILE = "${ca_pem}";
+        ETCDCTL_PEERS = "https://127.0.0.1:2379";
+      };
 
-          virtualisation.vlans = [ 1 2 ];
-          networking.bridges = {
-            cbr0.interfaces = [ "eth2" ];
-          };
-          networking.interfaces = {
-            cbr0 = {
-              ipAddress = "10.10.1.1";
-              prefixLength = 24;
-            };
-            eth2.ip4 = lib.mkOverride 0 [ ];
-          };
-          networking.localCommands = ''
-            ip route add 10.10.0.0/16 dev cbr0
-            ip route flush cache
-          '';
-          networking.extraHosts = "127.0.0.1 node";
+      networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+    };
 
-          networking.firewall.enable = false;
-          #networking.firewall.allowedTCPPorts = [ 4001 7001 ];
+    kubeConfig = {
+      virtualisation.diskSize = 2048;
+      programs.bash.enableCompletion = true;
 
-          environment.systemPackages = [ pkgs.redis ];
+      services.flannel = {
+        enable = true;
+        network = "10.10.0.0/16";
+        iface = "eth1";
+        etcd = {
+          endpoints = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"];
+          keyFile = etcd_client_key;
+          certFile = etcd_client_cert;
+          caFile = ca_pem;
         };
+      };
+
+      # vxlan
+      networking.firewall.allowedUDPPorts = [ 8472 ];
 
-    client =
-      { config, pkgs, nodes, ... }:
-        {
-          virtualisation.docker.enable = true;
-          virtualisation.docker.extraOptions = "--insecure-registry master:5000";
-          environment.systemPackages = [ pkgs.kubernetes ];
-          environment.etc."test/redis-master-pod.yaml".source = redisMaster;
-          environment.etc."test/pause".source = "${pkgs.kubernetes}/bin/kube-pause";
-          environment.etc."test/Dockerfile".source = pkgs.writeText "Dockerfile" ''
-            FROM scratch
-            ADD pause /
-            ENTRYPOINT ["/pause"]
-          '';
+      systemd.services.docker.after = ["flannel.service"];
+      systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
+      virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
+
+      services.kubernetes.verbose = true;
+      services.kubernetes.etcd = {
+        servers = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"];
+        keyFile = etcd_client_key;
+        certFile = etcd_client_cert;
+        caFile = ca_pem;
+      };
+
+      environment.systemPackages = [ pkgs.bind pkgs.tcpdump pkgs.utillinux ];
+    };
+
+    kubeMasterConfig = {pkgs, ...}: {
+      require = [kubeConfig];
+
+      # kube apiserver
+      networking.firewall.allowedTCPPorts = [ 443 ];
+
+      virtualisation.memorySize = 512;
+
+      services.kubernetes = {
+        roles = ["master"];
+        scheduler.leaderElect = true;
+        controllerManager.leaderElect = true;
+
+        apiserver = {
+          publicAddress = "0.0.0.0";
+          advertiseAddress = "192.168.1.8";
+          tlsKeyFile = apiserver_key;
+          tlsCertFile = apiserver_cert;
+          clientCaFile = ca_pem;
+          kubeletClientCaFile = ca_pem;
+          kubeletClientKeyFile = worker_key;
+          kubeletClientCertFile = worker_cert;
         };
-  };
+      };
+    };
 
-  testScript = ''
-    startAll;
-
-    $master->waitForUnit("kubernetes-apiserver.service");
-    $master->waitForUnit("kubernetes-scheduler.service");
-    $master->waitForUnit("kubernetes-controller-manager.service");
-    $master->waitForUnit("kubernetes-kubelet.service");
-    $master->waitForUnit("kubernetes-proxy.service");
-
-    $node->waitForUnit("kubernetes-kubelet.service");
-    $node->waitForUnit("kubernetes-proxy.service");
-
-    $master->waitUntilSucceeds("kubectl get minions | grep master");
-    $master->waitUntilSucceeds("kubectl get minions | grep node");
-
-    $client->waitForUnit("docker.service");
-    $client->succeed("tar cv --files-from /dev/null | docker import - nix");
-    $client->succeed("docker tag nix master:5000/nix");
-    $master->waitForUnit("docker-registry.service");
-    $client->succeed("docker push master:5000/nix");
-    $client->succeed("mkdir -p /root/pause");
-    $client->succeed("cp /etc/test/pause /root/pause/");
-    $client->succeed("cp /etc/test/Dockerfile /root/pause/");
-    $client->succeed("cd /root/pause && docker build -t master:5000/pause .");
-    $client->succeed("docker push master:5000/pause");
-
-    subtest "simple pod", sub {
-      $client->succeed("kubectl create -f ${redisMaster} -s http://master:8080");
-      $client->waitUntilSucceeds("kubectl get pods -s http://master:8080 | grep redis-master | grep -i running");
-    }
+    kubeWorkerConfig = { pkgs, ... }: {
+      require = [kubeConfig];
 
-  '';
-})
+      virtualisation.memorySize = 512;
+
+      # kubelet
+      networking.firewall.allowedTCPPorts = [ 10250 ];
+
+      services.kubernetes = {
+        roles = ["node"];
+        kubeconfig = {
+          server = "https://kubernetes:443";
+          caFile = ca_pem;
+          certFile = worker_cert;
+          keyFile = worker_key;
+        };
+        kubelet = {
+          tlsKeyFile = worker_key;
+          tlsCertFile = worker_cert;
+        };
+      };
+    };
+  in makeTest {
+    name = "kubernetes-cluster";
+
+    nodes = {
+      etcd1 = { config, pkgs, nodes, ... }: {
+        require = [etcdNodeConfig];
+        services.etcd = {
+          advertiseClientUrls = ["https://etcd1:2379"];
+          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
+          initialAdvertisePeerUrls = ["https://etcd1:2380"];
+        };
+      };
+
+      etcd2 = { config, pkgs, ... }: {
+        require = [etcdNodeConfig];
+        services.etcd = {
+          advertiseClientUrls = ["https://etcd2:2379"];
+          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
+          initialAdvertisePeerUrls = ["https://etcd2:2380"];
+        };
+      };
+
+      etcd3 = { config, pkgs, ... }: {
+        require = [etcdNodeConfig];
+        services.etcd = {
+          advertiseClientUrls = ["https://etcd3:2379"];
+          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
+          initialAdvertisePeerUrls = ["https://etcd3:2380"];
+        };
+      };
+
+      kubeMaster1 = { config, pkgs, lib, nodes, ... }: {
+        require = [kubeMasterConfig];
+      };
+
+      kubeMaster2 = { config, pkgs, lib, nodes, ... }: {
+        require = [kubeMasterConfig];
+      };
+
+      # Kubernetes TCP load balancer
+      kubernetes = { config, pkgs, ... }: {
+        # kubernetes
+        networking.firewall.allowedTCPPorts = [ 443 ];
+
+        services.haproxy.enable = true;
+        services.haproxy.config = ''
+          global
+              log 127.0.0.1 local0 notice
+              user haproxy
+              group haproxy
+
+          defaults
+              log global
+              retries 2
+              timeout connect 3000
+              timeout server 5000
+              timeout client 5000
+
+          listen kubernetes
+              bind 0.0.0.0:443
+              mode tcp
+              option ssl-hello-chk
+              balance roundrobin
+              server kube-master-1 kubeMaster1:443 check
+              server kube-master-2 kubeMaster2:443 check
+        '';
+      };
+
+      kubeWorker1 = { config, pkgs, lib, nodes, ... }: {
+        require = [kubeWorkerConfig];
+      };
+
+      kubeWorker2 = { config, pkgs, lib, nodes, ... }: {
+        require = [kubeWorkerConfig];
+      };
+    };
+
+    testScript = ''
+      startAll;
+
+      ${testSimplePod}
+    '';
+  };
+}