Merge pull request #41884 from johanot/k8s-improvements

nixos/kubernetes: improvements

5 files changed, 105 insertions, 63 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml
index 72f96f1ca1a2..f57fd75c782d 100644
--- a/nixos/doc/manual/release-notes/rl-1809.xml
+++ b/nixos/doc/manual/release-notes/rl-1809.xml
@@ -288,11 +288,24 @@ inherit (pkgs.nixos {
    </listitem>
    <listitem>
     <para>
-     Recommented way to access the Kubernetes Dashboard is with HTTPS (TLS)
+     Recommended way to access the Kubernetes Dashboard is via HTTPS (TLS)
      Therefore; public service port for the dashboard has changed to 443
      (container port 8443) and scheme to https.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     The option <varname>services.kubernetes.apiserver.address</varname>
+     was renamed to <varname>services.kubernetes.apiserver.bindAddress</varname>.
+     Note that the default value has changed from 127.0.0.1 to 0.0.0.0.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     The option <varname>services.kubernetes.apiserver.publicAddress</varname>
+     was not used and thus has been removed.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 </section>
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index 9b9e9e7109de..2df737452fbc 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -32,6 +32,8 @@ with lib;
     (mkRenamedOptionModule [ "services" "i2pd" "extIp" ] [ "services" "i2pd" "address" ])
     (mkRenamedOptionModule [ "services" "kibana" "host" ] [ "services" "kibana" "listenAddress" ])
     (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ])
+    (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"])
+    (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
     (mkRenamedOptionModule [ "services" "logstash" "address" ] [ "services" "logstash" "listenAddress" ])
     (mkRenamedOptionModule [ "services" "mpd" "network" "host" ] [ "services" "mpd" "network" "listenAddress" ])
     (mkRenamedOptionModule [ "services" "neo4j" "host" ] [ "services" "neo4j" "listenAddress" ])
diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/dashboard.nix
index 8c1f35ec651b..9c1f814b683c 100644
--- a/nixos/modules/services/cluster/kubernetes/dashboard.nix
+++ b/nixos/modules/services/cluster/kubernetes/dashboard.nix
@@ -4,16 +4,6 @@ with lib;
 
 let
   cfg = config.services.kubernetes.addons.dashboard;
-
-  name = "k8s.gcr.io/kubernetes-dashboard-amd64";
-  version = "v1.8.3";
-
-  image = pkgs.dockerTools.pullImage {
-    imageName = name;
-    imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0";
-    finalImageTag = version;
-    sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8";
-  };
 in {
   options.services.kubernetes.addons.dashboard = {
     enable = mkEnableOption "kubernetes dashboard addon";
@@ -23,10 +13,27 @@ in {
       type = types.bool;
       default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
     };
+
+    version = mkOption {
+      description = "Which version of the kubernetes dashboard to deploy";
+      type = types.str;
+      default = "v1.8.3";
+    };
+
+    image = mkOption {
+      description = "Docker image to seed for the kubernetes dashboard container.";
+      type = types.attrs;
+      default = {
+        imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
+        imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0";
+        finalImageTag = cfg.version;
+        sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8";
+      };
+    };
   };
 
   config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages = [image];
+    services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)];
 
     services.kubernetes.addonManager.addons = {
       kubernetes-dashboard-deployment = {
@@ -36,7 +43,7 @@ in {
           labels = {
             k8s-addon = "kubernetes-dashboard.addons.k8s.io";
             k8s-app = "kubernetes-dashboard";
-            version = version;
+            version = cfg.version;
             "kubernetes.io/cluster-service" = "true";
             "addonmanager.kubernetes.io/mode" = "Reconcile";
           };
@@ -52,7 +59,7 @@ in {
               labels = {
                 k8s-addon = "kubernetes-dashboard.addons.k8s.io";
                 k8s-app = "kubernetes-dashboard";
-                version = version;
+                version = cfg.version;
                 "kubernetes.io/cluster-service" = "true";
               };
               annotations = {
@@ -63,7 +70,7 @@ in {
               priorityClassName = "system-cluster-critical";
               containers = [{
                 name = "kubernetes-dashboard";
-                image = "${name}:${version}";
+                image = with cfg.image; "${imageName}:${finalImageTag}";
                 ports = [{
                   containerPort = 8443;
                   protocol = "TCP";
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index e624f41601b3..20f2308508c6 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -73,7 +73,9 @@ let
   mkKubeConfigOptions = prefix: {
     server = mkOption {
       description = "${prefix} kube-apiserver server address.";
-      default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
+      default = "http://${if cfg.apiserver.advertiseAddress != null
+                          then cfg.apiserver.advertiseAddress
+                          else "127.0.0.1"}:${toString cfg.apiserver.port}";
       type = types.str;
     };
 
@@ -103,12 +105,18 @@ let
     keyFile = mkDefault cfg.kubeconfig.keyFile;
   };
 
-  cniConfig = pkgs.buildEnv {
-    name = "kubernetes-cni-config";
-    paths = imap (i: entry:
-      pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
-    ) cfg.kubelet.cni.config;
-  };
+  cniConfig =
+    if cfg.kubelet.cni.config != [] && !(isNull cfg.kubelet.cni.configDir) then
+      throw "Verbatim CNI-config and CNI configDir cannot both be set."
+    else if !(isNull cfg.kubelet.cni.configDir) then
+      cfg.kubelet.cni.configDir
+    else
+      (pkgs.buildEnv {
+        name = "kubernetes-cni-config";
+        paths = imap (i: entry:
+          pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
+        ) cfg.kubelet.cni.config;
+      });
 
   manifests = pkgs.buildEnv {
     name = "kubernetes-manifests";
@@ -244,18 +252,13 @@ in {
         type = types.listOf types.str;
       };
 
-      address = mkOption {
-        description = "Kubernetes apiserver listening address.";
-        default = "127.0.0.1";
-        type = types.str;
-      };
-
-      publicAddress = mkOption {
+      bindAddress = mkOption {
         description = ''
-          Kubernetes apiserver public listening address used for read only and
-          secure port.
+          The IP address on which to listen for the --secure-port port.
+          The associated interface(s) must be reachable by the rest
+          of the cluster, and by CLI/web clients.
         '';
-        default = cfg.apiserver.address;
+        default = "0.0.0.0";
         type = types.str;
       };
 
@@ -670,6 +673,12 @@ in {
             }]
           '';
         };
+
+        configDir = mkOption {
+          description = "Path to Kubernetes CNI configuration directory.";
+          type = types.nullOr types.path;
+          default = null;
+        };
       };
 
       manifests = mkOption {
@@ -892,7 +901,7 @@ in {
 
     (mkIf cfg.apiserver.enable {
       systemd.services.kube-apiserver = {
-        description = "Kubernetes Kubelet Service";
+        description = "Kubernetes APIServer Service";
         wantedBy = [ "kubernetes.target" ];
         after = [ "network.target" "docker.service" ];
         serviceConfig = {
@@ -906,7 +915,7 @@ in {
             ${optionalString (cfg.etcd.keyFile != null)
               "--etcd-keyfile=${cfg.etcd.keyFile}"} \
             --insecure-port=${toString cfg.apiserver.port} \
-            --bind-address=${toString cfg.apiserver.address} \
+            --bind-address=${cfg.apiserver.bindAddress} \
             ${optionalString (cfg.apiserver.advertiseAddress != null)
               "--advertise-address=${cfg.apiserver.advertiseAddress}"} \
             --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/dns.nix
index 9751e5f7cf0a..43bbb50a48d4 100644
--- a/nixos/modules/services/cluster/kubernetes/dns.nix
+++ b/nixos/modules/services/cluster/kubernetes/dns.nix
@@ -4,28 +4,6 @@ with lib;
 
 let
   version = "1.14.10";
-
-  k8s-dns-kube-dns = pkgs.dockerTools.pullImage {
-    imageName = "k8s.gcr.io/k8s-dns-kube-dns-amd64";
-    imageDigest = "sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8";
-    finalImageTag = version;
-    sha256 = "0x583znk9smqn0fix7ld8sm5jgaxhqhx3fq97b1wkqm7iwhvl3pj";
-  };
-
-  k8s-dns-dnsmasq-nanny = pkgs.dockerTools.pullImage {
-    imageName = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64";
-    imageDigest = "sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8";
-    finalImageTag = version;
-    sha256 = "1fihml7s2mfwgac51cbqpylkwbivc8nyhgi4vb820s83zvl8a6y1";
-  };
-
-  k8s-dns-sidecar = pkgs.dockerTools.pullImage {
-    imageName = "k8s.gcr.io/k8s-dns-sidecar-amd64";
-    imageDigest = "sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4";
-    finalImageTag = version;
-    sha256 = "08l1bv5jgrhvjzpqpbinrkgvv52snc4fzyd8ya9v18ns2klyz7m0";
-  };
-
   cfg = config.services.kubernetes.addons.dns;
 in {
   options.services.kubernetes.addons.dns = {
@@ -48,13 +26,46 @@ in {
       default = "cluster.local";
       type = types.str;
     };
+
+    kube-dns = mkOption {
+      description = "Docker image to seed for the kube-dns main container.";
+      type = types.attrs;
+      default = {
+        imageName = "k8s.gcr.io/k8s-dns-kube-dns-amd64";
+        imageDigest = "sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8";
+        finalImageTag = version;
+        sha256 = "0x583znk9smqn0fix7ld8sm5jgaxhqhx3fq97b1wkqm7iwhvl3pj";
+      };
+    };
+
+    dnsmasq-nanny = mkOption {
+      description = "Docker image to seed for the kube-dns dnsmasq container.";
+      type = types.attrs;
+      default = {
+        imageName = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64";
+        imageDigest = "sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8";
+        finalImageTag = version;
+        sha256 = "1fihml7s2mfwgac51cbqpylkwbivc8nyhgi4vb820s83zvl8a6y1";
+      };
+    };
+
+    sidecar = mkOption {
+      description = "Docker image to seed for the kube-dns sidecar container.";
+      type = types.attrs;
+      default = {
+        imageName = "k8s.gcr.io/k8s-dns-sidecar-amd64";
+        imageDigest = "sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4";
+        finalImageTag = version;
+        sha256 = "08l1bv5jgrhvjzpqpbinrkgvv52snc4fzyd8ya9v18ns2klyz7m0";
+      };
+    };
   };
 
   config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages = [
-      k8s-dns-kube-dns
-      k8s-dns-dnsmasq-nanny
-      k8s-dns-sidecar
+    services.kubernetes.kubelet.seedDockerImages = with pkgs.dockerTools; [
+      (pullImage cfg.kube-dns)
+      (pullImage cfg.dnsmasq-nanny)
+      (pullImage cfg.sidecar)
     ];
 
     services.kubernetes.addonManager.addons = {
@@ -88,7 +99,7 @@ in {
               containers = [
                 {
                   name = "kubedns";
-                  image = "k8s.gcr.io/k8s-dns-kube-dns-amd64:${version}";
+                  image = with cfg.kube-dns; "${imageName}:${finalImageTag}";
                   resources = {
                     limits.memory = "170Mi";
                     requests = {
@@ -154,7 +165,7 @@ in {
                 }
                 {
                   name = "dnsmasq";
-                  image = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:${version}";
+                  image = with cfg.dnsmasq-nanny; "${imageName}:${finalImageTag}";
                   livenessProbe = {
                     httpGet = {
                       path = "/healthcheck/dnsmasq";
@@ -206,7 +217,7 @@ in {
                 }
                 {
                   name = "sidecar";
-                  image = "k8s.gcr.io/k8s-dns-sidecar-amd64:${version}";
+                  image = with cfg.sidecar; "${imageName}:${finalImageTag}";
                   livenessProbe = {
                     httpGet = {
                       path = "/metrics";