From 7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 26 May 2017 23:21:17 +0200 Subject: kubernetes module: flannel support, minor fixes - add flannel support - remove deprecated authorizationRBACSuperAdmin option - rename from deprecated poratalNet to serviceClusterIpRange - add nodeIp option for kubelet - kubelet, add br_netfilter to kernelModules - enable firewall by default - enable dns by default on node and on master - disable iptables for docker by default on nodes - dns, restart on failure - update tests and other minor changes --- .../services/cluster/kubernetes/default.nix | 117 ++++++++++++++++++--- nixos/tests/kubernetes/kubernetes-common.nix | 75 ++----------- nixos/tests/kubernetes/kubernetes-master.nix | 8 +- nixos/tests/kubernetes/multinode-kubectl.nix | 18 +--- nixos/tests/kubernetes/singlenode.nix | 47 ++++----- 5 files changed, 132 insertions(+), 133 deletions(-) diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 76b27ac0efba..bd37c49486e4 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -162,6 +162,17 @@ let }; } ]); + + # needed for flannel to pass options to docker + mkDockerOpts = pkgs.runCommand "mk-docker-opts" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out + cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh + + # bashInteractive needed for `compgen` + makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" + ''; in { ###### interface @@ -357,20 +368,17 @@ in { type = types.listOf types.attrs; }; - authorizationRBACSuperAdmin = mkOption { - description = "Role based authorization super admin."; - default = "admin"; - type = types.str; - }; - allowPrivileged = mkOption { description = "Whether to allow privileged containers on Kubernetes."; default = true; type = types.bool; }; - portalNet = mkOption { - description = "Kubernetes CIDR notation IP range from which to assign portal IPs."; + serviceClusterIpRange = mkOption { + description = '' + A CIDR notation IP range from which to assign service cluster IPs. + This must not overlap with any IP ranges assigned to nodes for pods. + ''; default = "10.10.10.10/24"; type = types.str; }; @@ -666,6 +674,12 @@ in { type = types.attrsOf (types.submodule [ taintOptions ]); }; + nodeIp = mkOption { + description = "IP address of the node. If set, kubelet will use this IP address for the node."; + default = null; + type = types.nullOr types.str; + }; + extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; @@ -761,6 +775,12 @@ in { type = types.str; }; + flannel.enable = mkOption { + description = "Whether to enable flannel networking"; + default = false; + type = types.bool; + }; + }; ###### implementation @@ -808,6 +828,8 @@ in { "--network-plugin=${cfg.kubelet.networkPlugin}"} \ --cni-conf-dir=${cniConfig} \ --hairpin-mode=hairpin-veth \ + ${optionalString (cfg.kubelet.nodeIp != null) + "--node-ip=${cfg.kubelet.nodeIp}"} \ ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.kubelet.extraOpts} ''; @@ -817,6 +839,8 @@ in { # Allways include cni plugins services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + + boot.kernelModules = ["br_netfilter"]; }) (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { @@ -879,10 +903,8 @@ in { (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy) }" } \ - ${optionalString (elem "RBAC" cfg.apiserver.authorizationMode) - "--authorization-rbac-super-user=${cfg.apiserver.authorizationRBACSuperAdmin}"} \ --secure-port=${toString cfg.apiserver.securePort} \ - --service-cluster-ip-range=${cfg.apiserver.portalNet} \ + --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \ ${optionalString (cfg.apiserver.runtimeConfig != "") "--runtime-config=${cfg.apiserver.runtimeConfig}"} \ --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \ @@ -981,10 +1003,9 @@ in { WorkingDirectory = cfg.dataDir; }; }; - }) - (mkIf cfg.kubelet.enable { - boot.kernelModules = ["br_netfilter"]; + # kube-proxy needs iptables + networking.firewall.enable = mkDefault true; }) (mkIf (any (el: el == "master") cfg.roles) { @@ -999,15 +1020,25 @@ in { services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]); }) + # if this node is only a master make it unschedulable by default (mkIf (all (el: el == "master") cfg.roles) { services.kubernetes.kubelet.unschedulable = mkDefault true; }) (mkIf (any (el: el == "node") cfg.roles) { - virtualisation.docker.enable = mkDefault true; - virtualisation.docker.logDriver = mkDefault "json-file"; + virtualisation.docker = { + enable = mkDefault true; + + # kubernetes needs access to logs + logDriver = mkDefault "json-file"; + + # iptables must be disabled for kubernetes + extraOptions = "--iptables=false --ip-masq=false"; + }; + services.kubernetes.kubelet.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; + services.kubernetes.dns.enable = mkDefault true; }) (mkIf cfg.addonManager.enable { @@ -1035,8 +1066,17 @@ in { Group = "kubernetes"; AmbientCapabilities = "cap_net_bind_service"; SendSIGHUP = true; + RestartSec = "30s"; + Restart = "always"; + StartLimitInterval = "1m"; }; }; + + networking.firewall.extraCommands = '' + # allow container to host communication for DNS traffic + ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept + ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept + ''; }) (mkIf ( @@ -1070,5 +1110,50 @@ in { }; users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes; }) + + (mkIf cfg.flannel.enable { + services.flannel = { + enable = mkDefault true; + network = mkDefault cfg.clusterCidr; + etcd = mkDefault { + endpoints = cfg.etcd.servers; + inherit (cfg.etcd) caFile certFile keyFile; + }; + }; + + services.kubernetes.kubelet = { + networkPlugin = mkDefault "cni"; + cni.config = mkDefault [{ + name = "mynet"; + type = "flannel"; + delegate = { + isDefaultGateway = true; + bridge = "docker0"; + }; + }]; + }; + + systemd.services."mk-docker-opts" = { + description = "Pre-Docker Actions"; + wantedBy = [ "flannel.service" ]; + before = [ "docker.service" ]; + after = [ "flannel.service" ]; + path = [ pkgs.gawk pkgs.gnugrep ]; + script = '' + mkdir -p /run/flannel + ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker + ''; + serviceConfig.Type = "oneshot"; + }; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; + + # read environment variables generated by mk-docker-opts + virtualisation.docker.extraOptions = "$DOCKER_OPTS"; + + networking.firewall.allowedUDPPorts = [ + 8285 # flannel udp + 8472 # flannel vxlan + ]; + }) ]; } diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix index bc28244ad5b4..9f9e730fa655 100644 --- a/nixos/tests/kubernetes/kubernetes-common.nix +++ b/nixos/tests/kubernetes/kubernetes-common.nix @@ -1,4 +1,5 @@ { config, pkgs, certs, servers }: + let etcd_key = "${certs}/etcd-key.pem"; etcd_cert = "${certs}/etcd.pem"; @@ -9,8 +10,6 @@ let worker_key = "${certs}/worker-key.pem"; worker_cert = "${certs}/worker.pem"; - mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh"; - rootCaFile = pkgs.writeScript "rootCaFile.pem" '' ${pkgs.lib.readFile "${certs}/ca.pem"} @@ -26,16 +25,9 @@ in environment.systemPackages = with pkgs; [ netcat bind etcd.bin ]; networking = { - firewall = { - enable = true; - allowedTCPPorts = [ - 10250 80 443 - ]; - allowedUDPPorts = [ - 8285 # flannel udp - 8472 # flannel vxlan - ]; - }; + firewall.allowedTCPPorts = [ + 10250 # kubelet + ]; extraHosts = '' # register "external" domains ${servers.master} etcd.kubernetes.nixos.xyz @@ -43,42 +35,7 @@ in ${mkHosts} ''; }; - virtualisation.docker.extraOptions = '' - --iptables=false $DOCKER_OPTS - ''; - - # lets create environment file for docker startup - network stuff - systemd.services."pre-docker" = { - description = "Pre-Docker Actions"; - wantedBy = [ "flannel.service" ]; - before = [ "docker.service" ]; - after = [ "flannel.service" ]; - path = [ pkgs.gawk pkgs.gnugrep ]; - script = '' - mkdir -p /run/flannel - # bashInteractive needed for `compgen` - ${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker - cat /run/flannel/docker # just for debugging - - # allow container to host communication for DNS traffic - ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept - ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept - ''; - serviceConfig.Type = "simple"; - }; - systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; - - services.flannel = { - enable = true; - network = "10.2.0.0/16"; - iface = "eth1"; - etcd = { - endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"]; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; - }; - }; + services.flannel.iface = "eth1"; environment.variables = { ETCDCTL_CERT_FILE = "${etcd_client_cert}"; ETCDCTL_KEY_FILE = "${etcd_client_key}"; @@ -88,20 +45,10 @@ in services.kubernetes = { kubelet = { - networkPlugin = "cni"; - cni.config = [{ - name = "mynet"; - type = "flannel"; - delegate = { - isDefaultGateway = true; - bridge = "docker0"; - }; - }]; tlsKeyFile = worker_key; tlsCertFile = worker_cert; hostname = "${config.networking.hostName}.nixos.xyz"; - extraOpts = "--node-ip ${config.networking.primaryIPAddress}"; - clusterDns = config.networking.primaryIPAddress; + nodeIp = config.networking.primaryIPAddress; }; etcd = { servers = ["https://etcd.kubernetes.nixos.xyz:2379"]; @@ -110,22 +57,16 @@ in caFile = ca_pem; }; kubeconfig = { - server = "https://kubernetes.nixos.xyz:4443"; + server = "https://kubernetes.nixos.xyz"; caFile = rootCaFile; certFile = worker_cert; keyFile = worker_key; }; + flannel.enable = true; - # make sure you cover kubernetes.apiserver.portalNet and flannel networks - clusterCidr = "10.0.0.0/8"; - - dns.enable = true; dns.port = 4453; }; services.dnsmasq.enable = true; services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; - - virtualisation.docker.enable = true; - virtualisation.docker.storageDriver = "overlay"; } diff --git a/nixos/tests/kubernetes/kubernetes-master.nix b/nixos/tests/kubernetes/kubernetes-master.nix index 15e7e52e4832..28cce6ea6534 100644 --- a/nixos/tests/kubernetes/kubernetes-master.nix +++ b/nixos/tests/kubernetes/kubernetes-master.nix @@ -25,7 +25,7 @@ in allowPing = true; allowedTCPPorts = [ 2379 2380 # etcd - 4443 # kubernetes + 443 # kubernetes apiserver ]; }; }; @@ -43,14 +43,13 @@ in initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"]; initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"]; }; + services.kubernetes = { roles = ["master"]; scheduler.leaderElect = true; - controllerManager.leaderElect = true; controllerManager.rootCaFile = rootCaFile; controllerManager.serviceAccountKeyFile = apiserver_key; apiserver = { - securePort = 4443; publicAddress = "192.168.1.1"; advertiseAddress = "192.168.1.1"; tlsKeyFile = apiserver_key; @@ -59,9 +58,6 @@ in kubeletClientCaFile = rootCaFile; kubeletClientKeyFile = worker_key; kubeletClientCertFile = worker_cert; - portalNet = "10.1.10.0/24"; # --service-cluster-ip-range - runtimeConfig = ""; - /*extraOpts = "--v=2";*/ }; }; } diff --git a/nixos/tests/kubernetes/multinode-kubectl.nix b/nixos/tests/kubernetes/multinode-kubectl.nix index 4ea4c272b225..c5dd999a01e9 100644 --- a/nixos/tests/kubernetes/multinode-kubectl.nix +++ b/nixos/tests/kubernetes/multinode-kubectl.nix @@ -8,7 +8,6 @@ let servers.master = "192.168.1.1"; servers.one = "192.168.1.10"; servers.two = "192.168.1.20"; - servers.three = "192.168.1.30"; certs = import ./certs.nix { inherit servers; }; @@ -39,7 +38,7 @@ let clusters = [{ name = "local"; cluster.certificate-authority = "/ca.pem"; - cluster.server = "https://${servers.master}:4443/"; + cluster.server = "https://${servers.master}"; }]; users = [{ name = "kubelet"; @@ -61,11 +60,9 @@ let $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready"); $one->execute("docker load < ${kubectlImage}"); $two->execute("docker load < ${kubectlImage}"); - $three->execute("docker load < ${kubectlImage}"); $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); @@ -116,19 +113,6 @@ in makeTest { } (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) ]; - - three = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.three; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; }; testScript = '' diff --git a/nixos/tests/kubernetes/singlenode.nix b/nixos/tests/kubernetes/singlenode.nix index 9bab51c49824..d924da155bdc 100644 --- a/nixos/tests/kubernetes/singlenode.nix +++ b/nixos/tests/kubernetes/singlenode.nix @@ -46,37 +46,30 @@ let $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running"); $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379"); ''; -in { - # This test runs kubernetes on a single node - singlenode = makeTest { - name = "kubernetes-singlenode"; +in makeTest { + name = "kubernetes-singlenode"; - nodes = { - kubernetes = - { config, pkgs, lib, nodes, ... }: - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 2048; + nodes = { + kubernetes = + { config, pkgs, lib, nodes, ... }: + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 2048; - programs.bash.enableCompletion = true; - environment.systemPackages = with pkgs; [ netcat bind ]; + programs.bash.enableCompletion = true; + environment.systemPackages = with pkgs; [ netcat bind ]; - services.kubernetes.roles = ["master" "node"]; - services.kubernetes.dns.port = 4453; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0"; + services.kubernetes.roles = ["master" "node"]; + services.kubernetes.dns.port = 4453; - networking.bridges.cbr0.interfaces = []; - networking.interfaces.cbr0 = {}; - - services.dnsmasq.enable = true; - services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; - }; - }; + services.dnsmasq.enable = true; + services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; + }; + }; - testScript = '' - startAll; + testScript = '' + startAll; - ${testSimplePod} - ''; - }; + ${testSimplePod} + ''; } -- cgit 1.4.1