From 7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 26 May 2017 23:21:17 +0200 Subject: kubernetes module: flannel support, minor fixes - add flannel support - remove deprecated authorizationRBACSuperAdmin option - rename from deprecated poratalNet to serviceClusterIpRange - add nodeIp option for kubelet - kubelet, add br_netfilter to kernelModules - enable firewall by default - enable dns by default on node and on master - disable iptables for docker by default on nodes - dns, restart on failure - update tests and other minor changes --- .../services/cluster/kubernetes/default.nix | 117 ++++++++++++++++++--- 1 file changed, 101 insertions(+), 16 deletions(-) (limited to 'nixos/modules') diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 76b27ac0efba..bd37c49486e4 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -162,6 +162,17 @@ let }; } ]); + + # needed for flannel to pass options to docker + mkDockerOpts = pkgs.runCommand "mk-docker-opts" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out + cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh + + # bashInteractive needed for `compgen` + makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" + ''; in { ###### interface @@ -357,20 +368,17 @@ in { type = types.listOf types.attrs; }; - authorizationRBACSuperAdmin = mkOption { - description = "Role based authorization super admin."; - default = "admin"; - type = types.str; - }; - allowPrivileged = mkOption { description = "Whether to allow privileged containers on Kubernetes."; default = true; type = types.bool; }; - portalNet = mkOption { - description = "Kubernetes CIDR notation IP range from which to assign portal IPs."; + serviceClusterIpRange = mkOption { + description = '' + A CIDR notation IP range from which to assign service cluster IPs. + This must not overlap with any IP ranges assigned to nodes for pods. + ''; default = "10.10.10.10/24"; type = types.str; }; @@ -666,6 +674,12 @@ in { type = types.attrsOf (types.submodule [ taintOptions ]); }; + nodeIp = mkOption { + description = "IP address of the node. If set, kubelet will use this IP address for the node."; + default = null; + type = types.nullOr types.str; + }; + extraOpts = mkOption { description = "Kubernetes kubelet extra command line options."; default = ""; @@ -761,6 +775,12 @@ in { type = types.str; }; + flannel.enable = mkOption { + description = "Whether to enable flannel networking"; + default = false; + type = types.bool; + }; + }; ###### implementation @@ -808,6 +828,8 @@ in { "--network-plugin=${cfg.kubelet.networkPlugin}"} \ --cni-conf-dir=${cniConfig} \ --hairpin-mode=hairpin-veth \ + ${optionalString (cfg.kubelet.nodeIp != null) + "--node-ip=${cfg.kubelet.nodeIp}"} \ ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ ${cfg.kubelet.extraOpts} ''; @@ -817,6 +839,8 @@ in { # Allways include cni plugins services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + + boot.kernelModules = ["br_netfilter"]; }) (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { @@ -879,10 +903,8 @@ in { (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy) }" } \ - ${optionalString (elem "RBAC" cfg.apiserver.authorizationMode) - "--authorization-rbac-super-user=${cfg.apiserver.authorizationRBACSuperAdmin}"} \ --secure-port=${toString cfg.apiserver.securePort} \ - --service-cluster-ip-range=${cfg.apiserver.portalNet} \ + --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \ ${optionalString (cfg.apiserver.runtimeConfig != "") "--runtime-config=${cfg.apiserver.runtimeConfig}"} \ --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \ @@ -981,10 +1003,9 @@ in { WorkingDirectory = cfg.dataDir; }; }; - }) - (mkIf cfg.kubelet.enable { - boot.kernelModules = ["br_netfilter"]; + # kube-proxy needs iptables + networking.firewall.enable = mkDefault true; }) (mkIf (any (el: el == "master") cfg.roles) { @@ -999,15 +1020,25 @@ in { services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]); }) + # if this node is only a master make it unschedulable by default (mkIf (all (el: el == "master") cfg.roles) { services.kubernetes.kubelet.unschedulable = mkDefault true; }) (mkIf (any (el: el == "node") cfg.roles) { - virtualisation.docker.enable = mkDefault true; - virtualisation.docker.logDriver = mkDefault "json-file"; + virtualisation.docker = { + enable = mkDefault true; + + # kubernetes needs access to logs + logDriver = mkDefault "json-file"; + + # iptables must be disabled for kubernetes + extraOptions = "--iptables=false --ip-masq=false"; + }; + services.kubernetes.kubelet.enable = mkDefault true; services.kubernetes.proxy.enable = mkDefault true; + services.kubernetes.dns.enable = mkDefault true; }) (mkIf cfg.addonManager.enable { @@ -1035,8 +1066,17 @@ in { Group = "kubernetes"; AmbientCapabilities = "cap_net_bind_service"; SendSIGHUP = true; + RestartSec = "30s"; + Restart = "always"; + StartLimitInterval = "1m"; }; }; + + networking.firewall.extraCommands = '' + # allow container to host communication for DNS traffic + ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept + ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept + ''; }) (mkIf ( @@ -1070,5 +1110,50 @@ in { }; users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes; }) + + (mkIf cfg.flannel.enable { + services.flannel = { + enable = mkDefault true; + network = mkDefault cfg.clusterCidr; + etcd = mkDefault { + endpoints = cfg.etcd.servers; + inherit (cfg.etcd) caFile certFile keyFile; + }; + }; + + services.kubernetes.kubelet = { + networkPlugin = mkDefault "cni"; + cni.config = mkDefault [{ + name = "mynet"; + type = "flannel"; + delegate = { + isDefaultGateway = true; + bridge = "docker0"; + }; + }]; + }; + + systemd.services."mk-docker-opts" = { + description = "Pre-Docker Actions"; + wantedBy = [ "flannel.service" ]; + before = [ "docker.service" ]; + after = [ "flannel.service" ]; + path = [ pkgs.gawk pkgs.gnugrep ]; + script = '' + mkdir -p /run/flannel + ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker + ''; + serviceConfig.Type = "oneshot"; + }; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; + + # read environment variables generated by mk-docker-opts + virtualisation.docker.extraOptions = "$DOCKER_OPTS"; + + networking.firewall.allowedUDPPorts = [ + 8285 # flannel udp + 8472 # flannel vxlan + ]; + }) ]; } -- cgit 1.4.1