diff options
author | Jaka Hudoklin <jakahudoklin@gmail.com> | 2017-05-26 23:21:17 +0200 |
---|---|---|
committer | Robin Gloster <mail@glob.in> | 2017-09-24 11:44:25 +0200 |
commit | 7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23 (patch) | |
tree | 75af6b69a4fa3525ab45ade3b1fdb2f604828574 /nixos/tests | |
parent | 8e14e978c8965db3378c57450d1177c03865554e (diff) | |
download | nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar.gz nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar.bz2 nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar.lz nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar.xz nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.tar.zst nixlib-7dfeac88aca9b83d8a4f5815aa4b88b34dd7ea23.zip |
kubernetes module: flannel support, minor fixes
- add flannel support - remove deprecated authorizationRBACSuperAdmin option - rename from deprecated poratalNet to serviceClusterIpRange - add nodeIp option for kubelet - kubelet, add br_netfilter to kernelModules - enable firewall by default - enable dns by default on node and on master - disable iptables for docker by default on nodes - dns, restart on failure - update tests and other minor changes
Diffstat (limited to 'nixos/tests')
-rw-r--r-- | nixos/tests/kubernetes/kubernetes-common.nix | 75 | ||||
-rw-r--r-- | nixos/tests/kubernetes/kubernetes-master.nix | 8 | ||||
-rw-r--r-- | nixos/tests/kubernetes/multinode-kubectl.nix | 18 | ||||
-rw-r--r-- | nixos/tests/kubernetes/singlenode.nix | 47 |
4 files changed, 31 insertions, 117 deletions
diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix index bc28244ad5b4..9f9e730fa655 100644 --- a/nixos/tests/kubernetes/kubernetes-common.nix +++ b/nixos/tests/kubernetes/kubernetes-common.nix @@ -1,4 +1,5 @@ { config, pkgs, certs, servers }: + let etcd_key = "${certs}/etcd-key.pem"; etcd_cert = "${certs}/etcd.pem"; @@ -9,8 +10,6 @@ let worker_key = "${certs}/worker-key.pem"; worker_cert = "${certs}/worker.pem"; - mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh"; - rootCaFile = pkgs.writeScript "rootCaFile.pem" '' ${pkgs.lib.readFile "${certs}/ca.pem"} @@ -26,16 +25,9 @@ in environment.systemPackages = with pkgs; [ netcat bind etcd.bin ]; networking = { - firewall = { - enable = true; - allowedTCPPorts = [ - 10250 80 443 - ]; - allowedUDPPorts = [ - 8285 # flannel udp - 8472 # flannel vxlan - ]; - }; + firewall.allowedTCPPorts = [ + 10250 # kubelet + ]; extraHosts = '' # register "external" domains ${servers.master} etcd.kubernetes.nixos.xyz @@ -43,42 +35,7 @@ in ${mkHosts} ''; }; - virtualisation.docker.extraOptions = '' - --iptables=false $DOCKER_OPTS - ''; - - # lets create environment file for docker startup - network stuff - systemd.services."pre-docker" = { - description = "Pre-Docker Actions"; - wantedBy = [ "flannel.service" ]; - before = [ "docker.service" ]; - after = [ "flannel.service" ]; - path = [ pkgs.gawk pkgs.gnugrep ]; - script = '' - mkdir -p /run/flannel - # bashInteractive needed for `compgen` - ${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker - cat /run/flannel/docker # just for debugging - - # allow container to host communication for DNS traffic - ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept - ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept - ''; - serviceConfig.Type = "simple"; - }; - systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; - - services.flannel = { - enable = true; - network = "10.2.0.0/16"; - iface = "eth1"; - etcd = { - endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"]; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; - }; - }; + services.flannel.iface = "eth1"; environment.variables = { ETCDCTL_CERT_FILE = "${etcd_client_cert}"; ETCDCTL_KEY_FILE = "${etcd_client_key}"; @@ -88,20 +45,10 @@ in services.kubernetes = { kubelet = { - networkPlugin = "cni"; - cni.config = [{ - name = "mynet"; - type = "flannel"; - delegate = { - isDefaultGateway = true; - bridge = "docker0"; - }; - }]; tlsKeyFile = worker_key; tlsCertFile = worker_cert; hostname = "${config.networking.hostName}.nixos.xyz"; - extraOpts = "--node-ip ${config.networking.primaryIPAddress}"; - clusterDns = config.networking.primaryIPAddress; + nodeIp = config.networking.primaryIPAddress; }; etcd = { servers = ["https://etcd.kubernetes.nixos.xyz:2379"]; @@ -110,22 +57,16 @@ in caFile = ca_pem; }; kubeconfig = { - server = "https://kubernetes.nixos.xyz:4443"; + server = "https://kubernetes.nixos.xyz"; caFile = rootCaFile; certFile = worker_cert; keyFile = worker_key; }; + flannel.enable = true; - # make sure you cover kubernetes.apiserver.portalNet and flannel networks - clusterCidr = "10.0.0.0/8"; - - dns.enable = true; dns.port = 4453; }; services.dnsmasq.enable = true; services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; - - virtualisation.docker.enable = true; - virtualisation.docker.storageDriver = "overlay"; } diff --git a/nixos/tests/kubernetes/kubernetes-master.nix b/nixos/tests/kubernetes/kubernetes-master.nix index 15e7e52e4832..28cce6ea6534 100644 --- a/nixos/tests/kubernetes/kubernetes-master.nix +++ b/nixos/tests/kubernetes/kubernetes-master.nix @@ -25,7 +25,7 @@ in allowPing = true; allowedTCPPorts = [ 2379 2380 # etcd - 4443 # kubernetes + 443 # kubernetes apiserver ]; }; }; @@ -43,14 +43,13 @@ in initialCluster = ["master=https://etcd.kubernetes.nixos.xyz:2380"]; initialAdvertisePeerUrls = ["https://etcd.kubernetes.nixos.xyz:2380"]; }; + services.kubernetes = { roles = ["master"]; scheduler.leaderElect = true; - controllerManager.leaderElect = true; controllerManager.rootCaFile = rootCaFile; controllerManager.serviceAccountKeyFile = apiserver_key; apiserver = { - securePort = 4443; publicAddress = "192.168.1.1"; advertiseAddress = "192.168.1.1"; tlsKeyFile = apiserver_key; @@ -59,9 +58,6 @@ in kubeletClientCaFile = rootCaFile; kubeletClientKeyFile = worker_key; kubeletClientCertFile = worker_cert; - portalNet = "10.1.10.0/24"; # --service-cluster-ip-range - runtimeConfig = ""; - /*extraOpts = "--v=2";*/ }; }; } diff --git a/nixos/tests/kubernetes/multinode-kubectl.nix b/nixos/tests/kubernetes/multinode-kubectl.nix index 4ea4c272b225..c5dd999a01e9 100644 --- a/nixos/tests/kubernetes/multinode-kubectl.nix +++ b/nixos/tests/kubernetes/multinode-kubectl.nix @@ -8,7 +8,6 @@ let servers.master = "192.168.1.1"; servers.one = "192.168.1.10"; servers.two = "192.168.1.20"; - servers.three = "192.168.1.30"; certs = import ./certs.nix { inherit servers; }; @@ -39,7 +38,7 @@ let clusters = [{ name = "local"; cluster.certificate-authority = "/ca.pem"; - cluster.server = "https://${servers.master}:4443/"; + cluster.server = "https://${servers.master}"; }]; users = [{ name = "kubelet"; @@ -61,11 +60,9 @@ let $master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready"); $master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready"); $master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready"); - $master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready"); $one->execute("docker load < ${kubectlImage}"); $two->execute("docker load < ${kubectlImage}"); - $three->execute("docker load < ${kubectlImage}"); $master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}"); @@ -116,19 +113,6 @@ in makeTest { } (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) ]; - - three = - { config, pkgs, lib, nodes, ... }: - mkMerge [ - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 4096; - networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}]; - networking.primaryIPAddress = mkForce servers.three; - services.kubernetes.roles = ["node"]; - } - (import ./kubernetes-common.nix { inherit pkgs config certs servers; }) - ]; }; testScript = '' diff --git a/nixos/tests/kubernetes/singlenode.nix b/nixos/tests/kubernetes/singlenode.nix index 9bab51c49824..d924da155bdc 100644 --- a/nixos/tests/kubernetes/singlenode.nix +++ b/nixos/tests/kubernetes/singlenode.nix @@ -46,37 +46,30 @@ let $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running"); $kubernetes->succeed("nc -z \$\(dig redis.default.svc.cluster.local +short\) 6379"); ''; -in { - # This test runs kubernetes on a single node - singlenode = makeTest { - name = "kubernetes-singlenode"; +in makeTest { + name = "kubernetes-singlenode"; - nodes = { - kubernetes = - { config, pkgs, lib, nodes, ... }: - { - virtualisation.memorySize = 768; - virtualisation.diskSize = 2048; + nodes = { + kubernetes = + { config, pkgs, lib, nodes, ... }: + { + virtualisation.memorySize = 768; + virtualisation.diskSize = 2048; - programs.bash.enableCompletion = true; - environment.systemPackages = with pkgs; [ netcat bind ]; + programs.bash.enableCompletion = true; + environment.systemPackages = with pkgs; [ netcat bind ]; - services.kubernetes.roles = ["master" "node"]; - services.kubernetes.dns.port = 4453; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0"; + services.kubernetes.roles = ["master" "node"]; + services.kubernetes.dns.port = 4453; - networking.bridges.cbr0.interfaces = []; - networking.interfaces.cbr0 = {}; - - services.dnsmasq.enable = true; - services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; - }; - }; + services.dnsmasq.enable = true; + services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"]; + }; + }; - testScript = '' - startAll; + testScript = '' + startAll; - ${testSimplePod} - ''; - }; + ${testSimplePod} + ''; } |