kubernetes: add tests

1 files changed, 131 insertions, 0 deletions

diff --git a/nixos/tests/kubernetes/kubernetes-common.nix b/nixos/tests/kubernetes/kubernetes-common.nix
new file mode 100644
index 000000000000..bc28244ad5b4
--- /dev/null
+++ b/nixos/tests/kubernetes/kubernetes-common.nix
@@ -0,0 +1,131 @@
+{ config, pkgs, certs, servers }:
+let
+  etcd_key = "${certs}/etcd-key.pem";
+  etcd_cert = "${certs}/etcd.pem";
+  ca_pem = "${certs}/ca.pem";
+  etcd_client_cert = "${certs}/etcd-client.crt";
+  etcd_client_key = "${certs}/etcd-client-key.pem";
+
+  worker_key = "${certs}/worker-key.pem";
+  worker_cert = "${certs}/worker.pem";
+
+  mkDockerOpts = "${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh";
+
+  rootCaFile = pkgs.writeScript "rootCaFile.pem" ''
+    ${pkgs.lib.readFile "${certs}/ca.pem"}
+
+    ${pkgs.lib.readFile ("${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt")}
+  '';
+
+  mkHosts =
+    pkgs.lib.concatMapStringsSep "\n" (v: "${v.ip} ${v.name}.nixos.xyz") (pkgs.lib.mapAttrsToList (n: v: {name = n; ip = v;}) servers);
+
+in
+{
+  programs.bash.enableCompletion = true;
+  environment.systemPackages = with pkgs; [ netcat bind etcd.bin ];
+
+  networking = {
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [
+        10250 80 443
+      ];
+      allowedUDPPorts = [
+        8285  # flannel udp
+        8472  # flannel vxlan
+      ];
+    };
+    extraHosts = ''
+      # register "external" domains
+      ${servers.master} etcd.kubernetes.nixos.xyz
+      ${servers.master} kubernetes.nixos.xyz
+      ${mkHosts}
+    '';
+  };
+  virtualisation.docker.extraOptions = ''
+    --iptables=false $DOCKER_OPTS
+  '';
+
+  # lets create environment file for docker startup - network stuff
+  systemd.services."pre-docker" = {
+    description = "Pre-Docker Actions";
+    wantedBy = [ "flannel.service" ];
+    before = [ "docker.service" ];
+    after = [ "flannel.service" ];
+    path = [ pkgs.gawk pkgs.gnugrep ];
+    script = ''
+      mkdir -p /run/flannel
+      # bashInteractive needed for `compgen`
+      ${pkgs.bashInteractive}/bin/bash ${mkDockerOpts} -d /run/flannel/docker
+      cat /run/flannel/docker  # just for debugging
+
+      # allow container to host communication for DNS traffic
+      ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -i docker0 --dport 53 -j nixos-fw-accept
+      ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -i docker0 --dport 53 -j nixos-fw-accept
+    '';
+    serviceConfig.Type = "simple";
+  };
+  systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
+
+  services.flannel = {
+    enable = true;
+    network = "10.2.0.0/16";
+    iface = "eth1";
+    etcd = {
+      endpoints = ["https://etcd.kubernetes.nixos.xyz:2379"];
+      keyFile = etcd_client_key;
+      certFile = etcd_client_cert;
+      caFile = ca_pem;
+    };
+  };
+  environment.variables = {
+    ETCDCTL_CERT_FILE = "${etcd_client_cert}";
+    ETCDCTL_KEY_FILE = "${etcd_client_key}";
+    ETCDCTL_CA_FILE = "${rootCaFile}";
+    ETCDCTL_PEERS = "https://etcd.kubernetes.nixos.xyz:2379";
+  };
+
+  services.kubernetes = {
+    kubelet = {
+      networkPlugin = "cni";
+      cni.config = [{
+        name = "mynet";
+        type = "flannel";
+        delegate = {
+          isDefaultGateway = true;
+          bridge = "docker0";
+        };
+      }];
+      tlsKeyFile = worker_key;
+      tlsCertFile = worker_cert;
+      hostname = "${config.networking.hostName}.nixos.xyz";
+      extraOpts = "--node-ip ${config.networking.primaryIPAddress}";
+      clusterDns = config.networking.primaryIPAddress;
+    };
+    etcd = {
+      servers = ["https://etcd.kubernetes.nixos.xyz:2379"];
+      keyFile = etcd_client_key;
+      certFile = etcd_client_cert;
+      caFile = ca_pem;
+    };
+    kubeconfig = {
+      server = "https://kubernetes.nixos.xyz:4443";
+      caFile = rootCaFile;
+      certFile = worker_cert;
+      keyFile = worker_key;
+    };
+
+    # make sure you cover kubernetes.apiserver.portalNet and flannel networks
+    clusterCidr = "10.0.0.0/8";
+
+    dns.enable = true;
+    dns.port = 4453;
+  };
+
+  services.dnsmasq.enable = true;
+  services.dnsmasq.servers = ["/${config.services.kubernetes.dns.domain}/127.0.0.1#4453"];
+
+  virtualisation.docker.enable = true;
+  virtualisation.docker.storageDriver = "overlay";
+}