From 709b6f664e472ae2075bbee83bfb1dd1ba95e0e7 Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Sat, 17 Mar 2018 01:35:35 -0400 Subject: nixos: kubernetes fixes * Fix reference CNI plugins * The plugins were split out of the upstream cni repo around version 0.6.0 * Fix RBAC and DNS tests * Fix broken apiVersion fields * Change plugin linking to look in ${package}/bin rather than ${package.plugins} * Initial work towards a working e2e test * Test still fails, but at least the expression evaluates now Continues @srhb's work in #37199 Fixes #37199 --- .../services/cluster/kubernetes/default.nix | 4 +- nixos/release.nix | 5 +- nixos/tests/kubernetes/certs.nix | 71 ++++++++++++++++------ nixos/tests/kubernetes/e2e.nix | 2 +- nixos/tests/kubernetes/rbac.nix | 4 +- .../networking/cluster/cni/default.nix | 5 +- .../networking/cluster/cni/plugins.nix | 33 ++++++++++ pkgs/top-level/all-packages.nix | 1 + 8 files changed, 96 insertions(+), 29 deletions(-) create mode 100644 pkgs/applications/networking/cluster/cni/plugins.nix diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 39c5d78d4191..aeb0a0d2432d 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -766,7 +766,7 @@ in { rm /opt/cni/bin/* || true ${concatMapStrings (package: '' echo "Linking cni package: ${package}" - ln -fs ${package.plugins}/* /opt/cni/bin + ln -fs ${package}/bin/* /opt/cni/bin '') cfg.kubelet.cni.packages} ''; serviceConfig = { @@ -828,7 +828,7 @@ in { }; # Allways include cni plugins - services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; boot.kernelModules = ["br_netfilter"]; diff --git a/nixos/release.nix b/nixos/release.nix index 9b4aa4b0399d..c04f55bef8eb 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -296,7 +296,10 @@ in rec { tests.kernel-copperhead = callTest tests/kernel-copperhead.nix {}; tests.kernel-latest = callTest tests/kernel-latest.nix {}; tests.kernel-lts = callTest tests/kernel-lts.nix {}; - tests.kubernetes = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/default.nix {}; + tests.kubernetes.dns = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/dns.nix {}; + ## kubernetes.e2e should eventually replace kubernetes.rbac when it works + #tests.kubernetes.e2e = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/e2e.nix {}; + tests.kubernetes.rbac = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/rbac.nix {}; tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; }; tests.ldap = callTest tests/ldap.nix {}; #tests.lightdm = callTest tests/lightdm.nix {}; diff --git a/nixos/tests/kubernetes/certs.nix b/nixos/tests/kubernetes/certs.nix index d3eff910c467..520c728b65ee 100644 --- a/nixos/tests/kubernetes/certs.nix +++ b/nixos/tests/kubernetes/certs.nix @@ -6,29 +6,62 @@ kubelets }: let - runWithCFSSL = name: cmd: - builtins.fromJSON (builtins.readFile ( - pkgs.runCommand "${name}-cfss.json" { - buildInputs = [ pkgs.cfssl ]; - } "cfssl ${cmd} > $out" - )); - - writeCFSSL = content: - pkgs.runCommand content.name { - buildInputs = [ pkgs.cfssl ]; - } '' - mkdir -p $out - cd $out - cat ${writeFile content} | cfssljson -bare ${content.name} - ''; + runWithCFSSL = name: cmd: + let secrets = pkgs.runCommand "${name}-cfss.json" { + buildInputs = [ pkgs.cfssl pkgs.jq ]; + outputs = [ "out" "cert" "key" "csr" ]; + } + '' + ( + echo "${cmd}" + cfssl ${cmd} > tmp + cat tmp | jq -r .key > $key + cat tmp | jq -r .cert > $cert + cat tmp | jq -r .csr > $csr + + touch $out + ) 2>&1 | fold -w 80 -s + ''; + in { + key = secrets.key; + cert = secrets.cert; + csr = secrets.csr; + }; + + writeCFSSL = content: + pkgs.runCommand content.name { + buildInputs = [ pkgs.cfssl pkgs.jq ]; + } '' + mkdir -p $out + cd $out + + json=${pkgs.lib.escapeShellArg (builtins.toJSON content)} + + # for a given $field in the $json, treat the associated value as a + # file path and substitute the contents thereof into the $json + # object. + expandFileField() { + local field=$1 + if jq -e --arg field "$field" 'has($field)'; then + local path="$(echo "$json" | jq -r ".$field")" + json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")" + fi + } + + expandFileField key + expandFileField ca + expandFileField cert + + echo "$json" | cfssljson -bare ${content.name} + ''; noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content; noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content; - writeFile = content: pkgs.writeText "content" ( - if pkgs.lib.isAttrs content then builtins.toJSON content - else toString content - ); + writeFile = content: + if pkgs.lib.isDerivation content + then content + else pkgs.writeText "content" (builtins.toJSON content); createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }: noCSR ( diff --git a/nixos/tests/kubernetes/e2e.nix b/nixos/tests/kubernetes/e2e.nix index d9d7ba9bb2cc..175d8413045e 100644 --- a/nixos/tests/kubernetes/e2e.nix +++ b/nixos/tests/kubernetes/e2e.nix @@ -2,7 +2,7 @@ with import ./base.nix { inherit system; }; let domain = "my.zyx"; - certs = import ./certs.nix { externalDomain = domain; }; + certs = import ./certs.nix { externalDomain = domain; kubelets = ["machine1" "machine2"]; }; kubeconfig = pkgs.writeText "kubeconfig.json" (builtins.toJSON { apiVersion = "v1"; kind = "Config"; diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix index 1966fed3a5fb..226808c4b263 100644 --- a/nixos/tests/kubernetes/rbac.nix +++ b/nixos/tests/kubernetes/rbac.nix @@ -12,7 +12,7 @@ let }); roRoleBinding = pkgs.writeText "ro-role-binding.json" (builtins.toJSON { - apiVersion = "rbac.authorization.k8s.io/v1beta1"; + apiVersion = "rbac.authorization.k8s.io/v1"; kind = "RoleBinding"; metadata = { name = "read-pods"; @@ -31,7 +31,7 @@ let }); roRole = pkgs.writeText "ro-role.json" (builtins.toJSON { - apiVersion = "rbac.authorization.k8s.io/v1beta1"; + apiVersion = "rbac.authorization.k8s.io/v1"; kind = "Role"; metadata = { name = "pod-reader"; diff --git a/pkgs/applications/networking/cluster/cni/default.nix b/pkgs/applications/networking/cluster/cni/default.nix index 3ce056c1e11e..0f232772f076 100644 --- a/pkgs/applications/networking/cluster/cni/default.nix +++ b/pkgs/applications/networking/cluster/cni/default.nix @@ -13,17 +13,14 @@ stdenv.mkDerivation rec { buildInputs = [ go ]; - outputs = ["out" "plugins"]; - buildPhase = '' patchShebangs build.sh ./build.sh ''; installPhase = '' - mkdir -p $out/bin $plugins + mkdir -p $out/bin mv bin/cnitool $out/bin - mv bin/* $plugins/ ''; meta = with stdenv.lib; { diff --git a/pkgs/applications/networking/cluster/cni/plugins.nix b/pkgs/applications/networking/cluster/cni/plugins.nix new file mode 100644 index 000000000000..9f6b6fcb7e11 --- /dev/null +++ b/pkgs/applications/networking/cluster/cni/plugins.nix @@ -0,0 +1,33 @@ +{ stdenv, lib, fetchFromGitHub, go }: + +stdenv.mkDerivation rec { + name = "cni-plugins-${version}"; + version = "0.7.0"; + + src = fetchFromGitHub { + owner = "containernetworking"; + repo = "plugins"; + rev = "v${version}"; + sha256 = "0m885v76azs7lrk6m6n53rwh0xadwvdcr90h0l3bxpdv87sj2mnf"; + }; + + buildInputs = [ go ]; + + buildPhase = '' + patchShebangs build.sh + ./build.sh + ''; + + installPhase = '' + mkdir -p $out/bin + mv bin/* $out/bin + ''; + + meta = with lib; { + description = "Some standard networking plugins, maintained by the CNI team"; + homepage = https://github.com/containernetworking/plugins; + license = licenses.asl20; + platforms = [ "x86_64-linux" ]; + maintainers = with maintainers; [ cstrahan ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 2704910be948..e4533d1f3b36 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -15053,6 +15053,7 @@ with pkgs; }; cni = callPackage ../applications/networking/cluster/cni {}; + cni-plugins = callPackage ../applications/networking/cluster/cni/plugins.nix {}; communi = libsForQt5.callPackage ../applications/networking/irc/communi { }; -- cgit 1.4.1