diff options
author | Alyssa Ross <hi@alyssa.is> | 2024-06-23 00:32:22 +0200 |
---|---|---|
committer | Alyssa Ross <hi@alyssa.is> | 2024-06-23 00:32:22 +0200 |
commit | 6402b188ddd100b3cd6afe7b8a3e553365203f43 (patch) | |
tree | 676b85e4a6ffee092e413e723f7dce8ba01bb48f /nixpkgs/nixos/tests/rke2 | |
parent | 5a1826585861b32ce2509c0643e793196d81893e (diff) | |
parent | d603719ec6e294f034936c0d0dc06f689d91b6c3 (diff) | |
download | nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar.gz nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar.bz2 nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar.lz nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar.xz nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.tar.zst nixlib-6402b188ddd100b3cd6afe7b8a3e553365203f43.zip |
Merge remote-tracking branch 'nixpkgs/nixos-unstable'
Diffstat (limited to 'nixpkgs/nixos/tests/rke2')
-rw-r--r-- | nixpkgs/nixos/tests/rke2/default.nix | 13 | ||||
-rw-r--r-- | nixpkgs/nixos/tests/rke2/multi-node.nix | 176 | ||||
-rw-r--r-- | nixpkgs/nixos/tests/rke2/single-node.nix | 75 |
3 files changed, 264 insertions, 0 deletions
diff --git a/nixpkgs/nixos/tests/rke2/default.nix b/nixpkgs/nixos/tests/rke2/default.nix new file mode 100644 index 000000000000..e8a5f382b735 --- /dev/null +++ b/nixpkgs/nixos/tests/rke2/default.nix @@ -0,0 +1,13 @@ +{ system ? builtins.currentSystem +, pkgs ? import ../../.. { inherit system; } +, lib ? pkgs.lib +}: +let + allRKE2 = lib.filterAttrs (n: _: lib.strings.hasPrefix "rke2" n) pkgs; +in +{ + # Run a single node rke2 cluster and verify a pod can run + singleNode = lib.mapAttrs (_: rke2: import ./single-node.nix { inherit system pkgs rke2; }) allRKE2; + # Run a multi-node rke2 cluster and verify pod networking works across nodes + multiNode = lib.mapAttrs (_: rke2: import ./multi-node.nix { inherit system pkgs rke2; }) allRKE2; +} diff --git a/nixpkgs/nixos/tests/rke2/multi-node.nix b/nixpkgs/nixos/tests/rke2/multi-node.nix new file mode 100644 index 000000000000..ddf0b60f6fba --- /dev/null +++ b/nixpkgs/nixos/tests/rke2/multi-node.nix @@ -0,0 +1,176 @@ +import ../make-test-python.nix ({ pkgs, lib, rke2, ... }: + let + pauseImage = pkgs.dockerTools.streamLayeredImage { + name = "test.local/pause"; + tag = "local"; + contents = pkgs.buildEnv { + name = "rke2-pause-image-env"; + paths = with pkgs; [ tini bashInteractive coreutils socat ]; + }; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + # A daemonset that responds 'server' on port 8000 + networkTestDaemonset = pkgs.writeText "test.yml" '' + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: test + labels: + name: test + spec: + selector: + matchLabels: + name: test + template: + metadata: + labels: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + resources: + limits: + memory: 20Mi + command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"] + ''; + tokenFile = pkgs.writeText "token" "p@s$w0rd"; + agentTokenFile = pkgs.writeText "agent-token" "p@s$w0rd"; + in + { + name = "${rke2.name}-multi-node"; + meta.maintainers = rke2.meta.maintainers; + + nodes = { + server1 = { pkgs, ... }: { + networking.firewall.enable = false; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.1"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { address = "192.168.1.1"; prefixLength = 24; } + ]; + + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.rke2 = { + enable = true; + role = "server"; + inherit tokenFile; + inherit agentTokenFile; + nodeName = "${rke2.name}-server1"; + package = rke2; + nodeIP = "192.168.1.1"; + disable = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + ]; + extraFlags = [ + "--cluster-reset" + ]; + }; + }; + + server2 = { pkgs, ... }: { + networking.firewall.enable = false; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.2"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { address = "192.168.1.2"; prefixLength = 24; } + ]; + + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.rke2 = { + enable = true; + role = "server"; + serverAddr = "https://192.168.1.1:6443"; + inherit tokenFile; + inherit agentTokenFile; + nodeName = "${rke2.name}-server2"; + package = rke2; + nodeIP = "192.168.1.2"; + disable = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + ]; + }; + }; + + agent1 = { pkgs, ... }: { + networking.firewall.enable = false; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.3"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { address = "192.168.1.3"; prefixLength = 24; } + ]; + + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.rke2 = { + enable = true; + role = "agent"; + tokenFile = agentTokenFile; + serverAddr = "https://192.168.1.2:6443"; + nodeName = "${rke2.name}-agent1"; + package = rke2; + nodeIP = "192.168.1.3"; + }; + }; + }; + + testScript = let + kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; + ctr = "${pkgs.containerd}/bin/ctr -a /run/k3s/containerd/containerd.sock"; + jq = "${pkgs.jq}/bin/jq"; + ping = "${pkgs.iputils}/bin/ping"; + in '' + machines = [server1, server2, agent1] + + for machine in machines: + machine.start() + machine.wait_for_unit("rke2") + + # wait for the agent to show up + server1.succeed("${kubectl} get node ${rke2.name}-agent1") + + for machine in machines: + machine.succeed("${pauseImage} | ${ctr} image import -") + + server1.succeed("${kubectl} cluster-info") + server1.wait_until_succeeds("${kubectl} get serviceaccount default") + + # Now create a pod on each node via a daemonset and verify they can talk to each other. + server1.succeed("${kubectl} apply -f ${networkTestDaemonset}") + server1.wait_until_succeeds( + f'[ "$(${kubectl} get ds test -o json | ${jq} .status.numberReady)" -eq {len(machines)} ]' + ) + + # Get pod IPs + pods = server1.succeed("${kubectl} get po -o json | ${jq} '.items[].metadata.name' -r").splitlines() + pod_ips = [ + server1.succeed(f"${kubectl} get po {n} -o json | ${jq} '.status.podIP' -cr").strip() for n in pods + ] + + # Verify each server can ping each pod ip + for pod_ip in pod_ips: + server1.succeed(f"${ping} -c 1 {pod_ip}") + agent1.succeed(f"${ping} -c 1 {pod_ip}") + + # Verify the pods can talk to each other + resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -") + assert resp.strip() == "server" + resp = server1.wait_until_succeeds(f"${kubectl} exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -") + assert resp.strip() == "server" + + # Cleanup + server1.succeed("${kubectl} delete -f ${networkTestDaemonset}") + for machine in machines: + machine.shutdown() + ''; + }) diff --git a/nixpkgs/nixos/tests/rke2/single-node.nix b/nixpkgs/nixos/tests/rke2/single-node.nix new file mode 100644 index 000000000000..5a512eacca0f --- /dev/null +++ b/nixpkgs/nixos/tests/rke2/single-node.nix @@ -0,0 +1,75 @@ +import ../make-test-python.nix ({ pkgs, lib, rke2, ... }: + let + pauseImage = pkgs.dockerTools.streamLayeredImage { + name = "test.local/pause"; + tag = "local"; + contents = pkgs.buildEnv { + name = "rke2-pause-image-env"; + paths = with pkgs; [ tini (hiPrio coreutils) busybox ]; + }; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + testPodYaml = pkgs.writeText "test.yaml" '' + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; + in + { + name = "${rke2.name}-single-node"; + meta.maintainers = rke2.meta.maintainers; + + nodes.machine = { pkgs, ... }: { + networking.firewall.enable = false; + networking.useDHCP = false; + networking.defaultGateway = "192.168.1.1"; + networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [ + { address = "192.168.1.1"; prefixLength = 24; } + ]; + + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; + + services.rke2 = { + enable = true; + role = "server"; + package = rke2; + nodeIP = "192.168.1.1"; + disable = [ + "rke2-coredns" + "rke2-metrics-server" + "rke2-ingress-nginx" + ]; + extraFlags = [ + "--cluster-reset" + ]; + }; + }; + + testScript = let + kubectl = "${pkgs.kubectl}/bin/kubectl --kubeconfig=/etc/rancher/rke2/rke2.yaml"; + ctr = "${pkgs.containerd}/bin/ctr -a /run/k3s/containerd/containerd.sock"; + in '' + start_all() + + machine.wait_for_unit("rke2") + machine.succeed("${kubectl} cluster-info") + machine.wait_until_succeeds( + "${pauseImage} | ${ctr} -n k8s.io image import -" + ) + + machine.wait_until_succeeds("${kubectl} get serviceaccount default") + machine.succeed("${kubectl} apply -f ${testPodYaml}") + machine.succeed("${kubectl} wait --for 'condition=Ready' pod/test") + machine.succeed("${kubectl} delete -f ${testPodYaml}") + + machine.shutdown() + ''; + }) |