1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with import ../../lib/qemu-flags.nix;
with pkgs.lib;
let
servers.master = "192.168.1.1";
servers.one = "192.168.1.10";
servers.two = "192.168.1.20";
servers.three = "192.168.1.30";
certs = import ./certs.nix { inherit servers; };
kubectlPod = pkgs.writeText "kubectl-pod.json" (builtins.toJSON {
kind = "Pod";
apiVersion = "v1";
metadata.name = "kubectl";
metadata.labels.name = "kubectl";
spec.containers = [{
name = "kubectl";
image = "kubectl:latest";
command = ["${pkgs.busybox}/bin/tail" "-f"];
imagePullPolicy = "Never";
tty = true;
}];
});
kubectlImage = pkgs.dockerTools.buildImage {
name = "kubectl";
tag = "latest";
contents = [ pkgs.kubernetes pkgs.busybox certs kubeconfig ];
config.Entrypoint = "${pkgs.busybox}/bin/sh";
};
kubeconfig = pkgs.writeTextDir "kubeconfig.json" (builtins.toJSON {
apiVersion = "v1";
kind = "Config";
clusters = [{
name = "local";
cluster.certificate-authority = "/ca.pem";
cluster.server = "https://${servers.master}:4443/";
}];
users = [{
name = "kubelet";
user = {
client-certificate = "/admin.crt";
client-key = "/admin-key.pem";
};
}];
contexts = [{
context = {
cluster = "local";
user = "kubelet";
};
current-context = "kubelet-context";
}];
});
test = ''
$master->waitUntilSucceeds("kubectl get node master.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node one.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node two.nixos.xyz | grep Ready");
$master->waitUntilSucceeds("kubectl get node three.nixos.xyz | grep Ready");
$one->execute("docker load < ${kubectlImage}");
$two->execute("docker load < ${kubectlImage}");
$three->execute("docker load < ${kubectlImage}");
$master->waitUntilSucceeds("kubectl create -f ${kubectlPod} || kubectl apply -f ${kubectlPod}");
$master->waitUntilSucceeds("kubectl get pod kubectl | grep Running");
$master->succeed("kubectl exec -ti kubectl -- kubectl --kubeconfig=/kubeconfig.json version");
'';
in makeTest {
name = "kubernetes-multinode-kubectl";
nodes = {
master =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.master; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.master;
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
(import ./kubernetes-master.nix { inherit pkgs config certs; })
];
one =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.one; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.one;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
two =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.two; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.two;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
three =
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = 768;
virtualisation.diskSize = 4096;
networking.interfaces.eth1.ip4 = mkForce [{address = servers.three; prefixLength = 24;}];
networking.primaryIPAddress = mkForce servers.three;
services.kubernetes.roles = ["node"];
}
(import ./kubernetes-common.nix { inherit pkgs config certs servers; })
];
};
testScript = ''
startAll;
${test}
'';
}
|