diff options
author | Jaka Hudoklin <jakahudoklin@gmail.com> | 2014-12-11 23:32:37 +0100 |
---|---|---|
committer | Jaka Hudoklin <jakahudoklin@gmail.com> | 2014-12-11 23:32:37 +0100 |
commit | 4be10897817f25336856a0a6bcd20582cb130943 (patch) | |
tree | 439b501a64fe24196409381aaf8d47edacf024c3 /nixos/modules/services/cluster | |
parent | 1465d9321dd025f625d28d0faa55b64f6021f941 (diff) | |
download | nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar.gz nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar.bz2 nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar.lz nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar.xz nixlib-4be10897817f25336856a0a6bcd20582cb130943.tar.zst nixlib-4be10897817f25336856a0a6bcd20582cb130943.zip |
nixos: move kubernetes & fleet to services/cluster
Diffstat (limited to 'nixos/modules/services/cluster')
-rw-r--r-- | nixos/modules/services/cluster/fleet.nix | 150 | ||||
-rw-r--r-- | nixos/modules/services/cluster/kubernetes.nix | 462 |
2 files changed, 612 insertions, 0 deletions
diff --git a/nixos/modules/services/cluster/fleet.nix b/nixos/modules/services/cluster/fleet.nix new file mode 100644 index 000000000000..04d95fbf186b --- /dev/null +++ b/nixos/modules/services/cluster/fleet.nix @@ -0,0 +1,150 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.fleet; + +in { + + ##### Interface + options.services.fleet = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable fleet service. + ''; + }; + + listen = mkOption { + type = types.listOf types.str; + default = [ "/var/run/fleet.sock" ]; + example = [ "/var/run/fleet.sock" "127.0.0.1:49153" ]; + description = '' + Fleet listening addresses. + ''; + }; + + etcdServers = mkOption { + type = types.listOf types.str; + default = [ "http://127.0.0.1:4001" ]; + description = '' + Fleet list of etcd endpoints to use. + ''; + }; + + publicIp = mkOption { + type = types.nullOr types.str; + default = ""; + description = '' + Fleet IP address that should be published with the local Machine's + state and any socket information. If not set, fleetd will attempt + to detect the IP it should publish based on the machine's IP + routing information. + ''; + }; + + etcdCafile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Fleet TLS ca file when SSL certificate authentication is enabled + in etcd endpoints. + ''; + }; + + etcdKeyfile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Fleet TLS key file when SSL certificate authentication is enabled + in etcd endpoints. + ''; + }; + + etcdCertfile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Fleet TLS cert file when SSL certificate authentication is enabled + in etcd endpoints. + ''; + }; + + metadata = mkOption { + type = types.attrsOf types.str; + default = {}; + apply = attrs: concatMapStringsSep "," (n: "${n}=${attrs."${n}"}") (attrNames attrs); + example = literalExample '' + { + region = "us-west"; + az = "us-west-1"; + } + ''; + description = '' + Key/value pairs that are published with the local to the fleet registry. + This data can be used directly by a client of fleet to make scheduling decisions. + ''; + }; + + extraConfig = mkOption { + type = types.attrsOf types.str; + apply = mapAttrs' (n: v: nameValuePair ("ETCD_" + n) v); + default = {}; + example = literalExample '' + { + VERBOSITY = 1; + ETCD_REQUEST_TIMEOUT = "2.0"; + AGENT_TTL = "40s"; + } + ''; + description = '' + Fleet extra config. See + <link xlink:href="https://github.com/coreos/fleet/blob/master/Documentation/deployment-and-configuration.md"/> + for configuration options. + ''; + }; + + }; + + ##### Implementation + config = mkIf cfg.enable { + systemd.services.fleet = { + description = "Fleet Init System Daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "fleet.socket" "etcd.service" "docker.service" ]; + requires = [ "fleet.socket" ]; + environment = { + FLEET_ETCD_SERVERS = concatStringsSep "," cfg.etcdServers; + FLEET_PUBLIC_IP = cfg.publicIp; + FLEET_ETCD_CAFILE = cfg.etcdCafile; + FLEET_ETCD_KEYFILE = cfg.etcdKeyfile; + FEELT_ETCD_CERTFILE = cfg.etcdCertfile; + FLEET_METADATA = cfg.metadata; + } // cfg.extraConfig; + serviceConfig = { + ExecStart = "${pkgs.fleet}/bin/fleetd"; + Group = "fleet"; + }; + }; + + systemd.sockets.fleet = { + description = "Fleet Socket for the API"; + wantedBy = [ "sockets.target" ]; + listenStreams = cfg.listen; + socketConfig = { + ListenStream = "/var/run/fleet.sock"; + SocketMode = "0660"; + SocketUser = "root"; + SocketGroup = "fleet"; + }; + }; + + services.etcd.enable = mkDefault true; + virtualisation.docker.enable = mkDefault true; + + environment.systemPackages = [ pkgs.fleet ]; + users.extraGroups.fleet.gid = config.ids.gids.fleet; + }; +} diff --git a/nixos/modules/services/cluster/kubernetes.nix b/nixos/modules/services/cluster/kubernetes.nix new file mode 100644 index 000000000000..7fd2d77aa825 --- /dev/null +++ b/nixos/modules/services/cluster/kubernetes.nix @@ -0,0 +1,462 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.kubernetes; + +in { + + ###### interface + + options.services.kubernetes = { + package = mkOption { + description = "Kubernetes package to use."; + type = types.package; + }; + + verbose = mkOption { + description = "Kubernetes enable verbose mode for debugging"; + default = false; + type = types.bool; + }; + + etcdServers = mkOption { + description = "Kubernetes list of etcd servers to watch."; + default = [ "127.0.0.1:4001" ]; + type = types.listOf types.str; + }; + + roles = mkOption { + description = '' + Kubernetes role that this machine should take. + + Master role will enable etcd, apiserver, scheduler and controller manager + services. Node role will enable etcd, docker, kubelet and proxy services. + ''; + default = []; + type = types.listOf (types.enum ["master" "node"]); + }; + + dataDir = mkOption { + description = "Kubernetes root directory for managing kubelet files."; + default = "/var/lib/kubernetes"; + type = types.path; + }; + + apiserver = { + enable = mkOption { + description = "Whether to enable kubernetes apiserver."; + default = false; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes apiserver listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + publicAddress = mkOption { + description = '' + Kubernetes apiserver public listening address used for read only and + secure port. + ''; + default = cfg.apiserver.address; + type = types.str; + }; + + port = mkOption { + description = "Kubernets apiserver listening port."; + default = 8080; + type = types.int; + }; + + readOnlyPort = mkOption { + description = "Kubernets apiserver read-only port."; + default = 7080; + type = types.int; + }; + + securePort = mkOption { + description = "Kubernetes apiserver secure port."; + default = 6443; + type = types.int; + }; + + tlsCertFile = mkOption { + description = "Kubernetes apiserver certificate file."; + default = ""; + type = types.str; + }; + + tlsPrivateKeyFile = mkOption { + description = "Kubernetes apiserver private key file."; + default = ""; + type = types.str; + }; + + tokenAuth = mkOption { + description = '' + Kubernetes apiserver token authentication file. See + <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authentication.md"/> + ''; + default = {}; + example = literalExample '' + { + alice = "abc123"; + bob = "xyz987"; + } + ''; + type = types.attrsOf types.str; + }; + + authorizationMode = mkOption { + description = '' + Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC). See + <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authorization.md"/> + ''; + default = "AlwaysAllow"; + type = types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC"]; + }; + + authorizationPolicy = mkOption { + description = '' + Kubernetes apiserver authorization policy file. See + <link xlink:href="https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/authorization.md"/> + ''; + default = []; + example = literalExample '' + [ + {user = "admin";} + {user = "scheduler"; readonly = true; kind= "pods";} + {user = "scheduler"; kind = "bindings";} + {user = "kubelet"; readonly = true; kind = "bindings";} + {user = "kubelet"; kind = "events";} + {user= "alice"; ns = "projectCaribou";} + {user = "bob"; readonly = true; ns = "projectCaribou";} + ] + ''; + type = types.listOf types.attrs; + }; + + allowPrivileged = mkOption { + description = "Whether to allow privileged containers on kubernetes."; + default = false; + type = types.bool; + }; + + portalNet = mkOption { + description = "Kubernetes CIDR notation IP range from which to assign portal IPs"; + default = "10.10.10.10/16"; + type = types.str; + }; + + extraOpts = mkOption { + description = "Kubernetes apiserver extra command line options."; + default = ""; + type = types.str; + }; + }; + + scheduler = { + enable = mkOption { + description = "Whether to enable kubernetes scheduler."; + default = false; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes scheduler listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + port = mkOption { + description = "Kubernets scheduler listening port."; + default = 10251; + type = types.int; + }; + + master = mkOption { + description = "Kubernetes apiserver address"; + default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}"; + type = types.str; + }; + + extraOpts = mkOption { + description = "Kubernetes scheduler extra command line options."; + default = ""; + type = types.str; + }; + }; + + controllerManager = { + enable = mkOption { + description = "Whether to enable kubernetes controller manager."; + default = false; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes controller manager listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + port = mkOption { + description = "Kubernets controller manager listening port."; + default = 10252; + type = types.int; + }; + + master = mkOption { + description = "Kubernetes apiserver address"; + default = "${cfg.apiserver.address}:${toString cfg.apiserver.port}"; + type = types.str; + }; + + machines = mkOption { + description = "Kubernetes apiserver list of machines to schedule to schedule onto"; + default = []; + type = types.listOf types.str; + }; + + extraOpts = mkOption { + description = "Kubernetes scheduler extra command line options."; + default = ""; + type = types.str; + }; + }; + + kubelet = { + enable = mkOption { + description = "Whether to enable kubernetes kubelet."; + default = false; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes kubelet info server listening address."; + default = "0.0.0.0"; + type = types.str; + }; + + port = mkOption { + description = "Kubernets kubelet info server listening port."; + default = 10250; + type = types.int; + }; + + hostname = mkOption { + description = "Kubernetes kubelet hostname override"; + default = config.networking.hostName; + type = types.str; + }; + + allowPrivileged = mkOption { + description = "Whether to allow kubernetes containers to request privileged mode."; + default = false; + type = types.bool; + }; + + extraOpts = mkOption { + description = "Kubernetes kubelet extra command line options."; + default = ""; + type = types.str; + }; + }; + + proxy = { + enable = mkOption { + description = "Whether to enable kubernetes proxy."; + default = false; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes proxy listening address."; + default = "0.0.0.0"; + type = types.str; + }; + + extraOpts = mkOption { + description = "Kubernetes proxy extra command line options."; + default = ""; + type = types.str; + }; + }; + }; + + ###### implementation + + config = mkMerge [ + (mkIf cfg.apiserver.enable { + systemd.services.kubernetes-apiserver = { + description = "Kubernetes Api Server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" "etcd.service" ]; + serviceConfig = { + ExecStart = let + authorizationPolicyFile = + pkgs.writeText "kubernetes-policy" + (builtins.toJSON cfg.apiserver.authorizationPolicy); + tokenAuthFile = + pkgs.writeText "kubernetes-auth" + (concatImapStringsSep "\n" (i: v: v + "," + (toString i)) + (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth)); + in ''${cfg.package}/bin/kube-apiserver \ + -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ + -address=${cfg.apiserver.address} \ + -port=${toString cfg.apiserver.port} \ + -read_only_port=${toString cfg.apiserver.readOnlyPort} \ + -public_address_override=${cfg.apiserver.publicAddress} \ + -allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \ + ${optionalString (cfg.apiserver.tlsCertFile!="") + "-tls_cert_file=${cfg.apiserver.tlsCertFile}"} \ + ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="") + "-tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \ + ${optionalString (cfg.apiserver.tokenAuth!=[]) + "-token_auth_file=${tokenAuthFile}"} \ + -authorization_mode=${cfg.apiserver.authorizationMode} \ + ${optionalString (cfg.apiserver.authorizationMode == "ABAC") + "-authorization_policy_file=${authorizationPolicyFile}"} \ + ${optionalString (cfg.apiserver.tlsCertFile!="" && cfg.apiserver.tlsCertFile!="") + "-secure_port=${toString cfg.apiserver.securePort}"} \ + -portal_net=${cfg.apiserver.portalNet} \ + -logtostderr=true \ + ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + ${cfg.apiserver.extraOpts} + ''; + User = "kubernetes"; + }; + postStart = '' + until ${pkgs.curl}/bin/curl -s -o /dev/null 'http://${cfg.apiserver.address}:${toString cfg.apiserver.port}/'; do + sleep 1; + done + ''; + }; + }) + + (mkIf cfg.scheduler.enable { + systemd.services.kubernetes-scheduler = { + description = "Kubernetes Scheduler Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" "kubernetes-apiserver.service" ]; + serviceConfig = { + ExecStart = ''${cfg.package}/bin/kube-scheduler \ + -address=${cfg.scheduler.address} \ + -port=${toString cfg.scheduler.port} \ + -master=${cfg.scheduler.master} \ + -logtostderr=true \ + ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + ${cfg.scheduler.extraOpts} + ''; + User = "kubernetes"; + }; + }; + }) + + (mkIf cfg.controllerManager.enable { + systemd.services.kubernetes-controller-manager = { + description = "Kubernetes Controller Manager Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" "kubernetes-apiserver.service" ]; + serviceConfig = { + ExecStart = ''${cfg.package}/bin/kube-controller-manager \ + -address=${cfg.controllerManager.address} \ + -port=${toString cfg.controllerManager.port} \ + -master=${cfg.controllerManager.master} \ + ${optionalString (cfg.controllerManager.machines != []) + "-machines=${concatStringsSep "," cfg.controllerManager.machines}"} \ + -logtostderr=true \ + ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + ${cfg.controllerManager.extraOpts} + ''; + User = "kubernetes"; + }; + }; + }) + + (mkIf cfg.kubelet.enable { + systemd.services.kubernetes-kubelet = { + description = "Kubernetes Kubelet Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" "etcd.service" "docker.service" ]; + serviceConfig = { + ExecStart = ''${cfg.package}/bin/kubelet \ + -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \ + -address=${cfg.kubelet.address} \ + -port=${toString cfg.kubelet.port} \ + -hostname_override=${cfg.kubelet.hostname} \ + -allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \ + -root_dir=${cfg.dataDir} \ + -logtostderr=true \ + ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + ${cfg.kubelet.extraOpts} + ''; + User = "kubernetes"; + PermissionsStartOnly = true; + WorkingDirectory = cfg.dataDir; + }; + }; + }) + + (mkIf cfg.proxy.enable { + systemd.services.kubernetes-proxy = { + description = "Kubernetes Proxy Service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-interfaces.target" "etcd.service" ]; + serviceConfig = { + ExecStart = ''${cfg.package}/bin/kube-proxy \ + -etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \ + -bind_address=${cfg.proxy.address} \ + -logtostderr=true \ + ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \ + ${cfg.proxy.extraOpts} + ''; + }; + }; + }) + + (mkIf (any (el: el == "master") cfg.roles) { + services.kubernetes.apiserver.enable = mkDefault true; + services.kubernetes.scheduler.enable = mkDefault true; + services.kubernetes.controllerManager.enable = mkDefault true; + }) + + (mkIf (any (el: el == "node") cfg.roles) { + virtualisation.docker.enable = mkDefault true; + services.kubernetes.kubelet.enable = mkDefault true; + services.kubernetes.proxy.enable = mkDefault true; + }) + + (mkIf (any (el: el == "node" || el == "master") cfg.roles) { + services.etcd.enable = mkDefault true; + }) + + (mkIf ( + cfg.apiserver.enable || + cfg.scheduler.enable || + cfg.controllerManager.enable || + cfg.kubelet.enable || + cfg.proxy.enable + ) { + services.kubernetes.package = mkDefault pkgs.kubernetes; + + environment.systemPackages = [ cfg.package ]; + + users.extraUsers = singleton { + name = "kubernetes"; + uid = config.ids.uids.kubernetes; + description = "Kubernetes user"; + extraGroups = [ "docker" ]; + group = "kubernetes"; + home = cfg.dataDir; + createHome = true; + }; + users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes; + }) + + ]; +} |