kubernetes module: refactor module system, kube-dns as module

4 files changed, 473 insertions, 192 deletions

diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/dashboard.nix
index 337c2634a374..75d71fccfda4 100644
--- a/nixos/modules/services/cluster/kubernetes/dashboard.nix
+++ b/nixos/modules/services/cluster/kubernetes/dashboard.nix
@@ -1,85 +1,160 @@
-{ cfg }: {
-    "dashboard-controller" = {
-        "apiVersion" = "extensions/v1beta1";
-        "kind" = "Deployment";
-        "metadata" = {
-            "labels" = {
-                "addonmanager.kubernetes.io/mode" = "Reconcile";
-                "k8s-app" = "kubernetes-dashboard";
-                "kubernetes.io/cluster-service" = "true";
-            };
-            "name" = "kubernetes-dashboard";
-            "namespace" = "kube-system";
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.kubernetes.addons.dashboard;
+
+  name = "gcr.io/google_containers/kubernetes-dashboard-amd64";
+	version = "v1.6.3";
+
+  image = pkgs.dockerTools.pullImage {
+    imageName = name;
+    imageTag = version;
+    sha256 = "1sf54d96nkgic9hir9c6p14gw24ns1k5d5a0r1sg414kjrvic0b4";
+  };
+in {
+  options.services.kubernetes.addons.dashboard = {
+    enable = mkEnableOption "kubernetes dashboard addon";
+
+    enableRBAC = mkOption {
+      description = "Whether to enable role based access control is enabled for kubernetes dashboard";
+      type = types.bool;
+      default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.kubernetes.kubelet.seedDockerImages = [image];
+
+    services.kubernetes.addonManager.addons = {
+      kubernetes-dashboard-deployment = {
+        kind = "Deployment";
+        apiVersion = "apps/v1beta1";
+        metadata = {
+          labels = {
+            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
+            k8s-app = "kubernetes-dashboard";
+            version = version;
+            "kubernetes.io/cluster-service" = "true";
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+          };
+          name = "kubernetes-dashboard";
+          namespace = "kube-system";
         };
-        "spec" = {
-            "selector" = {
-                "matchLabels" = {
-                    "k8s-app" = "kubernetes-dashboard";
-                };
+        spec = {
+          replicas = 1;
+          revisionHistoryLimit = 10;
+          selector.matchLabels."k8s-app" = "kubernetes-dashboard";
+          template = {
+            metadata = {
+              labels = {
+                k8s-addon = "kubernetes-dashboard.addons.k8s.io";
+                k8s-app = "kubernetes-dashboard";
+                version = version;
+                "kubernetes.io/cluster-service" = "true";
+              };
+              annotations = {
+                "scheduler.alpha.kubernetes.io/critical-pod" = "";
+                #"scheduler.alpha.kubernetes.io/tolerations" = ''[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'';
+              };
             };
-            "template" = {
-                "metadata" = {
-                    "annotations" = {
-                        "scheduler.alpha.kubernetes.io/critical-pod" = "";
-                    };
-                    "labels" = {
-                        "k8s-app" = "kubernetes-dashboard";
-                    };
+            spec = {
+              containers = [{
+                name = "kubernetes-dashboard";
+                image = "${name}:${version}";
+                ports = [{
+                  containerPort = 9090;
+                  protocol = "TCP";
+                }];
+                resources = {
+                  limits = {
+                    cpu = "100m";
+                    memory = "50Mi";
+                  };
+                  requests = {
+                    cpu = "100m";
+                    memory = "50Mi";
+                  };
                 };
-                "spec" = {
-                    "containers" = [{
-                        "image" = "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0";
-                        "livenessProbe" = {
-                            "httpGet" = {
-                                "path" = "/";
-                                "port" = 9090;
-                            };
-                            "initialDelaySeconds" = 30;
-                            "timeoutSeconds" = 30;
-                        };
-                        "name" = "kubernetes-dashboard";
-                        "ports" = [{
-                            "containerPort" = 9090;
-                        }];
-                        "resources" = {
-                            "limits" = {
-                                "cpu" = "100m";
-                                "memory" = "50Mi";
-                            };
-                            "requests" = {
-                                "cpu" = "100m";
-                                "memory" = "50Mi";
-                            };
-                        };
-                    }];
-                    "tolerations" = [{
-                        "key" = "CriticalAddonsOnly";
-                        "operator" = "Exists";
-                    }];
+                livenessProbe = {
+                  httpGet = {
+                    path = "/";
+                    port = 9090;
+                  };
+                  initialDelaySeconds = 30;
+                  timeoutSeconds = 30;
                 };
+              }];
+              serviceAccountName = "kubernetes-dashboard";
+              tolerations = [{
+                key = "node-role.kubernetes.io/master";
+                effect = "NoSchedule";
+              }];
             };
+          };
         };
-    };
-    "dashboard-service" = {
-        "apiVersion" = "v1";
-        "kind" = "Service";
-        "metadata" = {
-            "labels" = {
-                "addonmanager.kubernetes.io/mode" = "Reconcile";
-                "k8s-app" = "kubernetes-dashboard";
-                "kubernetes.io/cluster-service" = "true";
-            };
-            "name" = "kubernetes-dashboard";
-            "namespace" = "kube-system";
+      };
+
+      kubernetes-dashboard-svc = {
+        apiVersion = "v1";
+        kind = "Service";
+        metadata = {
+          labels = {
+            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
+            k8s-app = "kubernetes-dashboard";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "KubeDashboard";
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+          };
+          name = "kubernetes-dashboard";
+          namespace  = "kube-system";
         };
-        "spec" = {
-            "ports" = [{
-                "port" = 80;
-                "targetPort" = 9090;
-            }];
-            "selector" = {
-                "k8s-app" = "kubernetes-dashboard";
-            };
+        spec = {
+          ports = [{
+            port = 80;
+            targetPort = 9090;
+          }];
+          selector.k8s-app = "kubernetes-dashboard";
         };
-    };
+      };
+
+      kubernetes-dashboard-sa = {
+        apiVersion = "v1";
+        kind = "ServiceAccount";
+        metadata = {
+          labels = {
+            k8s-app = "kubernetes-dashboard";
+            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
+						"addonmanager.kubernetes.io/mode" = "Reconcile";
+          };
+          name = "kubernetes-dashboard";
+          namespace = "kube-system";
+        };
+      };
+    } // (optionalAttrs cfg.enableRBAC {
+      kubernetes-dashboard-crb = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRoleBinding";
+        metadata = {
+          name = "kubernetes-dashboard";
+          labels = {
+            k8s-app = "kubernetes-dashboard";
+            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+          };
+        };
+        roleRef = {
+          apiGroup = "rbac.authorization.k8s.io";
+          kind = "ClusterRole";
+          name = "cluster-admin";
+        };
+        subjects = [{
+          kind = "ServiceAccount";
+          name = "kubernetes-dashboard";
+          namespace = "kube-system";
+        }];
+      };
+    });
+  };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index d799ce292e01..1584fde5afb6 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -587,7 +587,7 @@ in {
 
       clusterDomain = mkOption {
         description = "Use alternative domain.";
-        default = "cluster.local";
+        default = config.services.kubernetes.addons.dns.clusterDomain;
         type = types.str;
       };
 
@@ -705,16 +705,10 @@ in {
         type = types.bool;
       };
 
-      versionTag = mkOption {
-        description = "Version tag of Kubernetes addon manager image.";
-        default = "v6.4-beta.1";
-        type = types.str;
-      };
-
       addons = mkOption {
         description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
         default = { };
-        type = types.attrsOf types.attrs;
+        type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
         example = literalExample ''
           {
             "my-service" = {
@@ -732,30 +726,6 @@ in {
       };
     };
 
-    dns = {
-      enable = mkEnableOption "Kubernetes DNS service.";
-
-      port = mkOption {
-        description = "Kubernetes DNS listening port.";
-        default = 53;
-        type = types.int;
-      };
-
-      domain = mkOption  {
-        description = "Kubernetes DNS domain under which to create names.";
-        default = cfg.kubelet.clusterDomain;
-        type = types.str;
-      };
-
-      kubeconfig = mkKubeConfigOptions "Kubernetes dns";
-
-      extraOpts = mkOption {
-        description = "Kubernetes DNS extra command line options.";
-        default = "";
-        type = types.str;
-      };
-    };
-
     path = mkOption {
       description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
       type = types.listOf types.package;
@@ -1052,8 +1022,8 @@ in {
       services.kubernetes.apiserver.enable = mkDefault true;
       services.kubernetes.scheduler.enable = mkDefault true;
       services.kubernetes.controllerManager.enable = mkDefault true;
-      services.kubernetes.dns.enable = mkDefault true;
       services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
+      services.kubernetes.addonManager.enable = mkDefault true;
     })
 
     # if this node is only a master make it unschedulable by default
@@ -1074,47 +1044,24 @@ in {
 
       services.kubernetes.kubelet.enable = mkDefault true;
       services.kubernetes.proxy.enable = mkDefault true;
-      services.kubernetes.dns.enable = mkDefault true;
     })
 
     (mkIf cfg.addonManager.enable {
-      services.kubernetes.kubelet.manifests = import ./kube-addon-manager.nix { inherit cfg addons; };
       environment.etc."kubernetes/addons".source = "${addons}/";
-    })
 
-    (mkIf cfg.dns.enable {
-      systemd.services.kube-dns = {
-        description = "Kubernetes DNS Service";
+      systemd.services.kube-addon-manager = {
+        description = "Kubernetes addon manager";
         wantedBy = [ "kubernetes.target" ];
         after = [ "kube-apiserver.service" ];
+        environment.ADDON_PATH = "/etc/kubernetes/addons/";
         serviceConfig = {
           Slice = "kubernetes.slice";
-          ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \
-            --kubecfg-file=${mkKubeConfig "kube-dns" cfg.dns.kubeconfig} \
-            --dns-port=${toString cfg.dns.port} \
-            --domain=${cfg.dns.domain} \
-            ${optionalString cfg.verbose "--v=6"} \
-            ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
-            ${cfg.dns.extraOpts}
-          '';
+          ExecStart = "${cfg.package}/bin/kube-addons";
           WorkingDirectory = cfg.dataDir;
           User = "kubernetes";
           Group = "kubernetes";
-          AmbientCapabilities = "cap_net_bind_service";
-          SendSIGHUP = true;
-          RestartSec = "30s";
-          Restart = "always";
-          StartLimitInterval = "1m";
         };
       };
-
-      networking.firewall.extraCommands = ''
-        # allow container to host communication for DNS traffic
-        ${pkgs.iptables}/bin/iptables -I nixos-fw -p tcp -m tcp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
-        ${pkgs.iptables}/bin/iptables -I nixos-fw -p udp -m udp -d ${cfg.clusterCidr} --dport 53 -j nixos-fw-accept
-      '';
-
-      services.kubernetes.dns.kubeconfig = kubeConfigDefaults;
     })
 
     (mkIf (
@@ -1122,8 +1069,7 @@ in {
         cfg.scheduler.enable ||
         cfg.controllerManager.enable ||
         cfg.kubelet.enable ||
-        cfg.proxy.enable ||
-        cfg.dns.enable
+        cfg.proxy.enable
     ) {
       systemd.targets.kubernetes = {
         description = "Kubernetes";
@@ -1147,6 +1093,9 @@ in {
         createHome = true;
       };
       users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
+
+			# dns addon is enabled by default
+      services.kubernetes.addons.dns.enable = mkDefault true;
     })
 
     (mkIf cfg.flannel.enable {
diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/dns.nix
new file mode 100644
index 000000000000..743fbdedfc54
--- /dev/null
+++ b/nixos/modules/services/cluster/kubernetes/dns.nix
@@ -0,0 +1,311 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  version = "1.14.4";
+
+  k8s-dns-kube-dns = pkgs.dockerTools.pullImage {
+    imageName = "gcr.io/google_containers/k8s-dns-kube-dns-amd64";
+    imageTag = version;
+    sha256 = "1zq8xl0rac99r925xmalmx1xc3pgzprh6csrzck9k8fnqp50smdk";
+  };
+
+  k8s-dns-dnsmasq-nanny = pkgs.dockerTools.pullImage {
+    imageName = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64";
+    imageTag = version;
+    sha256 = "1dzw1qb97w2km6c5p564g1r5lnvyblwg5h2cssn09v7x4w3lf732";
+  };
+
+  k8s-dns-sidecar = pkgs.dockerTools.pullImage {
+    imageName = "gcr.io/google_containers/k8s-dns-sidecar-amd64";
+    imageTag = version;
+    sha256 = "1k9k6xklc1b74slpldnjsciikhkfhi0nl2n1vn1xr1y40dggssy4";
+  };
+
+  cfg = config.services.kubernetes.addons.dns;
+in {
+  options.services.kubernetes.addons.dns = {
+    enable = mkEnableOption "kubernetes dns addon";
+
+    clusterIp = mkOption {
+      description = "Dns addon clusterIP";
+
+      # this default is also what kubernetes users
+      default = (
+        concatStringsSep "." (
+          take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
+        ))
+      ) + ".254";
+      type = types.str;
+    };
+
+    clusterDomain = mkOption {
+      description = "Dns cluster domain";
+      default = "cluster.local";
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.kubernetes.kubelet.seedDockerImages = [
+      k8s-dns-kube-dns
+      k8s-dns-dnsmasq-nanny
+      k8s-dns-sidecar
+    ];
+
+    services.kubernetes.addonManager.addons = {
+      kubedns-deployment = {
+        apiVersion = "apps/v1beta1";
+        kind = "Deployment";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            "k8s-app" = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+          };
+          name = "kube-dns";
+          namespace = "kube-system";
+        };
+        spec = {
+          selector.matchLabels."k8s-app" = "kube-dns";
+          strategy = {
+            rollingUpdate = {
+              maxSurge = "10%";
+              maxUnavailable = 0;
+            };
+          };
+          template = {
+            metadata = {
+              annotations."scheduler.alpha.kubernetes.io/critical-pod" = "";
+              labels.k8s-app = "kube-dns";
+            };
+            spec = {
+              containers = [
+                {
+                  name = "kubedns";
+                  args = [
+                    "--domain=${cfg.clusterDomain}"
+                    "--dns-port=10053"
+                    "--config-dir=/kube-dns-config"
+                    "--v=2"
+                  ];
+                  env = [
+                    {
+                      name = "PROMETHEUS_PORT";
+                      value = "10055";
+                    }
+                  ];
+                  image = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:${version}";
+                  livenessProbe = {
+                    failureThreshold = 5;
+                    httpGet = {
+                      path = "/healthcheck/kubedns";
+                      port = 10054;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 60;
+                    successThreshold = 1;
+                    timeoutSeconds = 5;
+                  };
+                  ports = [
+                    {
+                      containerPort = 10053;
+                      name = "dns-local";
+                      protocol = "UDP";
+                    }
+                    {
+                      containerPort = 10053;
+                      name = "dns-tcp-local";
+                      protocol = "TCP";
+                    }
+                    {
+                      containerPort = 10055;
+                      name = "metrics";
+                      protocol = "TCP";
+                    }
+                  ];
+                  readinessProbe = {
+                    httpGet = {
+                      path = "/readiness";
+                      port = 8081;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 3;
+                    timeoutSeconds = 5;
+                  };
+                  resources = {
+                    limits.memory = "170Mi";
+                    requests = {
+                      cpu = "100m";
+                      memory = "70Mi";
+                    };
+                  };
+                  volumeMounts = [
+                    {
+                      mountPath = "/kube-dns-config";
+                      name = "kube-dns-config";
+                    }
+                  ];
+                }
+                {
+                  args = [
+                    "-v=2"
+                    "-logtostderr"
+                    "-configDir=/etc/k8s/dns/dnsmasq-nanny"
+                    "-restartDnsmasq=true"
+                    "--"
+                    "-k"
+                    "--cache-size=1000"
+                    "--log-facility=-"
+                    "--server=/${cfg.clusterDomain}/127.0.0.1#10053"
+                    "--server=/in-addr.arpa/127.0.0.1#10053"
+                    "--server=/ip6.arpa/127.0.0.1#10053"
+                  ];
+                  image = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:${version}";
+                  livenessProbe = {
+                    failureThreshold = 5;
+                    httpGet = {
+                      path = "/healthcheck/dnsmasq";
+                      port = 10054;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 60;
+                    successThreshold = 1;
+                    timeoutSeconds = 5;
+                  };
+                  name = "dnsmasq";
+                  ports = [
+                    {
+                      containerPort = 53;
+                      name = "dns";
+                      protocol = "UDP";
+                    }
+                    {
+                      containerPort = 53;
+                      name = "dns-tcp";
+                      protocol = "TCP";
+                    }
+                  ];
+                  resources = {
+                    requests = {
+                      cpu = "150m";
+                      memory = "20Mi";
+                    };
+                  };
+                  volumeMounts = [
+                    {
+                      mountPath = "/etc/k8s/dns/dnsmasq-nanny";
+                      name = "kube-dns-config";
+                    }
+                  ];
+                }
+                {
+                  name = "sidecar";
+                  image = "gcr.io/google_containers/k8s-dns-sidecar-amd64:${version}";
+                  args = [
+                    "--v=2"
+                    "--logtostderr"
+                    "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.clusterDomain},5,A"
+                    "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.clusterDomain},5,A"
+                  ];
+                  livenessProbe = {
+                    failureThreshold = 5;
+                    httpGet = {
+                      path = "/metrics";
+                      port = 10054;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 60;
+                    successThreshold = 1;
+                    timeoutSeconds = 5;
+                  };
+                  ports = [
+                    {
+                      containerPort = 10054;
+                      name = "metrics";
+                      protocol = "TCP";
+                    }
+                  ];
+                  resources = {
+                    requests = {
+                      cpu = "10m";
+                      memory = "20Mi";
+                    };
+                  };
+                }
+              ];
+              dnsPolicy = "Default";
+              serviceAccountName = "kube-dns";
+              tolerations = [
+                {
+                  key = "CriticalAddonsOnly";
+                  operator = "Exists";
+                }
+              ];
+              volumes = [
+                {
+                  configMap = {
+                    name = "kube-dns";
+                    optional = true;
+                  };
+                  name = "kube-dns-config";
+                }
+              ];
+            };
+          };
+        };
+      };
+
+      kubedns-svc = {
+        apiVersion = "v1";
+        kind = "Service";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            "k8s-app" = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "KubeDNS";
+          };
+          name = "kube-dns";
+          namespace  = "kube-system";
+        };
+        spec = {
+          clusterIP = cfg.clusterIp;
+          ports = [
+            {name = "dns"; port = 53; protocol = "UDP";}
+            {name = "dns-tcp"; port = 53; protocol = "TCP";}
+          ];
+          selector.k8s-app = "kube-dns";
+        };
+      };
+
+      kubedns-sa = {
+        apiVersion = "v1";
+        kind = "ServiceAccount";
+        metadata = {
+          name = "kube-dns";
+          namespace = "kube-system";
+          labels = {
+            "kubernetes.io/cluster-service" = "true";
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+          };
+        };
+      };
+
+      kubedns-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          name = "kube-dns";
+          namespace = "kube-system";
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "EnsureExists";
+          };
+        };
+      };
+    };
+
+    services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
+  };
+}
diff --git a/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix b/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix
deleted file mode 100644
index f1a367dfff1b..000000000000
--- a/nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix
+++ /dev/null
@@ -1,54 +0,0 @@
-{ cfg, addons }: {
-    "kube-addon-manager" = {
-        "apiVersion" = "v1";
-        "kind" = "Pod";
-        "metadata" = {
-            "labels" = {
-                "component" = "kube-addon-manager";
-            };
-            "name" = "kube-addon-manager";
-            "namespace" = "kube-system";
-        };
-        "spec" = {
-            "containers" = [{
-                "command" = ["/bin/bash"
-                    "-c"
-                    "/opt/kube-addons.sh | tee /var/log/kube-addon-manager.log"
-                ];
-                "env" = [{
-                    "name" = "KUBECTL_OPTS";
-                    "value" = "--server=${cfg.kubeconfig.server}";
-                }];
-                "image" = "gcr.io/google-containers/kube-addon-manager:${cfg.addonManager.versionTag}";
-                "name" = "kube-addon-manager";
-                "resources" = {
-                    "requests" = {
-                        "cpu" = "5m";
-                        "memory" = "50Mi";
-                    };
-                };
-                "volumeMounts" = [{
-                    "mountPath" = "/etc/kubernetes/addons/";
-                    "name" = "addons";
-                    "readOnly" = true;
-                } {
-                    "mountPath" = "/var/log";
-                    "name" = "varlog";
-                    "readOnly" = false;
-                }];
-            }];
-            "hostNetwork" = true;
-            "volumes" = [{
-                "hostPath" = {
-                    "path" = "${addons}/";
-                };
-                "name" = "addons";
-            } {
-                "hostPath" = {
-                    "path" = "/var/log";
-                };
-                "name" = "varlog";
-            }];
-        };
-    };
-}