dd-agent: 5.11.2 -> 5.13.2 + service rework

3 files changed, 50 insertions, 157 deletions

diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
deleted file mode 100644
index 045128197421..000000000000
--- a/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
+++ /dev/null
@@ -1,8 +0,0 @@
-# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.
-[
-  "auto_conf"
-  "agent_metrics.yaml.default"
-  "disk.yaml.default"
-  "network.yaml.default"
-  "ntp.yaml.default"
-]
diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
index 8d3d4f2065ec..e55faed8e078 100644
--- a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
+++ b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
@@ -16,100 +16,24 @@ let
     forwarder_log_file: /var/log/datadog/forwarder.log
     dogstatsd_log_file: /var/log/datadog/dogstatsd.log
     pup_log_file:       /var/log/datadog/pup.log
-
-    # proxy_host: my-proxy.com
-    # proxy_port: 3128
-    # proxy_user: user
-    # proxy_password: password
-
-    # tags: mytag0, mytag1
     ${optionalString (cfg.tags != null ) "tags: ${concatStringsSep "," cfg.tags }"}
-
-    # collect_ec2_tags: no
-    # recent_point_threshold: 30
-    # use_mount: no
-    # listen_port: 17123
-    # graphite_listen_port: 17124
-    # non_local_traffic: no
-    # use_curl_http_client: False
-    # bind_host: localhost
-
-    # use_pup: no
-    # pup_port: 17125
-    # pup_interface: localhost
-    # pup_url: http://localhost:17125
-
-    # dogstatsd_port : 8125
-    # dogstatsd_interval : 10
-    # dogstatsd_normalize : yes
-    # statsd_forward_host: address_of_own_statsd_server
-    # statsd_forward_port: 8125
-
-    # device_blacklist_re: .*\/dev\/mapper\/lxc-box.*
-
-    # ganglia_host: localhost
-    # ganglia_port: 8651
+    ${cfg.extraDdConfig}
   '';
 
-  diskConfig = pkgs.writeText "disk.yaml" ''
-    init_config:
-
-    instances:
-      - use_mount: no
-  '';
-  
-  networkConfig = pkgs.writeText "network.yaml" ''
-    init_config:
-
-    instances:
-      # Network check only supports one configured instance
-      - collect_connection_state: false
-        excluded_interfaces:
-          - lo
-          - lo0
-  '';
-  
-  postgresqlConfig = pkgs.writeText "postgres.yaml" cfg.postgresqlConfig;
-  nginxConfig = pkgs.writeText "nginx.yaml" cfg.nginxConfig;
-  mongoConfig = pkgs.writeText "mongo.yaml" cfg.mongoConfig;
-  jmxConfig = pkgs.writeText "jmx.yaml" cfg.jmxConfig;
-  processConfig = pkgs.writeText "process.yaml" cfg.processConfig;
-  
   etcfiles =
-    let
-      defaultConfd = import ./dd-agent-defaults.nix;
-    in (map (f: { source = "${pkgs.dd-agent}/agent/conf.d-system/${f}";
-                  target = "dd-agent/conf.d/${f}";
-                }) defaultConfd) ++ [
-      { source = ddConf;
-        target = "dd-agent/datadog.conf";
-      }
-      { source = diskConfig;
-        target = "dd-agent/conf.d/disk.yaml";
-      }
-      { source = networkConfig;
-        target = "dd-agent/conf.d/network.yaml";
-      } ] ++
-    (optional (cfg.postgresqlConfig != null)
-      { source = postgresqlConfig;
-        target = "dd-agent/conf.d/postgres.yaml";
-      }) ++
-    (optional (cfg.nginxConfig != null)
-      { source = nginxConfig;
-        target = "dd-agent/conf.d/nginx.yaml";
-      }) ++
-    (optional (cfg.mongoConfig != null)
-      { source = mongoConfig;
-        target = "dd-agent/conf.d/mongo.yaml";
-      }) ++
-    (optional (cfg.processConfig != null)
-      { source = processConfig;
-        target = "dd-agent/conf.d/process.yaml";
-      }) ++
-    (optional (cfg.jmxConfig != null)
-      { source = jmxConfig;
-        target = "dd-agent/conf.d/jmx.yaml";
-      });
+    map (i: { source = if builtins.hasAttr "config" i
+                       then pkgs.writeText "${i.name}.yaml" i.config
+                       else "${pkgs.dd-agent}/agent/conf.d-system/${i.name}.yaml";
+              target = "dd-agent/conf.d/${i.name}.yaml";
+            }
+        ) cfg.integrations ++
+        [ { source = ddConf;
+            target = "dd-agent/datadog.conf";
+          }
+        ];
+
+  # restart triggers
+  etcSources = map (i: i.source) etcfiles;
 
 in {
   options.services.dd-agent = {
@@ -139,44 +63,46 @@ in {
       type = types.uniq (types.nullOr types.string);
     };
 
-    postgresqlConfig = mkOption {
-      description = "Datadog PostgreSQL integration configuration";
-      default = null;
-      type = types.uniq (types.nullOr types.string);
-    };
-
-    nginxConfig = mkOption {
-      description = "Datadog nginx integration configuration";
-      default = null;
-      type = types.uniq (types.nullOr types.string);
-    };
-    
-    mongoConfig = mkOption {
-      description = "MongoDB integration configuration";
-      default = null;
-      type = types.uniq (types.nullOr types.string);
-    };
-
-    jmxConfig = mkOption {
-      description = "JMX integration configuration";
-      default = null;
-      type = types.uniq (types.nullOr types.string);
+    agent = mkOption {
+      description = "The dd-agent package to use. Useful when overriding the package.";
+      default = pkgs.dd-agent;
+      type = types.package;
     };
 
-    processConfig = mkOption {
+    integrations = mkOption {
       description = ''
-        Process integration configuration
- 
-        See http://docs.datadoghq.com/integrations/process/
+        Any integrations to use. Default config used if none
+        specified. It is currently up to the user to make sure that
+        the dd-agent package used has all the dependencies chosen
+        integrations require in scope.
+      '';
+      type = types.listOf (types.attrsOf types.string);
+      default = [];
+      example = ''
+        [ { name = "elastic";
+            config = '''
+              init_config:
+
+              instances:
+                - url: http://localhost:9200
+            ''';
+          }
+          { name = "nginx"; }
+          { name = "ntp"; }
+          { name = "network"; }
+        ]
       '';
-      default = null;
-      type = types.uniq (types.nullOr types.string);
     };
 
+    extraDdConfig = mkOption {
+      description = "Extra settings to append to datadog agent config.";
+      default = "";
+      type = types.string;
+    };
   };
 
   config = mkIf cfg.enable {
-    environment.systemPackages = [ pkgs."dd-agent" pkgs.sysstat pkgs.procps ];
+    environment.systemPackages = [ cfg.agent pkgs.sysstat pkgs.procps ];
 
     users.extraUsers.datadog = {
       description = "Datadog Agent User";
@@ -190,7 +116,7 @@ in {
 
     systemd.services.dd-agent = {
       description = "Datadog agent monitor";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps ];
+      path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground";
@@ -199,28 +125,12 @@ in {
         Restart = "always";
         RestartSec = 2;
       };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
-    };
-
-    systemd.services.dogstatsd = {
-      description = "Datadog statsd";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.procps ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start";
-        User = "datadog";
-        Group = "datadog";
-        Type = "forking";
-        PIDFile = "/tmp/dogstatsd.pid";
-        Restart = "always";
-        RestartSec = 2;
-      };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
+      restartTriggers = [ pkgs.dd-agent ddConf ] ++ etcSources;
     };
 
-    systemd.services.dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) {
+    systemd.services.dd-jmxfetch = lib.mkIf (builtins.any (i: i.name == "jmx") cfg.integrations) {
       description = "Datadog JMX Fetcher";
-      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
+      path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch";
@@ -229,7 +139,7 @@ in {
         Restart = "always";
         RestartSec = 2;
       };
-      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig ];
+      restartTriggers = [ cfg.agent ddConf ] ++ etcSources;
     };
 
     environment.etc = etcfiles;
diff --git a/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults b/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
deleted file mode 100755
index 76724173171a..000000000000
--- a/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-dd=$(nix-build --no-out-link -A dd-agent ../../../..)
-echo '# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.' > dd-agent-defaults.nix
-echo '[' >> dd-agent-defaults.nix
-echo '  "auto_conf"' >> dd-agent-defaults.nix
-for f in $(find $dd/agent/conf.d-system -maxdepth 1 -type f | grep -v '\.example' | sort); do
-  echo "  \"$(basename $f)\"" >> dd-agent-defaults.nix
-done
-echo ']' >> dd-agent-defaults.nix