Revert "dd-agent: 5.11.2 -> 5.13.2 + service rework"

This reverts commit af096c8bff1e534be9c69f50eed13e6b48427d0e.

4 files changed, 169 insertions, 95 deletions

diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
new file mode 100644
index 000000000000..045128197421
--- /dev/null
+++ b/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix
@@ -0,0 +1,8 @@
+# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.
+[
+  "auto_conf"
+  "agent_metrics.yaml.default"
+  "disk.yaml.default"
+  "network.yaml.default"
+  "ntp.yaml.default"
+]
diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
index e55faed8e078..8d3d4f2065ec 100644
--- a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
+++ b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix
@@ -16,24 +16,100 @@ let
     forwarder_log_file: /var/log/datadog/forwarder.log
     dogstatsd_log_file: /var/log/datadog/dogstatsd.log
     pup_log_file:       /var/log/datadog/pup.log
+
+    # proxy_host: my-proxy.com
+    # proxy_port: 3128
+    # proxy_user: user
+    # proxy_password: password
+
+    # tags: mytag0, mytag1
     ${optionalString (cfg.tags != null ) "tags: ${concatStringsSep "," cfg.tags }"}
-    ${cfg.extraDdConfig}
+
+    # collect_ec2_tags: no
+    # recent_point_threshold: 30
+    # use_mount: no
+    # listen_port: 17123
+    # graphite_listen_port: 17124
+    # non_local_traffic: no
+    # use_curl_http_client: False
+    # bind_host: localhost
+
+    # use_pup: no
+    # pup_port: 17125
+    # pup_interface: localhost
+    # pup_url: http://localhost:17125
+
+    # dogstatsd_port : 8125
+    # dogstatsd_interval : 10
+    # dogstatsd_normalize : yes
+    # statsd_forward_host: address_of_own_statsd_server
+    # statsd_forward_port: 8125
+
+    # device_blacklist_re: .*\/dev\/mapper\/lxc-box.*
+
+    # ganglia_host: localhost
+    # ganglia_port: 8651
   '';
 
+  diskConfig = pkgs.writeText "disk.yaml" ''
+    init_config:
+
+    instances:
+      - use_mount: no
+  '';
+  
+  networkConfig = pkgs.writeText "network.yaml" ''
+    init_config:
+
+    instances:
+      # Network check only supports one configured instance
+      - collect_connection_state: false
+        excluded_interfaces:
+          - lo
+          - lo0
+  '';
+  
+  postgresqlConfig = pkgs.writeText "postgres.yaml" cfg.postgresqlConfig;
+  nginxConfig = pkgs.writeText "nginx.yaml" cfg.nginxConfig;
+  mongoConfig = pkgs.writeText "mongo.yaml" cfg.mongoConfig;
+  jmxConfig = pkgs.writeText "jmx.yaml" cfg.jmxConfig;
+  processConfig = pkgs.writeText "process.yaml" cfg.processConfig;
+  
   etcfiles =
-    map (i: { source = if builtins.hasAttr "config" i
-                       then pkgs.writeText "${i.name}.yaml" i.config
-                       else "${pkgs.dd-agent}/agent/conf.d-system/${i.name}.yaml";
-              target = "dd-agent/conf.d/${i.name}.yaml";
-            }
-        ) cfg.integrations ++
-        [ { source = ddConf;
-            target = "dd-agent/datadog.conf";
-          }
-        ];
-
-  # restart triggers
-  etcSources = map (i: i.source) etcfiles;
+    let
+      defaultConfd = import ./dd-agent-defaults.nix;
+    in (map (f: { source = "${pkgs.dd-agent}/agent/conf.d-system/${f}";
+                  target = "dd-agent/conf.d/${f}";
+                }) defaultConfd) ++ [
+      { source = ddConf;
+        target = "dd-agent/datadog.conf";
+      }
+      { source = diskConfig;
+        target = "dd-agent/conf.d/disk.yaml";
+      }
+      { source = networkConfig;
+        target = "dd-agent/conf.d/network.yaml";
+      } ] ++
+    (optional (cfg.postgresqlConfig != null)
+      { source = postgresqlConfig;
+        target = "dd-agent/conf.d/postgres.yaml";
+      }) ++
+    (optional (cfg.nginxConfig != null)
+      { source = nginxConfig;
+        target = "dd-agent/conf.d/nginx.yaml";
+      }) ++
+    (optional (cfg.mongoConfig != null)
+      { source = mongoConfig;
+        target = "dd-agent/conf.d/mongo.yaml";
+      }) ++
+    (optional (cfg.processConfig != null)
+      { source = processConfig;
+        target = "dd-agent/conf.d/process.yaml";
+      }) ++
+    (optional (cfg.jmxConfig != null)
+      { source = jmxConfig;
+        target = "dd-agent/conf.d/jmx.yaml";
+      });
 
 in {
   options.services.dd-agent = {
@@ -63,46 +139,44 @@ in {
       type = types.uniq (types.nullOr types.string);
     };
 
-    agent = mkOption {
-      description = "The dd-agent package to use. Useful when overriding the package.";
-      default = pkgs.dd-agent;
-      type = types.package;
+    postgresqlConfig = mkOption {
+      description = "Datadog PostgreSQL integration configuration";
+      default = null;
+      type = types.uniq (types.nullOr types.string);
+    };
+
+    nginxConfig = mkOption {
+      description = "Datadog nginx integration configuration";
+      default = null;
+      type = types.uniq (types.nullOr types.string);
+    };
+    
+    mongoConfig = mkOption {
+      description = "MongoDB integration configuration";
+      default = null;
+      type = types.uniq (types.nullOr types.string);
+    };
+
+    jmxConfig = mkOption {
+      description = "JMX integration configuration";
+      default = null;
+      type = types.uniq (types.nullOr types.string);
     };
 
-    integrations = mkOption {
+    processConfig = mkOption {
       description = ''
-        Any integrations to use. Default config used if none
-        specified. It is currently up to the user to make sure that
-        the dd-agent package used has all the dependencies chosen
-        integrations require in scope.
-      '';
-      type = types.listOf (types.attrsOf types.string);
-      default = [];
-      example = ''
-        [ { name = "elastic";
-            config = '''
-              init_config:
-
-              instances:
-                - url: http://localhost:9200
-            ''';
-          }
-          { name = "nginx"; }
-          { name = "ntp"; }
-          { name = "network"; }
-        ]
+        Process integration configuration
+ 
+        See http://docs.datadoghq.com/integrations/process/
       '';
+      default = null;
+      type = types.uniq (types.nullOr types.string);
     };
 
-    extraDdConfig = mkOption {
-      description = "Extra settings to append to datadog agent config.";
-      default = "";
-      type = types.string;
-    };
   };
 
   config = mkIf cfg.enable {
-    environment.systemPackages = [ cfg.agent pkgs.sysstat pkgs.procps ];
+    environment.systemPackages = [ pkgs."dd-agent" pkgs.sysstat pkgs.procps ];
 
     users.extraUsers.datadog = {
       description = "Datadog Agent User";
@@ -116,7 +190,7 @@ in {
 
     systemd.services.dd-agent = {
       description = "Datadog agent monitor";
-      path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps ];
+      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground";
@@ -125,12 +199,28 @@ in {
         Restart = "always";
         RestartSec = 2;
       };
-      restartTriggers = [ pkgs.dd-agent ddConf ] ++ etcSources;
+      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
+    };
+
+    systemd.services.dogstatsd = {
+      description = "Datadog statsd";
+      path = [ pkgs."dd-agent" pkgs.python pkgs.procps ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start";
+        User = "datadog";
+        Group = "datadog";
+        Type = "forking";
+        PIDFile = "/tmp/dogstatsd.pid";
+        Restart = "always";
+        RestartSec = 2;
+      };
+      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ];
     };
 
-    systemd.services.dd-jmxfetch = lib.mkIf (builtins.any (i: i.name == "jmx") cfg.integrations) {
+    systemd.services.dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) {
       description = "Datadog JMX Fetcher";
-      path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
+      path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ];
       wantedBy = [ "multi-user.target" ];
       serviceConfig = {
         ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch";
@@ -139,7 +229,7 @@ in {
         Restart = "always";
         RestartSec = 2;
       };
-      restartTriggers = [ cfg.agent ddConf ] ++ etcSources;
+      restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig ];
     };
 
     environment.etc = etcfiles;
diff --git a/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults b/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
new file mode 100755
index 000000000000..76724173171a
--- /dev/null
+++ b/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+dd=$(nix-build --no-out-link -A dd-agent ../../../..)
+echo '# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.' > dd-agent-defaults.nix
+echo '[' >> dd-agent-defaults.nix
+echo '  "auto_conf"' >> dd-agent-defaults.nix
+for f in $(find $dd/agent/conf.d-system -maxdepth 1 -type f | grep -v '\.example' | sort); do
+  echo "  \"$(basename $f)\"" >> dd-agent-defaults.nix
+done
+echo ']' >> dd-agent-defaults.nix
diff --git a/pkgs/tools/networking/dd-agent/default.nix b/pkgs/tools/networking/dd-agent/default.nix
index 2556287570fd..d1171a3c8355 100644
--- a/pkgs/tools/networking/dd-agent/default.nix
+++ b/pkgs/tools/networking/dd-agent/default.nix
@@ -1,11 +1,5 @@
 { stdenv, fetchFromGitHub, pythonPackages
-, sysstat, unzip, makeWrapper
-# We need extraBuildInputs as we want to be able to override this
-# package with python packages _and_ have the produced binaries
-# wrapper with their PYTHONPATH. This means overrideAttrs is not
-# strong enough (it overrides too late), we need to call it
-# beforehand.
-, extraBuildInputs ? [ pythonPackages.psutil ] }:
+, sysstat, unzip, makeWrapper }:
 let
   inherit (pythonPackages) python;
   docker_1_10 = pythonPackages.buildPythonPackage rec {
@@ -32,42 +26,33 @@ let
     # due to flake8
     doCheck = false;
   };
-  version = "5.13.2";
-
-  integrations = fetchFromGitHub {
-    owner = "datadog";
-    repo = "integrations-core";
-    rev = version;
-    sha256 = "1nbjmkq0wdfndmx0qap69h2rkwkkb0632j87h9d3j99bykyav3y3";
-  };
 
 in stdenv.mkDerivation rec {
+  version = "5.11.2";
   name = "dd-agent-${version}";
 
   src = fetchFromGitHub {
     owner  = "datadog";
     repo   = "dd-agent";
     rev    = version;
-    sha256 = "0x2bxi70l2yf0wi232qksvcscjdpjg8l7dmgg1286vqryyfazfjb";
+    sha256 = "1iqxvgpsqibqw3vk79158l2pnb6y4pjhjp2d6724lm5rpz4825lx";
   };
 
   buildInputs = [
     python
     unzip
     makeWrapper
-    pythonPackages.boto
-    docker_1_10
-    pythonPackages.kazoo
-    pythonPackages.ntplib
-    pythonPackages.consul
-    pythonPackages.python-etcd
-    pythonPackages.pyyaml
     pythonPackages.requests
+    pythonPackages.psycopg2
+    pythonPackages.psutil
+    pythonPackages.ntplib
     pythonPackages.simplejson
-    pythonPackages.supervisor
-    pythonPackages.tornado
-    pythonPackages.uptime
-  ] ++ extraBuildInputs;
+    pythonPackages.pyyaml
+    pythonPackages.pymongo_2_9_1
+    pythonPackages.python-etcd
+    pythonPackages.consul
+    docker_1_10
+  ];
   propagatedBuildInputs = with pythonPackages; [ python tornado ];
 
   buildCommand = ''
@@ -82,24 +67,6 @@ in stdenv.mkDerivation rec {
     # Move out default conf.d so that /etc/dd-agent/conf.d is used
     mv $out/agent/conf.d $out/agent/conf.d-system
 
-    # Sometime between 5.11.2 and 5.13.2 datadog moved out all its
-    # checks into separate repository. Copy them back in so dd-agent
-    # service can easily pick and choose by copying out configs into
-    # its etc files.
-    mkdir -p $out/agent/checks.d
-    for i in ${toString integrations}/* # */
-    do
-      if [ -f "$i/check.py" ]; then
-        if [ -f "$i/conf.yaml.default" -o -f "$i/conf.yaml.example" ]; then
-          local name=$(basename $i)
-          cp $i/check.py $out/agent/checks.d/$name.py
-          # Copy .default file first unless it doesn't exist then copy .default
-          cp $i/conf.yaml.default $out/agent/conf.d-system/$name.yaml &> /dev/null || \
-            cp $i/conf.yaml.example $out/agent/conf.d-system/$name.yaml
-        fi
-      fi
-    done
-
     cat > $out/bin/dd-jmxfetch <<EOF
     #!/usr/bin/env bash
     exec ${python}/bin/python $out/agent/jmxfetch.py $@