ELK: update kibana and the elastic beats to 5.4 (#26252)

* Add kibana5 and logstash5
* Upgrade the elastic beats to 5.4
* Make sure all elastic products use the same version
  (see elk5Version)
* Add a test for the ELK stack

2 files changed, 127 insertions, 29 deletions

diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix
index c9477b9e3ab0..b4abd2cd7e5e 100644
--- a/nixos/modules/services/logging/logstash.nix
+++ b/nixos/modules/services/logging/logstash.nix
@@ -4,17 +4,46 @@ with lib;
 
 let
   cfg = config.services.logstash;
+  atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4";
   pluginPath = lib.concatStringsSep ":" cfg.plugins;
   havePluginPath = lib.length cfg.plugins > 0;
   ops = lib.optionalString;
-  verbosityFlag = {
-    debug = "--debug";
-    info  = "--verbose";
-    warn  = ""; # intentionally empty
-    error = "--quiet";
-    fatal = "--silent";
-  }."${cfg.logLevel}";
-
+  verbosityFlag =
+    if atLeast54
+    then "--log.level " + cfg.logLevel
+    else {
+      debug = "--debug";
+      info  = "--verbose";
+      warn  = ""; # intentionally empty
+      error = "--quiet";
+      fatal = "--silent";
+    }."${cfg.logLevel}";
+
+  pluginsPath =
+    if atLeast54
+    then "--path.plugins ${pluginPath}"
+    else "--pluginpath ${pluginPath}";
+
+  logstashConf = pkgs.writeText "logstash.conf" ''
+    input {
+      ${cfg.inputConfig}
+    }
+
+    filter {
+      ${cfg.filterConfig}
+    }
+
+    output {
+      ${cfg.outputConfig}
+    }
+  '';
+
+  logstashSettingsYml = pkgs.writeText "logstash.yml" cfg.extraSettings;
+
+  logstashSettingsDir = pkgs.runCommand "logstash-settings" {inherit logstashSettingsYml;} ''
+    mkdir -p $out
+    ln -s $logstashSettingsYml $out/logstash.yml
+  '';
 in
 
 {
@@ -45,6 +74,15 @@ in
         description = "The paths to find other logstash plugins in.";
       };
 
+      dataDir = mkOption {
+        type = types.str;
+        default = "/var/lib/logstash";
+        description = ''
+          A path to directory writable by logstash that it uses to store data.
+          Plugins will also have access to this path.
+        '';
+      };
+
       logLevel = mkOption {
         type = types.enum [ "debug" "info" "warn" "error" "fatal" ];
         default = "warn";
@@ -116,6 +154,19 @@ in
         '';
       };
 
+      extraSettings = mkOption {
+        type = types.lines;
+        default = "";
+        description = "Extra Logstash settings in YAML format.";
+        example = ''
+          pipeline:
+            batch:
+              size: 125
+              delay: 5
+        '';
+      };
+
+
     };
   };
 
@@ -123,31 +174,34 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
+    assertions = [
+      { assertion = atLeast54 -> !cfg.enableWeb;
+        message = ''
+          The logstash web interface is only available for versions older than 5.4.
+          So either set services.logstash.enableWeb = false,
+          or set services.logstash.package to an older logstash.
+        '';
+      }
+    ];
+
     systemd.services.logstash = with pkgs; {
       description = "Logstash Daemon";
       wantedBy = [ "multi-user.target" ];
       environment = { JAVA_HOME = jre; };
       path = [ pkgs.bash ];
       serviceConfig = {
-        ExecStart =
-          "${cfg.package}/bin/logstash agent " +
-          "-w ${toString cfg.filterWorkers} " +
-          ops havePluginPath "--pluginpath ${pluginPath} " +
-          "${verbosityFlag} " +
-          "-f ${writeText "logstash.conf" ''
-            input {
-              ${cfg.inputConfig}
-            }
-
-            filter {
-              ${cfg.filterConfig}
-            }
-
-            output {
-              ${cfg.outputConfig}
-            }
-          ''} " +
-          ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}";
+        ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"'';
+        ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [
+          "${cfg.package}/bin/logstash"
+          (ops (!atLeast54) "agent")
+          "-w ${toString cfg.filterWorkers}"
+          (ops havePluginPath pluginsPath)
+          "${verbosityFlag}"
+          "-f ${logstashConf}"
+          (ops atLeast54 "--path.settings ${logstashSettingsDir}")
+          (ops atLeast54 "--path.data ${cfg.dataDir}")
+          (ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}")
+        ]);
       };
     };
   };
diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix
index d377a6feeb8e..9d7d2d799189 100644
--- a/nixos/modules/services/search/kibana.nix
+++ b/nixos/modules/services/search/kibana.nix
@@ -5,7 +5,11 @@ with lib;
 let
   cfg = config.services.kibana;
 
-  cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON (
+  atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4";
+
+  cfgFile = if atLeast54 then cfgFile5 else cfgFile4;
+
+  cfgFile4 = pkgs.writeText "kibana.json" (builtins.toJSON (
     (filterAttrsRecursive (n: v: v != null) ({
       host = cfg.listenAddress;
       port = cfg.port;
@@ -36,6 +40,27 @@ let
       ];
     } // cfg.extraConf)
   )));
+
+  cfgFile5 = pkgs.writeText "kibana.json" (builtins.toJSON (
+    (filterAttrsRecursive (n: v: v != null) ({
+      server.host = cfg.listenAddress;
+      server.port = cfg.port;
+      server.ssl.certificate = cfg.cert;
+      server.ssl.key = cfg.key;
+
+      kibana.index = cfg.index;
+      kibana.defaultAppId = cfg.defaultAppId;
+
+      elasticsearch.url = cfg.elasticsearch.url;
+      elasticsearch.username = cfg.elasticsearch.username;
+      elasticsearch.password = cfg.elasticsearch.password;
+
+      elasticsearch.ssl.certificate = cfg.elasticsearch.cert;
+      elasticsearch.ssl.key = cfg.elasticsearch.key;
+      elasticsearch.ssl.certificateAuthorities = cfg.elasticsearch.certificateAuthorities;
+    } // cfg.extraConf)
+  )));
+
 in {
   options.services.kibana = {
     enable = mkEnableOption "enable kibana service";
@@ -96,11 +121,29 @@ in {
       };
 
       ca = mkOption {
-        description = "CA file to auth against elasticsearch.";
+        description = ''
+          CA file to auth against elasticsearch.
+
+          It's recommended to use the <option>certificateAuthorities</option> option
+          when using kibana-5.4 or newer.
+        '';
         default = null;
         type = types.nullOr types.path;
       };
 
+      certificateAuthorities = mkOption {
+        description = ''
+          CA files to auth against elasticsearch.
+
+          Please use the <option>ca</option> option when using kibana &lt; 5.4
+          because those old versions don't support setting multiple CA's.
+
+          This defaults to the singleton list [ca] when the <option>ca</option> option is defined.
+        '';
+        default = if isNull cfg.elasticsearch.ca then [] else [ca];
+        type = types.listOf types.path;
+      };
+
       cert = mkOption {
         description = "Certificate file to auth against elasticsearch.";
         default = null;
@@ -118,6 +161,7 @@ in {
       description = "Kibana package to use";
       default = pkgs.kibana;
       defaultText = "pkgs.kibana";
+      example = "pkgs.kibana5";
       type = types.package;
     };