Merge pull request #44389 from Mic92/es6

elasticsearch: use 6.x as default version, remove unsupported releases

5 files changed, 62 insertions, 116 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml
index d831f8514466..8ee2a5f16238 100644
--- a/nixos/doc/manual/release-notes/rl-1809.xml
+++ b/nixos/doc/manual/release-notes/rl-1809.xml
@@ -192,6 +192,39 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
    </listitem>
    <listitem>
     <para>
+     The ELK stack: <varname>elasticsearch</varname>, <varname>logstash</varname> and <varname>kibana</varname>
+     has been upgraded from 2.* to 6.3.*.
+     The 2.* versions have been <link xlink:href="https://www.elastic.co/support/eol">unsupported since last year</link>
+     so they have been removed. You can still use the 5.* versions under the names
+     <varname>elasticsearch5</varname>, <varname>logstash5</varname> and
+     <varname>kibana5</varname>.
+    </para>
+    <para>
+     The elastic beats:
+     <varname>filebeat</varname>, <varname>heartbeat</varname>,
+     <varname>metricbeat</varname> and <varname>packetbeat</varname>
+     have had the same treatment: they now target 6.3.* as well.
+     The 5.* versions are available under the names:
+     <varname>filebeat5</varname>, <varname>heartbeat5</varname>,
+     <varname>metricbeat5</varname> and <varname>packetbeat5</varname>
+    </para>
+    <para>
+     The ELK-6.3 stack now comes with
+     <link xlink:href="https://www.elastic.co/products/x-pack/open">X-Pack by default</link>.
+     Since X-Pack is licensed under the
+     <link xlink:href="https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt">Elastic License</link>
+     the ELK packages now have an unfree license. To use them you need to specify
+     <literal>allowUnfree = true;</literal> in your nixpkgs configuration.
+    </para>
+    <para>
+     Fortunately there is also a free variant of the ELK stack without X-Pack.
+     The packages are available under the names:
+     <varname>elasticsearch-oss</varname>, <varname>logstash-oss</varname> and
+     <varname>kibana-oss</varname>.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
       Options
       <literal>boot.initrd.luks.devices.<replaceable>name</replaceable>.yubikey.ramfsMountPoint</literal>
       <literal>boot.initrd.luks.devices.<replaceable>name</replaceable>.yubikey.storage.mountPoint</literal>
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index f032f10e4557..b51dcd2976f4 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -256,6 +256,7 @@ with lib;
     (mkRemovedOptionModule [ "fonts" "fontconfig" "forceAutohint" ] "")
     (mkRemovedOptionModule [ "fonts" "fontconfig" "renderMonoTTFAsBitmap" ] "")
     (mkRemovedOptionModule [ "virtualisation" "xen" "qemu" ] "You don't need this option anymore, it will work without it.")
+    (mkRemovedOptionModule [ "services" "logstash" "enableWeb" ] "The web interface was removed from logstash")
     (mkRemovedOptionModule [ "boot" "zfs" "enableLegacyCrypto" ] "The corresponding package was removed from nixpkgs.")
 
     # ZSH
diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix
index 28d89a7463ab..aa019d855ea9 100644
--- a/nixos/modules/services/logging/logstash.nix
+++ b/nixos/modules/services/logging/logstash.nix
@@ -4,25 +4,12 @@ with lib;
 
 let
   cfg = config.services.logstash;
-  atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4";
   pluginPath = lib.concatStringsSep ":" cfg.plugins;
   havePluginPath = lib.length cfg.plugins > 0;
   ops = lib.optionalString;
-  verbosityFlag =
-    if atLeast54
-    then "--log.level " + cfg.logLevel
-    else {
-      debug = "--debug";
-      info  = "--verbose";
-      warn  = ""; # intentionally empty
-      error = "--quiet";
-      fatal = "--silent";
-    }."${cfg.logLevel}";
-
-  pluginsPath =
-    if atLeast54
-    then "--path.plugins ${pluginPath}"
-    else "--pluginpath ${pluginPath}";
+  verbosityFlag = "--log.level " + cfg.logLevel;
+
+  pluginsPath = "--path.plugins ${pluginPath}";
 
   logstashConf = pkgs.writeText "logstash.conf" ''
     input {
@@ -63,7 +50,7 @@ in
         type = types.package;
         default = pkgs.logstash;
         defaultText = "pkgs.logstash";
-        example = literalExample "pkgs.logstash";
+        example = literalExample "pkgs.logstash5";
         description = "Logstash package to use.";
       };
 
@@ -95,12 +82,6 @@ in
         description = "The quantity of filter workers to run.";
       };
 
-      enableWeb = mkOption {
-        type = types.bool;
-        default = false;
-        description = "Enable the logstash web interface.";
-      };
-
       listenAddress = mkOption {
         type = types.str;
         default = "127.0.0.1";
@@ -174,16 +155,6 @@ in
   ###### implementation
 
   config = mkIf cfg.enable {
-    assertions = [
-      { assertion = atLeast54 -> !cfg.enableWeb;
-        message = ''
-          The logstash web interface is only available for versions older than 5.4.
-          So either set services.logstash.enableWeb = false,
-          or set services.logstash.package to an older logstash.
-        '';
-      }
-    ];
-
     systemd.services.logstash = with pkgs; {
       description = "Logstash Daemon";
       wantedBy = [ "multi-user.target" ];
@@ -193,14 +164,12 @@ in
         ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"'';
         ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [
           "${cfg.package}/bin/logstash"
-          (ops (!atLeast54) "agent")
           "-w ${toString cfg.filterWorkers}"
           (ops havePluginPath pluginsPath)
           "${verbosityFlag}"
           "-f ${logstashConf}"
-          (ops atLeast54 "--path.settings ${logstashSettingsDir}")
-          (ops atLeast54 "--path.data ${cfg.dataDir}")
-          (ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}")
+          "--path.settings ${logstashSettingsDir}"
+          "--path.data ${cfg.dataDir}"
         ]);
       };
     };
diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix
index b0831dcd1ca8..6b688c640d59 100644
--- a/nixos/modules/services/search/elasticsearch.nix
+++ b/nixos/modules/services/search/elasticsearch.nix
@@ -5,22 +5,14 @@ with lib;
 let
   cfg = config.services.elasticsearch;
 
-  es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0;
-  es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0;
+  es6 = builtins.compareVersions cfg.package.version "6" >= 0;
 
   esConfig = ''
     network.host: ${cfg.listenAddress}
     cluster.name: ${cfg.cluster_name}
 
-    ${if es5 then ''
-      http.port: ${toString cfg.port}
-      transport.tcp.port: ${toString cfg.tcp_port}
-    '' else ''
-      network.port: ${toString cfg.port}
-      network.tcp.port: ${toString cfg.tcp_port}
-      # TODO: find a way to enable security manager
-      security.manager.enabled: false
-    ''}
+    http.port: ${toString cfg.port}
+    transport.tcp.port: ${toString cfg.tcp_port}
 
     ${cfg.extraConf}
   '';
@@ -32,7 +24,7 @@ let
     text = esConfig;
   };
 
-  loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml";
+  loggingConfigFilename = "log4j2.properties";
   loggingConfigFile = pkgs.writeTextFile {
     name = loggingConfigFilename;
     text = cfg.logging;
@@ -41,8 +33,7 @@ let
   esPlugins = pkgs.buildEnv {
     name = "elasticsearch-plugins";
     paths = cfg.plugins;
-    # Elasticsearch 5.x won't start when the plugins directory does not exist
-    postBuild = if es5 then "${pkgs.coreutils}/bin/mkdir -p $out/plugins" else "";
+    postBuild = "${pkgs.coreutils}/bin/mkdir -p $out/plugins";
   };
 
 in {
@@ -58,8 +49,8 @@ in {
 
     package = mkOption {
       description = "Elasticsearch package to use.";
-      default = pkgs.elasticsearch2;
-      defaultText = "pkgs.elasticsearch2";
+      default = pkgs.elasticsearch;
+      defaultText = "pkgs.elasticsearch";
       type = types.package;
     };
 
@@ -100,30 +91,18 @@ in {
 
     logging = mkOption {
       description = "Elasticsearch logging configuration.";
-      default =
-        if es5 then ''
-          logger.action.name = org.elasticsearch.action
-          logger.action.level = info
-
-          appender.console.type = Console
-          appender.console.name = console
-          appender.console.layout.type = PatternLayout
-          appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-
-          rootLogger.level = info
-          rootLogger.appenderRef.console.ref = console
-        '' else ''
-          rootLogger: INFO, console
-          logger:
-            action: INFO
-            com.amazonaws: WARN
-          appender:
-            console:
-              type: console
-              layout:
-                type: consolePattern
-                conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-        '';
+      default = ''
+        logger.action.name = org.elasticsearch.action
+        logger.action.level = info
+
+        appender.console.type = Console
+        appender.console.name = console
+        appender.console.layout.type = PatternLayout
+        appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+        rootLogger.level = info
+        rootLogger.appenderRef.console.ref = console
+      '';
       type = types.str;
     };
 
@@ -204,9 +183,9 @@ in {
 
         cp ${elasticsearchYml} ${configDir}/elasticsearch.yml
         # Make sure the logging configuration for old elasticsearch versions is removed:
-        rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"}
+        rm -f "${configDir}/logging.yml"
         cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename}
-        ${optionalString es5 "mkdir -p ${configDir}/scripts"}
+        mkdir -p ${configDir}/scripts
         ${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"}
 
         if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi
diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix
index 5885a72c6628..ca36bba58c02 100644
--- a/nixos/modules/services/search/kibana.nix
+++ b/nixos/modules/services/search/kibana.nix
@@ -5,43 +5,7 @@ with lib;
 let
   cfg = config.services.kibana;
 
-  atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4";
-
-  cfgFile = if atLeast54 then cfgFile5 else cfgFile4;
-
-  cfgFile4 = pkgs.writeText "kibana.json" (builtins.toJSON (
-    (filterAttrsRecursive (n: v: v != null) ({
-      host = cfg.listenAddress;
-      port = cfg.port;
-      ssl_cert_file = cfg.cert;
-      ssl_key_file = cfg.key;
-
-      kibana_index = cfg.index;
-      default_app_id = cfg.defaultAppId;
-
-      elasticsearch_url = cfg.elasticsearch.url;
-      kibana_elasticsearch_username = cfg.elasticsearch.username;
-      kibana_elasticsearch_password = cfg.elasticsearch.password;
-      kibana_elasticsearch_cert = cfg.elasticsearch.cert;
-      kibana_elasticsearch_key = cfg.elasticsearch.key;
-      ca = cfg.elasticsearch.ca;
-
-      bundled_plugin_ids = [
-        "plugins/dashboard/index"
-        "plugins/discover/index"
-        "plugins/doc/index"
-        "plugins/kibana/index"
-        "plugins/markdown_vis/index"
-        "plugins/metric_vis/index"
-        "plugins/settings/index"
-        "plugins/table_vis/index"
-        "plugins/vis_types/index"
-        "plugins/visualize/index"
-      ];
-    } // cfg.extraConf)
-  )));
-
-  cfgFile5 = pkgs.writeText "kibana.json" (builtins.toJSON (
+  cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON (
     (filterAttrsRecursive (n: v: v != null) ({
       server.host = cfg.listenAddress;
       server.port = cfg.port;