elk: 6.2.4 -> 6.3.2

* The ELK stack is upgraded to 6.3.2.

* `elasticsearch6`, `logstash6` and `kibana6` now come with X-Pack which is
  a suite of additional features. These are however licensed under the unfree
  "Elastic License".

* Fortunately they also provide OSS versions which are now packaged
  under: `elasticsearch6-oss`, `logstash6-oss` and `kibana6-oss`.
  Note that the naming of the attributes is consistent with upstream.

* The test `nix-build nixos/tests/elk.nix -A ELK-6` will test the OSS
  version by default. You can also run the test on the unfree ELK using:
  `NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true`

8 files changed, 145 insertions, 37 deletions

diff --git a/lib/licenses.nix b/lib/licenses.nix
index 1c953a2ba844..79124855f7fa 100644
--- a/lib/licenses.nix
+++ b/lib/licenses.nix
@@ -231,6 +231,12 @@ lib.mapAttrs (n: v: v // { shortName = n; }) rec {
     fullName = "Eiffel Forum License v2.0";
   };
 
+  elastic = {
+    fullName = "ELASTIC LICENSE";
+    url = https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt;
+    free = false;
+  };
+
   epl10 = spdx {
     spdxId = "EPL-1.0";
     fullName = "Eclipse Public License 1.0";
diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix
index d61f588205af..b0831dcd1ca8 100644
--- a/nixos/modules/services/search/elasticsearch.nix
+++ b/nixos/modules/services/search/elasticsearch.nix
@@ -25,18 +25,17 @@ let
     ${cfg.extraConf}
   '';
 
-  configDir = pkgs.buildEnv {
-    name = "elasticsearch-config";
-    paths = [
-      (pkgs.writeTextDir "elasticsearch.yml" esConfig)
-      (if es5 then (pkgs.writeTextDir "log4j2.properties" cfg.logging)
-              else (pkgs.writeTextDir "logging.yml" cfg.logging))
-    ];
-    postBuild = concatStringsSep "\n" (concatLists [
-      # Elasticsearch 5.x won't start when the scripts directory does not exist
-      (optional es5 "${pkgs.coreutils}/bin/mkdir -p $out/scripts")
-      (optional es6 "ln -s ${cfg.package}/config/jvm.options $out/jvm.options")
-    ]);
+  configDir = cfg.dataDir + "/config";
+
+  elasticsearchYml = pkgs.writeTextFile {
+    name = "elasticsearch.yml";
+    text = esConfig;
+  };
+
+  loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml";
+  loggingConfigFile = pkgs.writeTextFile {
+    name = loggingConfigFilename;
+    text = cfg.logging;
   };
 
   esPlugins = pkgs.buildEnv {
@@ -193,7 +192,24 @@ in {
         ln -sfT ${esPlugins}/plugins ${cfg.dataDir}/plugins
         ln -sfT ${cfg.package}/lib ${cfg.dataDir}/lib
         ln -sfT ${cfg.package}/modules ${cfg.dataDir}/modules
-        if [ "$(id -u)" = 0 ]; then chown -R elasticsearch ${cfg.dataDir}; fi
+
+        # elasticsearch needs to create the elasticsearch.keystore in the config directory
+        # so this directory needs to be writable.
+        mkdir -m 0700 -p ${configDir}
+
+        # Note that we copy config files from the nix store instead of symbolically linking them
+        # because otherwise X-Pack Security will raise the following exception:
+        # java.security.AccessControlException:
+        # access denied ("java.io.FilePermission" "/var/lib/elasticsearch/config/elasticsearch.yml" "read")
+
+        cp ${elasticsearchYml} ${configDir}/elasticsearch.yml
+        # Make sure the logging configuration for old elasticsearch versions is removed:
+        rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"}
+        cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename}
+        ${optionalString es5 "mkdir -p ${configDir}/scripts"}
+        ${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"}
+
+        if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi
       '';
     };
 
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index 8dba7a905fa6..4c5c441ca265 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -1,4 +1,4 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem, enableUnfree ? false }:
 with import ../lib/testing.nix { inherit system; };
 with pkgs.lib;
 let
@@ -99,9 +99,16 @@ in mapAttrs mkElkTest {
     logstash      = pkgs.logstash5;
     kibana        = pkgs.kibana5;
   };
-  "ELK-6" = {
-    elasticsearch = pkgs.elasticsearch6;
-    logstash      = pkgs.logstash6;
-    kibana        = pkgs.kibana6;
-  };
+  "ELK-6" =
+    if enableUnfree
+    then {
+      elasticsearch = pkgs.elasticsearch6;
+      logstash      = pkgs.logstash6;
+      kibana        = pkgs.kibana6;
+    }
+    else {
+      elasticsearch = pkgs.elasticsearch6-oss;
+      logstash      = pkgs.logstash6-oss;
+      kibana        = pkgs.kibana6-oss;
+    };
 }
diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
index 17620c9df6c8..1fd52f435574 100644
--- a/pkgs/development/tools/misc/kibana/6.x.nix
+++ b/pkgs/development/tools/misc/kibana/6.x.nix
@@ -1,4 +1,13 @@
-{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, makeWrapper
+, fetchzip
+, fetchurl
+, nodejs
+, coreutils
+, which
+}:
 
 with stdenv.lib;
 let
@@ -6,12 +15,31 @@ let
   info = splitString "-" stdenv.system;
   arch = elemAt info 0;
   plat = elemAt info 1;
-  shas = {
-    "x86_64-linux"  = "1br9nvwa3i5sfcbnrxp2x3dxxnsbs9iavz6zwgw0jlh5ngf5vysk";
-    "x86_64-darwin" = "1w4dck02i0rrl8m18kvy2zz02cb7bb9a2pdhkd1jfy1qz4ssnhii";
+  shas =
+    if enableUnfree
+    then {
+      "x86_64-linux"  = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0";
+      "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3";
+    }
+    else {
+      "x86_64-linux"  = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly";
+      "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b";
+    };
+
+  # For the correct phantomjs version see:
+  # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js
+  phantomjs = rec {
+    name = "phantomjs-${version}-linux-x86_64";
+    version = "2.1.1";
+    src = fetchzip {
+      inherit name;
+      url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2";
+      sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv";
+    };
   };
+
 in stdenv.mkDerivation rec {
-  name = "kibana-${version}";
+  name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
   version = elk6Version;
 
   src = fetchurl {
@@ -28,12 +56,19 @@ in stdenv.mkDerivation rec {
     makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
       --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
     sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+  '' +
+  # phantomjs is needed in the unfree version. When phantomjs doesn't exist in
+  # $out/libexec/kibana/data kibana will try to download and unpack it during
+  # runtime which will fail because the nix store is read-only. So we make sure
+  # it already exist in the nix store.
+  optionalString enableUnfree ''
+    ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name}
   '';
 
   meta = {
     description = "Visualize logs and time-stamped data";
     homepage = http://www.elasticsearch.org/overview/kibana;
-    license = licenses.asl20;
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
     maintainers = with maintainers; [ offline rickynils basvandijk ];
     platforms = with platforms; unix;
   };
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index ce911ae40d5e..111bc449d888 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "194z3j9zwlbc6j97iy1m1cl0xqks3ws2bjp2xrgy8cwpi7fclaw2";
+        sha256 = "0ymg6y6v0mdhs1rs11fn33xdp3r6v85563z0f4p7s22j1kd3nd6r";
       };
 
       goPackagePath = "github.com/elastic/beats";
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
index 06caf191587c..84872649c492 100644
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -1,14 +1,26 @@
-{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre_headless
+, utillinux
+, autoPatchelfHook
+, zlib
+}:
 
 with stdenv.lib;
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (rec {
   version = elk6Version;
-  name = "elasticsearch-${version}";
+  name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
 
   src = fetchurl {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
-    sha256 = "13hf00khq33yw6zv022vcrsf6vm43isx40x7ww8r1lqx3vmg3rli";
+    sha256 =
+      if enableUnfree
+      then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh"
+      else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4";
   };
 
   patches = [ ./es-home-6.x.patch ];
@@ -32,10 +44,22 @@ stdenv.mkDerivation rec {
     wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
   '';
 
+  passthru = { inherit enableUnfree; };
+
   meta = {
     description = "Open Source, Distributed, RESTful Search Engine";
-    license = licenses.asl20;
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
     platforms = platforms.unix;
     maintainers = with maintainers; [ apeschar basvandijk ];
   };
-}
+} // optionalAttrs enableUnfree {
+  dontPatchELF = true;
+  nativeBuildInputs = [ autoPatchelfHook ];
+  runtimeDependencies = [ zlib ];
+  postFixup = ''
+    for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
+      echo "patching $exe..."
+      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
+    done
+  '';
+})
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
index 6aa3fd836965..4f15ba90d39d 100644
--- a/pkgs/tools/misc/logstash/6.x.nix
+++ b/pkgs/tools/misc/logstash/6.x.nix
@@ -1,12 +1,23 @@
-{ stdenv, fetchurl, elk6Version, makeWrapper, jre  }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre
+}:
+
+with stdenv.lib;
 
 stdenv.mkDerivation rec {
   version = elk6Version;
-  name = "logstash-${version}";
+  name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
 
   src = fetchurl {
     url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
-    sha256 = "07j3jjg5ik4gjgvcx15qqqas9p1m3815jml82a5r1ip9l6vc4h20";
+    sha256 =
+      if enableUnfree
+      then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr"
+      else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w";
   };
 
   dontBuild         = true;
@@ -35,7 +46,7 @@ stdenv.mkDerivation rec {
   meta = with stdenv.lib; {
     description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
     homepage    = https://www.elastic.co/products/logstash;
-    license     = licenses.asl20;
+    license     = if enableUnfree then licenses.elastic else licenses.asl20;
     platforms   = platforms.unix;
     maintainers = with maintainers; [ wjlroe offline basvandijk ];
   };
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index c9f6c16e9928..6d2f490136c6 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2254,12 +2254,15 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   elk5Version = "5.6.9";
-  elk6Version = "6.2.4";
+  elk6Version = "6.3.2";
 
   elasticsearch = callPackage ../servers/search/elasticsearch { };
   elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
+  elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
+    enableUnfree = false;
+  };
 
   elasticsearchPlugins = recurseIntoAttrs (
     callPackage ../servers/search/elasticsearch/plugins.nix { }
@@ -3364,6 +3367,9 @@ with pkgs;
   kibana  = callPackage ../development/tools/misc/kibana { };
   kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
   kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
+  kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
+    enableUnfree = false;
+  };
 
   kismet = callPackage ../applications/networking/sniffers/kismet { };
 
@@ -3443,6 +3449,9 @@ with pkgs;
   logstash  = callPackage ../tools/misc/logstash { };
   logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
   logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
+  logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
+    enableUnfree = false;
+  };
 
   logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };