Merge pull request #44389 from Mic92/es6

elasticsearch: use 6.x as default version, remove unsupported releases

14 files changed, 163 insertions, 488 deletions

diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
deleted file mode 100644
index 1fd52f435574..000000000000
--- a/pkgs/development/tools/misc/kibana/6.x.nix
+++ /dev/null
@@ -1,75 +0,0 @@
-{ elk6Version
-, enableUnfree ? true
-, stdenv
-, makeWrapper
-, fetchzip
-, fetchurl
-, nodejs
-, coreutils
-, which
-}:
-
-with stdenv.lib;
-let
-  inherit (builtins) elemAt;
-  info = splitString "-" stdenv.system;
-  arch = elemAt info 0;
-  plat = elemAt info 1;
-  shas =
-    if enableUnfree
-    then {
-      "x86_64-linux"  = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0";
-      "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3";
-    }
-    else {
-      "x86_64-linux"  = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly";
-      "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b";
-    };
-
-  # For the correct phantomjs version see:
-  # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js
-  phantomjs = rec {
-    name = "phantomjs-${version}-linux-x86_64";
-    version = "2.1.1";
-    src = fetchzip {
-      inherit name;
-      url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2";
-      sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv";
-    };
-  };
-
-in stdenv.mkDerivation rec {
-  name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
-  version = elk6Version;
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
-    sha256 = shas."${stdenv.system}" or (throw "Unknown architecture");
-  };
-
-  buildInputs = [ makeWrapper ];
-
-  installPhase = ''
-    mkdir -p $out/libexec/kibana $out/bin
-    mv * $out/libexec/kibana/
-    rm -r $out/libexec/kibana/node
-    makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
-      --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
-    sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
-  '' +
-  # phantomjs is needed in the unfree version. When phantomjs doesn't exist in
-  # $out/libexec/kibana/data kibana will try to download and unpack it during
-  # runtime which will fail because the nix store is read-only. So we make sure
-  # it already exist in the nix store.
-  optionalString enableUnfree ''
-    ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name}
-  '';
-
-  meta = {
-    description = "Visualize logs and time-stamped data";
-    homepage = http://www.elasticsearch.org/overview/kibana;
-    license = if enableUnfree then licenses.elastic else licenses.asl20;
-    maintainers = with maintainers; [ offline rickynils basvandijk ];
-    platforms = with platforms; unix;
-  };
-}
diff --git a/pkgs/development/tools/misc/kibana/default.nix b/pkgs/development/tools/misc/kibana/default.nix
index fc1ac9323387..1fd52f435574 100644
--- a/pkgs/development/tools/misc/kibana/default.nix
+++ b/pkgs/development/tools/misc/kibana/default.nix
@@ -1,26 +1,49 @@
-{ stdenv, makeWrapper, fetchurl, nodejs, coreutils, which }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, makeWrapper
+, fetchzip
+, fetchurl
+, nodejs
+, coreutils
+, which
+}:
 
 with stdenv.lib;
 let
   inherit (builtins) elemAt;
-  archOverrides = {
-    "i686" = "x86";
-  };
   info = splitString "-" stdenv.system;
-  arch = (elemAt info 0);
-  elasticArch = archOverrides."${arch}" or arch;
+  arch = elemAt info 0;
   plat = elemAt info 1;
-  shas = {
-    "x86_64-linux"  = "1wnnrhhpgc58s09p99cmi8r2jmwsd5lmh2inb0k8nmizz5v1sjz0";
-    "i686-linux"    = "0sdx59jlfrf7r9793xpn2vxaxjdczgn3qfw8yny03dcs6fjaxi2y";
-    "x86_64-darwin" = "0rmp536kn001g52lxngpj6x6d0j3qj0r11d4djbz7h6s5ml03kza";
+  shas =
+    if enableUnfree
+    then {
+      "x86_64-linux"  = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0";
+      "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3";
+    }
+    else {
+      "x86_64-linux"  = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly";
+      "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b";
+    };
+
+  # For the correct phantomjs version see:
+  # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js
+  phantomjs = rec {
+    name = "phantomjs-${version}-linux-x86_64";
+    version = "2.1.1";
+    src = fetchzip {
+      inherit name;
+      url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2";
+      sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv";
+    };
   };
+
 in stdenv.mkDerivation rec {
-  name = "kibana-${version}";
-  version = "4.6.5";
+  name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
+  version = elk6Version;
 
   src = fetchurl {
-    url = "https://download.elastic.co/kibana/kibana/${name}-${plat}-${elasticArch}.tar.gz";
+    url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
     sha256 = shas."${stdenv.system}" or (throw "Unknown architecture");
   };
 
@@ -33,13 +56,20 @@ in stdenv.mkDerivation rec {
     makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
       --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
     sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+  '' +
+  # phantomjs is needed in the unfree version. When phantomjs doesn't exist in
+  # $out/libexec/kibana/data kibana will try to download and unpack it during
+  # runtime which will fail because the nix store is read-only. So we make sure
+  # it already exist in the nix store.
+  optionalString enableUnfree ''
+    ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name}
   '';
 
   meta = {
     description = "Visualize logs and time-stamped data";
     homepage = http://www.elasticsearch.org/overview/kibana;
-    license = licenses.asl20;
-    maintainers = with maintainers; [ offline rickynils ];
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
+    maintainers = with maintainers; [ offline rickynils basvandijk ];
     platforms = with platforms; unix;
   };
 }
diff --git a/pkgs/misc/logging/beats/5.x.nix b/pkgs/misc/logging/beats/5.x.nix
index 94d5449100db..e92613025ff9 100644
--- a/pkgs/misc/logging/beats/5.x.nix
+++ b/pkgs/misc/logging/beats/5.x.nix
@@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec {
       };
     } // extraArgs);
 in {
-  filebeat   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
-  heartbeat  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
-  metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
-  packetbeat = beat "packetbeat" {
+  filebeat5   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
+  heartbeat5  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
+  metricbeat5 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  packetbeat5 = beat "packetbeat" {
     buildInputs = [ libpcap ];
     meta.description = "Network packet analyzer that ships data to Elasticsearch";
     meta.longDescription = ''
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index 111bc449d888..59d7379e06c0 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec {
       };
     } // extraArgs);
 in {
-  filebeat   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
-  heartbeat  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
-  metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
-  packetbeat = beat "packetbeat" {
+  filebeat6   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
+  heartbeat6  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
+  metricbeat6 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  packetbeat6 = beat "packetbeat" {
     buildInputs = [ libpcap ];
     meta.description = "Network packet analyzer that ships data to Elasticsearch";
     meta.longDescription = ''
diff --git a/pkgs/servers/search/elasticsearch/2.x.nix b/pkgs/servers/search/elasticsearch/2.x.nix
deleted file mode 100644
index 28244d1b3dcd..000000000000
--- a/pkgs/servers/search/elasticsearch/2.x.nix
+++ /dev/null
@@ -1,40 +0,0 @@
-{ stdenv, fetchurl, makeWrapper, jre, utillinux }:
-
-with stdenv.lib;
-
-stdenv.mkDerivation rec {
-  version = "2.4.4";
-  name = "elasticsearch-${version}";
-
-  src = fetchurl {
-    url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${version}/${name}.tar.gz";
-    sha256 = "1qjq04sfqb35pf2xpvr8j5p27chfxpjp8ymrp1h5bfk5rbk9444q";
-  };
-
-  patches = [ ./es-home-2.x.patch ./es-classpath-2.x.patch ];
-
-  buildInputs = [ makeWrapper jre utillinux ];
-
-  installPhase = ''
-    mkdir -p $out
-    cp -R bin config lib modules $out
-
-    # don't want to have binary with name plugin
-    mv $out/bin/plugin $out/bin/elasticsearch-plugin
-    wrapProgram $out/bin/elasticsearch \
-      --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*" \
-      --prefix PATH : "${utillinux}/bin" \
-      --set JAVA_HOME "${jre}"
-    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre}"
-  '';
-
-  meta = {
-    description = "Open Source, Distributed, RESTful Search Engine";
-    license = licenses.asl20;
-    platforms = platforms.unix;
-    maintainers = [
-      maintainers.offline
-      maintainers.markWot
-    ];
-  };
-}
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
deleted file mode 100644
index 84872649c492..000000000000
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ /dev/null
@@ -1,65 +0,0 @@
-{ elk6Version
-, enableUnfree ? true
-, stdenv
-, fetchurl
-, makeWrapper
-, jre_headless
-, utillinux
-, autoPatchelfHook
-, zlib
-}:
-
-with stdenv.lib;
-
-stdenv.mkDerivation (rec {
-  version = elk6Version;
-  name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
-    sha256 =
-      if enableUnfree
-      then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh"
-      else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4";
-  };
-
-  patches = [ ./es-home-6.x.patch ];
-
-  postPatch = ''
-    sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env
-  '';
-
-  buildInputs = [ makeWrapper jre_headless utillinux ];
-
-  installPhase = ''
-    mkdir -p $out
-    cp -R bin config lib modules plugins $out
-
-    chmod -x $out/bin/*.*
-
-    wrapProgram $out/bin/elasticsearch \
-      --prefix PATH : "${utillinux}/bin/" \
-      --set JAVA_HOME "${jre_headless}"
-
-    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
-  '';
-
-  passthru = { inherit enableUnfree; };
-
-  meta = {
-    description = "Open Source, Distributed, RESTful Search Engine";
-    license = if enableUnfree then licenses.elastic else licenses.asl20;
-    platforms = platforms.unix;
-    maintainers = with maintainers; [ apeschar basvandijk ];
-  };
-} // optionalAttrs enableUnfree {
-  dontPatchELF = true;
-  nativeBuildInputs = [ autoPatchelfHook ];
-  runtimeDependencies = [ zlib ];
-  postFixup = ''
-    for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
-      echo "patching $exe..."
-      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
-    done
-  '';
-})
diff --git a/pkgs/servers/search/elasticsearch/default.nix b/pkgs/servers/search/elasticsearch/default.nix
index f4780603f502..84872649c492 100644
--- a/pkgs/servers/search/elasticsearch/default.nix
+++ b/pkgs/servers/search/elasticsearch/default.nix
@@ -1,40 +1,65 @@
-{ stdenv, fetchurl, makeWrapper, jre, utillinux }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre_headless
+, utillinux
+, autoPatchelfHook
+, zlib
+}:
 
 with stdenv.lib;
 
-stdenv.mkDerivation rec {
-  name = "elasticsearch-1.7.2";
+stdenv.mkDerivation (rec {
+  version = elk6Version;
+  name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
 
   src = fetchurl {
-    url = "https://download.elastic.co/elasticsearch/elasticsearch/${name}.tar.gz";
-    sha256 = "1lix4asvx1lbc227gzsrws3xqbcbqaal7v10w60kch0c4xg970bg";
+    url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
+    sha256 =
+      if enableUnfree
+      then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh"
+      else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4";
   };
 
-  patches = [ ./es-home.patch ];
+  patches = [ ./es-home-6.x.patch ];
 
-  buildInputs = [ makeWrapper jre utillinux ];
+  postPatch = ''
+    sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env
+  '';
+
+  buildInputs = [ makeWrapper jre_headless utillinux ];
 
   installPhase = ''
     mkdir -p $out
-    cp -R bin config lib $out
+    cp -R bin config lib modules plugins $out
 
-    # don't want to have binary with name plugin
-    mv $out/bin/plugin $out/bin/elasticsearch-plugin
+    chmod -x $out/bin/*.*
 
-    # set ES_CLASSPATH and JAVA_HOME
     wrapProgram $out/bin/elasticsearch \
-      --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \
-      --prefix PATH : "${utillinux}/bin" \
-      --set JAVA_HOME "${jre}"
-    wrapProgram $out/bin/elasticsearch-plugin \
-      --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \
-      --set JAVA_HOME "${jre}"
+      --prefix PATH : "${utillinux}/bin/" \
+      --set JAVA_HOME "${jre_headless}"
+
+    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
   '';
 
+  passthru = { inherit enableUnfree; };
+
   meta = {
     description = "Open Source, Distributed, RESTful Search Engine";
-    license = licenses.asl20;
+    license = if enableUnfree then licenses.elastic else licenses.asl20;
     platforms = platforms.unix;
-    maintainers = [ maintainers.offline ];
+    maintainers = with maintainers; [ apeschar basvandijk ];
   };
-}
+} // optionalAttrs enableUnfree {
+  dontPatchELF = true;
+  nativeBuildInputs = [ autoPatchelfHook ];
+  runtimeDependencies = [ zlib ];
+  postFixup = ''
+    for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
+      echo "patching $exe..."
+      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
+    done
+  '';
+})
diff --git a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch b/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch
deleted file mode 100644
index 46a3f0be71b7..000000000000
--- a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch
+++ /dev/null
@@ -1,38 +0,0 @@
-diff -rupN a/bin/elasticsearch b/bin/elasticsearch
---- a/bin/elasticsearch	2017-02-08 18:32:28.000298543 -0500
-+++ b/bin/elasticsearch	2017-02-08 19:10:45.692916675 -0500
-@@ -81,12 +81,7 @@ ES_HOME=`cd "$ES_HOME"; pwd`
- # If an include wasn't specified in the environment, then search for one...
- if [ "x$ES_INCLUDE" = "x" ]; then
-     # Locations (in order) to use when searching for an include file.
--    for include in /usr/share/elasticsearch/elasticsearch.in.sh \
--                   /usr/local/share/elasticsearch/elasticsearch.in.sh \
--                   /opt/elasticsearch/elasticsearch.in.sh \
--                   ~/.elasticsearch.in.sh \
--                   "$ES_HOME/bin/elasticsearch.in.sh" \
--                   "`dirname "$0"`"/elasticsearch.in.sh; do
-+    for include in "`dirname "$0"`"/elasticsearch.in.sh; do
-         if [ -r "$include" ]; then
-             . "$include"
-             break
-diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh
---- a/bin/elasticsearch.in.sh	2017-02-08 18:32:28.000298543 -0500
-+++ b/bin/elasticsearch.in.sh	2017-02-08 18:33:46.816634599 -0500
-@@ -1,17 +1,5 @@
- #!/bin/sh
- 
--# check in case a user was using this mechanism
--if [ "x$ES_CLASSPATH" != "x" ]; then
--    cat >&2 << EOF
--Error: Don't modify the classpath with ES_CLASSPATH. Best is to add
--additional elements via the plugin mechanism, or if code must really be
--added to the main classpath, add jars to lib/ (unsupported).
--EOF
--    exit 1
--fi
--
--ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.4.4.jar:$ES_HOME/lib/*"
--
- if [ "x$ES_MIN_MEM" = "x" ]; then
-     ES_MIN_MEM=256m
- fi
diff --git a/pkgs/servers/search/elasticsearch/es-home-2.x.patch b/pkgs/servers/search/elasticsearch/es-home-2.x.patch
deleted file mode 100644
index 0c80e8651145..000000000000
--- a/pkgs/servers/search/elasticsearch/es-home-2.x.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-diff -rupN a/bin/elasticsearch b/bin/elasticsearch
---- a/bin/elasticsearch	2015-11-18 21:48:18.000000000 +0100
-+++ b/bin/elasticsearch	2015-12-04 00:52:21.032475098 +0100
-@@ -72,7 +72,11 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`
-diff -rupN a/bin/plugin b/bin/plugin
---- a/bin/plugin	2015-11-18 21:48:18.000000000 +0100
-+++ b/bin/plugin	2015-12-04 00:52:55.947453619 +0100
-@@ -17,7 +17,10 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`
diff --git a/pkgs/servers/search/elasticsearch/es-home.patch b/pkgs/servers/search/elasticsearch/es-home.patch
deleted file mode 100644
index 2f2018fd69fa..000000000000
--- a/pkgs/servers/search/elasticsearch/es-home.patch
+++ /dev/null
@@ -1,37 +0,0 @@
-diff -rupN a/bin/elasticsearch b/bin/elasticsearch
---- a/bin/elasticsearch	2015-08-05 17:52:05.740819671 +0200
-+++ b/bin/elasticsearch	2015-08-05 17:22:34.664657364 +0200
-@@ -83,7 +83,10 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`
-diff -rupN a/bin/plugin b/bin/plugin
---- a/bin/plugin	2015-08-05 17:57:07.903088815 +0200
-+++ b/bin/plugin	2015-08-05 17:57:38.979808139 +0200
-@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`
-@@ -105,4 +105,4 @@
-
- export HOSTNAME=`hostname -s`
-
--eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home=\""$ES_HOME"\" $properties -cp \""$ES_HOME/lib/*"\" org.elasticsearch.plugins.PluginManager $args
-\ No newline at end of file
-+eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" $properties -cp "$ES_CLASSPATH/lib/*" org.elasticsearch.plugins.PluginManager $args
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index b93d8c2203d7..b3141ee807d1 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -1,4 +1,4 @@
-{ pkgs,  stdenv, fetchurl, unzip, elasticsearch }:
+{ pkgs,  stdenv, fetchurl, unzip, elasticsearch-oss, javaPackages, elk6Version }:
 
 with pkgs.lib;
 
@@ -6,8 +6,9 @@ let
   esPlugin = a@{
     pluginName,
     installPhase ? ''
-      mkdir -p $out/bin
-      ES_HOME=$out ${elasticsearch}/bin/elasticsearch-plugin --install ${pluginName} --url file://$src
+      mkdir -p $out/config
+      mkdir -p $out/plugins
+      ES_HOME=$out ${elasticsearch-oss}/bin/elasticsearch-plugin install --batch -v file://$src
     '',
     ...
   }:
@@ -16,33 +17,19 @@ let
       unpackPhase = "true";
       buildInputs = [ unzip ];
       meta = a.meta // {
-        platforms = elasticsearch.meta.platforms;
+        platforms = elasticsearch-oss.meta.platforms;
         maintainers = (a.meta.maintainers or []) ++ [ maintainers.offline ];
       };
     });
 in {
-  elasticsearch_river_jdbc = esPlugin rec {
-    name = "elasticsearch-river-jdbc-${version}";
-    pluginName = "elasticsearch-river-jdbc";
-    version = "1.5.0.5";
-    src = fetchurl {
-      url = "http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-river-jdbc/${version}/${name}-plugin.zip";
-      sha256 = "1p75l3vcnb90ar4j3dci2xf8dqnqyy31kc1r075fa2xqlsxgigcp";
-    };
-    meta = {
-      homepage = https://github.com/jprante/elasticsearch-river-jdbc;
-      description = "Plugin to fetch data from JDBC sources for indexing into Elasticsearch";
-      license = licenses.asl20;
-    };
-  };
 
   elasticsearch_analysis_lemmagen = esPlugin rec {
     name = "elasticsearch-analysis-lemmagen-${version}";
     pluginName = "elasticsearch-analysis-lemmagen";
-    version = "0.1";
+    version = "${elk6Version}";
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-analysis-lemmagen/releases/download/v${version}/${name}-plugin.zip";
-      sha256 = "bf7bf5ce3ccdd3afecd0e18cd6fce1ef56f824e41f4ef50553ae598caa5c366d";
+      sha256 = "1m4z05wixjrq4nlbdjyhvprkrwxjym8aba18scmzfn25fhbjgvkz";
     };
     meta = {
       homepage = https://github.com/vhyza/elasticsearch-analysis-lemmagen;
@@ -51,68 +38,28 @@ in {
     };
   };
 
-  elasticsearch_http_basic = stdenv.mkDerivation rec {
-    name = "elasticsearch-http-basic-${version}";
-    version = "1.5.0";
-
-    src = fetchurl {
-      url = "https://github.com/Asquera/elasticsearch-http-basic/releases/download/v${version}/${name}.jar";
-      sha256 = "0fif6sbn2ich39lrgm039y9d5bxkylx9pvly04wss8rdhspvdskb";
-    };
-
-    phases = ["installPhase"];
-    installPhase = "install -D $src $out/plugins/http-basic/${name}.jar";
-
-    meta = {
-      homepage = https://github.com/Asquera/elasticsearch-http-basic;
-      description = "HTTP Basic Authentication for Elasticsearch";
-      license = licenses.mit;
-      platforms = elasticsearch.meta.platforms;
-    };
-  };
-
-  elasticsearch_river_twitter = esPlugin rec {
-    name = pname + "-" + version;
-    pname = "elasticsearch-river-twitter";
-    pluginName = "elasticsearch/" + pname + "/" + version;
-    version = "2.5.0";
-
-    src = fetchurl {
-      url = "http://download.elasticsearch.org/elasticsearch/${pname}/${name}.zip";
-      sha256 = "0851yrmyrpp6whyxk34ykcj7b28f90w0nvkrhvl49dwqgr5s4mn4";
+  discovery-ec2 = esPlugin {
+    name = "elasticsearch-discovery-ec2-${version}";
+    pluginName = "discovery-ec2";
+    version = "${elk6Version}";
+    src = pkgs.fetchurl {
+      url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/discovery-ec2/discovery-ec2-${elk6Version}.zip";
+      sha256 = "1i7ksy69132sr84h51lamgq967yz3a3dw0b54nckxpqwad9pcpj0";
     };
-
     meta = {
-      homepage = https://github.com/elasticsearch/elasticsearch-river-twitter;
-      description = "Twitter River Plugin for ElasticSearch";
+      homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/discovery-ec2;
+      description = "The EC2 discovery plugin uses the AWS API for unicast discovery.";
       license = licenses.asl20;
-      maintainers = [ maintainers.edwtjo ];
-      platforms = elasticsearch.meta.platforms;
-    };
-  };
-
-  elasticsearch_kopf = esPlugin rec {
-    name = "elasticsearch-kopf-${version}";
-    pluginName = "elasticsearch-kopf";
-    version = "2.1.1";
-    src = fetchurl {
-      url = "https://github.com/lmenezes/elasticsearch-kopf/archive/v${version}.zip";
-      sha256 = "1nwwd92g0jxhfpkxb1a9z5a62naa1y7hvlx400dm6mwwav3mrf4v";
-    };
-    meta = {
-      homepage = https://github.com/lmenezes/elasticsearch-kopf;
-      description = "Web administration tool for ElasticSearch";
-      license = licenses.mit;
     };
   };
 
   search_guard = esPlugin rec {
     name = "elastic-search-guard-${version}";
     pluginName = "search-guard";
-    version = "0.5";
-    src = fetchurl {
-      url = "https://github.com/floragunncom/search-guard/releases/download/v${version}/${pluginName}-${version}.zip";
-      sha256 = "1zima4jmq1rrcqxhlrp2xian80vp244d2splby015n5cgqrp39fl";
+    version = "${elk6Version}-22.3";
+    src = fetchurl rec {
+      url = "mirror://maven/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
+      sha256 = "1r71h4h9bmxak1mq5gpm19xq5ji1gry1kp3sjmm8azy4ykdqdncx";
     };
     meta = {
       homepage = https://github.com/floragunncom/search-guard;
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
deleted file mode 100644
index 4f15ba90d39d..000000000000
--- a/pkgs/tools/misc/logstash/6.x.nix
+++ /dev/null
@@ -1,53 +0,0 @@
-{ elk6Version
-, enableUnfree ? true
-, stdenv
-, fetchurl
-, makeWrapper
-, jre
-}:
-
-with stdenv.lib;
-
-stdenv.mkDerivation rec {
-  version = elk6Version;
-  name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
-    sha256 =
-      if enableUnfree
-      then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr"
-      else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w";
-  };
-
-  dontBuild         = true;
-  dontPatchELF      = true;
-  dontStrip         = true;
-  dontPatchShebangs = true;
-
-  buildInputs = [
-    makeWrapper jre
-  ];
-
-  installPhase = ''
-    mkdir -p $out
-    cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
-
-    patchShebangs $out/bin/logstash
-    patchShebangs $out/bin/logstash-plugin
-
-    wrapProgram $out/bin/logstash \
-       --set JAVA_HOME "${jre}"
-
-    wrapProgram $out/bin/logstash-plugin \
-       --set JAVA_HOME "${jre}"
-  '';
-
-  meta = with stdenv.lib; {
-    description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
-    homepage    = https://www.elastic.co/products/logstash;
-    license     = if enableUnfree then licenses.elastic else licenses.asl20;
-    platforms   = platforms.unix;
-    maintainers = with maintainers; [ wjlroe offline basvandijk ];
-  };
-}
diff --git a/pkgs/tools/misc/logstash/default.nix b/pkgs/tools/misc/logstash/default.nix
index 079e5c7a93bf..4f15ba90d39d 100644
--- a/pkgs/tools/misc/logstash/default.nix
+++ b/pkgs/tools/misc/logstash/default.nix
@@ -1,12 +1,23 @@
-{ stdenv, fetchurl, makeWrapper, jre  }:
+{ elk6Version
+, enableUnfree ? true
+, stdenv
+, fetchurl
+, makeWrapper
+, jre
+}:
+
+with stdenv.lib;
 
 stdenv.mkDerivation rec {
-  version = "2.4.0";
-  name = "logstash-${version}";
+  version = elk6Version;
+  name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
 
   src = fetchurl {
-    url = "https://download.elasticsearch.org/logstash/logstash/logstash-${version}.tar.gz";
-    sha256 = "1k27hb6q1r26rp3y9pb2ry92kicw83mi352dzl2y4h0gbif46b32";
+    url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
+    sha256 =
+      if enableUnfree
+      then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr"
+      else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w";
   };
 
   dontBuild         = true;
@@ -20,12 +31,12 @@ stdenv.mkDerivation rec {
 
   installPhase = ''
     mkdir -p $out
-    cp -r {Gemfile*,vendor,lib,bin} $out
+    cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
 
-    wrapProgram $out/bin/logstash \
-       --set JAVA_HOME "${jre}"
+    patchShebangs $out/bin/logstash
+    patchShebangs $out/bin/logstash-plugin
 
-    wrapProgram $out/bin/rspec \
+    wrapProgram $out/bin/logstash \
        --set JAVA_HOME "${jre}"
 
     wrapProgram $out/bin/logstash-plugin \
@@ -35,8 +46,8 @@ stdenv.mkDerivation rec {
   meta = with stdenv.lib; {
     description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
     homepage    = https://www.elastic.co/products/logstash;
-    license     = licenses.asl20;
+    license     = if enableUnfree then licenses.elastic else licenses.asl20;
     platforms   = platforms.unix;
-    maintainers = [ maintainers.wjlroe maintainers.offline ];
+    maintainers = with maintainers; [ wjlroe offline basvandijk ];
   };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 390702597388..514f5ac551b7 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -849,23 +849,23 @@ with pkgs;
 
   bchunk = callPackage ../tools/cd-dvd/bchunk { };
 
-  inherit (callPackages ../misc/logging/beats/5.x.nix { })
-    filebeat
-    heartbeat
-    metricbeat
-    packetbeat;
-
-  inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in {
-    filebeat6   = beats6.filebeat;
-    heartbeat6  = beats6.heartbeat;
-    metricbeat6 = beats6.metricbeat;
-    packetbeat6 = beats6.packetbeat;
-  })
+  inherit (callPackages ../misc/logging/beats/6.x.nix { })
     filebeat6
     heartbeat6
     metricbeat6
     packetbeat6;
 
+  filebeat = filebeat6;
+  heartbeat = heartbeat6;
+  metricbeat = metricbeat6;
+  packetbeat = packetbeat6;
+
+  inherit (callPackages ../misc/logging/beats/5.x.nix { })
+    filebeat5
+    heartbeat5
+    metricbeat5
+    packetbeat5;
+
   bfr = callPackage ../tools/misc/bfr { };
 
   bibtool = callPackage ../tools/misc/bibtool { };
@@ -2303,13 +2303,13 @@ with pkgs;
   elk5Version = "5.6.9";
   elk6Version = "6.3.2";
 
-  elasticsearch = callPackage ../servers/search/elasticsearch { };
-  elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
-  elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
-  elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
+  elasticsearch6 = callPackage ../servers/search/elasticsearch { };
+  elasticsearch6-oss = callPackage ../servers/search/elasticsearch {
     enableUnfree = false;
   };
+  elasticsearch = elasticsearch6;
+  elasticsearch-oss = elasticsearch6-oss;
 
   elasticsearchPlugins = recurseIntoAttrs (
     callPackage ../servers/search/elasticsearch/plugins.nix { }
@@ -3424,12 +3424,13 @@ with pkgs;
 
   keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
 
-  kibana  = callPackage ../development/tools/misc/kibana { };
   kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
-  kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
-  kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
+  kibana6 = callPackage ../development/tools/misc/kibana/default.nix { };
+  kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix {
     enableUnfree = false;
   };
+  kibana = kibana6;
+  kibana-oss = kibana6-oss;
 
   kismet = callPackage ../applications/networking/sniffers/kismet { };
 
@@ -3506,12 +3507,12 @@ with pkgs;
 
   lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
 
-  logstash  = callPackage ../tools/misc/logstash { };
   logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
-  logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
-  logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
+  logstash6 = callPackage ../tools/misc/logstash { };
+  logstash6-oss = callPackage ../tools/misc/logstash {
     enableUnfree = false;
   };
+  logstash = logstash6;
 
   logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };