ELK: update kibana and the elastic beats to 5.4 (#26252)

* Add kibana5 and logstash5
* Upgrade the elastic beats to 5.4
* Make sure all elastic products use the same version
  (see elk5Version)
* Add a test for the ELK stack

9 files changed, 141 insertions, 123 deletions

diff --git a/pkgs/development/tools/misc/kibana/5.x.nix b/pkgs/development/tools/misc/kibana/5.x.nix
new file mode 100644
index 000000000000..4a4fc3e214f7
--- /dev/null
+++ b/pkgs/development/tools/misc/kibana/5.x.nix
@@ -0,0 +1,45 @@
+{ stdenv, makeWrapper, fetchurl, elk5Version, nodejs, coreutils, which }:
+
+with stdenv.lib;
+let
+  inherit (builtins) elemAt;
+  archOverrides = {
+    "i686" = "x86";
+  };
+  info = splitString "-" stdenv.system;
+  arch = (elemAt info 0);
+  elasticArch = archOverrides."${arch}" or arch;
+  plat = elemAt info 1;
+  shas = {
+    "x86_64-linux"  = "1g5i81wq77fk6pyaq3rpfqs2m23xsbz2cndh3rg4b59ibg5qv0sq";
+    "i686-linux"    = "0pxnpg3g8l6hy8qz404kbkk5rd9y65jrzd0y9j8wr5fd4pqs7vgv";
+    "x86_64-darwin" = "0pffl2hbck3s271jlzdibp5698djm5fdvj15w2knm815rs2kfbl5";
+  };
+in stdenv.mkDerivation rec {
+  name = "kibana-${version}";
+  version = elk5Version;
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${elasticArch}.tar.gz";
+    sha256 = shas."${stdenv.system}" or (throw "Unknown architecture");
+  };
+
+  buildInputs = [ makeWrapper ];
+
+  installPhase = ''
+    mkdir -p $out/libexec/kibana $out/bin
+    mv * $out/libexec/kibana/
+    rm -r $out/libexec/kibana/node
+    makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
+      --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
+    sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
+  '';
+
+  meta = {
+    description = "Visualize logs and time-stamped data";
+    homepage = http://www.elasticsearch.org/overview/kibana;
+    license = licenses.asl20;
+    maintainers = with maintainers; [ offline rickynils ];
+    platforms = with platforms; unix;
+  };
+}
diff --git a/pkgs/misc/logging/beats/default.nix b/pkgs/misc/logging/beats/default.nix
new file mode 100644
index 000000000000..d4c78389e664
--- /dev/null
+++ b/pkgs/misc/logging/beats/default.nix
@@ -0,0 +1,42 @@
+{ stdenv, fetchFromGitHub, elk5Version, buildGoPackage, libpcap }:
+
+let beat = package : extraArgs : buildGoPackage (rec {
+      name = "${package}-${version}";
+      version = elk5Version;
+
+      src = fetchFromGitHub {
+        owner = "elastic";
+        repo = "beats";
+        rev = "v${version}";
+        sha256 = "1if16sqbhgxc7ahn9pak8av9rq9l8ldk44hr4w4g7lhxnqhmhsji";
+      };
+
+      goPackagePath = "github.com/elastic/beats";
+
+      subPackages = [ package ];
+
+      meta = with stdenv.lib; {
+        homepage = https://www.elastic.co/products/beats;
+        license = licenses.asl20;
+        maintainers = with maintainers; [ fadenb basvandijk ];
+        platforms = platforms.linux;
+      };
+    } // extraArgs);
+in {
+  filebeat   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
+  heartbeat  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
+  metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  packetbeat = beat "packetbeat" {
+    buildInputs = [ libpcap ];
+    meta.description = "Network packet analyzer that ships data to Elasticsearch";
+    meta.longDescription = ''
+      Packetbeat is an open source network packet analyzer that ships the
+      data to Elasticsearch.
+
+      Think of it like a distributed real-time Wireshark with a lot more
+      analytics features. The Packetbeat shippers sniff the traffic between
+      your application processes, parse on the fly protocols like HTTP, MySQL,
+      PostgreSQL, Redis or Thrift and correlate the messages into transactions.
+    '';
+  };
+}
diff --git a/pkgs/misc/logging/filebeat/default.nix b/pkgs/misc/logging/filebeat/default.nix
deleted file mode 100644
index c33f34c6dc4d..000000000000
--- a/pkgs/misc/logging/filebeat/default.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-{ stdenv, fetchFromGitHub, buildGoPackage }:
-
-buildGoPackage rec {
-  name = "filebeat-${version}";
-  version = "5.2.2";
-
-  src = fetchFromGitHub {
-    owner = "elastic";
-    repo = "beats";
-    rev = "v${version}";
-    sha256 = "19hkq19xpi3c9y5g1yq77sm2d5vzybn6mxxf0s5l6sw4l98aak5q";
-  };
-
-  goPackagePath = "github.com/elastic/beats";
-
-  subPackages = [ "filebeat" ];
-
-  meta = with stdenv.lib; {
-    description = "Lightweight shipper for logfiles";
-    homepage = https://www.elastic.co/products/beats;
-    license = licenses.asl20;
-    maintainers = [ maintainers.fadenb ];
-    platforms = platforms.linux;
-  };
-}
diff --git a/pkgs/misc/logging/heartbeat/default.nix b/pkgs/misc/logging/heartbeat/default.nix
deleted file mode 100644
index faa558589f25..000000000000
--- a/pkgs/misc/logging/heartbeat/default.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-{ stdenv, fetchFromGitHub, buildGoPackage }:
-
-buildGoPackage rec {
-  name = "heartbeat-${version}";
-  version = "5.2.2";
-
-  src = fetchFromGitHub {
-    owner = "elastic";
-    repo = "beats";
-    rev = "v${version}";
-    sha256 = "19hkq19xpi3c9y5g1yq77sm2d5vzybn6mxxf0s5l6sw4l98aak5q";
-  };
-
-  goPackagePath = "github.com/elastic/beats";
-
-  subPackages = [ "heartbeat" ];
-
-  meta = with stdenv.lib; {
-    description = "Lightweight shipper for uptime monitoring";
-    homepage = https://www.elastic.co/products/beats;
-    license = licenses.asl20;
-    maintainers = [ maintainers.fadenb ];
-    platforms = platforms.linux;
-  };
-}
diff --git a/pkgs/misc/logging/metricbeat/default.nix b/pkgs/misc/logging/metricbeat/default.nix
deleted file mode 100644
index fcb2376a93d7..000000000000
--- a/pkgs/misc/logging/metricbeat/default.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-{ stdenv, fetchFromGitHub, buildGoPackage }:
-
-buildGoPackage rec {
-  name = "metricbeat-${version}";
-  version = "5.2.2";
-
-  src = fetchFromGitHub {
-    owner = "elastic";
-    repo = "beats";
-    rev = "v${version}";
-    sha256 = "19hkq19xpi3c9y5g1yq77sm2d5vzybn6mxxf0s5l6sw4l98aak5q";
-  };
-
-  goPackagePath = "github.com/elastic/beats";
-
-  subPackages = [ "metricbeat" ];
-
-  meta = with stdenv.lib; {
-    description = "Lightweight shipper for metrics";
-    homepage = https://www.elastic.co/products/beats;
-    license = licenses.asl20;
-    maintainers = [ maintainers.fadenb ];
-    platforms = platforms.linux;
-  };
-}
diff --git a/pkgs/misc/logging/packetbeat/default.nix b/pkgs/misc/logging/packetbeat/default.nix
deleted file mode 100644
index d4445bec9980..000000000000
--- a/pkgs/misc/logging/packetbeat/default.nix
+++ /dev/null
@@ -1,36 +0,0 @@
-{ stdenv, fetchFromGitHub, buildGoPackage, libpcap }:
-
-buildGoPackage rec {
-  name = "packetbeat-${version}";
-  version = "5.2.2";
-
-  src = fetchFromGitHub {
-    owner = "elastic";
-    repo = "beats";
-    rev = "v${version}";
-    sha256 = "19hkq19xpi3c9y5g1yq77sm2d5vzybn6mxxf0s5l6sw4l98aak5q";
-  };
-
-  goPackagePath = "github.com/elastic/beats";
-
-  subPackages = [ "packetbeat" ];
-
-  buildInputs = [ libpcap ];
-
-  meta = with stdenv.lib; {
-    description = "Network packet analyzer that ships data to Elasticsearch";
-    longDescription = ''
-      Packetbeat is an open source network packet analyzer that ships the
-      data to Elasticsearch.
-
-      Think of it like a distributed real-time Wireshark with a lot more
-      analytics features. The Packetbeat shippers sniff the traffic between
-      your application processes, parse on the fly protocols like HTTP, MySQL,
-      PostgreSQL, Redis or Thrift and correlate the messages into transactions.
-    '';
-    homepage = https://www.elastic.co/products/beats;
-    license = licenses.asl20;
-    maintainers = [ maintainers.fadenb ];
-    platforms = platforms.linux;
-  };
-}
diff --git a/pkgs/servers/search/elasticsearch/5.x.nix b/pkgs/servers/search/elasticsearch/5.x.nix
index 030c4730f38e..512337cf3826 100644
--- a/pkgs/servers/search/elasticsearch/5.x.nix
+++ b/pkgs/servers/search/elasticsearch/5.x.nix
@@ -1,9 +1,9 @@
-{ stdenv, fetchurl, makeWrapper, jre, utillinux, getopt }:
+{ stdenv, fetchurl, elk5Version, makeWrapper, jre, utillinux, getopt }:
 
 with stdenv.lib;
 
 stdenv.mkDerivation rec {
-  version = "5.4.0";
+  version = elk5Version;
   name = "elasticsearch-${version}";
 
   src = fetchurl {
diff --git a/pkgs/tools/misc/logstash/5.x.nix b/pkgs/tools/misc/logstash/5.x.nix
new file mode 100644
index 000000000000..13387b5c1a91
--- /dev/null
+++ b/pkgs/tools/misc/logstash/5.x.nix
@@ -0,0 +1,39 @@
+{ stdenv, fetchurl, elk5Version, makeWrapper, jre  }:
+
+stdenv.mkDerivation rec {
+  version = elk5Version;
+  name = "logstash-${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
+    sha256 = "019bhsnbbbg1a4g9jf02j3jb1xhhmrr3i7882s5l4pmkyn1d3gd1";
+  };
+
+  dontBuild         = true;
+  dontPatchELF      = true;
+  dontStrip         = true;
+  dontPatchShebangs = true;
+
+  buildInputs = [
+    makeWrapper jre
+  ];
+
+  installPhase = ''
+    mkdir -p $out
+    cp -r {Gemfile*,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
+
+    wrapProgram $out/bin/logstash \
+       --set JAVA_HOME "${jre}"
+
+    wrapProgram $out/bin/logstash-plugin \
+       --set JAVA_HOME "${jre}"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
+    homepage    = https://www.elastic.co/products/logstash;
+    license     = licenses.asl20;
+    platforms   = platforms.unix;
+    maintainers = [ maintainers.wjlroe maintainers.offline ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index ea00365659d6..5f838d3156c0 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -667,6 +667,12 @@ with pkgs;
 
   bchunk = callPackage ../tools/cd-dvd/bchunk { };
 
+  inherit (callPackages ../misc/logging/beats { })
+    filebeat
+    heartbeat
+    metricbeat
+    packetbeat;
+
   bfr = callPackage ../tools/misc/bfr { };
 
   bibtool = callPackage ../tools/misc/bibtool { };
@@ -968,8 +974,6 @@ with pkgs;
 
   fastJson = callPackage ../development/libraries/fastjson { };
 
-  filebeat = callPackage ../misc/logging/filebeat { };
-
   filebench = callPackage ../tools/misc/filebench { };
 
   fsmon = callPackage ../tools/misc/fsmon { };
@@ -1018,8 +1022,6 @@ with pkgs;
 
   gti = callPackage ../tools/misc/gti { };
 
-  heartbeat = callPackage ../misc/logging/heartbeat { };
-
   heatseeker = callPackage ../tools/misc/heatseeker { };
 
   hexio = callPackage ../development/tools/hexio { };
@@ -1050,8 +1052,6 @@ with pkgs;
 
   meson = callPackage ../development/tools/build-managers/meson { };
 
-  metricbeat = callPackage ../misc/logging/metricbeat { };
-
   mp3fs = callPackage ../tools/filesystems/mp3fs { };
 
   mpdcron = callPackage ../tools/audio/mpdcron { };
@@ -1732,6 +1732,9 @@ with pkgs;
 
   evemu = callPackage ../tools/system/evemu { };
 
+  # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
+  elk5Version = "5.4.0";
+
   elasticsearch = callPackage ../servers/search/elasticsearch { };
   elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
@@ -2625,7 +2628,8 @@ with pkgs;
 
   keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
 
-  kibana = callPackage ../development/tools/misc/kibana { };
+  kibana  = callPackage ../development/tools/misc/kibana { };
+  kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
 
   kismet = callPackage ../applications/networking/sniffers/kismet { };
 
@@ -2685,7 +2689,8 @@ with pkgs;
 
   lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
 
-  logstash = callPackage ../tools/misc/logstash { };
+  logstash  = callPackage ../tools/misc/logstash { };
+  logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
 
   logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };
 
@@ -3500,8 +3505,6 @@ with pkgs;
     nix = nixUnstable;
   };
 
-  packetbeat = callPackage ../misc/logging/packetbeat { };
-
   packetdrill = callPackage ../tools/networking/packetdrill { };
 
   pakcs = callPackage ../development/compilers/pakcs {};