From f644675bc9fc54c490f015fe57905a0eeb4b46bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:40:54 +0100 Subject: [PATCH 01/16] elasticsearch: drop 1.7.2 (not supported upstream) --- pkgs/servers/search/elasticsearch/default.nix | 40 ------------------- .../search/elasticsearch/es-home.patch | 37 ----------------- 2 files changed, 77 deletions(-) delete mode 100644 pkgs/servers/search/elasticsearch/default.nix delete mode 100644 pkgs/servers/search/elasticsearch/es-home.patch diff --git a/pkgs/servers/search/elasticsearch/default.nix b/pkgs/servers/search/elasticsearch/default.nix deleted file mode 100644 index f4780603f50..00000000000 --- a/pkgs/servers/search/elasticsearch/default.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ stdenv, fetchurl, makeWrapper, jre, utillinux }: - -with stdenv.lib; - -stdenv.mkDerivation rec { - name = "elasticsearch-1.7.2"; - - src = fetchurl { - url = "https://download.elastic.co/elasticsearch/elasticsearch/${name}.tar.gz"; - sha256 = "1lix4asvx1lbc227gzsrws3xqbcbqaal7v10w60kch0c4xg970bg"; - }; - - patches = [ ./es-home.patch ]; - - buildInputs = [ makeWrapper jre utillinux ]; - - installPhase = '' - mkdir -p $out - cp -R bin config lib $out - - # don't want to have binary with name plugin - mv $out/bin/plugin $out/bin/elasticsearch-plugin - - # set ES_CLASSPATH and JAVA_HOME - wrapProgram $out/bin/elasticsearch \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \ - --prefix PATH : "${utillinux}/bin" \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/elasticsearch-plugin \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*":"$out/lib/sigar/*" \ - --set JAVA_HOME "${jre}" - ''; - - meta = { - description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; - platforms = platforms.unix; - maintainers = [ maintainers.offline ]; - }; -} diff --git a/pkgs/servers/search/elasticsearch/es-home.patch b/pkgs/servers/search/elasticsearch/es-home.patch deleted file mode 100644 index 2f2018fd69f..00000000000 --- a/pkgs/servers/search/elasticsearch/es-home.patch +++ /dev/null @@ -1,37 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2015-08-05 17:52:05.740819671 +0200 -+++ b/bin/elasticsearch 2015-08-05 17:22:34.664657364 +0200 -@@ -83,7 +83,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -diff -rupN a/bin/plugin b/bin/plugin ---- a/bin/plugin 2015-08-05 17:57:07.903088815 +0200 -+++ b/bin/plugin 2015-08-05 17:57:38.979808139 +0200 -@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -@@ -105,4 +105,4 @@ - - export HOSTNAME=`hostname -s` - --eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home=\""$ES_HOME"\" $properties -cp \""$ES_HOME/lib/*"\" org.elasticsearch.plugins.PluginManager $args -\ No newline at end of file -+eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="$ES_HOME" $properties -cp "$ES_CLASSPATH/lib/*" org.elasticsearch.plugins.PluginManager $args From bedf471a089adb1b12d483f8d3672bb4070ebab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:42:53 +0100 Subject: [PATCH 02/16] elasticsearch6: new default version --- nixos/modules/services/search/elasticsearch.nix | 4 ++-- .../servers/search/elasticsearch/{6.x.nix => default.nix} | 0 pkgs/top-level/all-packages.nix | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) rename pkgs/servers/search/elasticsearch/{6.x.nix => default.nix} (100%) diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index b0831dcd1ca..5af82e01d21 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -58,8 +58,8 @@ in { package = mkOption { description = "Elasticsearch package to use."; - default = pkgs.elasticsearch2; - defaultText = "pkgs.elasticsearch2"; + default = pkgs.elasticsearch; + defaultText = "pkgs.elasticsearch"; type = types.package; }; diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/default.nix similarity index 100% rename from pkgs/servers/search/elasticsearch/6.x.nix rename to pkgs/servers/search/elasticsearch/default.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index fa555153e57..5db219dbcdd 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2272,13 +2272,15 @@ with pkgs; elk5Version = "5.6.9"; elk6Version = "6.3.2"; - elasticsearch = callPackage ../servers/search/elasticsearch { }; elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; - elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { }; - elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix { + + elasticsearch6 = callPackage ../servers/search/elasticsearch { }; + elasticsearch6-oss = callPackage ../servers/search/elasticsearch { enableUnfree = false; }; + elasticsearch = elasticsearch6; + elasticsearch-oss = elasticsearch6-oss; elasticsearchPlugins = recurseIntoAttrs ( callPackage ../servers/search/elasticsearch/plugins.nix { } From 1b2b62ba79b453dc648362cd2f6cc51e5a9a3ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:43:37 +0100 Subject: [PATCH 03/16] elasticsearch2: drop unsupported version --- pkgs/servers/search/elasticsearch/2.x.nix | 40 ------------------- .../elasticsearch/es-classpath-2.x.patch | 38 ------------------ .../search/elasticsearch/es-home-2.x.patch | 31 -------------- pkgs/top-level/all-packages.nix | 2 - 4 files changed, 111 deletions(-) delete mode 100644 pkgs/servers/search/elasticsearch/2.x.nix delete mode 100644 pkgs/servers/search/elasticsearch/es-classpath-2.x.patch delete mode 100644 pkgs/servers/search/elasticsearch/es-home-2.x.patch diff --git a/pkgs/servers/search/elasticsearch/2.x.nix b/pkgs/servers/search/elasticsearch/2.x.nix deleted file mode 100644 index 28244d1b3dc..00000000000 --- a/pkgs/servers/search/elasticsearch/2.x.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ stdenv, fetchurl, makeWrapper, jre, utillinux }: - -with stdenv.lib; - -stdenv.mkDerivation rec { - version = "2.4.4"; - name = "elasticsearch-${version}"; - - src = fetchurl { - url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${version}/${name}.tar.gz"; - sha256 = "1qjq04sfqb35pf2xpvr8j5p27chfxpjp8ymrp1h5bfk5rbk9444q"; - }; - - patches = [ ./es-home-2.x.patch ./es-classpath-2.x.patch ]; - - buildInputs = [ makeWrapper jre utillinux ]; - - installPhase = '' - mkdir -p $out - cp -R bin config lib modules $out - - # don't want to have binary with name plugin - mv $out/bin/plugin $out/bin/elasticsearch-plugin - wrapProgram $out/bin/elasticsearch \ - --prefix ES_CLASSPATH : "$out/lib/${name}.jar":"$out/lib/*" \ - --prefix PATH : "${utillinux}/bin" \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre}" - ''; - - meta = { - description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; - platforms = platforms.unix; - maintainers = [ - maintainers.offline - maintainers.markWot - ]; - }; -} diff --git a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch b/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch deleted file mode 100644 index 46a3f0be71b..00000000000 --- a/pkgs/servers/search/elasticsearch/es-classpath-2.x.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2017-02-08 18:32:28.000298543 -0500 -+++ b/bin/elasticsearch 2017-02-08 19:10:45.692916675 -0500 -@@ -81,12 +81,7 @@ ES_HOME=`cd "$ES_HOME"; pwd` - # If an include wasn't specified in the environment, then search for one... - if [ "x$ES_INCLUDE" = "x" ]; then - # Locations (in order) to use when searching for an include file. -- for include in /usr/share/elasticsearch/elasticsearch.in.sh \ -- /usr/local/share/elasticsearch/elasticsearch.in.sh \ -- /opt/elasticsearch/elasticsearch.in.sh \ -- ~/.elasticsearch.in.sh \ -- "$ES_HOME/bin/elasticsearch.in.sh" \ -- "`dirname "$0"`"/elasticsearch.in.sh; do -+ for include in "`dirname "$0"`"/elasticsearch.in.sh; do - if [ -r "$include" ]; then - . "$include" - break -diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh ---- a/bin/elasticsearch.in.sh 2017-02-08 18:32:28.000298543 -0500 -+++ b/bin/elasticsearch.in.sh 2017-02-08 18:33:46.816634599 -0500 -@@ -1,17 +1,5 @@ - #!/bin/sh - --# check in case a user was using this mechanism --if [ "x$ES_CLASSPATH" != "x" ]; then -- cat >&2 << EOF --Error: Don't modify the classpath with ES_CLASSPATH. Best is to add --additional elements via the plugin mechanism, or if code must really be --added to the main classpath, add jars to lib/ (unsupported). --EOF -- exit 1 --fi -- --ES_CLASSPATH="$ES_HOME/lib/elasticsearch-2.4.4.jar:$ES_HOME/lib/*" -- - if [ "x$ES_MIN_MEM" = "x" ]; then - ES_MIN_MEM=256m - fi diff --git a/pkgs/servers/search/elasticsearch/es-home-2.x.patch b/pkgs/servers/search/elasticsearch/es-home-2.x.patch deleted file mode 100644 index 0c80e865114..00000000000 --- a/pkgs/servers/search/elasticsearch/es-home-2.x.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2015-11-18 21:48:18.000000000 +0100 -+++ b/bin/elasticsearch 2015-12-04 00:52:21.032475098 +0100 -@@ -72,7 +72,11 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+ -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -diff -rupN a/bin/plugin b/bin/plugin ---- a/bin/plugin 2015-11-18 21:48:18.000000000 +0100 -+++ b/bin/plugin 2015-12-04 00:52:55.947453619 +0100 -@@ -17,7 +17,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5db219dbcdd..6753ab584ab 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2272,9 +2272,7 @@ with pkgs; elk5Version = "5.6.9"; elk6Version = "6.3.2"; - elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; - elasticsearch6 = callPackage ../servers/search/elasticsearch { }; elasticsearch6-oss = callPackage ../servers/search/elasticsearch { enableUnfree = false; From cba348c561f970482e4c3f792b2f72f78a83f46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:44:19 +0100 Subject: [PATCH 04/16] elasticsearchPlugins: updated for elasticsearch6 --- pkgs/servers/search/elasticsearch/plugins.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index b93d8c2203d..b810d5e865b 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -1,4 +1,4 @@ -{ pkgs, stdenv, fetchurl, unzip, elasticsearch }: +{ pkgs, stdenv, fetchurl, unzip, elasticsearch-oss, javaPackages, elk6Version }: with pkgs.lib; @@ -6,8 +6,9 @@ let esPlugin = a@{ pluginName, installPhase ? '' - mkdir -p $out/bin - ES_HOME=$out ${elasticsearch}/bin/elasticsearch-plugin --install ${pluginName} --url file://$src + mkdir -p $out/config + mkdir -p $out/plugins + ES_HOME=$out ${elasticsearch-oss}/bin/elasticsearch-plugin install --batch -v file://$src '', ... }: @@ -16,7 +17,7 @@ let unpackPhase = "true"; buildInputs = [ unzip ]; meta = a.meta // { - platforms = elasticsearch.meta.platforms; + platforms = elasticsearch-oss.meta.platforms; maintainers = (a.meta.maintainers or []) ++ [ maintainers.offline ]; }; }); From fa26b59091f56958236e8d0540bd76789e10b259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:48:25 +0100 Subject: [PATCH 05/16] elasticsearchPlugins.{river_jdbc,river_twitter}: remove River plugins are no longer supported in elasticsearch. --- pkgs/servers/search/elasticsearch/plugins.nix | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index b810d5e865b..8b1bad7768d 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -22,20 +22,6 @@ let }; }); in { - elasticsearch_river_jdbc = esPlugin rec { - name = "elasticsearch-river-jdbc-${version}"; - pluginName = "elasticsearch-river-jdbc"; - version = "1.5.0.5"; - src = fetchurl { - url = "http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-river-jdbc/${version}/${name}-plugin.zip"; - sha256 = "1p75l3vcnb90ar4j3dci2xf8dqnqyy31kc1r075fa2xqlsxgigcp"; - }; - meta = { - homepage = https://github.com/jprante/elasticsearch-river-jdbc; - description = "Plugin to fetch data from JDBC sources for indexing into Elasticsearch"; - license = licenses.asl20; - }; - }; elasticsearch_analysis_lemmagen = esPlugin rec { name = "elasticsearch-analysis-lemmagen-${version}"; @@ -72,26 +58,6 @@ in { }; }; - elasticsearch_river_twitter = esPlugin rec { - name = pname + "-" + version; - pname = "elasticsearch-river-twitter"; - pluginName = "elasticsearch/" + pname + "/" + version; - version = "2.5.0"; - - src = fetchurl { - url = "http://download.elasticsearch.org/elasticsearch/${pname}/${name}.zip"; - sha256 = "0851yrmyrpp6whyxk34ykcj7b28f90w0nvkrhvl49dwqgr5s4mn4"; - }; - - meta = { - homepage = https://github.com/elasticsearch/elasticsearch-river-twitter; - description = "Twitter River Plugin for ElasticSearch"; - license = licenses.asl20; - maintainers = [ maintainers.edwtjo ]; - platforms = elasticsearch.meta.platforms; - }; - }; - elasticsearch_kopf = esPlugin rec { name = "elasticsearch-kopf-${version}"; pluginName = "elasticsearch-kopf"; From 5d3b75ab5d5bd21584096f32769c933ed9dbc2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:51:40 +0100 Subject: [PATCH 06/16] elasticsearchPlugins.elasticsearch_http_basic: remove Deprecated by upstream, search_guard was recommended instead. --- pkgs/servers/search/elasticsearch/plugins.nix | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index 8b1bad7768d..f44e1a4e640 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -38,25 +38,6 @@ in { }; }; - elasticsearch_http_basic = stdenv.mkDerivation rec { - name = "elasticsearch-http-basic-${version}"; - version = "1.5.0"; - - src = fetchurl { - url = "https://github.com/Asquera/elasticsearch-http-basic/releases/download/v${version}/${name}.jar"; - sha256 = "0fif6sbn2ich39lrgm039y9d5bxkylx9pvly04wss8rdhspvdskb"; - }; - - phases = ["installPhase"]; - installPhase = "install -D $src $out/plugins/http-basic/${name}.jar"; - - meta = { - homepage = https://github.com/Asquera/elasticsearch-http-basic; - description = "HTTP Basic Authentication for Elasticsearch"; - license = licenses.mit; - platforms = elasticsearch.meta.platforms; - }; - }; elasticsearch_kopf = esPlugin rec { name = "elasticsearch-kopf-${version}"; From 0bbf90e1b466e503f616bdaa94b9810bad695459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:53:01 +0100 Subject: [PATCH 07/16] elasticsearchPlugins.elasticsearch_kopf: remove deprecated by upstream, https://github.com/lmenezes/cerebro is recommend instead (but not packaged in nixpkgs) --- pkgs/servers/search/elasticsearch/plugins.nix | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index f44e1a4e640..b7791e1c047 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -39,21 +39,6 @@ in { }; - elasticsearch_kopf = esPlugin rec { - name = "elasticsearch-kopf-${version}"; - pluginName = "elasticsearch-kopf"; - version = "2.1.1"; - src = fetchurl { - url = "https://github.com/lmenezes/elasticsearch-kopf/archive/v${version}.zip"; - sha256 = "1nwwd92g0jxhfpkxb1a9z5a62naa1y7hvlx400dm6mwwav3mrf4v"; - }; - meta = { - homepage = https://github.com/lmenezes/elasticsearch-kopf; - description = "Web administration tool for ElasticSearch"; - license = licenses.mit; - }; - }; - search_guard = esPlugin rec { name = "elastic-search-guard-${version}"; pluginName = "search-guard"; From dab5fd5f567933d03313909f810e0b608397ea18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:54:28 +0100 Subject: [PATCH 08/16] elasticsearchPlugins.elasticsearch_analysis_lemmagen: 0.1 -> 6.3.2 --- pkgs/servers/search/elasticsearch/plugins.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index b7791e1c047..51f18c213bf 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -26,10 +26,10 @@ in { elasticsearch_analysis_lemmagen = esPlugin rec { name = "elasticsearch-analysis-lemmagen-${version}"; pluginName = "elasticsearch-analysis-lemmagen"; - version = "0.1"; + version = "${elk6Version}"; src = fetchurl { url = "https://github.com/vhyza/elasticsearch-analysis-lemmagen/releases/download/v${version}/${name}-plugin.zip"; - sha256 = "bf7bf5ce3ccdd3afecd0e18cd6fce1ef56f824e41f4ef50553ae598caa5c366d"; + sha256 = "1m4z05wixjrq4nlbdjyhvprkrwxjym8aba18scmzfn25fhbjgvkz"; }; meta = { homepage = https://github.com/vhyza/elasticsearch-analysis-lemmagen; From 19fb8c7c6c7cb6f9acf4add346ad93f8ba783042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 09:55:22 +0100 Subject: [PATCH 09/16] elasticsarchPlugins.search_guard: 0.5 -> 6.3.2-22.3 --- pkgs/servers/search/elasticsearch/plugins.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index 51f18c213bf..fa0f9b32ce9 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -42,10 +42,10 @@ in { search_guard = esPlugin rec { name = "elastic-search-guard-${version}"; pluginName = "search-guard"; - version = "0.5"; - src = fetchurl { - url = "https://github.com/floragunncom/search-guard/releases/download/v${version}/${pluginName}-${version}.zip"; - sha256 = "1zima4jmq1rrcqxhlrp2xian80vp244d2splby015n5cgqrp39fl"; + version = "${elk6Version}-22.3"; + src = fetchurl rec { + url = "mirror://maven/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip"; + sha256 = "1r71h4h9bmxak1mq5gpm19xq5ji1gry1kp3sjmm8azy4ykdqdncx"; }; meta = { homepage = https://github.com/floragunncom/search-guard; From 806ab02d95deea9c302865b786893116dcfb6d17 Mon Sep 17 00:00:00 2001 From: David Smith Date: Fri, 3 Aug 2018 09:58:42 +0100 Subject: [PATCH 10/16] elasticsearchPlugins.discovery-ec2: init at 6.3.2 --- pkgs/servers/search/elasticsearch/plugins.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix index fa0f9b32ce9..b3141ee807d 100644 --- a/pkgs/servers/search/elasticsearch/plugins.nix +++ b/pkgs/servers/search/elasticsearch/plugins.nix @@ -38,6 +38,20 @@ in { }; }; + discovery-ec2 = esPlugin { + name = "elasticsearch-discovery-ec2-${version}"; + pluginName = "discovery-ec2"; + version = "${elk6Version}"; + src = pkgs.fetchurl { + url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/discovery-ec2/discovery-ec2-${elk6Version}.zip"; + sha256 = "1i7ksy69132sr84h51lamgq967yz3a3dw0b54nckxpqwad9pcpj0"; + }; + meta = { + homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/discovery-ec2; + description = "The EC2 discovery plugin uses the AWS API for unicast discovery."; + license = licenses.asl20; + }; + }; search_guard = esPlugin rec { name = "elastic-search-guard-${version}"; From cd6279020155ce96bf2611526b96591ad179a70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 11:09:50 +0100 Subject: [PATCH 11/16] nixos/elasticsearch: remove es5 version guard --- .../modules/services/search/elasticsearch.nix | 53 ++++++------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index 5af82e01d21..2c7e21a20b5 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -5,22 +5,14 @@ with lib; let cfg = config.services.elasticsearch; - es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0; es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0; esConfig = '' network.host: ${cfg.listenAddress} cluster.name: ${cfg.cluster_name} - ${if es5 then '' - http.port: ${toString cfg.port} - transport.tcp.port: ${toString cfg.tcp_port} - '' else '' - network.port: ${toString cfg.port} - network.tcp.port: ${toString cfg.tcp_port} - # TODO: find a way to enable security manager - security.manager.enabled: false - ''} + http.port: ${toString cfg.port} + transport.tcp.port: ${toString cfg.tcp_port} ${cfg.extraConf} ''; @@ -32,7 +24,7 @@ let text = esConfig; }; - loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml"; + loggingConfigFilename = "log4j2.properties"; loggingConfigFile = pkgs.writeTextFile { name = loggingConfigFilename; text = cfg.logging; @@ -41,8 +33,7 @@ let esPlugins = pkgs.buildEnv { name = "elasticsearch-plugins"; paths = cfg.plugins; - # Elasticsearch 5.x won't start when the plugins directory does not exist - postBuild = if es5 then "${pkgs.coreutils}/bin/mkdir -p $out/plugins" else ""; + postBuild = "${pkgs.coreutils}/bin/mkdir -p $out/plugins"; }; in { @@ -100,30 +91,18 @@ in { logging = mkOption { description = "Elasticsearch logging configuration."; - default = - if es5 then '' - logger.action.name = org.elasticsearch.action - logger.action.level = info + default = '' + logger.action.name = org.elasticsearch.action + logger.action.level = info - appender.console.type = Console - appender.console.name = console - appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + appender.console.type = Console + appender.console.name = console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - rootLogger.level = info - rootLogger.appenderRef.console.ref = console - '' else '' - rootLogger: INFO, console - logger: - action: INFO - com.amazonaws: WARN - appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - ''; + rootLogger.level = info + rootLogger.appenderRef.console.ref = console + ''; type = types.str; }; @@ -204,9 +183,9 @@ in { cp ${elasticsearchYml} ${configDir}/elasticsearch.yml # Make sure the logging configuration for old elasticsearch versions is removed: - rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"} + rm -f "${configDir}/logging.yml" cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename} - ${optionalString es5 "mkdir -p ${configDir}/scripts"} + mkdir -p ${configDir}/scripts ${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"} if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi From 13db07a092a05f1f692c1f3429fced80d2904526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 11:23:56 +0100 Subject: [PATCH 12/16] logstash: update default version to v6 --- nixos/modules/rename.nix | 1 + nixos/modules/services/logging/logstash.nix | 41 ++-------------- pkgs/tools/misc/logstash/6.x.nix | 53 --------------------- pkgs/tools/misc/logstash/default.nix | 33 ++++++++----- pkgs/top-level/all-packages.nix | 6 +-- 5 files changed, 31 insertions(+), 103 deletions(-) delete mode 100644 pkgs/tools/misc/logstash/6.x.nix diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index 40f66ccb26b..fe395c4256b 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -257,6 +257,7 @@ with lib; (mkRemovedOptionModule [ "fonts" "fontconfig" "forceAutohint" ] "") (mkRemovedOptionModule [ "fonts" "fontconfig" "renderMonoTTFAsBitmap" ] "") (mkRemovedOptionModule [ "virtualisation" "xen" "qemu" ] "You don't need this option anymore, it will work without it.") + (mkRemovedOptionModule [ "services" "logstash" "enableWeb" ] "The web interface was removed from logstash") # ZSH (mkRenamedOptionModule [ "programs" "zsh" "enableSyntaxHighlighting" ] [ "programs" "zsh" "syntaxHighlighting" "enable" ]) diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix index 28d89a7463a..aa019d855ea 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/nixos/modules/services/logging/logstash.nix @@ -4,25 +4,12 @@ with lib; let cfg = config.services.logstash; - atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; pluginPath = lib.concatStringsSep ":" cfg.plugins; havePluginPath = lib.length cfg.plugins > 0; ops = lib.optionalString; - verbosityFlag = - if atLeast54 - then "--log.level " + cfg.logLevel - else { - debug = "--debug"; - info = "--verbose"; - warn = ""; # intentionally empty - error = "--quiet"; - fatal = "--silent"; - }."${cfg.logLevel}"; + verbosityFlag = "--log.level " + cfg.logLevel; - pluginsPath = - if atLeast54 - then "--path.plugins ${pluginPath}" - else "--pluginpath ${pluginPath}"; + pluginsPath = "--path.plugins ${pluginPath}"; logstashConf = pkgs.writeText "logstash.conf" '' input { @@ -63,7 +50,7 @@ in type = types.package; default = pkgs.logstash; defaultText = "pkgs.logstash"; - example = literalExample "pkgs.logstash"; + example = literalExample "pkgs.logstash5"; description = "Logstash package to use."; }; @@ -95,12 +82,6 @@ in description = "The quantity of filter workers to run."; }; - enableWeb = mkOption { - type = types.bool; - default = false; - description = "Enable the logstash web interface."; - }; - listenAddress = mkOption { type = types.str; default = "127.0.0.1"; @@ -174,16 +155,6 @@ in ###### implementation config = mkIf cfg.enable { - assertions = [ - { assertion = atLeast54 -> !cfg.enableWeb; - message = '' - The logstash web interface is only available for versions older than 5.4. - So either set services.logstash.enableWeb = false, - or set services.logstash.package to an older logstash. - ''; - } - ]; - systemd.services.logstash = with pkgs; { description = "Logstash Daemon"; wantedBy = [ "multi-user.target" ]; @@ -193,14 +164,12 @@ in ExecStartPre = ''${pkgs.coreutils}/bin/mkdir -p "${cfg.dataDir}" ; ${pkgs.coreutils}/bin/chmod 700 "${cfg.dataDir}"''; ExecStart = concatStringsSep " " (filter (s: stringLength s != 0) [ "${cfg.package}/bin/logstash" - (ops (!atLeast54) "agent") "-w ${toString cfg.filterWorkers}" (ops havePluginPath pluginsPath) "${verbosityFlag}" "-f ${logstashConf}" - (ops atLeast54 "--path.settings ${logstashSettingsDir}") - (ops atLeast54 "--path.data ${cfg.dataDir}") - (ops cfg.enableWeb "-- web -a ${cfg.listenAddress} -p ${cfg.port}") + "--path.settings ${logstashSettingsDir}" + "--path.data ${cfg.dataDir}" ]); }; }; diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix deleted file mode 100644 index 4f15ba90d39..00000000000 --- a/pkgs/tools/misc/logstash/6.x.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ elk6Version -, enableUnfree ? true -, stdenv -, fetchurl -, makeWrapper -, jre -}: - -with stdenv.lib; - -stdenv.mkDerivation rec { - version = elk6Version; - name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; - sha256 = - if enableUnfree - then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr" - else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w"; - }; - - dontBuild = true; - dontPatchELF = true; - dontStrip = true; - dontPatchShebangs = true; - - buildInputs = [ - makeWrapper jre - ]; - - installPhase = '' - mkdir -p $out - cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out - - patchShebangs $out/bin/logstash - patchShebangs $out/bin/logstash-plugin - - wrapProgram $out/bin/logstash \ - --set JAVA_HOME "${jre}" - - wrapProgram $out/bin/logstash-plugin \ - --set JAVA_HOME "${jre}" - ''; - - meta = with stdenv.lib; { - description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; - homepage = https://www.elastic.co/products/logstash; - license = if enableUnfree then licenses.elastic else licenses.asl20; - platforms = platforms.unix; - maintainers = with maintainers; [ wjlroe offline basvandijk ]; - }; -} diff --git a/pkgs/tools/misc/logstash/default.nix b/pkgs/tools/misc/logstash/default.nix index 079e5c7a93b..4f15ba90d39 100644 --- a/pkgs/tools/misc/logstash/default.nix +++ b/pkgs/tools/misc/logstash/default.nix @@ -1,12 +1,23 @@ -{ stdenv, fetchurl, makeWrapper, jre }: +{ elk6Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre +}: + +with stdenv.lib; stdenv.mkDerivation rec { - version = "2.4.0"; - name = "logstash-${version}"; + version = elk6Version; + name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; src = fetchurl { - url = "https://download.elasticsearch.org/logstash/logstash/logstash-${version}.tar.gz"; - sha256 = "1k27hb6q1r26rp3y9pb2ry92kicw83mi352dzl2y4h0gbif46b32"; + url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; + sha256 = + if enableUnfree + then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr" + else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w"; }; dontBuild = true; @@ -20,14 +31,14 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out - cp -r {Gemfile*,vendor,lib,bin} $out + cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out + + patchShebangs $out/bin/logstash + patchShebangs $out/bin/logstash-plugin wrapProgram $out/bin/logstash \ --set JAVA_HOME "${jre}" - wrapProgram $out/bin/rspec \ - --set JAVA_HOME "${jre}" - wrapProgram $out/bin/logstash-plugin \ --set JAVA_HOME "${jre}" ''; @@ -35,8 +46,8 @@ stdenv.mkDerivation rec { meta = with stdenv.lib; { description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; homepage = https://www.elastic.co/products/logstash; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; platforms = platforms.unix; - maintainers = [ maintainers.wjlroe maintainers.offline ]; + maintainers = with maintainers; [ wjlroe offline basvandijk ]; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 6753ab584ab..e44799b9984 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3468,12 +3468,12 @@ with pkgs; lockfileProgs = callPackage ../tools/misc/lockfile-progs { }; - logstash = callPackage ../tools/misc/logstash { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; - logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; - logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix { + logstash6 = callPackage ../tools/misc/logstash { }; + logstash6-oss = callPackage ../tools/misc/logstash { enableUnfree = false; }; + logstash = logstash6; logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { }; From c3055ab3ce6eeed037d0a396432cc90748d734b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 11:24:38 +0100 Subject: [PATCH 13/16] kibana: update default version to v6 --- nixos/modules/services/search/kibana.nix | 38 +--------- pkgs/development/tools/misc/kibana/6.x.nix | 75 ------------------- .../development/tools/misc/kibana/default.nix | 60 +++++++++++---- pkgs/top-level/all-packages.nix | 7 +- 4 files changed, 50 insertions(+), 130 deletions(-) delete mode 100644 pkgs/development/tools/misc/kibana/6.x.nix diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix index 5885a72c662..ca36bba58c0 100644 --- a/nixos/modules/services/search/kibana.nix +++ b/nixos/modules/services/search/kibana.nix @@ -5,43 +5,7 @@ with lib; let cfg = config.services.kibana; - atLeast54 = versionAtLeast (builtins.parseDrvName cfg.package.name).version "5.4"; - - cfgFile = if atLeast54 then cfgFile5 else cfgFile4; - - cfgFile4 = pkgs.writeText "kibana.json" (builtins.toJSON ( - (filterAttrsRecursive (n: v: v != null) ({ - host = cfg.listenAddress; - port = cfg.port; - ssl_cert_file = cfg.cert; - ssl_key_file = cfg.key; - - kibana_index = cfg.index; - default_app_id = cfg.defaultAppId; - - elasticsearch_url = cfg.elasticsearch.url; - kibana_elasticsearch_username = cfg.elasticsearch.username; - kibana_elasticsearch_password = cfg.elasticsearch.password; - kibana_elasticsearch_cert = cfg.elasticsearch.cert; - kibana_elasticsearch_key = cfg.elasticsearch.key; - ca = cfg.elasticsearch.ca; - - bundled_plugin_ids = [ - "plugins/dashboard/index" - "plugins/discover/index" - "plugins/doc/index" - "plugins/kibana/index" - "plugins/markdown_vis/index" - "plugins/metric_vis/index" - "plugins/settings/index" - "plugins/table_vis/index" - "plugins/vis_types/index" - "plugins/visualize/index" - ]; - } // cfg.extraConf) - ))); - - cfgFile5 = pkgs.writeText "kibana.json" (builtins.toJSON ( + cfgFile = pkgs.writeText "kibana.json" (builtins.toJSON ( (filterAttrsRecursive (n: v: v != null) ({ server.host = cfg.listenAddress; server.port = cfg.port; diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix deleted file mode 100644 index 1fd52f43557..00000000000 --- a/pkgs/development/tools/misc/kibana/6.x.nix +++ /dev/null @@ -1,75 +0,0 @@ -{ elk6Version -, enableUnfree ? true -, stdenv -, makeWrapper -, fetchzip -, fetchurl -, nodejs -, coreutils -, which -}: - -with stdenv.lib; -let - inherit (builtins) elemAt; - info = splitString "-" stdenv.system; - arch = elemAt info 0; - plat = elemAt info 1; - shas = - if enableUnfree - then { - "x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0"; - "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3"; - } - else { - "x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly"; - "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b"; - }; - - # For the correct phantomjs version see: - # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js - phantomjs = rec { - name = "phantomjs-${version}-linux-x86_64"; - version = "2.1.1"; - src = fetchzip { - inherit name; - url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2"; - sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv"; - }; - }; - -in stdenv.mkDerivation rec { - name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; - version = elk6Version; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; - sha256 = shas."${stdenv.system}" or (throw "Unknown architecture"); - }; - - buildInputs = [ makeWrapper ]; - - installPhase = '' - mkdir -p $out/libexec/kibana $out/bin - mv * $out/libexec/kibana/ - rm -r $out/libexec/kibana/node - makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ - --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" - sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana - '' + - # phantomjs is needed in the unfree version. When phantomjs doesn't exist in - # $out/libexec/kibana/data kibana will try to download and unpack it during - # runtime which will fail because the nix store is read-only. So we make sure - # it already exist in the nix store. - optionalString enableUnfree '' - ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name} - ''; - - meta = { - description = "Visualize logs and time-stamped data"; - homepage = http://www.elasticsearch.org/overview/kibana; - license = if enableUnfree then licenses.elastic else licenses.asl20; - maintainers = with maintainers; [ offline rickynils basvandijk ]; - platforms = with platforms; unix; - }; -} diff --git a/pkgs/development/tools/misc/kibana/default.nix b/pkgs/development/tools/misc/kibana/default.nix index fc1ac932338..1fd52f43557 100644 --- a/pkgs/development/tools/misc/kibana/default.nix +++ b/pkgs/development/tools/misc/kibana/default.nix @@ -1,26 +1,49 @@ -{ stdenv, makeWrapper, fetchurl, nodejs, coreutils, which }: +{ elk6Version +, enableUnfree ? true +, stdenv +, makeWrapper +, fetchzip +, fetchurl +, nodejs +, coreutils +, which +}: with stdenv.lib; let inherit (builtins) elemAt; - archOverrides = { - "i686" = "x86"; - }; info = splitString "-" stdenv.system; - arch = (elemAt info 0); - elasticArch = archOverrides."${arch}" or arch; + arch = elemAt info 0; plat = elemAt info 1; - shas = { - "x86_64-linux" = "1wnnrhhpgc58s09p99cmi8r2jmwsd5lmh2inb0k8nmizz5v1sjz0"; - "i686-linux" = "0sdx59jlfrf7r9793xpn2vxaxjdczgn3qfw8yny03dcs6fjaxi2y"; - "x86_64-darwin" = "0rmp536kn001g52lxngpj6x6d0j3qj0r11d4djbz7h6s5ml03kza"; + shas = + if enableUnfree + then { + "x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0"; + "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3"; + } + else { + "x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly"; + "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b"; + }; + + # For the correct phantomjs version see: + # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js + phantomjs = rec { + name = "phantomjs-${version}-linux-x86_64"; + version = "2.1.1"; + src = fetchzip { + inherit name; + url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2"; + sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv"; + }; }; + in stdenv.mkDerivation rec { - name = "kibana-${version}"; - version = "4.6.5"; + name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; + version = elk6Version; src = fetchurl { - url = "https://download.elastic.co/kibana/kibana/${name}-${plat}-${elasticArch}.tar.gz"; + url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; sha256 = shas."${stdenv.system}" or (throw "Unknown architecture"); }; @@ -33,13 +56,20 @@ in stdenv.mkDerivation rec { makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana + '' + + # phantomjs is needed in the unfree version. When phantomjs doesn't exist in + # $out/libexec/kibana/data kibana will try to download and unpack it during + # runtime which will fail because the nix store is read-only. So we make sure + # it already exist in the nix store. + optionalString enableUnfree '' + ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name} ''; meta = { description = "Visualize logs and time-stamped data"; homepage = http://www.elasticsearch.org/overview/kibana; - license = licenses.asl20; - maintainers = with maintainers; [ offline rickynils ]; + license = if enableUnfree then licenses.elastic else licenses.asl20; + maintainers = with maintainers; [ offline rickynils basvandijk ]; platforms = with platforms; unix; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e44799b9984..5d1f4d347b3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3386,12 +3386,13 @@ with pkgs; keyfuzz = callPackage ../tools/inputmethods/keyfuzz { }; - kibana = callPackage ../development/tools/misc/kibana { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; - kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; - kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix { + kibana6 = callPackage ../development/tools/misc/kibana/default.nix { }; + kibana6-oss = callPackage ../development/tools/misc/kibana/default.nix { enableUnfree = false; }; + kibana = kibana6; + kibana-oss = kibana6-oss; kismet = callPackage ../applications/networking/sniffers/kismet { }; From d50253237e015bd93c5e777cf5b4b4c886f91bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 11:45:06 +0100 Subject: [PATCH 14/16] beats: upgrade default from v5 to v6 --- pkgs/misc/logging/beats/5.x.nix | 8 ++++---- pkgs/misc/logging/beats/6.x.nix | 8 ++++---- pkgs/top-level/all-packages.nix | 24 ++++++++++++------------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkgs/misc/logging/beats/5.x.nix b/pkgs/misc/logging/beats/5.x.nix index 94d5449100d..e92613025ff 100644 --- a/pkgs/misc/logging/beats/5.x.nix +++ b/pkgs/misc/logging/beats/5.x.nix @@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec { }; } // extraArgs); in { - filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; - heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; - metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; - packetbeat = beat "packetbeat" { + filebeat5 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat5 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat5 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat5 = beat "packetbeat" { buildInputs = [ libpcap ]; meta.description = "Network packet analyzer that ships data to Elasticsearch"; meta.longDescription = '' diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index 111bc449d88..59d7379e06c 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -23,10 +23,10 @@ let beat = package : extraArgs : buildGoPackage (rec { }; } // extraArgs); in { - filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; - heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; - metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; - packetbeat = beat "packetbeat" { + filebeat6 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat6 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat6 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat6 = beat "packetbeat" { buildInputs = [ libpcap ]; meta.description = "Network packet analyzer that ships data to Elasticsearch"; meta.longDescription = '' diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5d1f4d347b3..100a299b2ba 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -841,23 +841,23 @@ with pkgs; bchunk = callPackage ../tools/cd-dvd/bchunk { }; - inherit (callPackages ../misc/logging/beats/5.x.nix { }) - filebeat - heartbeat - metricbeat - packetbeat; - - inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in { - filebeat6 = beats6.filebeat; - heartbeat6 = beats6.heartbeat; - metricbeat6 = beats6.metricbeat; - packetbeat6 = beats6.packetbeat; - }) + inherit (callPackages ../misc/logging/beats/6.x.nix { }) filebeat6 heartbeat6 metricbeat6 packetbeat6; + filebeat = filebeat6; + heartbeat = heartbeat6; + metricbeat = metricbeat6; + packetbeat = packetbeat6; + + inherit (callPackages ../misc/logging/beats/5.x.nix { }) + filebeat5 + heartbeat5 + metricbeat5 + packetbeat5; + bfr = callPackage ../tools/misc/bfr { }; bibtool = callPackage ../tools/misc/bibtool { }; From c9b41aa81f86e15169d948535002a4d7a10a8f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Aug 2018 15:03:53 +0100 Subject: [PATCH 15/16] nixos/elasticsearch: fix evaluation on systems without unfree set --- nixos/modules/services/search/elasticsearch.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index 2c7e21a20b5..6b688c640d5 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -5,7 +5,7 @@ with lib; let cfg = config.services.elasticsearch; - es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0; + es6 = builtins.compareVersions cfg.package.version "6" >= 0; esConfig = '' network.host: ${cfg.listenAddress} From 3f889f30bee92c3f2d5eaf53e56aef68f61efc4a Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Fri, 3 Aug 2018 15:39:49 +0200 Subject: [PATCH 16/16] nixos/doc: mention the ELK changes in the 18.09 release notes --- nixos/doc/manual/release-notes/rl-1809.xml | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 33026c5cbae..b6514f41c01 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -154,6 +154,39 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull' which indicates that the nix output hash will be used as tag. + + + The ELK stack: elasticsearch, logstash and kibana + has been upgraded from 2.* to 6.3.*. + The 2.* versions have been unsupported since last year + so they have been removed. You can still use the 5.* versions under the names + elasticsearch5, logstash5 and + kibana5. + + + The elastic beats: + filebeat, heartbeat, + metricbeat and packetbeat + have had the same treatment: they now target 6.3.* as well. + The 5.* versions are available under the names: + filebeat5, heartbeat5, + metricbeat5 and packetbeat5 + + + The ELK-6.3 stack now comes with + X-Pack by default. + Since X-Pack is licensed under the + Elastic License + the ELK packages now have an unfree license. To use them you need to specify + allowUnfree = true; in your nixpkgs configuration. + + + Fortunately there is also a free variant of the ELK stack without X-Pack. + The packages are available under the names: + elasticsearch-oss, logstash-oss and + kibana-oss. + +