From 0cce8856b2009d919f9372bcd2fce3019ff525d3 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:06:36 +0100 Subject: [PATCH 1/7] elk-5: Remove tests These packages are EOL and are about to be removed --- nixos/tests/elk.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 95371ef4443..7837fbd2833 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -178,12 +178,6 @@ let ''; }; in mapAttrs mkElkTest { - "ELK-5" = { - elasticsearch = pkgs.elasticsearch5; - logstash = pkgs.logstash5; - kibana = pkgs.kibana5; - journalbeat = pkgs.journalbeat5; - }; "ELK-6" = if enableUnfree then { From a870cad373db675783d0d32d328a209ac12c41b7 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:02:04 +0100 Subject: [PATCH 2/7] elasticsearch5: Remove package It's EOL --- pkgs/servers/search/elasticsearch/5.x.nix | 44 ------------------- .../elasticsearch/es-classpath-5.x.patch | 43 ------------------ .../search/elasticsearch/es-home-5.x.patch | 31 ------------- pkgs/top-level/all-packages.nix | 6 --- 4 files changed, 124 deletions(-) delete mode 100644 pkgs/servers/search/elasticsearch/5.x.nix delete mode 100644 pkgs/servers/search/elasticsearch/es-classpath-5.x.patch delete mode 100644 pkgs/servers/search/elasticsearch/es-home-5.x.patch diff --git a/pkgs/servers/search/elasticsearch/5.x.nix b/pkgs/servers/search/elasticsearch/5.x.nix deleted file mode 100644 index c64687054ab..00000000000 --- a/pkgs/servers/search/elasticsearch/5.x.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ stdenv, fetchurl, elk5Version, makeWrapper, jre_headless -, utillinux, gnugrep, coreutils }: - -with stdenv.lib; - -stdenv.mkDerivation rec { - version = elk5Version; - pname = "elasticsearch"; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz"; - sha256 = "0zy7awb2cm2fk3c7zc7v8b8pl0jw49awqwpa1jvilmvx6dcml0vb"; - }; - - patches = [ ./es-home-5.x.patch ./es-classpath-5.x.patch ]; - - buildInputs = [ makeWrapper jre_headless utillinux ]; - - installPhase = '' - mkdir -p $out - cp -R bin config lib modules plugins $out - - chmod -x $out/bin/*.* - - wrapProgram $out/bin/elasticsearch \ - --prefix ES_CLASSPATH : "$out/lib/*" \ - --prefix PATH : "${makeBinPath [ utillinux gnugrep coreutils ]}" \ - --set JAVA_HOME "${jre_headless}" \ - --set ES_JVM_OPTIONS "$out/config/jvm.options" - - wrapProgram $out/bin/elasticsearch-plugin \ - --prefix ES_CLASSPATH : "$out/lib/*" \ - --set JAVA_HOME "${jre_headless}" - ''; - - meta = { - description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; - platforms = platforms.unix; - maintainers = [ - maintainers.apeschar - ]; - }; -} diff --git a/pkgs/servers/search/elasticsearch/es-classpath-5.x.patch b/pkgs/servers/search/elasticsearch/es-classpath-5.x.patch deleted file mode 100644 index ccdca1e53af..00000000000 --- a/pkgs/servers/search/elasticsearch/es-classpath-5.x.patch +++ /dev/null @@ -1,43 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2017-05-17 10:53:49.444487071 +0200 -+++ b/bin/elasticsearch 2017-05-17 10:55:52.755081523 +0200 -@@ -129,12 +129,7 @@ ES_JAVA_OPTS="$(parse_jvm_options "$ES_J - # If an include wasn't specified in the environment, then search for one... - if [ "x$ES_INCLUDE" = "x" ]; then - # Locations (in order) to use when searching for an include file. -- for include in /usr/share/elasticsearch/elasticsearch.in.sh \ -- /usr/local/share/elasticsearch/elasticsearch.in.sh \ -- /opt/elasticsearch/elasticsearch.in.sh \ -- ~/.elasticsearch.in.sh \ -- "$ES_HOME/bin/elasticsearch.in.sh" \ -- "`dirname "$0"`"/elasticsearch.in.sh; do -+ for include in "`dirname "$0"`"/elasticsearch.in.sh; do - if [ -r "$include" ]; then - . "$include" - break -diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh ---- a/bin/elasticsearch.in.sh 2017-04-28 19:41:47.000000000 +0200 -+++ b/bin/elasticsearch.in.sh 2017-05-17 10:56:49.303519788 +0200 -@@ -1,13 +1 @@ - #!/bin/bash -- --# check in case a user was using this mechanism --if [ "x$ES_CLASSPATH" != "x" ]; then -- cat >&2 << EOF --Error: Don't modify the classpath with ES_CLASSPATH. Best is to add --additional elements via the plugin mechanism, or if code must really be --added to the main classpath, add jars to lib/ (unsupported). --EOF -- exit 1 --fi -- --ES_CLASSPATH="$ES_HOME/lib/*" -diff -rupN a/bin/elasticsearch-plugin b/bin/elasticsearch-plugin ---- a/bin/elasticsearch-plugin 2018-04-13 01:21:55.000000000 +0900 -+++ b/bin/elasticsearch-plugin 2018-06-28 19:08:54.700969245 +0900 -@@ -88,4 +88,4 @@ if [ -e "$CONF_DIR" ]; then - path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR") - fi - --exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "${args[@]}" -+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_CLASSPATH" org.elasticsearch.plugins.PluginCli "${args[@]}" diff --git a/pkgs/servers/search/elasticsearch/es-home-5.x.patch b/pkgs/servers/search/elasticsearch/es-home-5.x.patch deleted file mode 100644 index cee0137312f..00000000000 --- a/pkgs/servers/search/elasticsearch/es-home-5.x.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff -rupN a/bin/elasticsearch b/bin/elasticsearch ---- a/bin/elasticsearch 2017-05-17 10:53:42.214686741 +0200 -+++ b/bin/elasticsearch 2017-05-17 10:53:49.444487071 +0200 -@@ -105,7 +105,11 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+ -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` -diff -rupN a/bin/elasticsearch-plugin b/bin/elasticsearch-plugin ---- a/bin/elasticsearch-plugin 2017-05-17 10:53:42.214686741 +0200 -+++ b/bin/elasticsearch-plugin 2017-05-17 10:53:49.445487044 +0200 -@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do - done - - # determine elasticsearch home --ES_HOME=`dirname "$SCRIPT"`/.. -+if [ -z "$ES_HOME" ]; then -+ echo "You must set the ES_HOME var" >&2 -+ exit 1 -+fi - - # make ELASTICSEARCH_HOME absolute - ES_HOME=`cd "$ES_HOME"; pwd` diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4a95ad0a392..4afd1054b52 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2895,9 +2895,6 @@ in elk6Version = "6.7.2"; elk7Version = "7.0.1"; - elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { - utillinux = utillinuxMinimal; - }; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { utillinux = utillinuxMinimal; }; @@ -2920,9 +2917,6 @@ in elasticsearch = elasticsearch-oss; } ); - elasticsearch5Plugins = elasticsearchPlugins.override { - elasticsearch = elasticsearch5; - }; elasticsearch6Plugins = elasticsearchPlugins.override { elasticsearch = elasticsearch6-oss; }; From 1b64fd7f53bd655651b5c1ad881f5392b1b49073 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:02:26 +0100 Subject: [PATCH 3/7] journalbeat5: Remove EOL package --- pkgs/tools/system/journalbeat/default.nix | 28 ----------------------- pkgs/top-level/all-packages.nix | 2 -- 2 files changed, 30 deletions(-) delete mode 100644 pkgs/tools/system/journalbeat/default.nix diff --git a/pkgs/tools/system/journalbeat/default.nix b/pkgs/tools/system/journalbeat/default.nix deleted file mode 100644 index 0a04581264e..00000000000 --- a/pkgs/tools/system/journalbeat/default.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ lib, systemd, buildGoPackage, fetchFromGitHub }: - -buildGoPackage rec { - pname = "journalbeat"; - version = "5.6.8"; - - src = fetchFromGitHub { - owner = "mheese"; - repo = "journalbeat"; - rev = "v${version}"; - sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a"; - }; - - goPackagePath = "github.com/mheese/journalbeat"; - - buildInputs = [ systemd.dev ]; - - postFixup = let libPath = lib.makeLibraryPath [ systemd.lib ]; in '' - patchelf --set-rpath ${libPath} "$bin/bin/journalbeat" - ''; - - meta = with lib; { - homepage = https://github.com/mheese/journalbeat; - description = "Journalbeat is a log shipper from systemd/journald to Logstash/Elasticsearch"; - license = licenses.asl20; - maintainers = with maintainers; [ mbrgm ]; - }; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4afd1054b52..0d7366865cb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1076,8 +1076,6 @@ in metricbeat5 packetbeat5; - journalbeat5 = callPackage ../tools/system/journalbeat { }; - inherit (callPackages ../misc/logging/beats/6.x.nix { # XXX: this is failing with Go 1.12. Error is related to cgo, an # update to this package might fix it. From 4e30b30a6540e7d8a324b6d5a172520ff5f6f2ba Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:02:56 +0100 Subject: [PATCH 4/7] logstash5: Remove EOL package --- nixos/modules/services/logging/logstash.nix | 2 +- pkgs/tools/misc/logstash/5.x.nix | 39 --------------------- pkgs/top-level/all-packages.nix | 2 -- 3 files changed, 1 insertion(+), 42 deletions(-) delete mode 100644 pkgs/tools/misc/logstash/5.x.nix diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix index 9b707e9deb5..4943e8d7db3 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/nixos/modules/services/logging/logstash.nix @@ -53,7 +53,7 @@ in type = types.package; default = pkgs.logstash; defaultText = "pkgs.logstash"; - example = literalExample "pkgs.logstash5"; + example = literalExample "pkgs.logstash"; description = "Logstash package to use."; }; diff --git a/pkgs/tools/misc/logstash/5.x.nix b/pkgs/tools/misc/logstash/5.x.nix deleted file mode 100644 index 2545dd1e6bc..00000000000 --- a/pkgs/tools/misc/logstash/5.x.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ stdenv, fetchurl, elk5Version, makeWrapper, jre }: - -stdenv.mkDerivation rec { - version = elk5Version; - pname = "logstash"; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz"; - sha256 = "0sax9p2bwjdrcvkm1mgvljdjn2qkyjd5i8rzajdn3n98gqin1la0"; - }; - - dontBuild = true; - dontPatchELF = true; - dontStrip = true; - dontPatchShebangs = true; - - buildInputs = [ - makeWrapper jre - ]; - - installPhase = '' - mkdir -p $out - cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out - - wrapProgram $out/bin/logstash \ - --set JAVA_HOME "${jre}" - - wrapProgram $out/bin/logstash-plugin \ - --set JAVA_HOME "${jre}" - ''; - - meta = with stdenv.lib; { - description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; - homepage = https://www.elastic.co/products/logstash; - license = licenses.asl20; - platforms = platforms.unix; - maintainers = [ maintainers.wjlroe maintainers.offline ]; - }; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 0d7366865cb..d2610687d2c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4207,7 +4207,6 @@ in keyfuzz = callPackage ../tools/inputmethods/keyfuzz { }; - kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix { enableUnfree = false; @@ -4309,7 +4308,6 @@ in lockfileProgs = callPackage ../tools/misc/lockfile-progs { }; - logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix { enableUnfree = false; From bded2f69d325463569f9b2b16311be46c5ef833e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:30:24 +0100 Subject: [PATCH 5/7] beats-5.*: Drop EOL packages --- pkgs/misc/logging/beats/5.x.nix | 42 --------------------------------- pkgs/top-level/all-packages.nix | 10 -------- 2 files changed, 52 deletions(-) delete mode 100644 pkgs/misc/logging/beats/5.x.nix diff --git a/pkgs/misc/logging/beats/5.x.nix b/pkgs/misc/logging/beats/5.x.nix deleted file mode 100644 index fb8bb5a85c8..00000000000 --- a/pkgs/misc/logging/beats/5.x.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ stdenv, fetchFromGitHub, elk5Version, buildGoPackage, libpcap }: - -let beat = package : extraArgs : buildGoPackage (rec { - name = "${package}-${version}"; - version = elk5Version; - - src = fetchFromGitHub { - owner = "elastic"; - repo = "beats"; - rev = "v${version}"; - sha256 = "0sslcwjdf7zb9xj2c98jid3rb09l96m22k3af48gppimxg1lkh9b"; - }; - - goPackagePath = "github.com/elastic/beats"; - - subPackages = [ package ]; - - meta = with stdenv.lib; { - homepage = https://www.elastic.co/products/beats; - license = licenses.asl20; - maintainers = with maintainers; [ fadenb basvandijk ]; - platforms = platforms.linux; - }; - } // extraArgs); -in { - filebeat5 = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; - heartbeat5 = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; - metricbeat5 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; - packetbeat5 = beat "packetbeat" { - buildInputs = [ libpcap ]; - meta.description = "Network packet analyzer that ships data to Elasticsearch"; - meta.longDescription = '' - Packetbeat is an open source network packet analyzer that ships the - data to Elasticsearch. - - Think of it like a distributed real-time Wireshark with a lot more - analytics features. The Packetbeat shippers sniff the traffic between - your application processes, parse on the fly protocols like HTTP, MySQL, - PostgreSQL, Redis or Thrift and correlate the messages into transactions. - ''; - }; -} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index d2610687d2c..854e3fd7db8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1066,16 +1066,6 @@ in bchunk = callPackage ../tools/cd-dvd/bchunk { }; - inherit (callPackages ../misc/logging/beats/5.x.nix { - # XXX: this is failing with Go 1.12. Error is related to cgo, an - # update to this package might fix it. - buildGoPackage = buildGo111Package; - }) - filebeat5 - heartbeat5 - metricbeat5 - packetbeat5; - inherit (callPackages ../misc/logging/beats/6.x.nix { # XXX: this is failing with Go 1.12. Error is related to cgo, an # update to this package might fix it. From a3032415f70bd865885425efec8e6bf623200915 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:04:29 +0100 Subject: [PATCH 6/7] kibana5: Remove EOL package --- nixos/modules/services/search/kibana.nix | 2 +- pkgs/development/tools/misc/kibana/5.x.nix | 45 ---------------------- 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 pkgs/development/tools/misc/kibana/5.x.nix diff --git a/nixos/modules/services/search/kibana.nix b/nixos/modules/services/search/kibana.nix index c096af731ad..75871a9df94 100644 --- a/nixos/modules/services/search/kibana.nix +++ b/nixos/modules/services/search/kibana.nix @@ -150,7 +150,7 @@ in { description = "Kibana package to use"; default = pkgs.kibana; defaultText = "pkgs.kibana"; - example = "pkgs.kibana5"; + example = "pkgs.kibana"; type = types.package; }; diff --git a/pkgs/development/tools/misc/kibana/5.x.nix b/pkgs/development/tools/misc/kibana/5.x.nix deleted file mode 100644 index 7d824fd81fb..00000000000 --- a/pkgs/development/tools/misc/kibana/5.x.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ stdenv, makeWrapper, fetchurl, elk5Version, nodejs, coreutils, which }: - -with stdenv.lib; -let - inherit (builtins) elemAt; - archOverrides = { - "i686" = "x86"; - }; - info = splitString "-" stdenv.hostPlatform.system; - arch = (elemAt info 0); - elasticArch = archOverrides."${arch}" or arch; - plat = elemAt info 1; - shas = { - "x86_64-linux" = "0hzr47hyw54b9j4c33n6f6n3pala6kjhyvinfszgikbghyhb7fsa"; - "i686-linux" = "0bka4h31cw10ii4pfygc81pwc3wr32pzw3v4k4bi8rnqbk280fmn"; - "x86_64-darwin" = "0jqc2g89rqkla0alqxr14sh4pccfn514jrwr7mkjivxdapygh1ll"; - }; -in stdenv.mkDerivation rec { - pname = "kibana"; - version = elk5Version; - - src = fetchurl { - url = "https://artifacts.elastic.co/downloads/kibana/${pname}-${version}-${plat}-${elasticArch}.tar.gz"; - sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture"); - }; - - buildInputs = [ makeWrapper ]; - - installPhase = '' - mkdir -p $out/libexec/kibana $out/bin - mv * $out/libexec/kibana/ - rm -r $out/libexec/kibana/node - makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ - --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" - sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana - ''; - - meta = { - description = "Visualize logs and time-stamped data"; - homepage = http://www.elasticsearch.org/overview/kibana; - license = licenses.asl20; - maintainers = with maintainers; [ offline ]; - platforms = with platforms; unix; - }; -} From 194aac9eed2b8c4fe6b300a869dc106f65542fd4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 6 Sep 2019 14:07:38 +0100 Subject: [PATCH 7/7] elk-stack: Add release note about elk-5 stack removal --- nixos/doc/manual/release-notes/rl-1909.xml | 10 ++++++++++ pkgs/top-level/all-packages.nix | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml index 066b322c96f..ee6ef883068 100644 --- a/nixos/doc/manual/release-notes/rl-1909.xml +++ b/nixos/doc/manual/release-notes/rl-1909.xml @@ -428,6 +428,16 @@ any updates from upstream and depends on outdated Python2-based modules. + + + Old unsupported versions (logstash5, + kibana5, + filebeat5, + heartbeat5, + metricbeat5, + packetbeat5) of the ELK-stack and Elastic beats have been removed. + + diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 854e3fd7db8..99b347a33c4 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2879,7 +2879,6 @@ in evemu = callPackage ../tools/system/evemu { }; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. - elk5Version = "5.6.16"; elk6Version = "6.7.2"; elk7Version = "7.0.1";