elk: 6.2.4 -> 6.3.2

* The ELK stack is upgraded to 6.3.2.

* `elasticsearch6`, `logstash6` and `kibana6` now come with X-Pack which is
  a suite of additional features. These are however licensed under the unfree
  "Elastic License".

* Fortunately they also provide OSS versions which are now packaged
  under: `elasticsearch6-oss`, `logstash6-oss` and `kibana6-oss`.
  Note that the naming of the attributes is consistent with upstream.

* The test `nix-build nixos/tests/elk.nix -A ELK-6` will test the OSS
  version by default. You can also run the test on the unfree ELK using:
  `NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true`
This commit is contained in:
Bas van Dijk 2018-06-24 13:22:12 +02:00
parent 28e11a0b6b
commit ebcdb822f8
8 changed files with 145 additions and 37 deletions

View file

@ -231,6 +231,12 @@ lib.mapAttrs (n: v: v // { shortName = n; }) rec {
fullName = "Eiffel Forum License v2.0";
};
elastic = {
fullName = "ELASTIC LICENSE";
url = https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt;
free = false;
};
epl10 = spdx {
spdxId = "EPL-1.0";
fullName = "Eclipse Public License 1.0";

View file

@ -25,18 +25,17 @@ let
${cfg.extraConf}
'';
configDir = pkgs.buildEnv {
name = "elasticsearch-config";
paths = [
(pkgs.writeTextDir "elasticsearch.yml" esConfig)
(if es5 then (pkgs.writeTextDir "log4j2.properties" cfg.logging)
else (pkgs.writeTextDir "logging.yml" cfg.logging))
];
postBuild = concatStringsSep "\n" (concatLists [
# Elasticsearch 5.x won't start when the scripts directory does not exist
(optional es5 "${pkgs.coreutils}/bin/mkdir -p $out/scripts")
(optional es6 "ln -s ${cfg.package}/config/jvm.options $out/jvm.options")
]);
configDir = cfg.dataDir + "/config";
elasticsearchYml = pkgs.writeTextFile {
name = "elasticsearch.yml";
text = esConfig;
};
loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml";
loggingConfigFile = pkgs.writeTextFile {
name = loggingConfigFilename;
text = cfg.logging;
};
esPlugins = pkgs.buildEnv {
@ -193,7 +192,24 @@ in {
ln -sfT ${esPlugins}/plugins ${cfg.dataDir}/plugins
ln -sfT ${cfg.package}/lib ${cfg.dataDir}/lib
ln -sfT ${cfg.package}/modules ${cfg.dataDir}/modules
if [ "$(id -u)" = 0 ]; then chown -R elasticsearch ${cfg.dataDir}; fi
# elasticsearch needs to create the elasticsearch.keystore in the config directory
# so this directory needs to be writable.
mkdir -m 0700 -p ${configDir}
# Note that we copy config files from the nix store instead of symbolically linking them
# because otherwise X-Pack Security will raise the following exception:
# java.security.AccessControlException:
# access denied ("java.io.FilePermission" "/var/lib/elasticsearch/config/elasticsearch.yml" "read")
cp ${elasticsearchYml} ${configDir}/elasticsearch.yml
# Make sure the logging configuration for old elasticsearch versions is removed:
rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"}
cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename}
${optionalString es5 "mkdir -p ${configDir}/scripts"}
${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"}
if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi
'';
};

View file

@ -1,4 +1,4 @@
{ system ? builtins.currentSystem }:
{ system ? builtins.currentSystem, enableUnfree ? false }:
with import ../lib/testing.nix { inherit system; };
with pkgs.lib;
let
@ -99,9 +99,16 @@ in mapAttrs mkElkTest {
logstash = pkgs.logstash5;
kibana = pkgs.kibana5;
};
"ELK-6" = {
elasticsearch = pkgs.elasticsearch6;
logstash = pkgs.logstash6;
kibana = pkgs.kibana6;
};
"ELK-6" =
if enableUnfree
then {
elasticsearch = pkgs.elasticsearch6;
logstash = pkgs.logstash6;
kibana = pkgs.kibana6;
}
else {
elasticsearch = pkgs.elasticsearch6-oss;
logstash = pkgs.logstash6-oss;
kibana = pkgs.kibana6-oss;
};
}

View file

@ -1,4 +1,13 @@
{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }:
{ elk6Version
, enableUnfree ? true
, stdenv
, makeWrapper
, fetchzip
, fetchurl
, nodejs
, coreutils
, which
}:
with stdenv.lib;
let
@ -6,12 +15,31 @@ let
info = splitString "-" stdenv.system;
arch = elemAt info 0;
plat = elemAt info 1;
shas = {
"x86_64-linux" = "1br9nvwa3i5sfcbnrxp2x3dxxnsbs9iavz6zwgw0jlh5ngf5vysk";
"x86_64-darwin" = "1w4dck02i0rrl8m18kvy2zz02cb7bb9a2pdhkd1jfy1qz4ssnhii";
shas =
if enableUnfree
then {
"x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0";
"x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3";
}
else {
"x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly";
"x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b";
};
# For the correct phantomjs version see:
# https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js
phantomjs = rec {
name = "phantomjs-${version}-linux-x86_64";
version = "2.1.1";
src = fetchzip {
inherit name;
url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2";
sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv";
};
};
in stdenv.mkDerivation rec {
name = "kibana-${version}";
name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}";
version = elk6Version;
src = fetchurl {
@ -28,12 +56,19 @@ in stdenv.mkDerivation rec {
makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
--prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
'' +
# phantomjs is needed in the unfree version. When phantomjs doesn't exist in
# $out/libexec/kibana/data kibana will try to download and unpack it during
# runtime which will fail because the nix store is read-only. So we make sure
# it already exist in the nix store.
optionalString enableUnfree ''
ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name}
'';
meta = {
description = "Visualize logs and time-stamped data";
homepage = http://www.elasticsearch.org/overview/kibana;
license = licenses.asl20;
license = if enableUnfree then licenses.elastic else licenses.asl20;
maintainers = with maintainers; [ offline rickynils basvandijk ];
platforms = with platforms; unix;
};

View file

@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "194z3j9zwlbc6j97iy1m1cl0xqks3ws2bjp2xrgy8cwpi7fclaw2";
sha256 = "0ymg6y6v0mdhs1rs11fn33xdp3r6v85563z0f4p7s22j1kd3nd6r";
};
goPackagePath = "github.com/elastic/beats";

View file

@ -1,14 +1,26 @@
{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux }:
{ elk6Version
, enableUnfree ? true
, stdenv
, fetchurl
, makeWrapper
, jre_headless
, utillinux
, autoPatchelfHook
, zlib
}:
with stdenv.lib;
stdenv.mkDerivation rec {
stdenv.mkDerivation (rec {
version = elk6Version;
name = "elasticsearch-${version}";
name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
sha256 = "13hf00khq33yw6zv022vcrsf6vm43isx40x7ww8r1lqx3vmg3rli";
sha256 =
if enableUnfree
then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh"
else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4";
};
patches = [ ./es-home-6.x.patch ];
@ -32,10 +44,22 @@ stdenv.mkDerivation rec {
wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
'';
passthru = { inherit enableUnfree; };
meta = {
description = "Open Source, Distributed, RESTful Search Engine";
license = licenses.asl20;
license = if enableUnfree then licenses.elastic else licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ apeschar basvandijk ];
};
}
} // optionalAttrs enableUnfree {
dontPatchELF = true;
nativeBuildInputs = [ autoPatchelfHook ];
runtimeDependencies = [ zlib ];
postFixup = ''
for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do
echo "patching $exe..."
patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe"
done
'';
})

View file

@ -1,12 +1,23 @@
{ stdenv, fetchurl, elk6Version, makeWrapper, jre }:
{ elk6Version
, enableUnfree ? true
, stdenv
, fetchurl
, makeWrapper
, jre
}:
with stdenv.lib;
stdenv.mkDerivation rec {
version = elk6Version;
name = "logstash-${version}";
name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
sha256 = "07j3jjg5ik4gjgvcx15qqqas9p1m3815jml82a5r1ip9l6vc4h20";
sha256 =
if enableUnfree
then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr"
else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w";
};
dontBuild = true;
@ -35,7 +46,7 @@ stdenv.mkDerivation rec {
meta = with stdenv.lib; {
description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
homepage = https://www.elastic.co/products/logstash;
license = licenses.asl20;
license = if enableUnfree then licenses.elastic else licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ wjlroe offline basvandijk ];
};

View file

@ -2254,12 +2254,15 @@ with pkgs;
# The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
elk5Version = "5.6.9";
elk6Version = "6.2.4";
elk6Version = "6.3.2";
elasticsearch = callPackage ../servers/search/elasticsearch { };
elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix {
enableUnfree = false;
};
elasticsearchPlugins = recurseIntoAttrs (
callPackage ../servers/search/elasticsearch/plugins.nix { }
@ -3364,6 +3367,9 @@ with pkgs;
kibana = callPackage ../development/tools/misc/kibana { };
kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
enableUnfree = false;
};
kismet = callPackage ../applications/networking/sniffers/kismet { };
@ -3443,6 +3449,9 @@ with pkgs;
logstash = callPackage ../tools/misc/logstash { };
logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
enableUnfree = false;
};
logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };