nixpkgs/pkgs/applications/networking/cluster/spark/default.nix

57 lines
1.9 KiB
Nix
Raw Normal View History

2021-01-15 05:42:41 +00:00
{ lib, stdenv, fetchzip, makeWrapper, jre, pythonPackages, coreutils, hadoop
2017-03-26 16:17:17 +00:00
, RSupport? true, R
2014-12-04 16:27:01 +00:00
}:
2021-01-15 05:42:41 +00:00
with lib;
stdenv.mkDerivation rec {
2019-08-13 21:52:01 +00:00
pname = "spark";
version = "2.4.4";
2015-12-26 17:29:08 +00:00
src = fetchzip {
2019-08-13 21:52:01 +00:00
url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
sha256 = "1a9w5k0207fysgpxx6db3a00fs5hdc2ncx99x4ccy2s0v5ndc66g";
};
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ jre pythonPackages.python pythonPackages.numpy ]
++ optional RSupport R;
2019-08-13 21:52:01 +00:00
untarDir = "${pname}-${version}-bin-without-hadoop";
installPhase = ''
mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java}
2014-12-04 16:27:01 +00:00
mv * $out/lib/${untarDir}
sed -e 's/INFO, console/WARN, console/' < \
$out/lib/${untarDir}/conf/log4j.properties.template > \
$out/lib/${untarDir}/conf/log4j.properties
2014-12-04 16:27:01 +00:00
cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
export JAVA_HOME="${jre}"
export SPARK_HOME="$out/lib/${untarDir}"
export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath)
2014-12-04 16:27:01 +00:00
export PYSPARK_PYTHON="${pythonPackages.python}/bin/${pythonPackages.python.executable}"
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
2017-03-26 16:17:17 +00:00
${optionalString RSupport
''export SPARKR_R_SHELL="${R}/bin/R"
export PATH=$PATH:"${R}/bin/R"''}
EOF
2014-12-04 16:27:01 +00:00
for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
makeWrapper "$n" "$out/bin/$(basename $n)"
2018-03-21 00:57:58 +00:00
substituteInPlace "$n" --replace dirname ${coreutils.out}/bin/dirname
done
ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
'';
meta = {
2017-03-26 16:17:17 +00:00
description = "Apache Spark is a fast and general engine for large-scale data processing";
homepage = "http://spark.apache.org";
2021-01-15 05:42:41 +00:00
license = lib.licenses.asl20;
platforms = lib.platforms.all;
maintainers = with maintainers; [ thoughtpolice offline kamilchm ];
repositories.git = "git://git.apache.org/spark.git";
};
}