From 3cdb8f581dbb85a0cb9552c0d84c44f736aecc63 Mon Sep 17 00:00:00 2001 From: Kamil Chmielewski Date: Wed, 18 Oct 2017 16:05:07 +0200 Subject: [PATCH] spark: provided-hadoop -> nixpkgs hadoop spark should use nixpkgs provided hadoop to make it consistent for bigger Nix hadoop + spark setup --- .../networking/cluster/spark/default.nix | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pkgs/applications/networking/cluster/spark/default.nix b/pkgs/applications/networking/cluster/spark/default.nix index 2b36d7c5647..6401194eac2 100644 --- a/pkgs/applications/networking/cluster/spark/default.nix +++ b/pkgs/applications/networking/cluster/spark/default.nix @@ -1,20 +1,16 @@ -{ stdenv, fetchzip, makeWrapper, jre, pythonPackages, coreutils +{ stdenv, fetchzip, makeWrapper, jre, pythonPackages, coreutils, hadoop , RSupport? true, R , mesosSupport ? true, mesos , version }: let - versionMap = { - "2.2.1" = { - hadoopVersion = "hadoop2.7"; - sparkSha256 = "10nxsf9a6hj1263sxv0cbdqxdb8mb4cl6iqq32ljq9ydvk32s99c"; - }; - }; + sha256 = { + "1.6.3" = "142hw73wf20d846l83ydx0yg7qj5qxywm4h7qrhwnd7lsy2sbnjf"; + "2.2.1" = "10nxsf9a6hj1263sxv0cbdqxdb8mb4cl6iqq32ljq9ydvk32s99c"; + }.${version}; in -with versionMap.${version}; - with stdenv.lib; stdenv.mkDerivation rec { @@ -22,15 +18,15 @@ stdenv.mkDerivation rec { name = "spark-${version}"; src = fetchzip { - url = "mirror://apache/spark/${name}/${name}-bin-${hadoopVersion}.tgz"; - sha256 = sparkSha256; + inherit sha256; + url = "mirror://apache/spark/${name}/${name}-bin-without-hadoop.tgz"; }; buildInputs = [ makeWrapper jre pythonPackages.python pythonPackages.numpy ] ++ optional RSupport R ++ optional mesosSupport mesos; - untarDir = "${name}-bin-${hadoopVersion}"; + untarDir = "${name}-bin-without-hadoop"; installPhase = '' mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java} mv * $out/lib/${untarDir} @@ -42,6 +38,7 @@ stdenv.mkDerivation rec { cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF export JAVA_HOME="${jre}" export SPARK_HOME="$out/lib/${untarDir}" + export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath) export PYSPARK_PYTHON="${pythonPackages.python}/bin/${pythonPackages.python.executable}" export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" ${optionalString RSupport @@ -63,7 +60,7 @@ stdenv.mkDerivation rec { homepage = "http://spark.apache.org"; license = stdenv.lib.licenses.asl20; platforms = stdenv.lib.platforms.all; - maintainers = with maintainers; [ thoughtpolice offline ]; + maintainers = with maintainers; [ thoughtpolice offline kamilchm ]; repositories.git = git://git.apache.org/spark.git; }; }