blob: bdcb0a8462591455741951d7d611cc90c80bb985 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
{ stdenv, fetchzip, makeWrapper, jre, pythonPackages
, mesosSupport ? true, mesos
, version
}:
let
versionMap = {
"1.6.0" = {
hadoopVersion = "cdh4";
sparkSha256 = "19ycx1r8g82vkvzmn9wxkssmv2damrg72yfmrgzpc6xyh071g91c";
};
"2.1.0" = {
hadoopVersion = "hadoop2.4";
sparkSha256 = "0pbsmbjwijsfgbnm56kgwnmnlqkz3w010ma0d7vzlkdklj40vqn2";
};
};
in
with versionMap.${version};
with stdenv.lib;
stdenv.mkDerivation rec {
name = "spark-${version}";
src = fetchzip {
url = "mirror://apache/spark/${name}/${name}-bin-${hadoopVersion}.tgz";
sha256 = sparkSha256;
};
buildInputs = [ makeWrapper jre pythonPackages.python pythonPackages.numpy ]
++ optional mesosSupport mesos;
untarDir = "${name}-bin-${hadoopVersion}";
installPhase = ''
mkdir -p $out/{lib/${untarDir}/conf,bin,/share/java}
mv * $out/lib/${untarDir}
sed -e 's/INFO, console/WARN, console/' < \
$out/lib/${untarDir}/conf/log4j.properties.template > \
$out/lib/${untarDir}/conf/log4j.properties
cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
export JAVA_HOME="${jre}"
export SPARK_HOME="$out/lib/${untarDir}"
export PYSPARK_PYTHON="${pythonPackages.python}/bin/${pythonPackages.python.executable}"
export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
${optionalString mesosSupport
''export MESOS_NATIVE_LIBRARY="$MESOS_NATIVE_LIBRARY"''}
EOF
for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
makeWrapper "$n" "$out/bin/$(basename $n)"
done
ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
'';
meta = {
description = "Lightning-fast cluster computing";
homepage = "http://spark.apache.org";
license = stdenv.lib.licenses.asl20;
platforms = stdenv.lib.platforms.all;
maintainers = with maintainers; [ thoughtpolice offline ];
repositories.git = git://git.apache.org/spark.git;
};
}
|