libtensorflow: add binary build and add automatic generation

This commit is contained in:
Nikolay Amiantov 2019-07-13 22:38:41 +03:00
parent 170dd558df
commit 0a1bf4734f
5 changed files with 113 additions and 90 deletions

View file

@ -1,88 +0,0 @@
{ stdenv
, fetchurl, python3, which
, patchelf, fetchFromGitHub, buildBazelPackage
, cudatoolkit, cudnn, nccl, nvidia_x11, symlinkJoin
, binutils, gcc, binutils-unwrapped, gcc-unwrapped
, glibcLocales
, features ? ["sse4.2" "avx" "avx2" "fma"]
, arch ? "x86-64"
, cudaSupport ? false
, computeCapabilities ? [] # example: [ "7.5" ], see https://developer.nvidia.com/cuda-gpus
}:
let
inherit (stdenv) lib;
cudatoolkit_joined = symlinkJoin {
name = "unsplit_cudatoolkit";
paths = [ cudatoolkit.out
cudatoolkit.lib ];};
gcc_joined = symlinkJoin {
name = "gcc-joined";
paths = [ binutils gcc binutils-unwrapped gcc-unwrapped ];
};
in
buildBazelPackage rec {
pname = "tensorflow";
version = "1.14.0";
name = "${pname}-${version}";
bazelFlags = [ "--incompatible_no_support_tools_in_action_inputs=false" ];
bazelTarget = "//tensorflow/tools/lib_package:libtensorflow";
fetchAttrs = {
sha256 = if cudaSupport then
"127xxwy3a2h1qsv2sqfhrh65g69hlb1q003vyyg7yjfqgfah9p2z"
else "1di1pnknr1hxdpn75lxf9c6dvb5kgllmgb9r9rgh5c2g9iil17zy";
};
CC_OPT_FLAGS = "-march=${arch} " + stdenv.lib.concatMapStringsSep " " (f: "-m"+f) features;
NIX_CFLAGS_COMPILE = CC_OPT_FLAGS;
TF_NEED_CUDA = if cudaSupport then "1" else "0";
TF_IGNORE_MAX_BAZEL_VERSION = "1";
TF_CUDA_COMPUTE_CAPABILITIES = stdenv.lib.concatStringsSep "," computeCapabilities;
nativeBuildInputs = [ glibcLocales python3 which ]
++ (lib.optionals cudaSupport [
cudatoolkit_joined cudnn nvidia_x11
]);
TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn}";
src = fetchFromGitHub {
repo = "tensorflow";
owner = "tensorflow";
rev = "v${version}";
sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi";
};
prePatch = ''
# doesn't work:
sed -i '/saved_model_portable_proto/d' tensorflow/cc/saved_model/BUILD
# calls ldconfig -p:
sed -i 's/+ _get_ld_config_paths()//' third_party/gpus/find_cuda_config.py
patchShebangs ./configure
patchShebangs tensorflow/tools/lib_package/concat_licenses.sh
patchShebangs third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
'';
GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${gcc_joined}/bin";
configurePhase = ''
runHook preConfigure
./configure
runHook postConfigure
'';
buildAttrs = {
installPhase = ''
mkdir -p $out
tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C $out
# Write pkgconfig file.
mkdir $out/lib/pkgconfig
cat > $out/lib/pkgconfig/tensorflow.pc << EOF
Name: TensorFlow
Version: ${version}
Description: Library for computation using data flow graphs for scalable machine learning
Requires:
Libs: -L$out/lib -ltensorflow
Cflags: -I$out/include/tensorflow
EOF
'';
};
meta = with lib; {
description = "C API for TensorFlow";
homepage = https://www.tensorflow.org/versions/master/install/install_c;
license = licenses.asl20;
platforms = with platforms; linux ++ darwin;
maintainers = with maintainers; [ basvandijk yorickvp ];
};
}

View file

@ -0,0 +1,72 @@
{ stdenv
, fetchurl
, patchelf
, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
}:
with stdenv.lib;
let
unavailable = throw "libtensorflow is not available for this platform!";
tfType = if cudaSupport then "gpu" else "cpu";
system =
if stdenv.isLinux then "linux"
else if stdenv.isDarwin then "darwin"
else unavailable;
platform =
if stdenv.isx86_64 then "x86_64"
else unavailable;
rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
packages = import ./binary-hashes.nix;
packageName = "${tfType}-${system}-${platform}";
url = packages.${packageName} or unavailable;
patchLibs =
if stdenv.isDarwin
then ''
install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib
install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib
''
else ''
patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
'';
in stdenv.mkDerivation rec {
pname = "libtensorflow";
inherit (packages) version;
src = fetchurl url;
# Patch library to use our libc, libstdc++ and others
buildCommand = ''
mkdir -pv $out
tar -C $out -xzf $src
chmod -R +w $out
${patchLibs}
# Write pkgconfig file.
mkdir $out/lib/pkgconfig
cat > $out/lib/pkgconfig/tensorflow.pc << EOF
Name: TensorFlow
Version: ${version}
Description: Library for computation using data flow graphs for scalable machine learning
Requires:
Libs: -L$out/lib -ltensorflow
Cflags: -I$out/include/tensorflow
EOF
'';
meta = {
description = "C API for TensorFlow";
homepage = https://www.tensorflow.org/install/lang_c;
license = licenses.asl20;
platforms = [ "x86_64-linux" "x86_64-darwin" ];
maintainers = with maintainers; [ basvandijk ];
};
}

View file

@ -0,0 +1,15 @@
{
version = "1.14.0";
"cpu-linux-x86_64" = {
url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz";
sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k";
};
"gpu-linux-x86_64" = {
url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz";
sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6";
};
"cpu-darwin-x86_64" = {
url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz";
sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92";
};
}

View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
version=1.14.0
hashfile=binary-hashes.nix
rm -f $hashfile
echo "{" >> $hashfile
echo "version = \"$version\";" >> $hashfile
for sys in "linux" "darwin"; do
for tfpref in "cpu" "gpu"; do
for platform in "x86_64"; do
if [ $sys = "darwin" ] && [ $tfpref = "gpu" ]; then
continue
fi
url=https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-$tfpref-$sys-$platform-$version.tar.gz
hash=$(nix-prefetch-url $url)
echo "\"${tfpref}-${sys}-${platform}\" = {" >> $hashfile
echo " url = \"$url\";" >> $hashfile
echo " sha256 = \"$hash\";" >> $hashfile
echo "};" >> $hashfile
done
done
done
echo "}" >> $hashfile

View file

@ -1093,11 +1093,11 @@ in
tensor = libsForQt5.callPackage ../applications/networking/instant-messengers/tensor { };
libtensorflow = callPackage ../development/libraries/libtensorflow {
libtensorflow = callPackage ../development/libraries/science/math/tensorflow/bin.nix {
cudaSupport = pkgs.config.cudaSupport or false;
inherit (linuxPackages) nvidia_x11;
cudatoolkit = cudatoolkit_10_0;
cudnn = cudnn_cudatoolkit_10_0;
nccl = nccl_cudatoolkit_10;
};
behdad-fonts = callPackage ../data/fonts/behdad-fonts { };