diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix deleted file mode 100644 index e4f6a868e9f..00000000000 --- a/pkgs/development/libraries/libtensorflow/default.nix +++ /dev/null @@ -1,88 +0,0 @@ -{ stdenv -, fetchurl, python3, which -, patchelf, fetchFromGitHub, buildBazelPackage -, cudatoolkit, cudnn, nccl, nvidia_x11, symlinkJoin -, binutils, gcc, binutils-unwrapped, gcc-unwrapped -, glibcLocales -, features ? ["sse4.2" "avx" "avx2" "fma"] -, arch ? "x86-64" -, cudaSupport ? false -, computeCapabilities ? [] # example: [ "7.5" ], see https://developer.nvidia.com/cuda-gpus -}: -let - inherit (stdenv) lib; - cudatoolkit_joined = symlinkJoin { - name = "unsplit_cudatoolkit"; - paths = [ cudatoolkit.out - cudatoolkit.lib ];}; - gcc_joined = symlinkJoin { - name = "gcc-joined"; - paths = [ binutils gcc binutils-unwrapped gcc-unwrapped ]; - }; -in -buildBazelPackage rec { - pname = "tensorflow"; - version = "1.14.0"; - name = "${pname}-${version}"; - bazelFlags = [ "--incompatible_no_support_tools_in_action_inputs=false" ]; - bazelTarget = "//tensorflow/tools/lib_package:libtensorflow"; - fetchAttrs = { - sha256 = if cudaSupport then - "127xxwy3a2h1qsv2sqfhrh65g69hlb1q003vyyg7yjfqgfah9p2z" - else "1di1pnknr1hxdpn75lxf9c6dvb5kgllmgb9r9rgh5c2g9iil17zy"; - }; - CC_OPT_FLAGS = "-march=${arch} " + stdenv.lib.concatMapStringsSep " " (f: "-m"+f) features; - NIX_CFLAGS_COMPILE = CC_OPT_FLAGS; - TF_NEED_CUDA = if cudaSupport then "1" else "0"; - TF_IGNORE_MAX_BAZEL_VERSION = "1"; - TF_CUDA_COMPUTE_CAPABILITIES = stdenv.lib.concatStringsSep "," computeCapabilities; - nativeBuildInputs = [ glibcLocales python3 which ] - ++ (lib.optionals cudaSupport [ - cudatoolkit_joined cudnn nvidia_x11 - ]); - TF_CUDA_PATHS = lib.optionalString cudaSupport "${cudatoolkit_joined},${cudnn}"; - src = fetchFromGitHub { - repo = "tensorflow"; - owner = "tensorflow"; - rev = "v${version}"; - sha256 = "06jvwlsm14b8rqwd8q8796r0vmn0wk64s4ps2zg0sapkmp9vvcmi"; - }; - prePatch = '' - # doesn't work: - sed -i '/saved_model_portable_proto/d' tensorflow/cc/saved_model/BUILD - # calls ldconfig -p: - sed -i 's/+ _get_ld_config_paths()//' third_party/gpus/find_cuda_config.py - patchShebangs ./configure - patchShebangs tensorflow/tools/lib_package/concat_licenses.sh - patchShebangs third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl - ''; - GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${gcc_joined}/bin"; - configurePhase = '' - runHook preConfigure - ./configure - runHook postConfigure - ''; - buildAttrs = { - installPhase = '' - mkdir -p $out - tar -xf bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz -C $out - # Write pkgconfig file. - mkdir $out/lib/pkgconfig - cat > $out/lib/pkgconfig/tensorflow.pc << EOF - Name: TensorFlow - Version: ${version} - Description: Library for computation using data flow graphs for scalable machine learning - Requires: - Libs: -L$out/lib -ltensorflow - Cflags: -I$out/include/tensorflow - EOF - ''; - }; - meta = with lib; { - description = "C API for TensorFlow"; - homepage = https://www.tensorflow.org/versions/master/install/install_c; - license = licenses.asl20; - platforms = with platforms; linux ++ darwin; - maintainers = with maintainers; [ basvandijk yorickvp ]; - }; -} diff --git a/pkgs/development/libraries/science/math/tensorflow/bin.nix b/pkgs/development/libraries/science/math/tensorflow/bin.nix new file mode 100644 index 00000000000..3db7a6f1d1e --- /dev/null +++ b/pkgs/development/libraries/science/math/tensorflow/bin.nix @@ -0,0 +1,72 @@ +{ stdenv +, fetchurl +, patchelf +, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11 +}: + +with stdenv.lib; +let + unavailable = throw "libtensorflow is not available for this platform!"; + + tfType = if cudaSupport then "gpu" else "cpu"; + + system = + if stdenv.isLinux then "linux" + else if stdenv.isDarwin then "darwin" + else unavailable; + + platform = + if stdenv.isx86_64 then "x86_64" + else unavailable; + + rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++ + optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]); + + packages = import ./binary-hashes.nix; + packageName = "${tfType}-${system}-${platform}"; + url = packages.${packageName} or unavailable; + + patchLibs = + if stdenv.isDarwin + then '' + install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib + install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib + '' + else '' + patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so + patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so + ''; + +in stdenv.mkDerivation rec { + pname = "libtensorflow"; + inherit (packages) version; + + src = fetchurl url; + + # Patch library to use our libc, libstdc++ and others + buildCommand = '' + mkdir -pv $out + tar -C $out -xzf $src + chmod -R +w $out + ${patchLibs} + + # Write pkgconfig file. + mkdir $out/lib/pkgconfig + cat > $out/lib/pkgconfig/tensorflow.pc << EOF + Name: TensorFlow + Version: ${version} + Description: Library for computation using data flow graphs for scalable machine learning + Requires: + Libs: -L$out/lib -ltensorflow + Cflags: -I$out/include/tensorflow + EOF + ''; + + meta = { + description = "C API for TensorFlow"; + homepage = https://www.tensorflow.org/install/lang_c; + license = licenses.asl20; + platforms = [ "x86_64-linux" "x86_64-darwin" ]; + maintainers = with maintainers; [ basvandijk ]; + }; +} diff --git a/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix new file mode 100644 index 00000000000..892dfa2a609 --- /dev/null +++ b/pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix @@ -0,0 +1,15 @@ +{ +version = "1.14.0"; +"cpu-linux-x86_64" = { + url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz"; + sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k"; +}; +"gpu-linux-x86_64" = { + url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz"; + sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6"; +}; +"cpu-darwin-x86_64" = { + url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz"; + sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92"; +}; +} diff --git a/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh new file mode 100755 index 00000000000..515f25df4c1 --- /dev/null +++ b/pkgs/development/libraries/science/math/tensorflow/prefetcher.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +version=1.14.0 +hashfile=binary-hashes.nix +rm -f $hashfile +echo "{" >> $hashfile +echo "version = \"$version\";" >> $hashfile +for sys in "linux" "darwin"; do + for tfpref in "cpu" "gpu"; do + for platform in "x86_64"; do + if [ $sys = "darwin" ] && [ $tfpref = "gpu" ]; then + continue + fi + url=https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-$tfpref-$sys-$platform-$version.tar.gz + hash=$(nix-prefetch-url $url) + echo "\"${tfpref}-${sys}-${platform}\" = {" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo "};" >> $hashfile + done + done +done +echo "}" >> $hashfile + diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a9342085d34..a16369a1e07 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1093,11 +1093,11 @@ in tensor = libsForQt5.callPackage ../applications/networking/instant-messengers/tensor { }; - libtensorflow = callPackage ../development/libraries/libtensorflow { + libtensorflow = callPackage ../development/libraries/science/math/tensorflow/bin.nix { + cudaSupport = pkgs.config.cudaSupport or false; inherit (linuxPackages) nvidia_x11; cudatoolkit = cudatoolkit_10_0; cudnn = cudnn_cudatoolkit_10_0; - nccl = nccl_cudatoolkit_10; }; behdad-fonts = callPackage ../data/fonts/behdad-fonts { };