From cc245fdcd5bc663609df74a0a5099b652be53176 Mon Sep 17 00:00:00 2001 From: Sam Stites Date: Tue, 17 Dec 2019 12:55:43 -0500 Subject: [PATCH 1/4] pytorch: 1.2.0 -> 1.4.1 Co-authored-by: Benjamin Hipple --- .../python-modules/pytorch/default.nix | 99 +++++++++++++------ 1 file changed, 67 insertions(+), 32 deletions(-) diff --git a/pkgs/development/python-modules/pytorch/default.nix b/pkgs/development/python-modules/pytorch/default.nix index a0a4aadfe58..0400a4773eb 100644 --- a/pkgs/development/python-modules/pytorch/default.nix +++ b/pkgs/development/python-modules/pytorch/default.nix @@ -1,23 +1,26 @@ -{ stdenv, fetchurl, fetchgit, buildPythonPackage, python, pythonOlder, +{ stdenv, fetchurl, fetchgit, fetchpatch, buildPythonPackage, python, pythonOlder, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null, - mklSupport ? false, mkl ? null, + mklDnnSupport ? true, useSystemNccl ? true, openMPISupport ? false, openmpi ? null, - buildNamedTensor ? false, buildBinaries ? false, + buildDocs ? false, cudaArchList ? null, - fetchFromGitHub, lib, numpy, pyyaml, cffi, click, typing, cmake, hypothesis, numactl, + fetchFromGitHub, lib, numpy, pyyaml, cffi, click, typing, cmake, hypothesis, numactl, psutil, linkFarm, symlinkJoin, + # virtual pkg that consistently instantiates blas across nixpkgs + # See https://github.com/NixOS/nixpkgs/pull/83888 + blas, + # ninja (https://ninja-build.org) must be available to run C++ extensions tests, ninja, # dependencies for torch.utils.tensorboard - tensorboardSupport ? true, pillow, six, future, tensorflow-tensorboard, + pillow, six, future, tensorflow-tensorboard, protobuf, utillinux, which, isPy3k }: assert !openMPISupport || openmpi != null; -assert !tensorboardSupport || tensorflow-tensorboard != null; # assert that everything needed for cuda is present and that the correct cuda versions are used assert !cudaSupport || cudatoolkit != null; @@ -28,17 +31,11 @@ assert !cudaSupport || (let majorIs = lib.versions.major cudatoolkit.version; let hasDependency = dep: pkg: lib.lists.any (inp: inp == dep) pkg.buildInputs; matchesCudatoolkit = hasDependency cudatoolkit; - matchesMkl = hasDependency mkl; in # confirm that cudatoolkits are sync'd across dependencies assert !(openMPISupport && cudaSupport) || matchesCudatoolkit openmpi; assert !cudaSupport || matchesCudatoolkit magma; -# confirm that mkl is sync'd across dependencies -assert !mklSupport || mkl != null; -assert !(mklSupport && cudaSupport) || matchesMkl magma; -assert !mklSupport || (numpy.blasImplementation == "mkl" && numpy.blas == mkl); - let cudatoolkit_joined = symlinkJoin { name = "${cudatoolkit.name}-unsplit"; @@ -108,7 +105,7 @@ let "LD_LIBRARY_PATH=${cudaStub}\${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH "; in buildPythonPackage rec { - version = "1.2.0"; + version = "1.4.1"; pname = "pytorch"; disabled = !isPy3k; @@ -122,11 +119,9 @@ in buildPythonPackage rec { repo = "pytorch"; rev = "v${version}"; fetchSubmodules = true; - sha256 = "1biyq2p48chakf2xw7hazzqmr5ps1nx475ql8vkmxjg5zaa071cz"; + sha256 = "1aa1il4f98pswfj20cv27yfb91l1jcq4515i7mvq7sh5647yzwms"; }; - dontUseCmakeConfigure = true; - preConfigure = lib.optionalString cudaSupport '' export TORCH_CUDA_ARCH_LIST="${lib.strings.concatStringsSep ";" final_cudaArchList}" export CC=${cudatoolkit.cc}/bin/gcc CXX=${cudatoolkit.cc}/bin/g++ @@ -134,6 +129,36 @@ in buildPythonPackage rec { export CUDNN_INCLUDE_DIR=${cudnn}/include ''; + patches = [ + # Prevents a race condition which would be introduced by pull 30333. + # See https://github.com/pytorch/pytorch/issues/32277 + # Can be removed >1.5.0. + (fetchpatch { + url = "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/30332.patch"; + sha256 = "1v9dwbhz3rdxcx6sz8y8j9n3bj6nqs78b1r8yg89yc15n6l4cqx2"; + }) + + # Fixes errors with gcc-9 compilation. Cherry-picked on advice from ezyang. + # See https://github.com/pytorch/pytorch/issues/32277 + # Can be removed >1.5.0. + (fetchpatch { + url = "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/30333.patch"; + sha256 = "139413fl37h2fnil0cv99a67mqqnsh02k74b92by1qyr6pcfyg3q"; + }) + ]; + + # Use pytorch's custom configurations + dontUseCmakeConfigure = true; + + BUILD_NAMEDTENSOR = true; + BUILD_DOCS = buildDocs; + USE_MKLDNN = mklDnnSupport; + preBuild = '' + export MAX_JOBS=$NIX_BUILD_CORES + ${python.interpreter} setup.py build --cmake-only + ${cmake}/bin/cmake build + ''; + preFixup = '' function join_by { local IFS="$1"; shift; echo "$*"; } function strip2 { @@ -149,14 +174,14 @@ in buildPythonPackage rec { done ''; + # Override the (weirdly) wrong version set by default. See # https://github.com/NixOS/nixpkgs/pull/52437#issuecomment-449718038 # https://github.com/pytorch/pytorch/blob/v1.0.0/setup.py#L267 PYTORCH_BUILD_VERSION = version; PYTORCH_BUILD_NUMBER = 0; - BUILD_NAMEDTENSOR = buildNamedTensor; # experimental feature - USE_SYSTEM_NCCL=true; # don't build pytorch's third_party NCCL + USE_SYSTEM_NCCL=useSystemNccl; # don't build pytorch's third_party NCCL # Suppress a weird warning in mkl-dnn, part of ideep in pytorch # (upstream seems to have fixed this in the wrong place?) @@ -165,7 +190,7 @@ in buildPythonPackage rec { # # Also of interest: pytorch ignores CXXFLAGS uses CFLAGS for both C and C++: # https://github.com/pytorch/pytorch/blob/v1.2.0/setup.py#L17 - NIX_CFLAGS_COMPILE = lib.optionals (numpy.blas == mkl) [ "-Wno-error=array-bounds" ]; + NIX_CFLAGS_COMPILE = lib.optionals (blas.implementation == "mkl") [ "-Wno-error=array-bounds" ]; nativeBuildInputs = [ cmake @@ -174,9 +199,8 @@ in buildPythonPackage rec { ninja ] ++ lib.optionals cudaSupport [ cudatoolkit_joined ]; - buildInputs = [ - numpy.blas - ] ++ lib.optionals cudaSupport [ cudnn magma nccl ] + buildInputs = [ blas ] + ++ lib.optionals cudaSupport [ cudnn magma nccl ] ++ lib.optionals stdenv.isLinux [ numactl ]; propagatedBuildInputs = [ @@ -184,23 +208,34 @@ in buildPythonPackage rec { click numpy pyyaml - ] ++ lib.optionals openMPISupport [ openmpi ] - ++ lib.optional (pythonOlder "3.5") typing - ++ lib.optionals tensorboardSupport [pillow six future tensorflow-tensorboard]; + # the following are required for tensorboard support + pillow six future tensorflow-tensorboard protobuf + ] ++ lib.optionals openMPISupport [ openmpi ]; - checkInputs = [ hypothesis ninja ]; + checkInputs = [ hypothesis ninja psutil ]; doCheck = false; # tests take a long time for channel release, so doCheck should be overridden only when developing - checkPhase = "${cudaStubEnv}python test/run_test.py" - + " --exclude utils" # utils requires git, which is not allowed in the check phase + checkPhase = with lib.versions; with lib.strings; concatStringsSep " " [ + # MKL 2019.5-only workaround. See: https://github.com/NixOS/nixpkgs/issues/75611 + (optionalString (blas.implementation == "mkl" && majorMinor blas.version == "2019.5") "KMP_INIT_AT_FORK=FALSE ") + cudaStubEnv + "${python.interpreter} test/run_test.py" + "--exclude" + (concatStringsSep " " [ + "utils" # utils requires git, which is not allowed in the check phase - # Other tests which have been disabled in previous nix derivations of pytorch. - # --exclude dataloader sparse torch utils thd_distributed distributed cpp_extensions - ; + # "dataloader" # psutils correctly finds and triggers multiprocessing, but is too sandboxed to run -- resulting in numerous errors + # ^^^^^^^^^^^^ NOTE: while test_dataloader does return errors, these are acceptable errors and do not interfere with the build + + # tensorboard has acceptable failures for pytorch 1.3.x due to dependencies on tensorboard-plugins + (optionalString (majorMinor version == "1.3" ) "tensorboard") + ]) + ]; postInstall = '' mkdir $dev cp -r $out/${python.sitePackages}/torch/lib $dev/lib cp -r $out/${python.sitePackages}/torch/include $dev/include + cp -r $out/${python.sitePackages}/torch/share $dev/share ''; postFixup = stdenv.lib.optionalString stdenv.isDarwin '' @@ -233,6 +268,6 @@ in buildPythonPackage rec { homepage = "https://pytorch.org/"; license = lib.licenses.bsd3; platforms = with lib.platforms; linux ++ lib.optionals (!cudaSupport) darwin; - maintainers = with lib.maintainers; [ teh thoughtpolice stites tscholak ]; # tscholak esp. for darwin-related builds + maintainers = with lib.maintainers; [ teh thoughtpolice tscholak ]; # tscholak esp. for darwin-related builds }; } From fb5bb25c10bed6dd577c5b1118429bbda4aa8554 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 2 May 2020 20:40:27 -0400 Subject: [PATCH 2/4] python3Packages.pytorch: fixup builds with MKL - Pass `blas.provider` into `buildInputs`, so that CMake can find the actual `mkl` for inspection of its cmake files and headers. - Add `USE_MKL` correctly when the blas provider is `mkl`. - Use the MKLDNN and MKLDNN_CBLAS flags by default, since `mkldnn` is FOSS and always available.. - Remove a patch for MKL 2019, since we've moved to 2020. - Add a pythonImportsCheck for "torch" as a basic sanity-check - Removed some unused variables at the top of the file --- .../python-modules/pytorch/default.nix | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/pytorch/default.nix b/pkgs/development/python-modules/pytorch/default.nix index 0400a4773eb..749a2c08096 100644 --- a/pkgs/development/python-modules/pytorch/default.nix +++ b/pkgs/development/python-modules/pytorch/default.nix @@ -1,11 +1,10 @@ -{ stdenv, fetchurl, fetchgit, fetchpatch, buildPythonPackage, python, pythonOlder, +{ stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null, mklDnnSupport ? true, useSystemNccl ? true, openMPISupport ? false, openmpi ? null, - buildBinaries ? false, buildDocs ? false, cudaArchList ? null, - fetchFromGitHub, lib, numpy, pyyaml, cffi, click, typing, cmake, hypothesis, numactl, psutil, + numpy, pyyaml, cffi, click, typing, cmake, dnnl, hypothesis, numactl, psutil, linkFarm, symlinkJoin, # virtual pkg that consistently instantiates blas across nixpkgs @@ -152,7 +151,15 @@ in buildPythonPackage rec { BUILD_NAMEDTENSOR = true; BUILD_DOCS = buildDocs; + + USE_MKL = blas.implementation == "mkl"; + + # Unlike MKL, MKLDNN is FOSS, so we enable support for it by default. Note + # that this was renamed to dnnl and then renamed again to oneDNN upstream, but + # pytorch still calls it by the old name mkldnn. USE_MKLDNN = mklDnnSupport; + USE_MKLDNN_CBLAS = mklDnnSupport; + preBuild = '' export MAX_JOBS=$NIX_BUILD_CORES ${python.interpreter} setup.py build --cmake-only @@ -174,7 +181,6 @@ in buildPythonPackage rec { done ''; - # Override the (weirdly) wrong version set by default. See # https://github.com/NixOS/nixpkgs/pull/52437#issuecomment-449718038 # https://github.com/pytorch/pytorch/blob/v1.0.0/setup.py#L267 @@ -199,7 +205,7 @@ in buildPythonPackage rec { ninja ] ++ lib.optionals cudaSupport [ cudatoolkit_joined ]; - buildInputs = [ blas ] + buildInputs = [ blas blas.provider dnnl ] ++ lib.optionals cudaSupport [ cudnn magma nccl ] ++ lib.optionals stdenv.isLinux [ numactl ]; @@ -214,10 +220,13 @@ in buildPythonPackage rec { checkInputs = [ hypothesis ninja psutil ]; - doCheck = false; # tests take a long time for channel release, so doCheck should be overridden only when developing + # Tests take a long time and may be flaky, so just sanity-check imports + doCheck = false; + pythonImportsCheck = [ + "torch" + ]; + checkPhase = with lib.versions; with lib.strings; concatStringsSep " " [ - # MKL 2019.5-only workaround. See: https://github.com/NixOS/nixpkgs/issues/75611 - (optionalString (blas.implementation == "mkl" && majorMinor blas.version == "2019.5") "KMP_INIT_AT_FORK=FALSE ") cudaStubEnv "${python.interpreter} test/run_test.py" "--exclude" From 47105a1689b06f89461e2d2379fd83e5e6849cb9 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 9 May 2020 12:50:17 -0400 Subject: [PATCH 3/4] python3Packages.ignite: 0.2.1 -> 0.3.0 --- .../python-modules/ignite/default.nix | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/ignite/default.nix b/pkgs/development/python-modules/ignite/default.nix index cd682072707..785ec581491 100644 --- a/pkgs/development/python-modules/ignite/default.nix +++ b/pkgs/development/python-modules/ignite/default.nix @@ -2,6 +2,7 @@ , buildPythonPackage , fetchFromGitHub , pytest +, matplotlib , mock , pytorch , pynvml @@ -11,24 +12,24 @@ buildPythonPackage rec { pname = "ignite"; - version = "0.2.1"; + version = "0.3.0"; src = fetchFromGitHub { owner = "pytorch"; repo = pname; rev = "v${version}"; - sha256 = "15k6dd11yxn4923llcpmw4srl1by5ljhh7aw5pnkn4n4qpywh6cm"; + sha256 = "0i863kxi1r1hspj19lhn6r8256vdazjcyvis0s33fgzrf7kxi08x"; }; - checkInputs = [ pytest mock ]; - - checkPhase = '' - pytest -k 'not visdom and not tensorboard and not mlflow and not polyaxon' tests/ - ''; - # these packages are not currently in nixpkgs - + checkInputs = [ pytest matplotlib mock ]; propagatedBuildInputs = [ pytorch scikitlearn tqdm pynvml ]; + # Some packages are not in NixPkgs; other tests try to build distributed + # models, which doesn't work in the sandbox. + checkPhase = '' + pytest -k 'not visdom and not tensorboard and not mlflow and not polyaxon and not conftest and not engines and not distrib_' tests/ + ''; + meta = with lib; { description = "High-level training library for PyTorch"; homepage = "https://pytorch.org/ignite"; From ecef2975e3420b560b017c8b856313df6556e556 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 9 May 2020 14:45:49 -0400 Subject: [PATCH 4/4] python3Packages.pytorch: reference dnnl by new name oneDNN --- pkgs/development/python-modules/pytorch/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pytorch/default.nix b/pkgs/development/python-modules/pytorch/default.nix index 749a2c08096..2318738411f 100644 --- a/pkgs/development/python-modules/pytorch/default.nix +++ b/pkgs/development/python-modules/pytorch/default.nix @@ -4,7 +4,7 @@ openMPISupport ? false, openmpi ? null, buildDocs ? false, cudaArchList ? null, - numpy, pyyaml, cffi, click, typing, cmake, dnnl, hypothesis, numactl, psutil, + numpy, pyyaml, cffi, click, typing, cmake, oneDNN, hypothesis, numactl, psutil, linkFarm, symlinkJoin, # virtual pkg that consistently instantiates blas across nixpkgs @@ -205,7 +205,7 @@ in buildPythonPackage rec { ninja ] ++ lib.optionals cudaSupport [ cudatoolkit_joined ]; - buildInputs = [ blas blas.provider dnnl ] + buildInputs = [ blas blas.provider oneDNN ] ++ lib.optionals cudaSupport [ cudnn magma nccl ] ++ lib.optionals stdenv.isLinux [ numactl ];