From 6dba41fbcb4239a628ac5bdf0035882a679b8648 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Sun, 10 Jan 2021 13:40:19 +0100 Subject: [PATCH 1/4] mpi: use mpi attribute consistently as the default MPI implementations Use the attribute mpi to provide a system wide default MPI implementation. The default is openmpi (as before). This now allows for overriding the MPI implentation by using the overlay mechanism. Build all packages with mpich instead of the default openmpi can now be achived like this: self: super: { mpi = super.mpich; } All derivations that have been using "mpi ? null" to provide optional building with MPI have been change in the following way to allow for optional builds with MPI: { ... , mpi , useMpi ? false } --- .../graphics/paraview/default.nix | 4 +- .../science/biology/migrate/default.nix | 4 +- .../science/biology/neuron/default.nix | 9 +++-- .../science/biology/raxml/default.nix | 10 ++--- .../science/chemistry/openmolcas/default.nix | 4 +- .../chemistry/quantum-espresso/default.nix | 7 ++-- .../science/chemistry/siesta/default.nix | 9 +++-- .../science/electronics/openems/default.nix | 5 +-- .../science/math/cntk/default.nix | 6 +-- .../science/math/getdp/default.nix | 4 +- .../science/math/scotch/default.nix | 4 +- .../molecular-dynamics/gromacs/default.nix | 4 +- .../molecular-dynamics/lammps/default.nix | 4 +- .../science/physics/elmerfem/default.nix | 4 +- pkgs/development/libraries/boost/generic.nix | 7 ++-- .../libraries/globalarrays/default.nix | 4 +- .../development/libraries/precice/default.nix | 4 +- .../python-modules/cntk/default.nix | 8 ++-- .../python-modules/pytorch/default.nix | 8 ++-- .../python-modules/tensorflow/default.nix | 4 +- pkgs/development/r-modules/default.nix | 10 ++--- .../tools/poetry2nix/poetry2nix/overrides.nix | 6 +-- pkgs/tools/misc/hdf5/default.nix | 11 +++--- pkgs/tools/misc/hpcg/default.nix | 4 +- pkgs/tools/system/ior/default.nix | 4 +- pkgs/top-level/all-packages.nix | 39 +++++++------------ pkgs/top-level/python-packages.nix | 5 +-- 27 files changed, 90 insertions(+), 102 deletions(-) diff --git a/pkgs/applications/graphics/paraview/default.nix b/pkgs/applications/graphics/paraview/default.nix index 6e4738dd07c..7bbff5499f6 100644 --- a/pkgs/applications/graphics/paraview/default.nix +++ b/pkgs/applications/graphics/paraview/default.nix @@ -1,6 +1,6 @@ { boost, cmake, fetchFromGitHub, ffmpeg, qtbase, qtx11extras, qttools, qtxmlpatterns, qtsvg, gdal, gfortran, libXt, makeWrapper, - mkDerivation, ninja, openmpi, python3, lib, stdenv, tbb, libGLU, libGL }: + mkDerivation, ninja, mpi, python3, lib, stdenv, tbb, libGLU, libGL }: mkDerivation rec { pname = "paraview"; @@ -65,7 +65,7 @@ mkDerivation rec { buildInputs = [ libGLU libGL libXt - openmpi + mpi tbb boost ffmpeg diff --git a/pkgs/applications/science/biology/migrate/default.nix b/pkgs/applications/science/biology/migrate/default.nix index 0e2aa1c1a79..ba55898e4f9 100644 --- a/pkgs/applications/science/biology/migrate/default.nix +++ b/pkgs/applications/science/biology/migrate/default.nix @@ -1,4 +1,4 @@ -{ gccStdenv, fetchurl, zlib, openmpi }: +{ gccStdenv, fetchurl, zlib, mpi }: gccStdenv.mkDerivation rec { version = "3.7.2"; @@ -9,7 +9,7 @@ gccStdenv.mkDerivation rec { sha256 = "1p2364ffjc56i82snzvjpy6pkf6wvqwvlvlqxliscx2c303fxs8v"; }; - buildInputs = [ zlib openmpi ]; + buildInputs = [ zlib mpi ]; setSourceRoot = ''sourceRoot=$(echo */src)''; buildFlags = [ "thread" "mpis" ]; preInstall = "mkdir -p $out/man/man1"; diff --git a/pkgs/applications/science/biology/neuron/default.nix b/pkgs/applications/science/biology/neuron/default.nix index 154965b1521..7bfef3a82fe 100644 --- a/pkgs/applications/science/biology/neuron/default.nix +++ b/pkgs/applications/science/biology/neuron/default.nix @@ -8,7 +8,8 @@ , readline , which , python ? null -, mpi ? null +, useMpi ? false +, mpi , iv }: @@ -17,7 +18,8 @@ stdenv.mkDerivation rec { version = "7.5"; nativeBuildInputs = [ which pkg-config automake autoconf libtool ]; - buildInputs = [ ncurses readline python mpi iv ]; + buildInputs = [ ncurses readline python iv ] + ++ lib.optional useMpi mpi; src = fetchurl { url = "https://www.neuron.yale.edu/ftp/neuron/versions/v${version}/nrn-${version}.tar.gz"; @@ -54,7 +56,7 @@ stdenv.mkDerivation rec { configureFlags = with lib; [ "--with-readline=${readline}" "--with-iv=${iv}" ] ++ optionals (python != null) [ "--with-nrnpython=${python.interpreter}" ] - ++ (if mpi != null then ["--with-mpi" "--with-paranrn"] + ++ (if useMpi then ["--with-mpi" "--with-paranrn"] else ["--without-mpi"]); @@ -84,4 +86,3 @@ stdenv.mkDerivation rec { platforms = platforms.x86_64 ++ platforms.i686; }; } - diff --git a/pkgs/applications/science/biology/raxml/default.nix b/pkgs/applications/science/biology/raxml/default.nix index 4f9b5aca1b2..6e747e318f5 100644 --- a/pkgs/applications/science/biology/raxml/default.nix +++ b/pkgs/applications/science/biology/raxml/default.nix @@ -1,7 +1,7 @@ { lib, stdenv , fetchFromGitHub -, pkgs -, mpi ? false +, useMpi ? false +, mpi }: stdenv.mkDerivation rec { @@ -15,16 +15,16 @@ stdenv.mkDerivation rec { sha256 = "1jqjzhch0rips0vp04prvb8vmc20c5pdmsqn8knadcf91yy859fh"; }; - buildInputs = lib.optionals mpi [ pkgs.openmpi ]; + buildInputs = lib.optionals useMpi [ mpi ]; # TODO darwin, AVX and AVX2 makefile targets - buildPhase = if mpi then '' + buildPhase = if useMpi then '' make -f Makefile.MPI.gcc '' else '' make -f Makefile.SSE3.PTHREADS.gcc ''; - installPhase = if mpi then '' + installPhase = if useMpi then '' mkdir -p $out/bin && cp raxmlHPC-MPI $out/bin '' else '' mkdir -p $out/bin && cp raxmlHPC-PTHREADS-SSE3 $out/bin diff --git a/pkgs/applications/science/chemistry/openmolcas/default.nix b/pkgs/applications/science/chemistry/openmolcas/default.nix index 1297e595b9a..4bd88456b8f 100644 --- a/pkgs/applications/science/chemistry/openmolcas/default.nix +++ b/pkgs/applications/science/chemistry/openmolcas/default.nix @@ -1,6 +1,6 @@ { lib, stdenv, fetchFromGitLab, cmake, gfortran, perl , openblas, hdf5-cpp, python3, texlive -, armadillo, openmpi, globalarrays, openssh +, armadillo, mpi, globalarrays, openssh , makeWrapper, fetchpatch } : @@ -33,7 +33,7 @@ in stdenv.mkDerivation { hdf5-cpp python armadillo - openmpi + mpi globalarrays openssh ]; diff --git a/pkgs/applications/science/chemistry/quantum-espresso/default.nix b/pkgs/applications/science/chemistry/quantum-espresso/default.nix index c7b1f901046..6d70e9f984f 100644 --- a/pkgs/applications/science/chemistry/quantum-espresso/default.nix +++ b/pkgs/applications/science/chemistry/quantum-espresso/default.nix @@ -1,6 +1,7 @@ { lib, stdenv, fetchurl , gfortran, fftw, blas, lapack -, mpi ? null +, useMpi ? false +, mpi }: stdenv.mkDerivation rec { @@ -21,9 +22,9 @@ stdenv.mkDerivation rec { ''; buildInputs = [ fftw blas lapack gfortran ] - ++ (lib.optionals (mpi != null) [ mpi ]); + ++ (lib.optionals useMpi [ mpi ]); -configureFlags = if (mpi != null) then [ "LD=${mpi}/bin/mpif90" ] else [ "LD=${gfortran}/bin/gfortran" ]; +configureFlags = if useMpi then [ "LD=${mpi}/bin/mpif90" ] else [ "LD=${gfortran}/bin/gfortran" ]; makeFlags = [ "all" ]; diff --git a/pkgs/applications/science/chemistry/siesta/default.nix b/pkgs/applications/science/chemistry/siesta/default.nix index 0df953f7106..02ff4c1ca44 100644 --- a/pkgs/applications/science/chemistry/siesta/default.nix +++ b/pkgs/applications/science/chemistry/siesta/default.nix @@ -1,6 +1,7 @@ { lib, stdenv, fetchurl -, gfortran, blas, lapack -, mpi ? null, scalapack +, gfortran, blas, lapack, scalapack +, useMpi ? false +, mpi }: stdenv.mkDerivation { @@ -17,7 +18,7 @@ stdenv.mkDerivation { }; buildInputs = [ blas lapack gfortran ] - ++ (lib.optionals (mpi != null) [ mpi scalapack ]); + ++ lib.optionals useMpi [ mpi scalapack ]; enableParallelBuilding = true; @@ -29,7 +30,7 @@ stdenv.mkDerivation { cp gfortran.make arch.make ''; - preBuild = if (mpi != null) then '' + preBuild = if useMpi then '' makeFlagsArray=( CC="mpicc" FC="mpifort" FPPFLAGS="-DMPI" MPI_INTERFACE="libmpi_f90.a" MPI_INCLUDE="." diff --git a/pkgs/applications/science/electronics/openems/default.nix b/pkgs/applications/science/electronics/openems/default.nix index e1063f8e26c..64afe3222c2 100644 --- a/pkgs/applications/science/electronics/openems/default.nix +++ b/pkgs/applications/science/electronics/openems/default.nix @@ -11,16 +11,15 @@ , cmake , octave , gl2ps +, mpi , withQcsxcad ? true , withMPI ? false , withHyp2mat ? true , qcsxcad ? null -, openmpi ? null , hyp2mat ? null }: assert withQcsxcad -> qcsxcad != null; -assert withMPI -> openmpi != null; assert withHyp2mat -> hyp2mat != null; stdenv.mkDerivation { @@ -50,7 +49,7 @@ stdenv.mkDerivation { csxcad (octave.override { inherit hdf5; }) ] ++ lib.optionals withQcsxcad [ qcsxcad ] - ++ lib.optionals withMPI [ openmpi ] + ++ lib.optionals withMPI [ mpi ] ++ lib.optionals withHyp2mat [ hyp2mat ]; postFixup = '' diff --git a/pkgs/applications/science/math/cntk/default.nix b/pkgs/applications/science/math/cntk/default.nix index a348210179e..e15e2a43d77 100644 --- a/pkgs/applications/science/math/cntk/default.nix +++ b/pkgs/applications/science/math/cntk/default.nix @@ -1,5 +1,5 @@ { lib, stdenv, fetchgit, fetchFromGitHub, cmake -, openblas, blas, lapack, opencv3, libzip, boost, protobuf, openmpi +, openblas, blas, lapack, opencv3, libzip, boost, protobuf, mpi , onebitSGDSupport ? false , cudaSupport ? false, addOpenGLRunpath, cudatoolkit, nvidia_x11 , cudnnSupport ? cudaSupport, cudnn @@ -33,7 +33,7 @@ in stdenv.mkDerivation rec { # Force OpenMPI to use g++ in PATH. OMPI_CXX = "g++"; - buildInputs = [ openblas opencv3 libzip boost protobuf openmpi ] + buildInputs = [ openblas opencv3 libzip boost protobuf mpi ] ++ lib.optional cudaSupport cudatoolkit ++ lib.optional cudnnSupport cudnn; @@ -43,7 +43,7 @@ in stdenv.mkDerivation rec { "--with-openblas=${openblas}" "--with-boost=${boost.dev}" "--with-protobuf=${protobuf}" - "--with-mpi=${openmpi}" + "--with-mpi=${mpi}" "--cuda=${if cudaSupport then "yes" else "no"}" # FIXME "--asgd=no" diff --git a/pkgs/applications/science/math/getdp/default.nix b/pkgs/applications/science/math/getdp/default.nix index 915c7e1147f..39d9c866cae 100644 --- a/pkgs/applications/science/math/getdp/default.nix +++ b/pkgs/applications/science/math/getdp/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, cmake, gfortran, blas, lapack, openmpi, petsc, python3 }: +{ lib, stdenv, fetchurl, cmake, gfortran, blas, lapack, mpi, petsc, python3 }: stdenv.mkDerivation rec { name = "getdp-${version}"; @@ -9,7 +9,7 @@ stdenv.mkDerivation rec { }; nativeBuildInputs = [ cmake gfortran ]; - buildInputs = [ blas lapack openmpi petsc python3 ]; + buildInputs = [ blas lapack mpi petsc python3 ]; meta = with lib; { description = "A General Environment for the Treatment of Discrete Problems"; diff --git a/pkgs/applications/science/math/scotch/default.nix b/pkgs/applications/science/math/scotch/default.nix index 6f8753ff38a..b6613f25cbc 100644 --- a/pkgs/applications/science/math/scotch/default.nix +++ b/pkgs/applications/science/math/scotch/default.nix @@ -1,11 +1,11 @@ -{ lib, stdenv, fetchurl, bison, openmpi, flex, zlib}: +{ lib, stdenv, fetchurl, bison, mpi, flex, zlib}: stdenv.mkDerivation rec { version = "6.0.4"; pname = "scotch"; src_name = "scotch_${version}"; - buildInputs = [ bison openmpi flex zlib ]; + buildInputs = [ bison mpi flex zlib ]; src = fetchurl { url = "https://gforge.inria.fr/frs/download.php/file/34618/${src_name}.tar.gz"; diff --git a/pkgs/applications/science/molecular-dynamics/gromacs/default.nix b/pkgs/applications/science/molecular-dynamics/gromacs/default.nix index c40faddbbf9..bdec2ccc669 100644 --- a/pkgs/applications/science/molecular-dynamics/gromacs/default.nix +++ b/pkgs/applications/science/molecular-dynamics/gromacs/default.nix @@ -3,10 +3,10 @@ , cmake , hwloc , fftw -, openmpi , perl , singlePrec ? true , mpiEnabled ? false +, mpi , cpuAcceleration ? null }: @@ -33,7 +33,7 @@ in stdenv.mkDerivation rec { nativeBuildInputs = [ cmake ]; buildInputs = [ fftw perl hwloc ] - ++ (lib.optionals mpiEnabled [ openmpi ]); + ++ (lib.optionals mpiEnabled [ mpi ]); cmakeFlags = [ "-DGMX_SIMD:STRING=${SIMD cpuAcceleration}" diff --git a/pkgs/applications/science/molecular-dynamics/lammps/default.nix b/pkgs/applications/science/molecular-dynamics/lammps/default.nix index 123afef03e8..51ce64115ea 100644 --- a/pkgs/applications/science/molecular-dynamics/lammps/default.nix +++ b/pkgs/applications/science/molecular-dynamics/lammps/default.nix @@ -1,6 +1,7 @@ { lib, stdenv, fetchFromGitHub , libpng, gzip, fftw, blas, lapack -, mpi ? null +, withMPI ? false +, mpi }: let packages = [ "asphere" "body" "class2" "colloid" "compress" "coreshell" @@ -8,7 +9,6 @@ let packages = [ "opt" "peri" "qeq" "replica" "rigid" "shock" "snap" "srd" "user-reaxc" ]; lammps_includes = "-DLAMMPS_EXCEPTIONS -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64"; - withMPI = (mpi != null); in stdenv.mkDerivation rec { # LAMMPS has weird versioning converted to ISO 8601 format diff --git a/pkgs/applications/science/physics/elmerfem/default.nix b/pkgs/applications/science/physics/elmerfem/default.nix index 6fa25ce66ae..5033b28bd3b 100644 --- a/pkgs/applications/science/physics/elmerfem/default.nix +++ b/pkgs/applications/science/physics/elmerfem/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, cmake, git, gfortran, openmpi, blas, liblapack, qt4, qwt6_qt4, pkg-config }: +{ lib, stdenv, fetchFromGitHub, cmake, git, gfortran, mpi, blas, liblapack, qt4, qwt6_qt4, pkg-config }: stdenv.mkDerivation rec { pname = "elmerfem"; @@ -14,7 +14,7 @@ stdenv.mkDerivation rec { hardeningDisable = [ "format" ]; nativeBuildInputs = [ cmake pkg-config git ]; - buildInputs = [ gfortran openmpi blas liblapack qt4 qwt6_qt4 ]; + buildInputs = [ gfortran mpi blas liblapack qt4 qwt6_qt4 ]; preConfigure = '' patchShebangs ./ diff --git a/pkgs/development/libraries/boost/generic.nix b/pkgs/development/libraries/boost/generic.nix index 5a2b437d41d..abff1268bfa 100644 --- a/pkgs/development/libraries/boost/generic.nix +++ b/pkgs/development/libraries/boost/generic.nix @@ -14,7 +14,8 @@ , enableNumpy ? false , taggedLayout ? ((enableRelease && enableDebug) || (enableSingleThreaded && enableMultiThreaded) || (enableShared && enableStatic)) , patches ? [] -, mpi ? null +, useMpi ? false +, mpi , extraB2Args ? [] # Attributes inherit from specific versions @@ -94,7 +95,7 @@ let ++ optional (variant == "release") "debug-symbols=off" ++ optional (toolset != null) "toolset=${toolset}" ++ optional (!enablePython) "--without-python" - ++ optional (mpi != null || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" + ++ optional (useMpi || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" ++ optionals (stdenv.hostPlatform.libc == "msvcrt") [ "threadapi=win32" ] ++ extraB2Args @@ -140,7 +141,7 @@ stdenv.mkDerivation { substituteInPlace tools/build/src/tools/clang-darwin.jam \ --replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)"; fi; - '' + optionalString (mpi != null) '' + '' + optionalString useMpi '' cat << EOF >> user-config.jam using mpi : ${mpi}/bin/mpiCC ; EOF diff --git a/pkgs/development/libraries/globalarrays/default.nix b/pkgs/development/libraries/globalarrays/default.nix index 049e262bb17..1f70946cbce 100644 --- a/pkgs/development/libraries/globalarrays/default.nix +++ b/pkgs/development/libraries/globalarrays/default.nix @@ -1,5 +1,5 @@ { lib, stdenv, fetchpatch, fetchFromGitHub, autoreconfHook -, blas, gfortran, openssh, openmpi +, blas, gfortran, openssh, mpi } : let @@ -17,7 +17,7 @@ in stdenv.mkDerivation { }; nativeBuildInputs = [ autoreconfHook ]; - buildInputs = [ openmpi blas gfortran openssh ]; + buildInputs = [ mpi blas gfortran openssh ]; preConfigure = '' configureFlagsArray+=( "--enable-i8" \ diff --git a/pkgs/development/libraries/precice/default.nix b/pkgs/development/libraries/precice/default.nix index e9e67737d32..10a80ebb91e 100644 --- a/pkgs/development/libraries/precice/default.nix +++ b/pkgs/development/libraries/precice/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, openmpi, python3, petsc }: +{ lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, mpi, python3, petsc }: stdenv.mkDerivation rec { pname = "precice"; @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { NIX_CFLAGS_COMPILE = lib.optional stdenv.isDarwin [ "-D_GNU_SOURCE" ]; nativeBuildInputs = [ cmake gcc ]; - buildInputs = [ boost eigen libxml2 openmpi python3 python3.pkgs.numpy ]; + buildInputs = [ boost eigen libxml2 mpi python3 python3.pkgs.numpy ]; meta = { description = "preCICE stands for Precise Code Interaction Coupling Environment"; diff --git a/pkgs/development/python-modules/cntk/default.nix b/pkgs/development/python-modules/cntk/default.nix index ce388bb1d5d..b1bba8cf1e0 100644 --- a/pkgs/development/python-modules/cntk/default.nix +++ b/pkgs/development/python-modules/cntk/default.nix @@ -3,7 +3,7 @@ , pkgs , numpy , scipy -, openmpi +, mpi , enum34 , protobuf , pip @@ -17,8 +17,8 @@ in buildPythonPackage { inherit (cntk) name version src; - nativeBuildInputs = [ swig openmpi ]; - buildInputs = [ cntk openmpi ]; + nativeBuildInputs = [ swig mpi ]; + buildInputs = [ cntk mpi ]; propagatedBuildInputs = [ numpy scipy enum34 protobuf pip ]; CNTK_LIB_PATH = "${cntk}/lib"; @@ -28,7 +28,7 @@ buildPythonPackage { postPatch = '' cd bindings/python - sed -i 's,"libmpi.so.12","${openmpi}/lib/libmpi.so",g' cntk/train/distributed.py + sed -i 's,"libmpi.so.12","${mpi}/lib/libmpi.so",g' cntk/train/distributed.py # Remove distro and libs checks; they aren't compatible with NixOS and besides we guarantee # compatibility by providing a package. diff --git a/pkgs/development/python-modules/pytorch/default.nix b/pkgs/development/python-modules/pytorch/default.nix index f072972937a..822586bf190 100644 --- a/pkgs/development/python-modules/pytorch/default.nix +++ b/pkgs/development/python-modules/pytorch/default.nix @@ -1,7 +1,7 @@ { stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null, mklDnnSupport ? true, useSystemNccl ? true, - openMPISupport ? false, openmpi ? null, + MPISupport ? false, mpi, buildDocs ? false, cudaArchList ? null, @@ -29,8 +29,6 @@ isPy3k, pythonOlder }: -assert !openMPISupport || openmpi != null; - # assert that everything needed for cuda is present and that the correct cuda versions are used assert !cudaSupport || cudatoolkit != null; assert cudnn == null || cudatoolkit != null; @@ -38,7 +36,7 @@ assert !cudaSupport || (let majorIs = lib.versions.major cudatoolkit.version; in majorIs == "9" || majorIs == "10" || majorIs == "11"); # confirm that cudatoolkits are sync'd across dependencies -assert !(openMPISupport && cudaSupport) || openmpi.cudatoolkit == cudatoolkit; +assert !(MPISupport && cudaSupport) || mpi.cudatoolkit == cudatoolkit; assert !cudaSupport || magma.cudatoolkit == cudatoolkit; let @@ -224,7 +222,7 @@ in buildPythonPackage rec { typing-extensions # the following are required for tensorboard support pillow six future tensorflow-tensorboard protobuf - ] ++ lib.optionals openMPISupport [ openmpi ] + ] ++ lib.optionals MPISupport [ mpi ] ++ lib.optionals (pythonOlder "3.7") [ dataclasses ]; checkInputs = [ hypothesis ninja psutil ]; diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 342a87a6e8d..9f64a689e2b 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -11,7 +11,7 @@ # Common deps , git, pybind11, which, binutils, glibcLocales, cython, perl # Common libraries -, jemalloc, openmpi, gast, grpc, sqlite, boringssl, jsoncpp +, jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp , curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib # Upsteam by default includes cuda support since tensorflow 1.15. We could do # that in nix as well. It would make some things easier and less confusing, but @@ -129,7 +129,7 @@ let buildInputs = [ jemalloc - openmpi + mpi glibcLocales git diff --git a/pkgs/development/r-modules/default.nix b/pkgs/development/r-modules/default.nix index e79504bdaab..e827c9b807f 100644 --- a/pkgs/development/r-modules/default.nix +++ b/pkgs/development/r-modules/default.nix @@ -237,7 +237,7 @@ let BayesSAE = [ pkgs.gsl_1 ]; BayesVarSel = [ pkgs.gsl_1 ]; BayesXsrc = [ pkgs.readline.dev pkgs.ncurses ]; - bigGP = [ pkgs.openmpi ]; + bigGP = [ pkgs.mpi ]; bio3d = [ pkgs.zlib ]; BiocCheck = [ pkgs.which ]; Biostrings = [ pkgs.zlib ]; @@ -284,8 +284,8 @@ let n1qn1 = [ pkgs.gfortran ]; odbc = [ pkgs.unixODBC ]; pander = [ pkgs.pandoc pkgs.which ]; - pbdMPI = [ pkgs.openmpi ]; - pbdPROF = [ pkgs.openmpi ]; + pbdMPI = [ pkgs.mpi ]; + pbdPROF = [ pkgs.mpi ]; pbdZMQ = lib.optionals stdenv.isDarwin [ pkgs.which ]; pdftools = [ pkgs.poppler.dev ]; phytools = [ pkgs.which ]; @@ -309,14 +309,14 @@ let RGtk2 = [ pkgs.gtk2.dev ]; rhdf5 = [ pkgs.zlib ]; Rhdf5lib = [ pkgs.zlib ]; - Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.openmpi pkgs.pcre.dev ]; + Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.mpi pkgs.pcre.dev ]; Rhtslib = [ pkgs.zlib.dev pkgs.automake pkgs.autoconf pkgs.bzip2.dev pkgs.lzma.dev pkgs.curl.dev ]; rjags = [ pkgs.jags ]; rJava = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.pcre.dev pkgs.jdk pkgs.libzip ]; Rlibeemd = [ pkgs.gsl_1 ]; rmatio = [ pkgs.zlib.dev ]; Rmpfr = [ pkgs.gmp pkgs.mpfr.dev ]; - Rmpi = [ pkgs.openmpi ]; + Rmpi = [ pkgs.mpi ]; RMySQL = [ pkgs.zlib pkgs.libmysqlclient pkgs.openssl.dev ]; RNetCDF = [ pkgs.netcdf pkgs.udunits ]; RODBC = [ pkgs.libiodbc ]; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 31065758914..c3cea53189b 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -245,7 +245,7 @@ self: super: horovod = super.horovod.overridePythonAttrs ( old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ]; } ); @@ -528,14 +528,14 @@ self: super: { } { mpi = { - mpicc = "${pkgs.openmpi.outPath}/bin/mpicc"; + mpicc = "${pkgs.mpi.outPath}/bin/mpicc"; }; } ); }; in { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ]; enableParallelBuilding = true; preBuild = '' ln -sf ${cfg} mpi.cfg diff --git a/pkgs/tools/misc/hdf5/default.nix b/pkgs/tools/misc/hdf5/default.nix index 0a5032074c1..fc7bb635040 100644 --- a/pkgs/tools/misc/hdf5/default.nix +++ b/pkgs/tools/misc/hdf5/default.nix @@ -5,13 +5,14 @@ , gfortran ? null , zlib ? null , szip ? null -, mpi ? null +, mpiSupport ? false +, mpi , enableShared ? !stdenv.hostPlatform.isStatic }: # cpp and mpi options are mutually exclusive # (--enable-unsupported could be used to force the build) -assert !cpp || mpi == null; +assert !cpp || !mpiSupport; let inherit (lib) optional optionals; in @@ -24,7 +25,7 @@ stdenv.mkDerivation rec { }; passthru = { - mpiSupport = (mpi != null); + inherit mpiSupport; inherit mpi; }; @@ -38,13 +39,13 @@ stdenv.mkDerivation rec { propagatedBuildInputs = [] ++ optional (zlib != null) zlib - ++ optional (mpi != null) mpi; + ++ optional mpiSupport mpi; configureFlags = [] ++ optional cpp "--enable-cxx" ++ optional (gfortran != null) "--enable-fortran" ++ optional (szip != null) "--with-szlib=${szip}" - ++ optionals (mpi != null) ["--enable-parallel" "CC=${mpi}/bin/mpicc"] + ++ optionals mpiSupport ["--enable-parallel" "CC=${mpi}/bin/mpicc"] ++ optional enableShared "--enable-shared"; patches = [ diff --git a/pkgs/tools/misc/hpcg/default.nix b/pkgs/tools/misc/hpcg/default.nix index 29799641880..d6896527ad2 100644 --- a/pkgs/tools/misc/hpcg/default.nix +++ b/pkgs/tools/misc/hpcg/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, openmpi } : +{ lib, stdenv, fetchurl, mpi } : stdenv.mkDerivation rec { pname = "hpcg"; @@ -13,7 +13,7 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; - buildInputs = [ openmpi ]; + buildInputs = [ mpi ]; makeFlags = [ "arch=Linux_MPI" ]; diff --git a/pkgs/tools/system/ior/default.nix b/pkgs/tools/system/ior/default.nix index 326602e3d4d..c2616797da3 100644 --- a/pkgs/tools/system/ior/default.nix +++ b/pkgs/tools/system/ior/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, openmpi, perl, autoreconfHook }: +{ lib, stdenv, fetchFromGitHub, mpi, perl, autoreconfHook }: stdenv.mkDerivation rec { pname = "ior"; @@ -12,7 +12,7 @@ stdenv.mkDerivation rec { }; nativeBuildInputs = [ autoreconfHook ]; - buildInputs = [ openmpi perl ]; + buildInputs = [ mpi perl ]; enableParallelBuilding = true; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index bd86fed4cb2..360d963fed8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4975,12 +4975,11 @@ in hdf5 = callPackage ../tools/misc/hdf5 { gfortran = null; szip = null; - mpi = null; }; hdf5-mpi = appendToName "mpi" (hdf5.override { szip = null; - mpi = pkgs.openmpi; + mpiSupport = true; }); hdf5-cpp = appendToName "cpp" (hdf5.override { @@ -7216,6 +7215,8 @@ in openmpi = callPackage ../development/libraries/openmpi { }; + mpi = openmpi; # this attribute should used to build MPI applications + ucx = callPackage ../development/libraries/ucx {}; openmodelica = callPackage ../applications/science/misc/openmodelica { @@ -22092,9 +22093,7 @@ in fractal = callPackage ../applications/networking/instant-messengers/fractal { }; - freecad = libsForQt5.callPackage ../applications/graphics/freecad { - mpi = openmpi; - }; + freecad = libsForQt5.callPackage ../applications/graphics/freecad { }; freemind = callPackage ../applications/misc/freemind { jdk = jdk8; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731 @@ -22480,7 +22479,7 @@ in hpcg = callPackage ../tools/misc/hpcg/default.nix { }; - hpl = callPackage ../tools/misc/hpl { mpi = openmpi; }; + hpl = callPackage ../tools/misc/hpl { }; hpmyroom = libsForQt5.callPackage ../applications/networking/hpmyroom { }; @@ -27254,15 +27253,11 @@ in quantum-espresso = callPackage ../applications/science/chemistry/quantum-espresso { }; - quantum-espresso-mpi = callPackage ../applications/science/chemistry/quantum-espresso { - mpi = openmpi; - }; + quantum-espresso-mpi = callPackage ../applications/science/chemistry/quantum-espresso { useMpi = true; }; siesta = callPackage ../applications/science/chemistry/siesta { }; - siesta-mpi = callPackage ../applications/science/chemistry/siesta { - mpi = openmpi; - }; + siesta-mpi = callPackage ../applications/science/chemistry/siesta { useMpi = true; }; ### SCIENCE/GEOMETRY @@ -27393,7 +27388,7 @@ in }; neuron-mpi = appendToName "mpi" (neuron.override { - mpi = pkgs.openmpi; + useMpi = true; }); neuron-full = neuron-mpi.override { inherit python; }; @@ -27445,7 +27440,7 @@ in raxml = callPackage ../applications/science/biology/raxml { }; raxml-mpi = appendToName "mpi" (raxml.override { - mpi = true; + useMpi = true; }); sambamba = callPackage ../applications/science/biology/sambamba { }; @@ -27567,9 +27562,7 @@ in planarity = callPackage ../development/libraries/science/math/planarity { }; - scalapack = callPackage ../development/libraries/science/math/scalapack { - mpi = openmpi; - }; + scalapack = callPackage ../development/libraries/science/math/scalapack { }; rankwidth = callPackage ../development/libraries/science/math/rankwidth { }; @@ -27599,9 +27592,7 @@ in petsc = callPackage ../development/libraries/science/math/petsc { }; - parmetis = callPackage ../development/libraries/science/math/parmetis { - mpi = openmpi; - }; + parmetis = callPackage ../development/libraries/science/math/parmetis { }; QuadProgpp = callPackage ../development/libraries/science/math/QuadProgpp { }; @@ -27631,17 +27622,13 @@ in ### SCIENCE/MOLECULAR-DYNAMICS - dl-poly-classic-mpi = callPackage ../applications/science/molecular-dynamics/dl-poly-classic { - mpi = openmpi; - }; + dl-poly-classic-mpi = callPackage ../applications/science/molecular-dynamics/dl-poly-classic { }; lammps = callPackage ../applications/science/molecular-dynamics/lammps { fftw = fftw; }; - lammps-mpi = lowPrio (lammps.override { - mpi = openmpi; - }); + lammps-mpi = lowPrio (lammps.override { withMPI = true; }); gromacs = callPackage ../applications/science/molecular-dynamics/gromacs { singlePrec = true; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 84c11b5cd0f..c5c17744266 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2184,7 +2184,6 @@ in { fenics = callPackage ../development/libraries/science/math/fenics { inherit (pkgs) pkg-config; - mpi = pkgs.openmpi; pytest = self.pytest_4; }; @@ -3486,7 +3485,7 @@ in { labelbox = callPackage ../development/python-modules/labelbox { }; - lammps-cython = callPackage ../development/python-modules/lammps-cython { mpi = pkgs.openmpi; }; + lammps-cython = callPackage ../development/python-modules/lammps-cython { mpi = pkgs.mpi; }; langcodes = callPackage ../development/python-modules/langcodes { }; @@ -4120,7 +4119,7 @@ in { mpd = callPackage ../development/python-modules/mpd { }; - mpi4py = callPackage ../development/python-modules/mpi4py { mpi = pkgs.openmpi; }; + mpi4py = callPackage ../development/python-modules/mpi4py { mpi = pkgs.mpi; }; mplleaflet = callPackage ../development/python-modules/mplleaflet { }; From b6a09f88874ea6ca3c8792b3e867e5d3071594bd Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Sun, 10 Jan 2021 15:08:59 +0100 Subject: [PATCH 2/4] scalapack: fix building with mpich --- pkgs/development/libraries/science/math/scalapack/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/development/libraries/science/math/scalapack/default.nix b/pkgs/development/libraries/science/math/scalapack/default.nix index 4ef10439cc5..3b84a9d99cb 100644 --- a/pkgs/development/libraries/science/math/scalapack/default.nix +++ b/pkgs/development/libraries/science/math/scalapack/default.nix @@ -36,6 +36,9 @@ stdenv.mkDerivation rec { # make sure the test starts even if we have less than 4 cores export OMPI_MCA_rmaps_base_oversubscribe=1 + # Fix to make mpich run in a sandbox + export HYDRA_IFACE=lo + # Run single threaded export OMP_NUM_THREADS=1 From 68fa960f6a79bb60d0ab0a700c443117382a95e2 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Sun, 10 Jan 2021 22:44:04 +0100 Subject: [PATCH 3/4] nixos/release-notes: add comment about generic mpi attribute --- nixos/doc/manual/release-notes/rl-2103.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml index 94e42369b60..cabc60a157a 100644 --- a/nixos/doc/manual/release-notes/rl-2103.xml +++ b/nixos/doc/manual/release-notes/rl-2103.xml @@ -418,6 +418,26 @@ http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/e SDK licenses if your project requires it. See the androidenv documentation for more details. + + + The attribute mpi is now consistently used to + provide a default, system-wide MPI implementation. + The default implementation is openmpi, which has been used before by + all derivations affects by this change. + Note that all packages that have used mpi ? null in the input + for optional MPI builds, have been changed to the boolean input paramater + useMpi to enable building with MPI. + + Building all packages with mpich instead + of the default openmpi can now be achived like this: + +self: super: +{ + mpi = super.mpich; +} + + + The Searx module has been updated with the ability to configure the From 72f96278f73dba04bb73fd3dea0943d21e225cf1 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Sun, 10 Jan 2021 22:44:50 +0100 Subject: [PATCH 4/4] nixpkgs/manual: add a paragraph about MPI switching mechanism --- doc/using/overlays.xml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/using/overlays.xml b/doc/using/overlays.xml index caacb0a0462..9ffbb4edd98 100644 --- a/doc/using/overlays.xml +++ b/doc/using/overlays.xml @@ -291,5 +291,40 @@ stdenv.mkDerivation { } +
+ Switching the MPI implementation + + All programs that are built with + MPI + support use the generic attribute mpi + as an input. At the moment Nixpkgs natively provides two different + MPI implementations: + + + + Open MPI + (default), attribute name openmpi + + + + + MPICH, + attribute name mpich + + + + + + To provide MPI enabled applications that use MPICH, instead + of the default Open MPI, simply use the following overlay: + + +self: super: + +{ + mpi = self.mpich; +} + +