Merge pull request #108983 from markuskowa/dev-mpi

Use mpi attribute consistently to provide a default MPI implementation
This commit is contained in:
markuskowa 2021-01-23 22:51:09 +01:00 committed by GitHub
commit 2f34b4b883
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 148 additions and 102 deletions

View file

@ -291,5 +291,40 @@ stdenv.mkDerivation {
} }
</programlisting> </programlisting>
</section> </section>
<section xml:id="sec-overlays-alternatives-mpi">
<title>Switching the MPI implementation</title>
<para>
All programs that are built with
<link xlink:href="https://en.wikipedia.org/wiki/Message_Passing_Interface">MPI</link>
support use the generic attribute <varname>mpi</varname>
as an input. At the moment Nixpkgs natively provides two different
MPI implementations:
<itemizedlist>
<listitem>
<para>
<link xlink:href="https://www.open-mpi.org/">Open MPI</link>
(default), attribute name <varname>openmpi</varname>
</para>
</listitem>
<listitem>
<para>
<link xlink:href="https://www.mpich.org/">MPICH</link>,
attribute name <varname>mpich</varname>
</para>
</listitem>
</itemizedlist>
</para>
<para>
To provide MPI enabled applications that use <literal>MPICH</literal>, instead
of the default <literal>Open MPI</literal>, simply use the following overlay:
</para>
<programlisting>
self: super:
{
mpi = self.mpich;
}
</programlisting>
</section>
</section> </section>
</chapter> </chapter>

View file

@ -418,6 +418,26 @@ http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/e
SDK licenses if your project requires it. See the androidenv documentation for more details. SDK licenses if your project requires it. See the androidenv documentation for more details.
</para> </para>
</listitem> </listitem>
<listitem>
<para>
The attribute <varname>mpi</varname> is now consistently used to
provide a default, system-wide MPI implementation.
The default implementation is openmpi, which has been used before by
all derivations affects by this change.
Note that all packages that have used <varname>mpi ? null</varname> in the input
for optional MPI builds, have been changed to the boolean input paramater
<varname>useMpi</varname> to enable building with MPI.
Building all packages with <varname>mpich</varname> instead
of the default <varname>openmpi</varname> can now be achived like this:
<programlisting>
self: super:
{
mpi = super.mpich;
}
</programlisting>
</para>
</listitem>
<listitem> <listitem>
<para> <para>
The Searx module has been updated with the ability to configure the The Searx module has been updated with the ability to configure the

View file

@ -1,6 +1,6 @@
{ boost, cmake, fetchFromGitHub, ffmpeg, qtbase, qtx11extras, { boost, cmake, fetchFromGitHub, ffmpeg, qtbase, qtx11extras,
qttools, qtxmlpatterns, qtsvg, gdal, gfortran, libXt, makeWrapper, qttools, qtxmlpatterns, qtsvg, gdal, gfortran, libXt, makeWrapper,
mkDerivation, ninja, openmpi, python3, lib, stdenv, tbb, libGLU, libGL }: mkDerivation, ninja, mpi, python3, lib, stdenv, tbb, libGLU, libGL }:
mkDerivation rec { mkDerivation rec {
pname = "paraview"; pname = "paraview";
@ -65,7 +65,7 @@ mkDerivation rec {
buildInputs = [ buildInputs = [
libGLU libGL libGLU libGL
libXt libXt
openmpi mpi
tbb tbb
boost boost
ffmpeg ffmpeg

View file

@ -1,4 +1,4 @@
{ gccStdenv, fetchurl, zlib, openmpi }: { gccStdenv, fetchurl, zlib, mpi }:
gccStdenv.mkDerivation rec { gccStdenv.mkDerivation rec {
version = "3.7.2"; version = "3.7.2";
@ -9,7 +9,7 @@ gccStdenv.mkDerivation rec {
sha256 = "1p2364ffjc56i82snzvjpy6pkf6wvqwvlvlqxliscx2c303fxs8v"; sha256 = "1p2364ffjc56i82snzvjpy6pkf6wvqwvlvlqxliscx2c303fxs8v";
}; };
buildInputs = [ zlib openmpi ]; buildInputs = [ zlib mpi ];
setSourceRoot = ''sourceRoot=$(echo */src)''; setSourceRoot = ''sourceRoot=$(echo */src)'';
buildFlags = [ "thread" "mpis" ]; buildFlags = [ "thread" "mpis" ];
preInstall = "mkdir -p $out/man/man1"; preInstall = "mkdir -p $out/man/man1";

View file

@ -8,7 +8,8 @@
, readline , readline
, which , which
, python ? null , python ? null
, mpi ? null , useMpi ? false
, mpi
, iv , iv
}: }:
@ -17,7 +18,8 @@ stdenv.mkDerivation rec {
version = "7.5"; version = "7.5";
nativeBuildInputs = [ which pkg-config automake autoconf libtool ]; nativeBuildInputs = [ which pkg-config automake autoconf libtool ];
buildInputs = [ ncurses readline python mpi iv ]; buildInputs = [ ncurses readline python iv ]
++ lib.optional useMpi mpi;
src = fetchurl { src = fetchurl {
url = "https://www.neuron.yale.edu/ftp/neuron/versions/v${version}/nrn-${version}.tar.gz"; url = "https://www.neuron.yale.edu/ftp/neuron/versions/v${version}/nrn-${version}.tar.gz";
@ -54,7 +56,7 @@ stdenv.mkDerivation rec {
configureFlags = with lib; configureFlags = with lib;
[ "--with-readline=${readline}" "--with-iv=${iv}" ] [ "--with-readline=${readline}" "--with-iv=${iv}" ]
++ optionals (python != null) [ "--with-nrnpython=${python.interpreter}" ] ++ optionals (python != null) [ "--with-nrnpython=${python.interpreter}" ]
++ (if mpi != null then ["--with-mpi" "--with-paranrn"] ++ (if useMpi then ["--with-mpi" "--with-paranrn"]
else ["--without-mpi"]); else ["--without-mpi"]);
@ -84,4 +86,3 @@ stdenv.mkDerivation rec {
platforms = platforms.x86_64 ++ platforms.i686; platforms = platforms.x86_64 ++ platforms.i686;
}; };
} }

View file

@ -1,7 +1,7 @@
{ lib, stdenv { lib, stdenv
, fetchFromGitHub , fetchFromGitHub
, pkgs , useMpi ? false
, mpi ? false , mpi
}: }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
@ -15,16 +15,16 @@ stdenv.mkDerivation rec {
sha256 = "1jqjzhch0rips0vp04prvb8vmc20c5pdmsqn8knadcf91yy859fh"; sha256 = "1jqjzhch0rips0vp04prvb8vmc20c5pdmsqn8knadcf91yy859fh";
}; };
buildInputs = lib.optionals mpi [ pkgs.openmpi ]; buildInputs = lib.optionals useMpi [ mpi ];
# TODO darwin, AVX and AVX2 makefile targets # TODO darwin, AVX and AVX2 makefile targets
buildPhase = if mpi then '' buildPhase = if useMpi then ''
make -f Makefile.MPI.gcc make -f Makefile.MPI.gcc
'' else '' '' else ''
make -f Makefile.SSE3.PTHREADS.gcc make -f Makefile.SSE3.PTHREADS.gcc
''; '';
installPhase = if mpi then '' installPhase = if useMpi then ''
mkdir -p $out/bin && cp raxmlHPC-MPI $out/bin mkdir -p $out/bin && cp raxmlHPC-MPI $out/bin
'' else '' '' else ''
mkdir -p $out/bin && cp raxmlHPC-PTHREADS-SSE3 $out/bin mkdir -p $out/bin && cp raxmlHPC-PTHREADS-SSE3 $out/bin

View file

@ -1,6 +1,6 @@
{ lib, stdenv, fetchFromGitLab, cmake, gfortran, perl { lib, stdenv, fetchFromGitLab, cmake, gfortran, perl
, openblas, hdf5-cpp, python3, texlive , openblas, hdf5-cpp, python3, texlive
, armadillo, openmpi, globalarrays, openssh , armadillo, mpi, globalarrays, openssh
, makeWrapper, fetchpatch , makeWrapper, fetchpatch
} : } :
@ -33,7 +33,7 @@ in stdenv.mkDerivation {
hdf5-cpp hdf5-cpp
python python
armadillo armadillo
openmpi mpi
globalarrays globalarrays
openssh openssh
]; ];

View file

@ -1,6 +1,7 @@
{ lib, stdenv, fetchurl { lib, stdenv, fetchurl
, gfortran, fftw, blas, lapack , gfortran, fftw, blas, lapack
, mpi ? null , useMpi ? false
, mpi
}: }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
@ -21,9 +22,9 @@ stdenv.mkDerivation rec {
''; '';
buildInputs = [ fftw blas lapack gfortran ] buildInputs = [ fftw blas lapack gfortran ]
++ (lib.optionals (mpi != null) [ mpi ]); ++ (lib.optionals useMpi [ mpi ]);
configureFlags = if (mpi != null) then [ "LD=${mpi}/bin/mpif90" ] else [ "LD=${gfortran}/bin/gfortran" ]; configureFlags = if useMpi then [ "LD=${mpi}/bin/mpif90" ] else [ "LD=${gfortran}/bin/gfortran" ];
makeFlags = [ "all" ]; makeFlags = [ "all" ];

View file

@ -1,6 +1,7 @@
{ lib, stdenv, fetchurl { lib, stdenv, fetchurl
, gfortran, blas, lapack , gfortran, blas, lapack, scalapack
, mpi ? null, scalapack , useMpi ? false
, mpi
}: }:
stdenv.mkDerivation { stdenv.mkDerivation {
@ -17,7 +18,7 @@ stdenv.mkDerivation {
}; };
buildInputs = [ blas lapack gfortran ] buildInputs = [ blas lapack gfortran ]
++ (lib.optionals (mpi != null) [ mpi scalapack ]); ++ lib.optionals useMpi [ mpi scalapack ];
enableParallelBuilding = true; enableParallelBuilding = true;
@ -29,7 +30,7 @@ stdenv.mkDerivation {
cp gfortran.make arch.make cp gfortran.make arch.make
''; '';
preBuild = if (mpi != null) then '' preBuild = if useMpi then ''
makeFlagsArray=( makeFlagsArray=(
CC="mpicc" FC="mpifort" CC="mpicc" FC="mpifort"
FPPFLAGS="-DMPI" MPI_INTERFACE="libmpi_f90.a" MPI_INCLUDE="." FPPFLAGS="-DMPI" MPI_INTERFACE="libmpi_f90.a" MPI_INCLUDE="."

View file

@ -11,16 +11,15 @@
, cmake , cmake
, octave , octave
, gl2ps , gl2ps
, mpi
, withQcsxcad ? true , withQcsxcad ? true
, withMPI ? false , withMPI ? false
, withHyp2mat ? true , withHyp2mat ? true
, qcsxcad ? null , qcsxcad ? null
, openmpi ? null
, hyp2mat ? null , hyp2mat ? null
}: }:
assert withQcsxcad -> qcsxcad != null; assert withQcsxcad -> qcsxcad != null;
assert withMPI -> openmpi != null;
assert withHyp2mat -> hyp2mat != null; assert withHyp2mat -> hyp2mat != null;
stdenv.mkDerivation { stdenv.mkDerivation {
@ -50,7 +49,7 @@ stdenv.mkDerivation {
csxcad csxcad
(octave.override { inherit hdf5; }) ] (octave.override { inherit hdf5; }) ]
++ lib.optionals withQcsxcad [ qcsxcad ] ++ lib.optionals withQcsxcad [ qcsxcad ]
++ lib.optionals withMPI [ openmpi ] ++ lib.optionals withMPI [ mpi ]
++ lib.optionals withHyp2mat [ hyp2mat ]; ++ lib.optionals withHyp2mat [ hyp2mat ];
postFixup = '' postFixup = ''

View file

@ -1,5 +1,5 @@
{ lib, stdenv, fetchgit, fetchFromGitHub, cmake { lib, stdenv, fetchgit, fetchFromGitHub, cmake
, openblas, blas, lapack, opencv3, libzip, boost, protobuf, openmpi , openblas, blas, lapack, opencv3, libzip, boost, protobuf, mpi
, onebitSGDSupport ? false , onebitSGDSupport ? false
, cudaSupport ? false, addOpenGLRunpath, cudatoolkit, nvidia_x11 , cudaSupport ? false, addOpenGLRunpath, cudatoolkit, nvidia_x11
, cudnnSupport ? cudaSupport, cudnn , cudnnSupport ? cudaSupport, cudnn
@ -33,7 +33,7 @@ in stdenv.mkDerivation rec {
# Force OpenMPI to use g++ in PATH. # Force OpenMPI to use g++ in PATH.
OMPI_CXX = "g++"; OMPI_CXX = "g++";
buildInputs = [ openblas opencv3 libzip boost protobuf openmpi ] buildInputs = [ openblas opencv3 libzip boost protobuf mpi ]
++ lib.optional cudaSupport cudatoolkit ++ lib.optional cudaSupport cudatoolkit
++ lib.optional cudnnSupport cudnn; ++ lib.optional cudnnSupport cudnn;
@ -43,7 +43,7 @@ in stdenv.mkDerivation rec {
"--with-openblas=${openblas}" "--with-openblas=${openblas}"
"--with-boost=${boost.dev}" "--with-boost=${boost.dev}"
"--with-protobuf=${protobuf}" "--with-protobuf=${protobuf}"
"--with-mpi=${openmpi}" "--with-mpi=${mpi}"
"--cuda=${if cudaSupport then "yes" else "no"}" "--cuda=${if cudaSupport then "yes" else "no"}"
# FIXME # FIXME
"--asgd=no" "--asgd=no"

View file

@ -1,4 +1,4 @@
{ lib, stdenv, fetchurl, cmake, gfortran, blas, lapack, openmpi, petsc, python3 }: { lib, stdenv, fetchurl, cmake, gfortran, blas, lapack, mpi, petsc, python3 }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
name = "getdp-${version}"; name = "getdp-${version}";
@ -9,7 +9,7 @@ stdenv.mkDerivation rec {
}; };
nativeBuildInputs = [ cmake gfortran ]; nativeBuildInputs = [ cmake gfortran ];
buildInputs = [ blas lapack openmpi petsc python3 ]; buildInputs = [ blas lapack mpi petsc python3 ];
meta = with lib; { meta = with lib; {
description = "A General Environment for the Treatment of Discrete Problems"; description = "A General Environment for the Treatment of Discrete Problems";

View file

@ -1,11 +1,11 @@
{ lib, stdenv, fetchurl, bison, openmpi, flex, zlib}: { lib, stdenv, fetchurl, bison, mpi, flex, zlib}:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
version = "6.0.4"; version = "6.0.4";
pname = "scotch"; pname = "scotch";
src_name = "scotch_${version}"; src_name = "scotch_${version}";
buildInputs = [ bison openmpi flex zlib ]; buildInputs = [ bison mpi flex zlib ];
src = fetchurl { src = fetchurl {
url = "https://gforge.inria.fr/frs/download.php/file/34618/${src_name}.tar.gz"; url = "https://gforge.inria.fr/frs/download.php/file/34618/${src_name}.tar.gz";

View file

@ -3,10 +3,10 @@
, cmake , cmake
, hwloc , hwloc
, fftw , fftw
, openmpi
, perl , perl
, singlePrec ? true , singlePrec ? true
, mpiEnabled ? false , mpiEnabled ? false
, mpi
, cpuAcceleration ? null , cpuAcceleration ? null
}: }:
@ -33,7 +33,7 @@ in stdenv.mkDerivation rec {
nativeBuildInputs = [ cmake ]; nativeBuildInputs = [ cmake ];
buildInputs = [ fftw perl hwloc ] buildInputs = [ fftw perl hwloc ]
++ (lib.optionals mpiEnabled [ openmpi ]); ++ (lib.optionals mpiEnabled [ mpi ]);
cmakeFlags = [ cmakeFlags = [
"-DGMX_SIMD:STRING=${SIMD cpuAcceleration}" "-DGMX_SIMD:STRING=${SIMD cpuAcceleration}"

View file

@ -1,6 +1,7 @@
{ lib, stdenv, fetchFromGitHub { lib, stdenv, fetchFromGitHub
, libpng, gzip, fftw, blas, lapack , libpng, gzip, fftw, blas, lapack
, mpi ? null , withMPI ? false
, mpi
}: }:
let packages = [ let packages = [
"asphere" "body" "class2" "colloid" "compress" "coreshell" "asphere" "body" "class2" "colloid" "compress" "coreshell"
@ -8,7 +9,6 @@ let packages = [
"opt" "peri" "qeq" "replica" "rigid" "shock" "snap" "srd" "user-reaxc" "opt" "peri" "qeq" "replica" "rigid" "shock" "snap" "srd" "user-reaxc"
]; ];
lammps_includes = "-DLAMMPS_EXCEPTIONS -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64"; lammps_includes = "-DLAMMPS_EXCEPTIONS -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64";
withMPI = (mpi != null);
in in
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
# LAMMPS has weird versioning converted to ISO 8601 format # LAMMPS has weird versioning converted to ISO 8601 format

View file

@ -1,4 +1,4 @@
{ lib, stdenv, fetchFromGitHub, cmake, git, gfortran, openmpi, blas, liblapack, qt4, qwt6_qt4, pkg-config }: { lib, stdenv, fetchFromGitHub, cmake, git, gfortran, mpi, blas, liblapack, qt4, qwt6_qt4, pkg-config }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
pname = "elmerfem"; pname = "elmerfem";
@ -14,7 +14,7 @@ stdenv.mkDerivation rec {
hardeningDisable = [ "format" ]; hardeningDisable = [ "format" ];
nativeBuildInputs = [ cmake pkg-config git ]; nativeBuildInputs = [ cmake pkg-config git ];
buildInputs = [ gfortran openmpi blas liblapack qt4 qwt6_qt4 ]; buildInputs = [ gfortran mpi blas liblapack qt4 qwt6_qt4 ];
preConfigure = '' preConfigure = ''
patchShebangs ./ patchShebangs ./

View file

@ -14,7 +14,8 @@
, enableNumpy ? false , enableNumpy ? false
, taggedLayout ? ((enableRelease && enableDebug) || (enableSingleThreaded && enableMultiThreaded) || (enableShared && enableStatic)) , taggedLayout ? ((enableRelease && enableDebug) || (enableSingleThreaded && enableMultiThreaded) || (enableShared && enableStatic))
, patches ? [] , patches ? []
, mpi ? null , useMpi ? false
, mpi
, extraB2Args ? [] , extraB2Args ? []
# Attributes inherit from specific versions # Attributes inherit from specific versions
@ -94,7 +95,7 @@ let
++ optional (variant == "release") "debug-symbols=off" ++ optional (variant == "release") "debug-symbols=off"
++ optional (toolset != null) "toolset=${toolset}" ++ optional (toolset != null) "toolset=${toolset}"
++ optional (!enablePython) "--without-python" ++ optional (!enablePython) "--without-python"
++ optional (mpi != null || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" ++ optional (useMpi || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam"
++ optionals (stdenv.hostPlatform.libc == "msvcrt") [ ++ optionals (stdenv.hostPlatform.libc == "msvcrt") [
"threadapi=win32" "threadapi=win32"
] ++ extraB2Args ] ++ extraB2Args
@ -140,7 +141,7 @@ stdenv.mkDerivation {
substituteInPlace tools/build/src/tools/clang-darwin.jam \ substituteInPlace tools/build/src/tools/clang-darwin.jam \
--replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)"; --replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)";
fi; fi;
'' + optionalString (mpi != null) '' '' + optionalString useMpi ''
cat << EOF >> user-config.jam cat << EOF >> user-config.jam
using mpi : ${mpi}/bin/mpiCC ; using mpi : ${mpi}/bin/mpiCC ;
EOF EOF

View file

@ -1,5 +1,5 @@
{ lib, stdenv, fetchpatch, fetchFromGitHub, autoreconfHook { lib, stdenv, fetchpatch, fetchFromGitHub, autoreconfHook
, blas, gfortran, openssh, openmpi , blas, gfortran, openssh, mpi
} : } :
let let
@ -17,7 +17,7 @@ in stdenv.mkDerivation {
}; };
nativeBuildInputs = [ autoreconfHook ]; nativeBuildInputs = [ autoreconfHook ];
buildInputs = [ openmpi blas gfortran openssh ]; buildInputs = [ mpi blas gfortran openssh ];
preConfigure = '' preConfigure = ''
configureFlagsArray+=( "--enable-i8" \ configureFlagsArray+=( "--enable-i8" \

View file

@ -1,4 +1,4 @@
{ lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, openmpi, python3, petsc }: { lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, mpi, python3, petsc }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
pname = "precice"; pname = "precice";
@ -21,7 +21,7 @@ stdenv.mkDerivation rec {
NIX_CFLAGS_COMPILE = lib.optional stdenv.isDarwin [ "-D_GNU_SOURCE" ]; NIX_CFLAGS_COMPILE = lib.optional stdenv.isDarwin [ "-D_GNU_SOURCE" ];
nativeBuildInputs = [ cmake gcc ]; nativeBuildInputs = [ cmake gcc ];
buildInputs = [ boost eigen libxml2 openmpi python3 python3.pkgs.numpy ]; buildInputs = [ boost eigen libxml2 mpi python3 python3.pkgs.numpy ];
meta = { meta = {
description = "preCICE stands for Precise Code Interaction Coupling Environment"; description = "preCICE stands for Precise Code Interaction Coupling Environment";

View file

@ -36,6 +36,9 @@ stdenv.mkDerivation rec {
# make sure the test starts even if we have less than 4 cores # make sure the test starts even if we have less than 4 cores
export OMPI_MCA_rmaps_base_oversubscribe=1 export OMPI_MCA_rmaps_base_oversubscribe=1
# Fix to make mpich run in a sandbox
export HYDRA_IFACE=lo
# Run single threaded # Run single threaded
export OMP_NUM_THREADS=1 export OMP_NUM_THREADS=1

View file

@ -3,7 +3,7 @@
, pkgs , pkgs
, numpy , numpy
, scipy , scipy
, openmpi , mpi
, enum34 , enum34
, protobuf , protobuf
, pip , pip
@ -17,8 +17,8 @@ in
buildPythonPackage { buildPythonPackage {
inherit (cntk) name version src; inherit (cntk) name version src;
nativeBuildInputs = [ swig openmpi ]; nativeBuildInputs = [ swig mpi ];
buildInputs = [ cntk openmpi ]; buildInputs = [ cntk mpi ];
propagatedBuildInputs = [ numpy scipy enum34 protobuf pip ]; propagatedBuildInputs = [ numpy scipy enum34 protobuf pip ];
CNTK_LIB_PATH = "${cntk}/lib"; CNTK_LIB_PATH = "${cntk}/lib";
@ -28,7 +28,7 @@ buildPythonPackage {
postPatch = '' postPatch = ''
cd bindings/python cd bindings/python
sed -i 's,"libmpi.so.12","${openmpi}/lib/libmpi.so",g' cntk/train/distributed.py sed -i 's,"libmpi.so.12","${mpi}/lib/libmpi.so",g' cntk/train/distributed.py
# Remove distro and libs checks; they aren't compatible with NixOS and besides we guarantee # Remove distro and libs checks; they aren't compatible with NixOS and besides we guarantee
# compatibility by providing a package. # compatibility by providing a package.

View file

@ -1,7 +1,7 @@
{ stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python, { stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python,
cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null,
mklDnnSupport ? true, useSystemNccl ? true, mklDnnSupport ? true, useSystemNccl ? true,
openMPISupport ? false, openmpi ? null, MPISupport ? false, mpi,
buildDocs ? false, buildDocs ? false,
cudaArchList ? null, cudaArchList ? null,
@ -29,8 +29,6 @@
isPy3k, pythonOlder }: isPy3k, pythonOlder }:
assert !openMPISupport || openmpi != null;
# assert that everything needed for cuda is present and that the correct cuda versions are used # assert that everything needed for cuda is present and that the correct cuda versions are used
assert !cudaSupport || cudatoolkit != null; assert !cudaSupport || cudatoolkit != null;
assert cudnn == null || cudatoolkit != null; assert cudnn == null || cudatoolkit != null;
@ -38,7 +36,7 @@ assert !cudaSupport || (let majorIs = lib.versions.major cudatoolkit.version;
in majorIs == "9" || majorIs == "10" || majorIs == "11"); in majorIs == "9" || majorIs == "10" || majorIs == "11");
# confirm that cudatoolkits are sync'd across dependencies # confirm that cudatoolkits are sync'd across dependencies
assert !(openMPISupport && cudaSupport) || openmpi.cudatoolkit == cudatoolkit; assert !(MPISupport && cudaSupport) || mpi.cudatoolkit == cudatoolkit;
assert !cudaSupport || magma.cudatoolkit == cudatoolkit; assert !cudaSupport || magma.cudatoolkit == cudatoolkit;
let let
@ -224,7 +222,7 @@ in buildPythonPackage rec {
typing-extensions typing-extensions
# the following are required for tensorboard support # the following are required for tensorboard support
pillow six future tensorflow-tensorboard protobuf pillow six future tensorflow-tensorboard protobuf
] ++ lib.optionals openMPISupport [ openmpi ] ] ++ lib.optionals MPISupport [ mpi ]
++ lib.optionals (pythonOlder "3.7") [ dataclasses ]; ++ lib.optionals (pythonOlder "3.7") [ dataclasses ];
checkInputs = [ hypothesis ninja psutil ]; checkInputs = [ hypothesis ninja psutil ];

View file

@ -11,7 +11,7 @@
# Common deps # Common deps
, git, pybind11, which, binutils, glibcLocales, cython, perl , git, pybind11, which, binutils, glibcLocales, cython, perl
# Common libraries # Common libraries
, jemalloc, openmpi, gast, grpc, sqlite, boringssl, jsoncpp , jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp
, curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib , curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib
# Upsteam by default includes cuda support since tensorflow 1.15. We could do # Upsteam by default includes cuda support since tensorflow 1.15. We could do
# that in nix as well. It would make some things easier and less confusing, but # that in nix as well. It would make some things easier and less confusing, but
@ -129,7 +129,7 @@ let
buildInputs = [ buildInputs = [
jemalloc jemalloc
openmpi mpi
glibcLocales glibcLocales
git git

View file

@ -237,7 +237,7 @@ let
BayesSAE = [ pkgs.gsl_1 ]; BayesSAE = [ pkgs.gsl_1 ];
BayesVarSel = [ pkgs.gsl_1 ]; BayesVarSel = [ pkgs.gsl_1 ];
BayesXsrc = [ pkgs.readline.dev pkgs.ncurses ]; BayesXsrc = [ pkgs.readline.dev pkgs.ncurses ];
bigGP = [ pkgs.openmpi ]; bigGP = [ pkgs.mpi ];
bio3d = [ pkgs.zlib ]; bio3d = [ pkgs.zlib ];
BiocCheck = [ pkgs.which ]; BiocCheck = [ pkgs.which ];
Biostrings = [ pkgs.zlib ]; Biostrings = [ pkgs.zlib ];
@ -284,8 +284,8 @@ let
n1qn1 = [ pkgs.gfortran ]; n1qn1 = [ pkgs.gfortran ];
odbc = [ pkgs.unixODBC ]; odbc = [ pkgs.unixODBC ];
pander = [ pkgs.pandoc pkgs.which ]; pander = [ pkgs.pandoc pkgs.which ];
pbdMPI = [ pkgs.openmpi ]; pbdMPI = [ pkgs.mpi ];
pbdPROF = [ pkgs.openmpi ]; pbdPROF = [ pkgs.mpi ];
pbdZMQ = lib.optionals stdenv.isDarwin [ pkgs.which ]; pbdZMQ = lib.optionals stdenv.isDarwin [ pkgs.which ];
pdftools = [ pkgs.poppler.dev ]; pdftools = [ pkgs.poppler.dev ];
phytools = [ pkgs.which ]; phytools = [ pkgs.which ];
@ -309,14 +309,14 @@ let
RGtk2 = [ pkgs.gtk2.dev ]; RGtk2 = [ pkgs.gtk2.dev ];
rhdf5 = [ pkgs.zlib ]; rhdf5 = [ pkgs.zlib ];
Rhdf5lib = [ pkgs.zlib ]; Rhdf5lib = [ pkgs.zlib ];
Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.openmpi pkgs.pcre.dev ]; Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.mpi pkgs.pcre.dev ];
Rhtslib = [ pkgs.zlib.dev pkgs.automake pkgs.autoconf pkgs.bzip2.dev pkgs.lzma.dev pkgs.curl.dev ]; Rhtslib = [ pkgs.zlib.dev pkgs.automake pkgs.autoconf pkgs.bzip2.dev pkgs.lzma.dev pkgs.curl.dev ];
rjags = [ pkgs.jags ]; rjags = [ pkgs.jags ];
rJava = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.pcre.dev pkgs.jdk pkgs.libzip ]; rJava = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.pcre.dev pkgs.jdk pkgs.libzip ];
Rlibeemd = [ pkgs.gsl_1 ]; Rlibeemd = [ pkgs.gsl_1 ];
rmatio = [ pkgs.zlib.dev ]; rmatio = [ pkgs.zlib.dev ];
Rmpfr = [ pkgs.gmp pkgs.mpfr.dev ]; Rmpfr = [ pkgs.gmp pkgs.mpfr.dev ];
Rmpi = [ pkgs.openmpi ]; Rmpi = [ pkgs.mpi ];
RMySQL = [ pkgs.zlib pkgs.libmysqlclient pkgs.openssl.dev ]; RMySQL = [ pkgs.zlib pkgs.libmysqlclient pkgs.openssl.dev ];
RNetCDF = [ pkgs.netcdf pkgs.udunits ]; RNetCDF = [ pkgs.netcdf pkgs.udunits ];
RODBC = [ pkgs.libiodbc ]; RODBC = [ pkgs.libiodbc ];

View file

@ -245,7 +245,7 @@ self: super:
horovod = super.horovod.overridePythonAttrs ( horovod = super.horovod.overridePythonAttrs (
old: { old: {
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ];
} }
); );
@ -528,14 +528,14 @@ self: super:
{ } { }
{ {
mpi = { mpi = {
mpicc = "${pkgs.openmpi.outPath}/bin/mpicc"; mpicc = "${pkgs.mpi.outPath}/bin/mpicc";
}; };
} }
); );
}; };
in in
{ {
propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ];
enableParallelBuilding = true; enableParallelBuilding = true;
preBuild = '' preBuild = ''
ln -sf ${cfg} mpi.cfg ln -sf ${cfg} mpi.cfg

View file

@ -5,13 +5,14 @@
, gfortran ? null , gfortran ? null
, zlib ? null , zlib ? null
, szip ? null , szip ? null
, mpi ? null , mpiSupport ? false
, mpi
, enableShared ? !stdenv.hostPlatform.isStatic , enableShared ? !stdenv.hostPlatform.isStatic
}: }:
# cpp and mpi options are mutually exclusive # cpp and mpi options are mutually exclusive
# (--enable-unsupported could be used to force the build) # (--enable-unsupported could be used to force the build)
assert !cpp || mpi == null; assert !cpp || !mpiSupport;
let inherit (lib) optional optionals; in let inherit (lib) optional optionals; in
@ -24,7 +25,7 @@ stdenv.mkDerivation rec {
}; };
passthru = { passthru = {
mpiSupport = (mpi != null); inherit mpiSupport;
inherit mpi; inherit mpi;
}; };
@ -38,13 +39,13 @@ stdenv.mkDerivation rec {
propagatedBuildInputs = [] propagatedBuildInputs = []
++ optional (zlib != null) zlib ++ optional (zlib != null) zlib
++ optional (mpi != null) mpi; ++ optional mpiSupport mpi;
configureFlags = [] configureFlags = []
++ optional cpp "--enable-cxx" ++ optional cpp "--enable-cxx"
++ optional (gfortran != null) "--enable-fortran" ++ optional (gfortran != null) "--enable-fortran"
++ optional (szip != null) "--with-szlib=${szip}" ++ optional (szip != null) "--with-szlib=${szip}"
++ optionals (mpi != null) ["--enable-parallel" "CC=${mpi}/bin/mpicc"] ++ optionals mpiSupport ["--enable-parallel" "CC=${mpi}/bin/mpicc"]
++ optional enableShared "--enable-shared"; ++ optional enableShared "--enable-shared";
patches = [ patches = [

View file

@ -1,4 +1,4 @@
{ lib, stdenv, fetchurl, openmpi } : { lib, stdenv, fetchurl, mpi } :
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
pname = "hpcg"; pname = "hpcg";
@ -13,7 +13,7 @@ stdenv.mkDerivation rec {
enableParallelBuilding = true; enableParallelBuilding = true;
buildInputs = [ openmpi ]; buildInputs = [ mpi ];
makeFlags = [ "arch=Linux_MPI" ]; makeFlags = [ "arch=Linux_MPI" ];

View file

@ -1,4 +1,4 @@
{ lib, stdenv, fetchFromGitHub, openmpi, perl, autoreconfHook }: { lib, stdenv, fetchFromGitHub, mpi, perl, autoreconfHook }:
stdenv.mkDerivation rec { stdenv.mkDerivation rec {
pname = "ior"; pname = "ior";
@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
}; };
nativeBuildInputs = [ autoreconfHook ]; nativeBuildInputs = [ autoreconfHook ];
buildInputs = [ openmpi perl ]; buildInputs = [ mpi perl ];
enableParallelBuilding = true; enableParallelBuilding = true;

View file

@ -4975,12 +4975,11 @@ in
hdf5 = callPackage ../tools/misc/hdf5 { hdf5 = callPackage ../tools/misc/hdf5 {
gfortran = null; gfortran = null;
szip = null; szip = null;
mpi = null;
}; };
hdf5-mpi = appendToName "mpi" (hdf5.override { hdf5-mpi = appendToName "mpi" (hdf5.override {
szip = null; szip = null;
mpi = pkgs.openmpi; mpiSupport = true;
}); });
hdf5-cpp = appendToName "cpp" (hdf5.override { hdf5-cpp = appendToName "cpp" (hdf5.override {
@ -7220,6 +7219,8 @@ in
openmpi = callPackage ../development/libraries/openmpi { }; openmpi = callPackage ../development/libraries/openmpi { };
mpi = openmpi; # this attribute should used to build MPI applications
ucx = callPackage ../development/libraries/ucx {}; ucx = callPackage ../development/libraries/ucx {};
openmodelica = callPackage ../applications/science/misc/openmodelica { openmodelica = callPackage ../applications/science/misc/openmodelica {
@ -22101,9 +22102,7 @@ in
fractal = callPackage ../applications/networking/instant-messengers/fractal { }; fractal = callPackage ../applications/networking/instant-messengers/fractal { };
freecad = libsForQt5.callPackage ../applications/graphics/freecad { freecad = libsForQt5.callPackage ../applications/graphics/freecad { };
mpi = openmpi;
};
freemind = callPackage ../applications/misc/freemind { freemind = callPackage ../applications/misc/freemind {
jdk = jdk8; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731 jdk = jdk8; # TODO: remove override https://github.com/NixOS/nixpkgs/pull/89731
@ -22489,7 +22488,7 @@ in
hpcg = callPackage ../tools/misc/hpcg/default.nix { }; hpcg = callPackage ../tools/misc/hpcg/default.nix { };
hpl = callPackage ../tools/misc/hpl { mpi = openmpi; }; hpl = callPackage ../tools/misc/hpl { };
hpmyroom = libsForQt5.callPackage ../applications/networking/hpmyroom { }; hpmyroom = libsForQt5.callPackage ../applications/networking/hpmyroom { };
@ -27263,15 +27262,11 @@ in
quantum-espresso = callPackage ../applications/science/chemistry/quantum-espresso { }; quantum-espresso = callPackage ../applications/science/chemistry/quantum-espresso { };
quantum-espresso-mpi = callPackage ../applications/science/chemistry/quantum-espresso { quantum-espresso-mpi = callPackage ../applications/science/chemistry/quantum-espresso { useMpi = true; };
mpi = openmpi;
};
siesta = callPackage ../applications/science/chemistry/siesta { }; siesta = callPackage ../applications/science/chemistry/siesta { };
siesta-mpi = callPackage ../applications/science/chemistry/siesta { siesta-mpi = callPackage ../applications/science/chemistry/siesta { useMpi = true; };
mpi = openmpi;
};
### SCIENCE/GEOMETRY ### SCIENCE/GEOMETRY
@ -27402,7 +27397,7 @@ in
}; };
neuron-mpi = appendToName "mpi" (neuron.override { neuron-mpi = appendToName "mpi" (neuron.override {
mpi = pkgs.openmpi; useMpi = true;
}); });
neuron-full = neuron-mpi.override { inherit python; }; neuron-full = neuron-mpi.override { inherit python; };
@ -27454,7 +27449,7 @@ in
raxml = callPackage ../applications/science/biology/raxml { }; raxml = callPackage ../applications/science/biology/raxml { };
raxml-mpi = appendToName "mpi" (raxml.override { raxml-mpi = appendToName "mpi" (raxml.override {
mpi = true; useMpi = true;
}); });
sambamba = callPackage ../applications/science/biology/sambamba { }; sambamba = callPackage ../applications/science/biology/sambamba { };
@ -27576,9 +27571,7 @@ in
planarity = callPackage ../development/libraries/science/math/planarity { }; planarity = callPackage ../development/libraries/science/math/planarity { };
scalapack = callPackage ../development/libraries/science/math/scalapack { scalapack = callPackage ../development/libraries/science/math/scalapack { };
mpi = openmpi;
};
rankwidth = callPackage ../development/libraries/science/math/rankwidth { }; rankwidth = callPackage ../development/libraries/science/math/rankwidth { };
@ -27608,9 +27601,7 @@ in
petsc = callPackage ../development/libraries/science/math/petsc { }; petsc = callPackage ../development/libraries/science/math/petsc { };
parmetis = callPackage ../development/libraries/science/math/parmetis { parmetis = callPackage ../development/libraries/science/math/parmetis { };
mpi = openmpi;
};
QuadProgpp = callPackage ../development/libraries/science/math/QuadProgpp { }; QuadProgpp = callPackage ../development/libraries/science/math/QuadProgpp { };
@ -27640,17 +27631,13 @@ in
### SCIENCE/MOLECULAR-DYNAMICS ### SCIENCE/MOLECULAR-DYNAMICS
dl-poly-classic-mpi = callPackage ../applications/science/molecular-dynamics/dl-poly-classic { dl-poly-classic-mpi = callPackage ../applications/science/molecular-dynamics/dl-poly-classic { };
mpi = openmpi;
};
lammps = callPackage ../applications/science/molecular-dynamics/lammps { lammps = callPackage ../applications/science/molecular-dynamics/lammps {
fftw = fftw; fftw = fftw;
}; };
lammps-mpi = lowPrio (lammps.override { lammps-mpi = lowPrio (lammps.override { withMPI = true; });
mpi = openmpi;
});
gromacs = callPackage ../applications/science/molecular-dynamics/gromacs { gromacs = callPackage ../applications/science/molecular-dynamics/gromacs {
singlePrec = true; singlePrec = true;

View file

@ -2188,7 +2188,6 @@ in {
fenics = callPackage ../development/libraries/science/math/fenics { fenics = callPackage ../development/libraries/science/math/fenics {
inherit (pkgs) pkg-config; inherit (pkgs) pkg-config;
mpi = pkgs.openmpi;
pytest = self.pytest_4; pytest = self.pytest_4;
}; };
@ -3490,7 +3489,7 @@ in {
labelbox = callPackage ../development/python-modules/labelbox { }; labelbox = callPackage ../development/python-modules/labelbox { };
lammps-cython = callPackage ../development/python-modules/lammps-cython { mpi = pkgs.openmpi; }; lammps-cython = callPackage ../development/python-modules/lammps-cython { mpi = pkgs.mpi; };
langcodes = callPackage ../development/python-modules/langcodes { }; langcodes = callPackage ../development/python-modules/langcodes { };
@ -4124,7 +4123,7 @@ in {
mpd = callPackage ../development/python-modules/mpd { }; mpd = callPackage ../development/python-modules/mpd { };
mpi4py = callPackage ../development/python-modules/mpi4py { mpi = pkgs.openmpi; }; mpi4py = callPackage ../development/python-modules/mpi4py { mpi = pkgs.mpi; };
mplleaflet = callPackage ../development/python-modules/mplleaflet { }; mplleaflet = callPackage ../development/python-modules/mplleaflet { };