diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d7c01b8986b..be54040ca2e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -72,12 +72,14 @@ /pkgs/common-updater/scripts/update-source-version @jtojnar # Python-related code and docs -/maintainers/scripts/update-python-libraries @FRidh -/pkgs/top-level/python-packages.nix @FRidh @jonringer -/pkgs/development/interpreters/python @FRidh -/pkgs/development/python-modules @FRidh @jonringer -/doc/languages-frameworks/python.section.md @FRidh -/pkgs/development/tools/poetry2nix @adisbladis +/maintainers/scripts/update-python-libraries @FRidh +/pkgs/top-level/python-packages.nix @FRidh @jonringer +/pkgs/development/interpreters/python @FRidh +/pkgs/development/python-modules @FRidh @jonringer +/doc/languages-frameworks/python.section.md @FRidh +/pkgs/development/tools/poetry2nix @adisbladis +/pkgs/development/interpreters/python/hooks @FRidh @jonringer @DavHau +/pkgs/development/interpreters/python/conda @DavHau # Haskell /doc/languages-frameworks/haskell.section.md @cdepillabout @sternenseemann @maralorn diff --git a/pkgs/development/interpreters/python/conda/default.nix b/pkgs/development/interpreters/python/conda/default.nix new file mode 100644 index 00000000000..5249ccce0d9 --- /dev/null +++ b/pkgs/development/interpreters/python/conda/default.nix @@ -0,0 +1,25 @@ +{ pkgs }: { + + # List of libraries that are needed for conda binary packages. + # When installing a conda binary package, just extend + # the `buildInputs` with `condaAutopatchLibs`. + condaPatchelfLibs = builtins.map (p: p.lib or p) ([ + pkgs.alsaLib + pkgs.cups + pkgs.gcc-unwrapped + pkgs.libGL + ] ++ (with pkgs.xorg; [ + libSM + libICE + libX11 + libXau + libXdamage + libXi + libXrender + libXrandr + libXcomposite + libXcursor + libXtst + libXScrnSaver]) + ); +} diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index 328ed715bae..30134a05628 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -44,6 +44,8 @@ with pkgs; toPythonModule toPythonApplication buildSetupcfg + condaInstallHook + condaUnpackHook eggUnpackHook eggBuildHook eggInstallHook diff --git a/pkgs/development/interpreters/python/hooks/conda-install-hook.sh b/pkgs/development/interpreters/python/hooks/conda-install-hook.sh new file mode 100644 index 00000000000..af9ed60a038 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/conda-install-hook.sh @@ -0,0 +1,27 @@ +# Setup hook to use in case a conda binary package is installed +echo "Sourcing conda install hook" + +condaInstallPhase(){ + echo "Executing condaInstallPhase" + runHook preInstall + + # There are two different formats of conda packages. + # It either contains only a site-packages directory + # or multiple top level directories. + siteDir=@pythonSitePackages@ + if [ -e ./site-packages ]; then + mkdir -p $out/$siteDir + cp -r ./site-packages/* $out/$siteDir + else + cp -r . $out + rm $out/env-vars + fi + + runHook postInstall + echo "Finished executing condaInstallPhase" +} + +if [ -z "${installPhase-}" ]; then + echo "Using condaInstallPhase" + installPhase=condaInstallPhase +fi diff --git a/pkgs/development/interpreters/python/hooks/conda-unpack-hook.sh b/pkgs/development/interpreters/python/hooks/conda-unpack-hook.sh new file mode 100644 index 00000000000..6204c13b722 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/conda-unpack-hook.sh @@ -0,0 +1,18 @@ +# Setup hook to use in case a conda binary package is fetched +echo "Sourcing conda unpack hook" + +condaUnpackPhase(){ + echo "Executing condaUnpackPhase" + runHook preUnpack + + # use lbzip2 for parallel decompression (bz2 is slow) + lbzip2 -dc -n $NIX_BUILD_CORES $src | tar --exclude='info' -x + + # runHook postUnpack # Calls find...? + echo "Finished executing condaUnpackPhase" +} + +if [ -z "${unpackPhase-}" ]; then + echo "Using condaUnpackPhase" + unpackPhase=condaUnpackPhase +fi diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index 1a64c79232b..1a0618225a3 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -16,6 +16,21 @@ let setuppy = ../run_setup.py; in rec { + condaInstallHook = callPackage ({ gnutar, lbzip2 }: + makeSetupHook { + name = "conda-install-hook"; + deps = [ gnutar lbzip2 ]; + substitutions = { + inherit pythonSitePackages; + }; + } ./conda-install-hook.sh) {}; + + condaUnpackHook = callPackage ({}: + makeSetupHook { + name = "conda-unpack-hook"; + deps = []; + } ./conda-unpack-hook.sh) {}; + eggBuildHook = callPackage ({ }: makeSetupHook { name = "egg-build-hook.sh"; diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index cfba04da938..872123338f8 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -121,4 +121,37 @@ let # in assert myPackages.foobar == myPackages.numpy; myPackages.python.withPackages(ps: with ps; [ foobar ]); }; -in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests) + condaTests = let + requests = callPackage ({ + autoPatchelfHook, + fetchurl, + pythonCondaPackages, + }: + python.pkgs.buildPythonPackage { + pname = "requests"; + version = "2.24.0"; + format = "other"; + src = fetchurl { + url = "https://repo.anaconda.com/pkgs/main/noarch/requests-2.24.0-py_0.tar.bz2"; + sha256 = "02qzaf6gwsqbcs69pix1fnjxzgnngwzvrsy65h1d521g750mjvvp"; + }; + nativeBuildInputs = [ autoPatchelfHook ] ++ (with python.pkgs; [ + condaUnpackHook condaInstallHook + ]); + buildInputs = [ + pythonCondaPackages.condaPatchelfLibs + ]; + propagatedBuildInputs = with python.pkgs; [ + chardet idna urllib3 certifi + ]; + } + ) {}; + pythonWithRequests = requests.pythonModule.withPackages (ps: [ requests ]); + in + { + condaExamplePackage = runCommand "import-requests" {} '' + ${pythonWithRequests.interpreter} -c "import requests" > $out + ''; + }; + +in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests // condaTests) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7ef62643fbc..9dee838bb66 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -12336,6 +12336,8 @@ in pythonManylinuxPackages = callPackage ./../development/interpreters/python/manylinux { }; + pythonCondaPackages = callPackage ./../development/interpreters/python/conda { }; + update-python-libraries = callPackage ../development/interpreters/python/update-python-libraries { }; # Should eventually be moved inside Python interpreters. diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 05c322bb932..91ef14b2f78 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -112,6 +112,8 @@ in { inherit buildSetupcfg; inherit (callPackage ../development/interpreters/python/hooks { }) + condaInstallHook + condaUnpackHook eggUnpackHook eggBuildHook eggInstallHook