From bf5d6fb9b1c12a919853f15571b5a012f87fd928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 6 Jan 2014 22:24:05 +0000 Subject: [PATCH] Refactor buildPythonPackage to modularize building process. Before we used `easy_install` command to handle installation in one shot, now this is split into two phases: - buildPhase: python setup.py build - installPhase: python setup.py install Each of those commands have the ability to pass extra parameters through buildPythonPackage parameters as `setupPyInstallFlags` and `setupPyBuildFlags`. Phases now correctly execute post/pre hooks. In configurePhase we inject setuptools dependency before distutils is imported to apply monkeypatching by setuptools that is needed for special features to apply. We don't have to reorder default phases anymore, as test phase comes after build and that works. I rewrote offineDistutils into distutils-cfg with a bit cleaner syntax and ability to specify extraCfg to the config file. Plone packages are failing and garbas said he will adopt them to the new functions. The rest of the packages I fixed and these commits shouldn't break any package (according to my testings) and they introduce 16 new jobs and fix 38 that were broken before. --- .../python-modules/distutils-cfg/default.nix | 31 +++++ .../python-modules/generic/default.nix | 121 ++++++++++-------- .../offline-distutils/default.nix | 21 --- pkgs/top-level/python-packages.nix | 13 +- 4 files changed, 106 insertions(+), 80 deletions(-) create mode 100644 pkgs/development/python-modules/distutils-cfg/default.nix delete mode 100644 pkgs/development/python-modules/offline-distutils/default.nix diff --git a/pkgs/development/python-modules/distutils-cfg/default.nix b/pkgs/development/python-modules/distutils-cfg/default.nix new file mode 100644 index 00000000000..7b0f3628698 --- /dev/null +++ b/pkgs/development/python-modules/distutils-cfg/default.nix @@ -0,0 +1,31 @@ +# global distutils configuration, see http://docs.python.org/2/install/index.html#distutils-configuration-files + +{ stdenv, python, writeText, extraCfg ? "" }: + + +let + distutilsCfg = writeText "distutils.cfg" '' + [easy_install] + + # don't allow network connections during build to ensure purity + allow-hosts = None + + # make sure we always unzip installed packages otherwise setup hooks won't work + zip_ok = 0 + + ${extraCfg} + ''; +in stdenv.mkDerivation { + name = "distutils.cfg-python${python.version}"; + + buildInputs = [ python ]; + + unpackPhase = "true"; + + installPhase = '' + dest="$out/lib/${python.libPrefix}/site-packages/distutils" + mkdir -p $dest + ln -s ${python}/lib/${python.libPrefix}/distutils/* $dest + ln -s ${distutilsCfg} $dest/distutils.cfg + ''; +} diff --git a/pkgs/development/python-modules/generic/default.nix b/pkgs/development/python-modules/generic/default.nix index 6a7d5e5f3d5..51af437265d 100644 --- a/pkgs/development/python-modules/generic/default.nix +++ b/pkgs/development/python-modules/generic/default.nix @@ -1,84 +1,95 @@ /* This function provides a generic Python package builder. It is - intended to work with packages that use `setuptools' + intended to work with packages that use `distutils/setuptools' (http://pypi.python.org/pypi/setuptools/), which represents a large number of Python packages nowadays. */ -{ python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }: +{ python, setuptools, wrapPython, lib, recursivePthLoader, distutils-cfg }: -{ name, namePrefix ? python.libPrefix + "-" +{ name + +# by default prefix name with python version name, e.g. "python3.3-" +, namePrefix ? python.libPrefix + "-" , buildInputs ? [] +# TODO: document +, distutilsExtraCfg ? "" + +# TODO: say what it does , propagatedBuildInputs ? [] -, # List of packages that should be added to the PYTHONPATH - # environment variable in programs built by this function. Packages - # in the standard `propagatedBuildInputs' variable are also added. - # The difference is that `pythonPath' is not propagated to the user - # environment. This is preferrable for programs because it doesn't - # pollute the user environment. - pythonPath ? [] +# passed to "python setup.py install" +, setupPyInstallFlags ? [] -, installCommand ? - '' - easy_install --always-unzip --prefix="$out" . - '' - -, preConfigure ? "true" - -, buildPhase ? "true" +# passed to "python setup.py build" +, setupPyBuildFlags ? [] +# enable tests by default , doCheck ? true -, checkPhase ? - '' - runHook preCheck - ${python}/bin/${python.executable} setup.py test - runHook postCheck - '' +# List of packages that should be added to the PYTHONPATH +# environment variable in programs built by this function. Packages +# in the standard `propagatedBuildInputs' variable are also added. +# The difference is that `pythonPath' is not propagated to the user +# environment. This is preferrable for programs because it doesn't +# pollute the user environment. +, pythonPath ? [] -, preInstall ? "" -, postInstall ? "" - -, meta ? {} +, meta ? {} , ... } @ attrs: -# Keep extra attributes from ATTR, e.g., `patchPhase', etc. +# Keep extra attributes from `attrs`, e.g., `patchPhase', etc. python.stdenv.mkDerivation (attrs // { - inherit doCheck buildPhase checkPhase; + inherit doCheck; name = namePrefix + name; - # default to python's platforms and add maintainer(s) to every - # package - meta = { - platforms = python.meta.platforms; - } // meta // { - maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ]; - }; - - # checkPhase after installPhase to run tests on installed packages - phases = "unpackPhase patchPhase configurePhase buildPhase installPhase checkPhase fixupPhase distPhase"; - - buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath; + buildInputs = [ python wrapPython setuptools (distutils-cfg.override { extraCfg = distutilsExtraCfg; }) ] ++ buildInputs ++ pythonPath; propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader ]; pythonPath = [ setuptools ] ++ pythonPath; - preConfigure = '' + configurePhase = attrs.configurePhase or '' + runHook preConfigure + + # TODO: document export DETERMINISTIC_BUILD=1 - PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH" - ${preConfigure} + + # we need to prepend following line to monkeypatch distutils commands + sed -i '0,/import distutils/s//import setuptools;import distutils/' setup.py + sed -i '0,/from distutils/s//import setuptools;from distutils/' setup.py + + + runHook postConfigure ''; - installPhase = preInstall + '' + checkPhase = attrs.checkPhase or '' + runHook preCheck + + ${python}/bin/${python.executable} setup.py test -q + + runHook postCheck + ''; + + buildPhase = attrs.buildPhase or '' + runHook preBuild + + ${python}/bin/${python.executable} setup.py build ${lib.concatStringsSep " " setupPyBuildFlags} + + runHook postBuild + ''; + + installPhase = attrs.installPhase or '' + runHook preInstall + mkdir -p "$out/lib/${python.libPrefix}/site-packages" - echo "installing \`${name}' with \`easy_install'..." export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH" - ${installCommand} + + ${python}/bin/${python.executable} setup.py install --install-lib=$out/lib/${python.libPrefix}/site-packages \ + --prefix="$out" ${lib.concatStringsSep " " setupPyInstallFlags} # A pth file might have been generated to load the package from # within its own site-packages, rename this package not to @@ -94,17 +105,18 @@ python.stdenv.mkDerivation (attrs // { # corresponding site.py needs to be included in the PYTHONPATH. rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py* - ${postInstall} + runHook postInstall ''; postFixup = '' wrapPythonPrograms - # If a user installs a Python package, she probably also wants its + # If a user installs a Python package, they probably also wants its # dependencies in the user environment (since Python modules don't # have something like an RPATH, so the only way to find the # dependencies is to have them in the PYTHONPATH variable). + # TODO: better docs if test -e $out/nix-support/propagated-build-inputs; then ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages fi @@ -116,4 +128,13 @@ python.stdenv.mkDerivation (attrs // { fi done ''; + + meta = { + # default to python's platforms + platforms = python.meta.platforms; + } // meta // { + # add extra maintainer(s) to every package + maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ]; + }; + }) diff --git a/pkgs/development/python-modules/offline-distutils/default.nix b/pkgs/development/python-modules/offline-distutils/default.nix deleted file mode 100644 index 269f67bf5cd..00000000000 --- a/pkgs/development/python-modules/offline-distutils/default.nix +++ /dev/null @@ -1,21 +0,0 @@ -# Used during module installation to prevent easy_install and python -# setup.py install/test from downloading - -{ stdenv, python }: - -stdenv.mkDerivation { - name = "python-offline-distutils-${python.version}"; - - buildInputs = [ python ]; - - unpackPhase = "true"; - installPhase = '' - dst="$out/lib/${python.libPrefix}/site-packages" - ensureDir $dst/distutils - ln -s ${python}/lib/${python.libPrefix}/distutils/* $dst/distutils/ - cat < $dst/distutils/distutils.cfg -[easy_install] -allow-hosts = None -EOF - ''; -} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 3aff5627732..0ccd5d5da26 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -20,10 +20,10 @@ pythonPackages = modules // import ./python-packages-generated.nix { callPackage = pkgs.lib.callPackageWith (pkgs // pythonPackages); - buildPythonPackage = import ../development/python-modules/generic { - inherit (pkgs) lib; - inherit python wrapPython setuptools recursivePthLoader offlineDistutils; - }; + # global distutils config used by buildPythonPackage + distutils-cfg = callPackage ../development/python-modules/distutils-cfg { }; + + buildPythonPackage = callPackage ../development/python-modules/generic { }; wrapPython = pkgs.makeSetupHook { deps = pkgs.makeWrapper; @@ -33,11 +33,6 @@ pythonPackages = modules // import ./python-packages-generated.nix { # specials - offlineDistutils = import ../development/python-modules/offline-distutils { - inherit (pkgs) stdenv; - inherit python; - }; - recursivePthLoader = import ../development/python-modules/recursive-pth-loader { inherit (pkgs) stdenv; inherit python;