Refactor buildPythonPackage to modularize building process.

Before we used `easy_install` command to handle installation
in one shot, now this is split into two phases:

 - buildPhase: python setup.py build
 - installPhase: python setup.py install

Each of those commands have the ability to pass extra
parameters through buildPythonPackage parameters as
`setupPyInstallFlags` and `setupPyBuildFlags`.

Phases now correctly execute post/pre hooks.

In configurePhase we inject setuptools dependency before distutils
is imported to apply monkeypatching by setuptools that is needed
for special features to apply.

We don't have to reorder default phases anymore, as test
phase comes after build and that works.

I rewrote offineDistutils into distutils-cfg with a bit cleaner
syntax and ability to specify extraCfg to the config file.

Plone packages are failing and garbas said he will adopt them to
the new functions. The rest of the packages I fixed and these commits
shouldn't break any package (according to my testings) and they introduce
16 new jobs and fix 38 that were broken before.
This commit is contained in:
Domen Kožar 2014-01-06 22:24:05 +00:00
parent e9923c6499
commit bf5d6fb9b1
4 changed files with 106 additions and 80 deletions

View file

@ -0,0 +1,31 @@
# global distutils configuration, see http://docs.python.org/2/install/index.html#distutils-configuration-files
{ stdenv, python, writeText, extraCfg ? "" }:
let
distutilsCfg = writeText "distutils.cfg" ''
[easy_install]
# don't allow network connections during build to ensure purity
allow-hosts = None
# make sure we always unzip installed packages otherwise setup hooks won't work
zip_ok = 0
${extraCfg}
'';
in stdenv.mkDerivation {
name = "distutils.cfg-python${python.version}";
buildInputs = [ python ];
unpackPhase = "true";
installPhase = ''
dest="$out/lib/${python.libPrefix}/site-packages/distutils"
mkdir -p $dest
ln -s ${python}/lib/${python.libPrefix}/distutils/* $dest
ln -s ${distutilsCfg} $dest/distutils.cfg
'';
}

View file

@ -1,84 +1,95 @@
/* This function provides a generic Python package builder. It is
intended to work with packages that use `setuptools'
intended to work with packages that use `distutils/setuptools'
(http://pypi.python.org/pypi/setuptools/), which represents a large
number of Python packages nowadays. */
{ python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }:
{ python, setuptools, wrapPython, lib, recursivePthLoader, distutils-cfg }:
{ name, namePrefix ? python.libPrefix + "-"
{ name
# by default prefix name with python version name, e.g. "python3.3-"
, namePrefix ? python.libPrefix + "-"
, buildInputs ? []
# TODO: document
, distutilsExtraCfg ? ""
# TODO: say what it does
, propagatedBuildInputs ? []
, # List of packages that should be added to the PYTHONPATH
# environment variable in programs built by this function. Packages
# in the standard `propagatedBuildInputs' variable are also added.
# The difference is that `pythonPath' is not propagated to the user
# environment. This is preferrable for programs because it doesn't
# pollute the user environment.
pythonPath ? []
# passed to "python setup.py install"
, setupPyInstallFlags ? []
, installCommand ?
''
easy_install --always-unzip --prefix="$out" .
''
, preConfigure ? "true"
, buildPhase ? "true"
# passed to "python setup.py build"
, setupPyBuildFlags ? []
# enable tests by default
, doCheck ? true
, checkPhase ?
''
runHook preCheck
${python}/bin/${python.executable} setup.py test
runHook postCheck
''
# List of packages that should be added to the PYTHONPATH
# environment variable in programs built by this function. Packages
# in the standard `propagatedBuildInputs' variable are also added.
# The difference is that `pythonPath' is not propagated to the user
# environment. This is preferrable for programs because it doesn't
# pollute the user environment.
, pythonPath ? []
, preInstall ? ""
, postInstall ? ""
, meta ? {}
, meta ? {}
, ... } @ attrs:
# Keep extra attributes from ATTR, e.g., `patchPhase', etc.
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
python.stdenv.mkDerivation (attrs // {
inherit doCheck buildPhase checkPhase;
inherit doCheck;
name = namePrefix + name;
# default to python's platforms and add maintainer(s) to every
# package
meta = {
platforms = python.meta.platforms;
} // meta // {
maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
};
# checkPhase after installPhase to run tests on installed packages
phases = "unpackPhase patchPhase configurePhase buildPhase installPhase checkPhase fixupPhase distPhase";
buildInputs = [ python wrapPython setuptools ] ++ buildInputs ++ pythonPath;
buildInputs = [ python wrapPython setuptools (distutils-cfg.override { extraCfg = distutilsExtraCfg; }) ] ++ buildInputs ++ pythonPath;
propagatedBuildInputs = propagatedBuildInputs ++ [ recursivePthLoader ];
pythonPath = [ setuptools ] ++ pythonPath;
preConfigure = ''
configurePhase = attrs.configurePhase or ''
runHook preConfigure
# TODO: document
export DETERMINISTIC_BUILD=1
PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
${preConfigure}
# we need to prepend following line to monkeypatch distutils commands
sed -i '0,/import distutils/s//import setuptools;import distutils/' setup.py
sed -i '0,/from distutils/s//import setuptools;from distutils/' setup.py
runHook postConfigure
'';
installPhase = preInstall + ''
checkPhase = attrs.checkPhase or ''
runHook preCheck
${python}/bin/${python.executable} setup.py test -q
runHook postCheck
'';
buildPhase = attrs.buildPhase or ''
runHook preBuild
${python}/bin/${python.executable} setup.py build ${lib.concatStringsSep " " setupPyBuildFlags}
runHook postBuild
'';
installPhase = attrs.installPhase or ''
runHook preInstall
mkdir -p "$out/lib/${python.libPrefix}/site-packages"
echo "installing \`${name}' with \`easy_install'..."
export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
${installCommand}
${python}/bin/${python.executable} setup.py install --install-lib=$out/lib/${python.libPrefix}/site-packages \
--prefix="$out" ${lib.concatStringsSep " " setupPyInstallFlags}
# A pth file might have been generated to load the package from
# within its own site-packages, rename this package not to
@ -94,17 +105,18 @@ python.stdenv.mkDerivation (attrs // {
# corresponding site.py needs to be included in the PYTHONPATH.
rm -f "$out/lib/${python.libPrefix}"/site-packages/site.py*
${postInstall}
runHook postInstall
'';
postFixup =
''
wrapPythonPrograms
# If a user installs a Python package, she probably also wants its
# If a user installs a Python package, they probably also wants its
# dependencies in the user environment (since Python modules don't
# have something like an RPATH, so the only way to find the
# dependencies is to have them in the PYTHONPATH variable).
# TODO: better docs
if test -e $out/nix-support/propagated-build-inputs; then
ln -s $out/nix-support/propagated-build-inputs $out/nix-support/propagated-user-env-packages
fi
@ -116,4 +128,13 @@ python.stdenv.mkDerivation (attrs // {
fi
done
'';
meta = {
# default to python's platforms
platforms = python.meta.platforms;
} // meta // {
# add extra maintainer(s) to every package
maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
};
})

View file

@ -1,21 +0,0 @@
# Used during module installation to prevent easy_install and python
# setup.py install/test from downloading
{ stdenv, python }:
stdenv.mkDerivation {
name = "python-offline-distutils-${python.version}";
buildInputs = [ python ];
unpackPhase = "true";
installPhase = ''
dst="$out/lib/${python.libPrefix}/site-packages"
ensureDir $dst/distutils
ln -s ${python}/lib/${python.libPrefix}/distutils/* $dst/distutils/
cat <<EOF > $dst/distutils/distutils.cfg
[easy_install]
allow-hosts = None
EOF
'';
}

View file

@ -20,10 +20,10 @@ pythonPackages = modules // import ./python-packages-generated.nix {
callPackage = pkgs.lib.callPackageWith (pkgs // pythonPackages);
buildPythonPackage = import ../development/python-modules/generic {
inherit (pkgs) lib;
inherit python wrapPython setuptools recursivePthLoader offlineDistutils;
};
# global distutils config used by buildPythonPackage
distutils-cfg = callPackage ../development/python-modules/distutils-cfg { };
buildPythonPackage = callPackage ../development/python-modules/generic { };
wrapPython = pkgs.makeSetupHook
{ deps = pkgs.makeWrapper;
@ -33,11 +33,6 @@ pythonPackages = modules // import ./python-packages-generated.nix {
# specials
offlineDistutils = import ../development/python-modules/offline-distutils {
inherit (pkgs) stdenv;
inherit python;
};
recursivePthLoader = import ../development/python-modules/recursive-pth-loader {
inherit (pkgs) stdenv;
inherit python;