Python: improve cross-compilation

This changeset allows for cross-compilation of Python packages. Packages
built with buildPythonPackage are not allowed to refer to the build
machine. Executables that have shebangs will refer to the host.
This commit is contained in:
Frederik Rietdijk 2019-01-02 20:09:44 +01:00
parent 613498af97
commit f665828fa3
14 changed files with 68 additions and 41 deletions

View file

@ -1,7 +1,6 @@
# This function provides generic bits to install a Python wheel.
{ python
, bootstrapped-pip
}:
{ buildInputs ? []
@ -10,7 +9,7 @@
, ... } @ attrs:
attrs // {
buildInputs = buildInputs ++ [ bootstrapped-pip ];
buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
configurePhase = attrs.configurePhase or ''
runHook preConfigure
@ -24,7 +23,7 @@ attrs // {
export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
pushd dist
${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
popd
runHook postInstall

View file

@ -2,7 +2,6 @@
{ lib
, python
, bootstrapped-pip
}:
{
@ -26,13 +25,13 @@ in attrs // {
buildPhase = attrs.buildPhase or ''
runHook preBuild
cp ${setuppy} nix_run_setup
${python.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
runHook postBuild
'';
installCheckPhase = attrs.checkPhase or ''
runHook preCheck
${python.interpreter} nix_run_setup test
${python.pythonForBuild.interpreter} nix_run_setup test
runHook postCheck
'';
@ -47,9 +46,9 @@ in attrs // {
if test -e setup.py; then
tmp_path=$(mktemp -d)
export PATH="$tmp_path/bin:$PATH"
export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
mkdir -p $tmp_path/${python.sitePackages}
${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
fi
${postShellHook}
'';

View file

@ -10,17 +10,16 @@
, ensureNewerSourcesForZipFilesHook
, toPythonModule
, namePrefix
, bootstrapped-pip
, flit
, writeScript
, update-python-libraries
}:
let
setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; };
setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
wheel-specific = import ./build-python-package-wheel.nix { };
common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
common = import ./build-python-package-common.nix { inherit python; };
mkPythonDerivation = import ./mk-python-derivation.nix {
inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
inherit toPythonModule namePrefix writeScript update-python-libraries;

View file

@ -32,6 +32,9 @@ assert x11Support -> tcl != null
with stdenv.lib;
let
pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
implementation = "cpython";

View file

@ -21,6 +21,7 @@
, sourceVersion
, sha256
, passthruFun
, bash
}:
assert x11Support -> tcl != null
@ -46,7 +47,8 @@ let
nativeBuildInputs = [
nukeReferences
] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
buildPackages.stdenv.cc crossPython
buildPackages.stdenv.cc
pythonForBuild
];
buildInputs = filter (p: p != null) [
@ -56,11 +58,11 @@ let
hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then
pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
"$out/bin/python"
else crossPython.interpreter;
else pythonForBuild.interpreter;
in with passthru; stdenv.mkDerivation {
pname = "python3";
@ -215,14 +217,25 @@ in with passthru; stdenv.mkDerivation {
# We rebuild three times, once for each optimization level
# Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
# This is automatically used when `SOURCE_DATE_EPOCH` is set.
find $out -name "*.py" | ${pythonForBuild} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuild} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuild} -OO -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'';
preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
# Ensure patch-shebangs uses shebangs of host interpreter.
export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
'';
# Enforce that we don't have references to the OpenSSL -dev package, which we
# explicitly specify in our configure flags above.
disallowedReferences = [ openssl.dev ];
disallowedReferences = [
openssl.dev
] ++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
# Ensure we don't have references to build-time packages.
# These typically end up in shebangs.
pythonForBuild buildPackages.bash
];
inherit passthru;

View file

@ -21,7 +21,6 @@ with pkgs;
overrides = packageOverrides;
};
in rec {
sitePackages = "lib/${libPrefix}/site-packages";
isPy27 = pythonVersion == "2.7";
isPy33 = pythonVersion == "3.3"; # TODO: remove
isPy34 = pythonVersion == "3.4"; # TODO: remove
@ -35,7 +34,7 @@ with pkgs;
withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;};
pkgs = pythonPackages;
interpreter = "${self}/bin/${executable}";
inherit executable implementation libPrefix pythonVersion;
inherit executable implementation libPrefix pythonVersion sitePackages;
inherit sourceVersion;
pythonAtLeast = lib.versionAtLeast pythonVersion;
pythonOlder = lib.versionOlder pythonVersion;
@ -112,8 +111,8 @@ in {
inherit passthruFun;
};
pypy3 = callPackage ./pypy {
self = pypy3;
pypy35 = callPackage ./pypy {
self = pypy35;
sourceVersion = {
major = "6";
minor = "0";

View file

@ -77,7 +77,7 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
buildInputs = [ wrapPython ]
++ lib.optional (lib.hasSuffix "zip" (attrs.src.name or "")) unzip
++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
# ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
++ buildInputs
++ pythonPath;
@ -100,9 +100,12 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
# Check if we have two packages with the same name in the closure and fail.
# If this happens, something went wrong with the dependencies specs.
# Intentionally kept in a subdirectory, see catch_conflicts/README.md.
${python.interpreter} ${./catch_conflicts}/catch_conflicts.py
${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
'' + attrs.postFixup or '''';
# Python packages built through cross-compilation are always for the host platform.
disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
meta = {
# default to python's platforms
platforms = python.meta.platforms;

View file

@ -9,7 +9,8 @@ makeSetupHook {
deps = makeWrapper;
substitutions.sitePackages = python.sitePackages;
substitutions.executable = python.interpreter;
substitutions.python = python;
substitutions.python = python.pythonForBuild;
substitutions.pythonHost = python;
substitutions.magicalSedExpression = let
# Looks weird? Of course, it's between single quoted shell strings.
# NOTE: Order DOES matter here, so single character quotes need to be

View file

@ -16,8 +16,8 @@ buildPythonPath() {
declare -A pythonPathsSeen=()
program_PYTHONPATH=
program_PATH=
pythonPathsSeen["@python@"]=1
addToSearchPath program_PATH @python@/bin
pythonPathsSeen["@pythonHost@"]=1
addToSearchPath program_PATH @pythonHost@/bin
for path in $pythonPath; do
_addToPythonPath $path
done
@ -53,7 +53,13 @@ wrapPythonProgramsIn() {
# Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
# Python to use besides one with this hook anyway.
if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then
sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^"
sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#!@executable@^"
fi
if head -n1 "$f" | grep -q '#!.*'; then
# Cross-compilation hack: ensure shebangs are for the host
echo "Rewriting $(head -n 1 $f) to #!@pythonHost@"
sed -i "$f" -e "1 s^#!@python@^#!@pythonHost@^"
fi
# catch /python and /.python-wrapped

View file

@ -1,5 +1,4 @@
{ stdenv
, bootstrapped-pip
, buildPythonPackage
, python
, fetchPypi
@ -24,11 +23,11 @@ buildPythonPackage rec {
# That is because while the default install phase succeeds to build the package,
# it fails to generate the file "auto_paridecl.pxd".
installPhase = ''
mkdir -p "$out/lib/${python.libPrefix}/site-packages"
export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
mkdir -p "$out/lib/${python.sitePackages}"
export PYTHONPATH="$out/lib/${python.sitePackages}:$PYTHONPATH"
# install "." instead of "*.whl"
${bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
'';
buildInputs = [

View file

@ -1,4 +1,4 @@
{ lib, buildPythonPackage, fetchPypi, libyaml }:
{ lib, buildPythonPackage, fetchPypi, libyaml, buildPackages }:
buildPythonPackage rec {
pname = "PyYAML";
@ -9,6 +9,8 @@ buildPythonPackage rec {
sha256 = "3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf";
};
nativeBuildInputs = [ buildPackages.stdenv.cc ];
propagatedBuildInputs = [ libyaml ];
meta = with lib; {

View file

@ -17,19 +17,24 @@ stdenv.mkDerivation rec {
sha256 = "86bb4d8e1b0fabad1f4642b64c335b673e53e7a381de03c9a89fe678152c4c64";
};
nativeBuildInputs = [ unzip wrapPython ];
buildInputs = [ python ];
nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ];
doCheck = false; # requires pytest
installPhase = ''
dst=$out/${python.sitePackages}
mkdir -p $dst
export PYTHONPATH="$dst:$PYTHONPATH"
${python.interpreter} setup.py install --prefix=$out
${python.pythonForBuild.interpreter} setup.py install --prefix=$out
wrapPythonPrograms
'';
pythonPath = [];
dontPatchShebangs = true;
# Python packages built through cross-compilation are always for the host platform.
disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ];
meta = with stdenv.lib; {
description = "Utilities to facilitate the installation of Python packages";
homepage = https://pypi.python.org/pypi/setuptools;

View file

@ -7937,6 +7937,7 @@ in
python3 = python37;
pypy = pypy2;
pypy2 = pypy27;
pypy3 = pypy35;
# Python interpreter that is build with all modules, including tkinter.
# These are for compatibility and should not be used inside Nixpkgs.
@ -7954,7 +7955,7 @@ in
python3Packages = python3.pkgs;
pythonInterpreters = callPackage ./../development/interpreters/python {};
inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy3;
inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy35;
# Python package sets.
python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs);

View file

@ -43,7 +43,6 @@ let
else ff;
buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
inherit bootstrapped-pip;
flit = self.flit;
# We want Python libraries to be named like e.g. "python3.6-${name}"
inherit namePrefix;
@ -51,7 +50,6 @@ let
}));
buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
inherit bootstrapped-pip;
flit = self.flit;
namePrefix = "";
toPythonModule = x: x; # Application does not provide modules.