Merge pull request #104201 from obsidiansystems/splice-python

python: Splice packages to better support cross
This commit is contained in:
John Ericson 2020-11-19 14:07:03 -05:00 committed by GitHub
commit 93b430bc6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 124 additions and 21 deletions

View file

@ -217,4 +217,31 @@ rec {
}; };
in self; in self;
/* Like the above, but aims to support cross compilation. It's still ugly, but
hopefully it helps a little bit. */
makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: f:
let
spliced = splicePackages {
pkgsBuildBuild = otherSplices.selfBuildBuild;
pkgsBuildHost = otherSplices.selfBuildHost;
pkgsBuildTarget = otherSplices.selfBuildTarget;
pkgsHostHost = otherSplices.selfHostHost;
pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`;
pkgsTargetTarget = otherSplices.selfTargetTarget;
} // keep self;
self = f self // {
newScope = scope: newScope (spliced // scope);
callPackage = newScope spliced; # == self.newScope {};
# N.B. the other stages of the package set spliced in are *not*
# overridden.
overrideScope = g: makeScopeWithSplicing
splicePackages
newScope
otherSplices
keep
(lib.fixedPoints.extends g f);
packages = f;
};
in self;
} }

View file

@ -101,7 +101,7 @@ let
noDepEntry fullDepEntry packEntry stringAfter; noDepEntry fullDepEntry packEntry stringAfter;
inherit (self.customisation) overrideDerivation makeOverridable inherit (self.customisation) overrideDerivation makeOverridable
callPackageWith callPackagesWith extendDerivation hydraJob callPackageWith callPackagesWith extendDerivation hydraJob
makeScope; makeScope makeScopeWithSplicing;
inherit (self.meta) addMetaAttrs dontDistribute setName updateName inherit (self.meta) addMetaAttrs dontDistribute setName updateName
appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio
hiPrioSet; hiPrioSet;

View file

@ -19,6 +19,10 @@
# For the Python package set # For the Python package set
, packageOverrides ? (self: super: {}) , packageOverrides ? (self: super: {})
, buildPackages , buildPackages
, pkgsBuildBuild
, pkgsBuildTarget
, pkgsHostHost
, pkgsTargetTarget
, sourceVersion , sourceVersion
, sha256 , sha256
, passthruFun , passthruFun
@ -35,7 +39,8 @@ with stdenv.lib;
let let
pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}; pythonAttr = "python${sourceVersion.major}${sourceVersion.minor}";
pythonForBuild = buildPackages.${pythonAttr};
passthru = passthruFun rec { passthru = passthruFun rec {
inherit self sourceVersion packageOverrides; inherit self sourceVersion packageOverrides;
@ -45,6 +50,10 @@ let
pythonVersion = with sourceVersion; "${major}.${minor}"; pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages"; sitePackages = "lib/${libPrefix}/site-packages";
inherit hasDistutilsCxxPatch pythonForBuild; inherit hasDistutilsCxxPatch pythonForBuild;
pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
} // { } // {
inherit ucsEncoding; inherit ucsEncoding;
}; };

View file

@ -20,7 +20,11 @@
# For the Python package set # For the Python package set
, packageOverrides ? (self: super: {}) , packageOverrides ? (self: super: {})
, buildPackages , buildPackages
, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"} , pkgsBuildBuild
, pkgsBuildTarget
, pkgsHostHost
, pkgsTargetTarget
, pythonForBuild ? buildPackages.${pythonAttr}
, sourceVersion , sourceVersion
, sha256 , sha256
, passthruFun , passthruFun
@ -36,6 +40,7 @@
# Not using optimizations on Darwin # Not using optimizations on Darwin
# configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found. # configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
, enableOptimizations ? (!stdenv.isDarwin) , enableOptimizations ? (!stdenv.isDarwin)
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}: }:
# Note: this package is used for bootstrapping fetchurl, and thus # Note: this package is used for bootstrapping fetchurl, and thus
@ -54,6 +59,7 @@ with stdenv.lib;
let let
passthru = passthruFun rec { passthru = passthruFun rec {
inherit self sourceVersion packageOverrides; inherit self sourceVersion packageOverrides;
implementation = "cpython"; implementation = "cpython";
@ -62,6 +68,10 @@ let
pythonVersion = with sourceVersion; "${major}.${minor}"; pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages"; sitePackages = "lib/${libPrefix}/site-packages";
inherit hasDistutilsCxxPatch pythonForBuild; inherit hasDistutilsCxxPatch pythonForBuild;
pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
}; };
version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";

View file

@ -14,13 +14,70 @@ with pkgs;
, packageOverrides , packageOverrides
, sitePackages , sitePackages
, hasDistutilsCxxPatch , hasDistutilsCxxPatch
, pythonForBuild , pythonPackagesBuildBuild
, self , pythonForBuild # provides pythonPackagesBuildHost
, pythonPackagesBuildTarget
, pythonPackagesHostHost
, self # is pythonPackagesHostTarget
, pythonPackagesTargetTarget
}: let }: let
pythonPackages = callPackage ../../../top-level/python-packages.nix { pythonPackages = callPackage
python = self; ({ pkgs, stdenv, python, overrides }: let
overrides = packageOverrides; pythonPackagesFun = import ../../../top-level/python-packages.nix {
}; inherit stdenv pkgs;
python = self;
};
otherSplices = {
selfBuildBuild = pythonPackagesBuildBuild;
selfBuildHost = pythonForBuild.pkgs;
selfBuildTarget = pythonPackagesBuildTarget;
selfHostHost = pythonPackagesHostHost;
selfTargetTarget = pythonPackagesTargetTarget;
};
keep = self: {
# TODO maybe only define these here so nothing is needed to be kept in sync.
inherit (self)
isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder
python bootstrapped-pip buildPythonPackage buildPythonApplication
fetchPypi
hasPythonModule requiredPythonModules makePythonPath disabledIf
toPythonModule toPythonApplication
buildSetupcfg
eggUnpackHook
eggBuildHook
eggInstallHook
flitBuildHook
pipBuildHook
pipInstallHook
pytestCheckHook
pythonCatchConflictsHook
pythonImportsCheckHook
pythonNamespacesHook
pythonRecompileBytecodeHook
pythonRemoveBinBytecodeHook
pythonRemoveTestsDirHook
setuptoolsBuildHook
setuptoolsCheckHook
venvShellHook
wheelUnpackHook
wrapPython
pythonPackages
recursivePthLoader
;
};
in lib.makeScopeWithSplicing
pkgs.splicePackages
pkgs.newScope
otherSplices
keep
(lib.extends overrides pythonPackagesFun))
{
overrides = packageOverrides;
};
in rec { in rec {
isPy27 = pythonVersion == "2.7"; isPy27 = pythonVersion == "2.7";
isPy35 = pythonVersion == "3.5"; isPy35 = pythonVersion == "3.5";
@ -48,7 +105,6 @@ with pkgs;
python = self; python = self;
}; };
}; };
in { in {
python27 = callPackage ./cpython/2.7 { python27 = callPackage ./cpython/2.7 {

View file

@ -1,7 +1,6 @@
# Hooks for building Python packages. # Hooks for building Python packages.
{ python { python
, lib , lib
, callPackage
, makeSetupHook , makeSetupHook
, disabledIf , disabledIf
, isPy3k , isPy3k
@ -9,6 +8,7 @@
}: }:
let let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter; pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages; pythonSitePackages = python.sitePackages;
pythonCheckInterpreter = python.interpreter; pythonCheckInterpreter = python.interpreter;

View file

@ -22,9 +22,15 @@ let
implementation = "pypy"; implementation = "pypy";
libPrefix = "pypy${pythonVersion}"; libPrefix = "pypy${pythonVersion}";
executable = "pypy${if isPy3k then "3" else ""}"; executable = "pypy${if isPy3k then "3" else ""}";
pythonForBuild = self; # No cross-compiling for now.
sitePackages = "site-packages"; sitePackages = "site-packages";
hasDistutilsCxxPatch = false; hasDistutilsCxxPatch = false;
# No cross-compiling for now.
pythonForBuild = self;
pythonPackagesBuildBuild = {};
pythonPackagesBuildTarget = {};
pythonPackagesHostHost = {};
pythonPackagesTargetTarget = {};
}; };
pname = passthru.executable; pname = passthru.executable;
version = with sourceVersion; "${major}.${minor}.${patch}"; version = with sourceVersion; "${major}.${minor}.${patch}";

View file

@ -9,19 +9,16 @@
{ pkgs { pkgs
, stdenv , stdenv
, python , python
, overrides ? (self: super: {})
}: }:
with pkgs.lib; with pkgs.lib;
let self:
packages = ( self:
let let
inherit (self) callPackage;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder; inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
callPackage = pkgs.newScope self;
namePrefix = python.libPrefix + "-"; namePrefix = python.libPrefix + "-";
bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { }; bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
@ -102,7 +99,7 @@ in {
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder; inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
inherit python bootstrapped-pip buildPythonPackage buildPythonApplication; inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
inherit fetchPypi callPackage; inherit fetchPypi;
inherit hasPythonModule requiredPythonModules makePythonPath disabledIf; inherit hasPythonModule requiredPythonModules makePythonPath disabledIf;
inherit toPythonModule toPythonApplication; inherit toPythonModule toPythonApplication;
inherit buildSetupcfg; inherit buildSetupcfg;
@ -7976,6 +7973,4 @@ in {
zxcvbn = callPackage ../development/python-modules/zxcvbn { }; zxcvbn = callPackage ../development/python-modules/zxcvbn { };
}); }
in fix' (extends overrides packages)