Merge pull request #104201 from obsidiansystems/splice-python

python: Splice packages to better support cross
This commit is contained in:
John Ericson 2020-11-19 14:07:03 -05:00 committed by GitHub
commit 93b430bc6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 124 additions and 21 deletions

View file

@ -217,4 +217,31 @@ rec {
};
in self;
/* Like the above, but aims to support cross compilation. It's still ugly, but
hopefully it helps a little bit. */
makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: f:
let
spliced = splicePackages {
pkgsBuildBuild = otherSplices.selfBuildBuild;
pkgsBuildHost = otherSplices.selfBuildHost;
pkgsBuildTarget = otherSplices.selfBuildTarget;
pkgsHostHost = otherSplices.selfHostHost;
pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`;
pkgsTargetTarget = otherSplices.selfTargetTarget;
} // keep self;
self = f self // {
newScope = scope: newScope (spliced // scope);
callPackage = newScope spliced; # == self.newScope {};
# N.B. the other stages of the package set spliced in are *not*
# overridden.
overrideScope = g: makeScopeWithSplicing
splicePackages
newScope
otherSplices
keep
(lib.fixedPoints.extends g f);
packages = f;
};
in self;
}

View file

@ -101,7 +101,7 @@ let
noDepEntry fullDepEntry packEntry stringAfter;
inherit (self.customisation) overrideDerivation makeOverridable
callPackageWith callPackagesWith extendDerivation hydraJob
makeScope;
makeScope makeScopeWithSplicing;
inherit (self.meta) addMetaAttrs dontDistribute setName updateName
appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio
hiPrioSet;

View file

@ -19,6 +19,10 @@
# For the Python package set
, packageOverrides ? (self: super: {})
, buildPackages
, pkgsBuildBuild
, pkgsBuildTarget
, pkgsHostHost
, pkgsTargetTarget
, sourceVersion
, sha256
, passthruFun
@ -35,7 +39,8 @@ with stdenv.lib;
let
pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
pythonAttr = "python${sourceVersion.major}${sourceVersion.minor}";
pythonForBuild = buildPackages.${pythonAttr};
passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
@ -45,6 +50,10 @@ let
pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages";
inherit hasDistutilsCxxPatch pythonForBuild;
pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
} // {
inherit ucsEncoding;
};

View file

@ -20,7 +20,11 @@
# For the Python package set
, packageOverrides ? (self: super: {})
, buildPackages
, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}
, pkgsBuildBuild
, pkgsBuildTarget
, pkgsHostHost
, pkgsTargetTarget
, pythonForBuild ? buildPackages.${pythonAttr}
, sourceVersion
, sha256
, passthruFun
@ -36,6 +40,7 @@
# Not using optimizations on Darwin
# configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
, enableOptimizations ? (!stdenv.isDarwin)
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
# Note: this package is used for bootstrapping fetchurl, and thus
@ -54,6 +59,7 @@ with stdenv.lib;
let
passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
implementation = "cpython";
@ -62,6 +68,10 @@ let
pythonVersion = with sourceVersion; "${major}.${minor}";
sitePackages = "lib/${libPrefix}/site-packages";
inherit hasDistutilsCxxPatch pythonForBuild;
pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr};
pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr};
pythonPackagesHostHost = pkgsHostHost.${pythonAttr};
pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {};
};
version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";

View file

@ -14,13 +14,70 @@ with pkgs;
, packageOverrides
, sitePackages
, hasDistutilsCxxPatch
, pythonForBuild
, self
, pythonPackagesBuildBuild
, pythonForBuild # provides pythonPackagesBuildHost
, pythonPackagesBuildTarget
, pythonPackagesHostHost
, self # is pythonPackagesHostTarget
, pythonPackagesTargetTarget
}: let
pythonPackages = callPackage ../../../top-level/python-packages.nix {
python = self;
overrides = packageOverrides;
};
pythonPackages = callPackage
({ pkgs, stdenv, python, overrides }: let
pythonPackagesFun = import ../../../top-level/python-packages.nix {
inherit stdenv pkgs;
python = self;
};
otherSplices = {
selfBuildBuild = pythonPackagesBuildBuild;
selfBuildHost = pythonForBuild.pkgs;
selfBuildTarget = pythonPackagesBuildTarget;
selfHostHost = pythonPackagesHostHost;
selfTargetTarget = pythonPackagesTargetTarget;
};
keep = self: {
# TODO maybe only define these here so nothing is needed to be kept in sync.
inherit (self)
isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder
python bootstrapped-pip buildPythonPackage buildPythonApplication
fetchPypi
hasPythonModule requiredPythonModules makePythonPath disabledIf
toPythonModule toPythonApplication
buildSetupcfg
eggUnpackHook
eggBuildHook
eggInstallHook
flitBuildHook
pipBuildHook
pipInstallHook
pytestCheckHook
pythonCatchConflictsHook
pythonImportsCheckHook
pythonNamespacesHook
pythonRecompileBytecodeHook
pythonRemoveBinBytecodeHook
pythonRemoveTestsDirHook
setuptoolsBuildHook
setuptoolsCheckHook
venvShellHook
wheelUnpackHook
wrapPython
pythonPackages
recursivePthLoader
;
};
in lib.makeScopeWithSplicing
pkgs.splicePackages
pkgs.newScope
otherSplices
keep
(lib.extends overrides pythonPackagesFun))
{
overrides = packageOverrides;
};
in rec {
isPy27 = pythonVersion == "2.7";
isPy35 = pythonVersion == "3.5";
@ -48,7 +105,6 @@ with pkgs;
python = self;
};
};
in {
python27 = callPackage ./cpython/2.7 {

View file

@ -1,7 +1,6 @@
# Hooks for building Python packages.
{ python
, lib
, callPackage
, makeSetupHook
, disabledIf
, isPy3k
@ -9,6 +8,7 @@
}:
let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
pythonCheckInterpreter = python.interpreter;

View file

@ -22,9 +22,15 @@ let
implementation = "pypy";
libPrefix = "pypy${pythonVersion}";
executable = "pypy${if isPy3k then "3" else ""}";
pythonForBuild = self; # No cross-compiling for now.
sitePackages = "site-packages";
hasDistutilsCxxPatch = false;
# No cross-compiling for now.
pythonForBuild = self;
pythonPackagesBuildBuild = {};
pythonPackagesBuildTarget = {};
pythonPackagesHostHost = {};
pythonPackagesTargetTarget = {};
};
pname = passthru.executable;
version = with sourceVersion; "${major}.${minor}.${patch}";

View file

@ -9,19 +9,16 @@
{ pkgs
, stdenv
, python
, overrides ? (self: super: {})
}:
with pkgs.lib;
let
packages = ( self:
self:
let
inherit (self) callPackage;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
callPackage = pkgs.newScope self;
namePrefix = python.libPrefix + "-";
bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
@ -102,7 +99,7 @@ in {
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder;
inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
inherit fetchPypi callPackage;
inherit fetchPypi;
inherit hasPythonModule requiredPythonModules makePythonPath disabledIf;
inherit toPythonModule toPythonApplication;
inherit buildSetupcfg;
@ -7976,6 +7973,4 @@ in {
zxcvbn = callPackage ../development/python-modules/zxcvbn { };
});
in fix' (extends overrides packages)
}