From a6218c058b3e994779147dd7286606c750be7d73 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 18 Nov 2020 12:12:34 -0500 Subject: [PATCH 1/3] lib: Create `makeScopeWithSplicing` It's ugly as hell, but I suppose it is needed to codify how to make spliced package sets. --- lib/customisation.nix | 27 +++++++++++++++++++++++++++ lib/default.nix | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index dc5dd769197..37a7951896b 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -217,4 +217,31 @@ rec { }; in self; + /* Like the above, but aims to support cross compilation. It's still ugly, but + hopefully it helps a little bit. */ + makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: f: + let + spliced = splicePackages { + pkgsBuildBuild = otherSplices.selfBuildBuild; + pkgsBuildHost = otherSplices.selfBuildHost; + pkgsBuildTarget = otherSplices.selfBuildTarget; + pkgsHostHost = otherSplices.selfHostHost; + pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`; + pkgsTargetTarget = otherSplices.selfTargetTarget; + } // keep self; + self = f self // { + newScope = scope: newScope (spliced // scope); + callPackage = newScope spliced; # == self.newScope {}; + # N.B. the other stages of the package set spliced in are *not* + # overridden. + overrideScope = g: makeScopeWithSplicing + splicePackages + newScope + otherSplices + keep + (lib.fixedPoints.extends g f); + packages = f; + }; + in self; + } diff --git a/lib/default.nix b/lib/default.nix index e3c1ed71346..f985266ed93 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -101,7 +101,7 @@ let noDepEntry fullDepEntry packEntry stringAfter; inherit (self.customisation) overrideDerivation makeOverridable callPackageWith callPackagesWith extendDerivation hydraJob - makeScope; + makeScope makeScopeWithSplicing; inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio hiPrioSet; From 330d4680022c92d8893f926624e51480c9e254ee Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 18 Nov 2020 14:13:05 -0500 Subject: [PATCH 2/3] python: Use packages from previous stage in setup-hooks This makes sense as these are tools we want to run at build time. --- pkgs/development/interpreters/python/hooks/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index d14eb9cbb09..456aea4c5d8 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -1,7 +1,6 @@ # Hooks for building Python packages. { python , lib -, callPackage , makeSetupHook , disabledIf , isPy3k @@ -9,6 +8,7 @@ }: let + callPackage = python.pythonForBuild.pkgs.callPackage; pythonInterpreter = python.pythonForBuild.interpreter; pythonSitePackages = python.sitePackages; pythonCheckInterpreter = python.interpreter; From b57c5d44569e236edbc142c0f95772fca2bb6dad Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 18 Nov 2020 12:13:11 -0500 Subject: [PATCH 3/3] python: Use `makeScopeWithSplicing` Now non-`buildInputs` that are python packages should be resolved correctly. --- .../python/cpython/2.7/default.nix | 11 ++- .../interpreters/python/cpython/default.nix | 12 +++- .../interpreters/python/default.nix | 70 +++++++++++++++++-- .../interpreters/python/pypy/default.nix | 8 ++- pkgs/top-level/python-packages.nix | 13 ++-- 5 files changed, 95 insertions(+), 19 deletions(-) diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index 2cfaa69a4c8..26bd8a8f360 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -19,6 +19,10 @@ # For the Python package set , packageOverrides ? (self: super: {}) , buildPackages +, pkgsBuildBuild +, pkgsBuildTarget +, pkgsHostHost +, pkgsTargetTarget , sourceVersion , sha256 , passthruFun @@ -35,7 +39,8 @@ with stdenv.lib; let - pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}; + pythonAttr = "python${sourceVersion.major}${sourceVersion.minor}"; + pythonForBuild = buildPackages.${pythonAttr}; passthru = passthruFun rec { inherit self sourceVersion packageOverrides; @@ -45,6 +50,10 @@ let pythonVersion = with sourceVersion; "${major}.${minor}"; sitePackages = "lib/${libPrefix}/site-packages"; inherit hasDistutilsCxxPatch pythonForBuild; + pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr}; + pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr}; + pythonPackagesHostHost = pkgsHostHost.${pythonAttr}; + pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {}; } // { inherit ucsEncoding; }; diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index 77512c02d55..02777063a77 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -20,7 +20,11 @@ # For the Python package set , packageOverrides ? (self: super: {}) , buildPackages -, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"} +, pkgsBuildBuild +, pkgsBuildTarget +, pkgsHostHost +, pkgsTargetTarget +, pythonForBuild ? buildPackages.${pythonAttr} , sourceVersion , sha256 , passthruFun @@ -36,6 +40,7 @@ # Not using optimizations on Darwin # configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found. , enableOptimizations ? (!stdenv.isDarwin) +, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}" }: # Note: this package is used for bootstrapping fetchurl, and thus @@ -54,6 +59,7 @@ with stdenv.lib; let + passthru = passthruFun rec { inherit self sourceVersion packageOverrides; implementation = "cpython"; @@ -62,6 +68,10 @@ let pythonVersion = with sourceVersion; "${major}.${minor}"; sitePackages = "lib/${libPrefix}/site-packages"; inherit hasDistutilsCxxPatch pythonForBuild; + pythonPackagesBuildBuild = pkgsBuildBuild.${pythonAttr}; + pythonPackagesBuildTarget = pkgsBuildTarget.${pythonAttr}; + pythonPackagesHostHost = pkgsHostHost.${pythonAttr}; + pythonPackagesTargetTarget = pkgsTargetTarget.${pythonAttr} or {}; }; version = with sourceVersion; "${major}.${minor}.${patch}${suffix}"; diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index ae666d44f57..19a7f44de36 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -14,13 +14,70 @@ with pkgs; , packageOverrides , sitePackages , hasDistutilsCxxPatch - , pythonForBuild - , self + , pythonPackagesBuildBuild + , pythonForBuild # provides pythonPackagesBuildHost + , pythonPackagesBuildTarget + , pythonPackagesHostHost + , self # is pythonPackagesHostTarget + , pythonPackagesTargetTarget }: let - pythonPackages = callPackage ../../../top-level/python-packages.nix { - python = self; - overrides = packageOverrides; - }; + pythonPackages = callPackage + ({ pkgs, stdenv, python, overrides }: let + pythonPackagesFun = import ../../../top-level/python-packages.nix { + inherit stdenv pkgs; + python = self; + }; + otherSplices = { + selfBuildBuild = pythonPackagesBuildBuild; + selfBuildHost = pythonForBuild.pkgs; + selfBuildTarget = pythonPackagesBuildTarget; + selfHostHost = pythonPackagesHostHost; + selfTargetTarget = pythonPackagesTargetTarget; + }; + keep = self: { + # TODO maybe only define these here so nothing is needed to be kept in sync. + inherit (self) + isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder + python bootstrapped-pip buildPythonPackage buildPythonApplication + fetchPypi + hasPythonModule requiredPythonModules makePythonPath disabledIf + toPythonModule toPythonApplication + buildSetupcfg + + eggUnpackHook + eggBuildHook + eggInstallHook + flitBuildHook + pipBuildHook + pipInstallHook + pytestCheckHook + pythonCatchConflictsHook + pythonImportsCheckHook + pythonNamespacesHook + pythonRecompileBytecodeHook + pythonRemoveBinBytecodeHook + pythonRemoveTestsDirHook + setuptoolsBuildHook + setuptoolsCheckHook + venvShellHook + wheelUnpackHook + + wrapPython + + pythonPackages + + recursivePthLoader + ; + }; + in lib.makeScopeWithSplicing + pkgs.splicePackages + pkgs.newScope + otherSplices + keep + (lib.extends overrides pythonPackagesFun)) + { + overrides = packageOverrides; + }; in rec { isPy27 = pythonVersion == "2.7"; isPy35 = pythonVersion == "3.5"; @@ -48,7 +105,6 @@ with pkgs; python = self; }; }; - in { python27 = callPackage ./cpython/2.7 { diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 482ba5df54c..0647ce87864 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -22,9 +22,15 @@ let implementation = "pypy"; libPrefix = "pypy${pythonVersion}"; executable = "pypy${if isPy3k then "3" else ""}"; - pythonForBuild = self; # No cross-compiling for now. sitePackages = "site-packages"; hasDistutilsCxxPatch = false; + + # No cross-compiling for now. + pythonForBuild = self; + pythonPackagesBuildBuild = {}; + pythonPackagesBuildTarget = {}; + pythonPackagesHostHost = {}; + pythonPackagesTargetTarget = {}; }; pname = passthru.executable; version = with sourceVersion; "${major}.${minor}.${patch}"; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ae79ca29f69..909cd5cfe0a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -9,19 +9,16 @@ { pkgs , stdenv , python -, overrides ? (self: super: {}) }: with pkgs.lib; -let - packages = ( self: +self: let + inherit (self) callPackage; inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder; - callPackage = pkgs.newScope self; - namePrefix = python.libPrefix + "-"; bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { }; @@ -102,7 +99,7 @@ in { inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder; inherit python bootstrapped-pip buildPythonPackage buildPythonApplication; - inherit fetchPypi callPackage; + inherit fetchPypi; inherit hasPythonModule requiredPythonModules makePythonPath disabledIf; inherit toPythonModule toPythonApplication; inherit buildSetupcfg; @@ -7960,6 +7957,4 @@ in { zxcvbn = callPackage ../development/python-modules/zxcvbn { }; -}); - -in fix' (extends overrides packages) +}