Merge pull request #112804 from danieldk/cargo-build-hook

This commit is contained in:
Jörg Thalheim 2021-02-14 18:08:26 +00:00 committed by GitHub
commit b5b47d6445
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 288 additions and 168 deletions

View file

@ -248,18 +248,65 @@ hooks that can be used to integrate Cargo in non-Rust packages.
Since network access is not allowed in sandboxed builds, Rust crate
dependencies need to be retrieved using a fetcher. `rustPlatform`
provides the `fetchCargoTarball` fetcher, which vendors all
dependencies of a crate. This fetcher can be used jointly with
`cargoSetupHook` to vendor dependencies in derivations that do not use
`buildRustPackage`.
dependencies of a crate. For example, given a source path `src`
containing `Cargo.toml` and `Cargo.lock`, `fetchCargoTarball`
can be used as follows:
In the following partial example, `fetchCargoTarball` and
`cargoSetupHook` are used to vendor dependencies in the Python
`tokenizers` derivation. The `tokenizers` Python package is in the
`source/bindings/python` directory of the project's source archive. We
use `fetchCargoTarball` to retrieve the dependencies specified in
`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
the `cargoDeps` attribute, which is used by `cargoSetupHook` to
configure Cargo.
```nix
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
};
```
The `src` attribute is required, as well as a hash specified through
one of the `sha256` or `hash` attributes. The following optional
attributes can also be used:
* `name`: the name that is used for the dependencies tarball. If
`name` is not specified, then the name `cargo-deps` will be used.
* `sourceRoot`: when the `Cargo.lock`/`Cargo.toml` are in a
subdirectory, `sourceRoot` specifies the relative path to these
files.
* `patches`: patches to apply before vendoring. This is useful when
the `Cargo.lock`/`Cargo.toml` files need to be patched before
vendoring.
### Hooks
`rustPlatform` provides the following hooks to automate Cargo builds:
* `cargoSetupHook`: configure Cargo to use depenencies vendored
through `fetchCargoTarball`. This hook uses the `cargoDeps`
environment variable to find the vendored dependencies. If a project
already vendors its dependencies, the variable `cargoVendorDir` can
be used instead. When the `Cargo.toml`/`Cargo.lock` files are not in
`sourceRoot`, then the optional `cargoRoot` is used to specify the
Cargo root directory relative to `sourceRoot`.
* `cargoBuildHook`: use Cargo to build a crate. If the crate to be
built is a crate in e.g. a Cargo workspace, the relative path to the
crate to build can be set through the optional `buildAndTestSubdir`
environment variable. Additional Cargo build flags can be passed
through `cargoBuildFlags`.
* `maturinBuildHook`: use [Maturin](https://github.com/PyO3/maturin)
to build a Python wheel. Similar to `cargoBuildHook`, the optional
variable `buildAndTestSubdir` can be used to build a crate in a
Cargo workspace. Additional maturin flags can be passed through
`maturinBuildFlags`.
### Examples
#### Python package using `setuptools-rust`
For Python packages using `setuptools-rust`, you can use
`fetchCargoTarball` and `cargoSetupHook` to retrieve and set up Cargo
dependencies. The build itself is then performed by
`buildPythonPackage`.
The following example outlines how the `tokenizers` Python package is
built. Since the Python package is in the `source/bindings/python`
directory of the *tokenizers* project's source archive, we use
`sourceRoot` to point the tooling to this directory:
```nix
{ fetchFromGitHub
@ -297,9 +344,9 @@ buildPythonPackage rec {
}
```
In some projects, the Rust crate is not in the main source directory
of the projects. In such cases, the `cargoRoot` attribute can be used
to specify the crate's directory relative to `sourceRoot`. In the
In some projects, the Rust crate is not in the main Python source
directory. In such cases, the `cargoRoot` attribute can be used to
specify the crate's directory relative to `sourceRoot`. In the
following example, the crate is in `src/rust`, as specified in the
`cargoRoot` attribute. Note that we also need to specify the correct
path for `fetchCargoTarball`.
@ -335,6 +382,47 @@ buildPythonPackage rec {
}
```
#### Python package using `maturin`
Python packages that use [Maturin](https://github.com/PyO3/maturin)
can be built with `fetchCargoTarball`, `cargoSetupHook`, and
`maturinBuildHook`. For example, the following (partial) derivation
builds the `retworkx` Python package. `fetchCargoTarball` and
`cargoSetupHook` are used to fetch and set up the crate dependencies.
`maturinBuildHook` is used to perform the build.
```nix
{ lib
, buildPythonPackage
, rustPlatform
, fetchFromGitHub
}:
buildPythonPackage rec {
pname = "retworkx";
version = "0.6.0";
src = fetchFromGitHub {
owner = "Qiskit";
repo = "retworkx";
rev = version;
sha256 = "11n30ldg3y3y6qxg3hbj837pnbwjkqw3nxq6frds647mmmprrd20";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0=";
};
format = "pyproject";
nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ];
# ...
}
```
## Compiling Rust crates using Nix instead of Cargo
### Simple operation

View file

@ -3,6 +3,7 @@
, buildPackages
, cacert
, cargo
, cargoBuildHook
, cargoSetupHook
, fetchCargoTarball
, runCommandNoCC
@ -37,7 +38,6 @@
, cargoBuildFlags ? []
, buildType ? "release"
, meta ? {}
, target ? rust.toRustTargetSpec stdenv.hostPlatform
, cargoVendorDir ? null
, checkType ? buildType
, depsExtraArgs ? {}
@ -71,6 +71,7 @@ let
# against the src fixed-output derivation to check consistency.
validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");
target = rust.toRustTargetSpec stdenv.hostPlatform;
targetIsJSON = lib.hasSuffix ".json" target;
useSysroot = targetIsJSON && !__internal_dontAddSysroot;
@ -86,10 +87,6 @@ let
originalCargoToml = src + /Cargo.toml; # profile info is later extracted
};
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
releaseDir = "target/${shortTarget}/${buildType}";
tmpDir = "${releaseDir}-tmp";
@ -102,11 +99,17 @@ assert useSysroot -> !(args.doCheck or true);
stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs useSysroot {
RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or "");
} // {
inherit cargoDeps;
inherit buildAndTestSubdir cargoDeps releaseDir tmpDir;
cargoBuildFlags = lib.concatStringsSep " " cargoBuildFlags;
cargoBuildType = "--${buildType}";
patchRegistryDeps = ./patch-registry-deps;
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
nativeBuildInputs = nativeBuildInputs ++
[ cacert git cargo cargoBuildHook cargoSetupHook rustc ];
buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;
patches = cargoPatches ++ patches;
@ -125,38 +128,6 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
runHook postConfigure
'';
buildPhase = with builtins; args.buildPhase or ''
${lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"}
runHook preBuild
(
set -x
env \
"CC_${rust.toRustTarget stdenv.buildPlatform}"="${ccForBuild}" \
"CXX_${rust.toRustTarget stdenv.buildPlatform}"="${cxxForBuild}" \
"CC_${rust.toRustTarget stdenv.hostPlatform}"="${ccForHost}" \
"CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \
cargo build -j $NIX_BUILD_CORES \
${lib.optionalString (buildType == "release") "--release"} \
--target ${target} \
--frozen ${concatStringsSep " " cargoBuildFlags}
)
runHook postBuild
${lib.optionalString (buildAndTestSubdir != null) "popd"}
# This needs to be done after postBuild: packages like `cargo` do a pushd/popd in
# the pre/postBuild-hooks that need to be taken into account before gathering
# all binaries to install.
mkdir -p $tmpDir
cp -r $releaseDir/* $tmpDir/
bins=$(find $tmpDir \
-maxdepth 1 \
-type f \
-executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \))
'';
checkPhase = args.checkPhase or (let
argstr = "${lib.optionalString (checkType == "release") "--release"} --target ${target} --frozen";
threads = if cargoParallelTestThreads then "$NIX_BUILD_CORES" else "1";
@ -173,11 +144,19 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
strictDeps = true;
inherit releaseDir tmpDir;
installPhase = args.installPhase or ''
runHook preInstall
# This needs to be done after postBuild: packages like `cargo` do a pushd/popd in
# the pre/postBuild-hooks that need to be taken into account before gathering
# all binaries to install.
mkdir -p $tmpDir
cp -r $releaseDir/* $tmpDir/
bins=$(find $tmpDir \
-maxdepth 1 \
-type f \
-executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \))
# rename the output dir to a architecture independent one
mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${tmpDir}$')
for target in "''${targets[@]}"; do

View file

@ -21,7 +21,7 @@ in
, src ? null
, srcs ? []
, patches ? []
, sourceRoot
, sourceRoot ? ""
, hash ? ""
, sha256 ? ""
, cargoUpdateHook ? ""

View file

@ -0,0 +1,33 @@
cargoBuildHook() {
echo "Executing cargoBuildHook"
runHook preBuild
if [ ! -z "${buildAndTestSubdir-}" ]; then
pushd "${buildAndTestSubdir}"
fi
(
set -x
env \
"CC_@rustBuildPlatform@=@ccForBuild@" \
"CXX_@rustBuildPlatform@=@cxxForBuild@" \
"CC_@rustTargetPlatform@=@ccForHost@" \
"CXX_@rustTargetPlatform@=@cxxForHost@" \
cargo build -j $NIX_BUILD_CORES \
--target @rustTargetPlatformSpec@ \
--frozen \
${cargoBuildType} \
${cargoBuildFlags}
)
runHook postBuild
if [ ! -z "${buildAndTestSubdir-}" ]; then
popd
fi
echo "Finished cargoBuildHook"
}
buildPhase=cargoBuildHook

View file

@ -1,8 +1,10 @@
{ buildPackages
, callPackage
, cargo
, diffutils
, lib
, makeSetupHook
, maturin
, rust
, stdenv
, target ? rust.toRustTargetSpec stdenv.hostPlatform
@ -16,9 +18,24 @@ let
shortTarget = if targetIsJSON then
(lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
else target;
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform;
rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform;
rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform;
in {
cargoBuildHook = callPackage ({ }:
makeSetupHook {
name = "cargo-build-hook.sh";
deps = [ cargo ];
substitutions = {
inherit ccForBuild ccForHost cxxForBuild cxxForHost
rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec;
};
} ./cargo-build-hook.sh) {};
cargoSetupHook = callPackage ({ }:
makeSetupHook {
name = "cargo-setup-hook.sh";
@ -46,4 +63,14 @@ in {
'';
};
} ./cargo-setup-hook.sh) {};
maturinBuildHook = callPackage ({ }:
makeSetupHook {
name = "maturin-build-hook.sh";
deps = [ cargo maturin ];
substitutions = {
inherit ccForBuild ccForHost cxxForBuild cxxForHost
rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec;
};
} ./maturin-build-hook.sh) {};
}

View file

@ -0,0 +1,39 @@
maturinBuildHook() {
echo "Executing maturinBuildHook"
runHook preBuild
if [ ! -z "${buildAndTestSubdir-}" ]; then
pushd "${buildAndTestSubdir}"
fi
(
set -x
env \
"CC_@rustBuildPlatform@=@ccForBuild@" \
"CXX_@rustBuildPlatform@=@cxxForBuild@" \
"CC_@rustTargetPlatform@=@ccForHost@" \
"CXX_@rustTargetPlatform@=@cxxForHost@" \
maturin build \
--cargo-extra-args="-j $NIX_BUILD_CORES --frozen" \
--target @rustTargetPlatformSpec@ \
--manylinux off \
--strip \
--release \
${maturinBuildFlags-}
)
runHook postBuild
if [ ! -z "${buildAndTestSubdir-}" ]; then
popd
fi
# Move the wheel to dist/ so that regular Python tooling can find it.
mkdir -p dist
mv target/wheels/*.whl dist/
echo "Finished maturinBuildHook"
}
buildPhase=maturinBuildHook

View file

@ -12,7 +12,7 @@ rec {
};
buildRustPackage = callPackage ../../../build-support/rust {
inherit rustc cargo cargoSetupHook fetchCargoTarball;
inherit rustc cargo cargoBuildHook cargoSetupHook fetchCargoTarball;
};
rustcSrc = callPackage ./rust-src.nix {
@ -24,5 +24,7 @@ rec {
};
# Hooks
inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook;
inherit (callPackage ../../../build-support/rust/hooks {
inherit cargo;
}) cargoBuildHook cargoSetupHook maturinBuildHook;
}

View file

@ -1,10 +1,9 @@
{ stdenv
, lib
, rustPlatform
, fetchFromGitHub
, pipInstallHook
, buildPythonPackage
, rustPlatform
, pythonImportsCheckHook
, maturin
, pkg-config
, openssl
, publicsuffix-list
@ -13,7 +12,7 @@
, Security
}:
rustPlatform.buildRustPackage rec {
buildPythonPackage rec {
pname = "adblock";
version = "0.4.0";
disabled = isPy27;
@ -25,33 +24,27 @@ rustPlatform.buildRustPackage rec {
rev = version;
sha256 = "10d6ks2fyzbizq3kb69q478idj0h86k6ygjb6wl3zq3mf65ma4zg";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-gEFmj3/KvhvvsOK2nX2L1RUD4Wfp3nYzEzVnQZIsIDY=";
};
format = "pyproject";
cargoSha256 = "0di05j942rrm2crpdpp9czhh65fmidyrvdp2n3pipgnagy7nchc0";
nativeBuildInputs = [ pipInstallHook maturin pkg-config pythonImportsCheckHook ];
nativeBuildInputs = [ pkg-config pythonImportsCheckHook ]
++ (with rustPlatform; [ cargoSetupHook maturinBuildHook ]);
buildInputs = [ openssl ]
++ lib.optionals stdenv.isDarwin [ CoreFoundation Security ];
PSL_PATH = "${publicsuffix-list}/share/publicsuffix/public_suffix_list.dat";
buildPhase = ''
runHook preBuild
maturin build --release --manylinux off --strip
runHook postBuild
'';
# There are no rust tests
doCheck = false;
pythonImportsCheck = [ "adblock" ];
installPhase = ''
runHook preInstall
install -Dm644 -t dist target/wheels/*.whl
pipInstallPhase
runHook postInstall
'';
pythonImportsCheck = [ "adblock" ];
passthru.meta = with lib; {
description = "Python wrapper for Brave's adblocking library, which is written in Rust";

View file

@ -1,14 +1,13 @@
{ lib
, stdenv
, rustPlatform
, fetchFromGitHub
, pipInstallHook
, buildPythonPackage
, rustPlatform
, llvmPackages
, pkg-config
, maturin
, pcsclite
, nettle
, python
, requests
, vcrpy
, numpy
@ -17,7 +16,7 @@
, PCSC
}:
rustPlatform.buildRustPackage rec {
buildPythonPackage rec {
pname = "johnnycanencrypt";
version = "0.5.0";
disabled = pythonOlder "3.7";
@ -28,7 +27,16 @@ rustPlatform.buildRustPackage rec {
rev = "v${version}";
sha256 = "192wfrlyylrpzq70yki421mi1smk8q2cyki2a1d03q7h6apib3j4";
};
cargoPatches = [ ./Cargo.lock.patch ];
cargoDeps = rustPlatform.fetchCargoTarball {
inherit patches src;
name = "${pname}-${version}";
hash = "sha256-2XhXCKyXVlFgbcOoMy/A5ajiIVxBii56YeI29mO720U=";
};
format = "pyproject";
patches = [ ./Cargo.lock.patch ];
cargoSha256 = "0ifvpdizcdp2c5x2x2j1bhhy5a75q0pk7a63dmh52mlpmh45fy6r";
@ -42,10 +50,10 @@ rustPlatform.buildRustPackage rec {
nativeBuildInputs = [
llvmPackages.clang
pkg-config
python
maturin
pipInstallHook
];
] ++ (with rustPlatform; [
cargoSetupHook
maturinBuildHook
]);
buildInputs = [
pcsclite
@ -67,17 +75,6 @@ rustPlatform.buildRustPackage rec {
sed '/project-url = /d' -i Cargo.toml
'';
buildPhase = ''
runHook preBuild
maturin build --release --manylinux off --strip --cargo-extra-args="-j $NIX_BUILD_CORES --frozen"
runHook postBuild
'';
installPhase = ''
install -Dm644 -t dist target/wheels/*.whl
pipInstallPhase
'';
preCheck = ''
export TESTDIR=$(mktemp -d)
cp -r tests/ $TESTDIR

View file

@ -1,16 +1,14 @@
{ lib
, buildPythonPackage
, rustPlatform
, python
, fetchFromGitHub
, pipInstallHook
, maturin
, pip
# Check inputs
, pytestCheckHook
, numpy
}:
rustPlatform.buildRustPackage rec {
buildPythonPackage rec {
pname = "retworkx";
version = "0.6.0";
@ -21,11 +19,15 @@ rustPlatform.buildRustPackage rec {
sha256 = "11n30ldg3y3y6qxg3hbj837pnbwjkqw3nxq6frds647mmmprrd20";
};
cargoSha256 = "1vg4yf0k6yypqf9z46zz818mz7fdrgxj7zl6zjf7pnm2r8mq3qw5";
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0=";
};
propagatedBuildInputs = [ python ];
format = "pyproject";
nativeBuildInputs = [ pipInstallHook maturin pip ];
nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ];
# Needed b/c need to check AFTER python wheel is installed (using Rust Build, not buildPythonPackage)
doCheck = false;
@ -33,17 +35,6 @@ rustPlatform.buildRustPackage rec {
installCheckInputs = [ pytestCheckHook numpy ];
buildPhase = ''
runHook preBuild
maturin build --release --manylinux off --strip
runHook postBuild
'';
installPhase = ''
install -Dm644 -t dist target/wheels/*.whl
pipInstallPhase
'';
preCheck = ''
export TESTDIR=$(mktemp -d)
cp -r tests/ $TESTDIR

View file

@ -1,63 +1,34 @@
{ lib
, rustPlatform
, fetchFromGitHub
, maturin
, buildPythonPackage
, isPy38
, python
}:
let
pname = "wasmer";
version = "1.0.0";
wheel = rustPlatform.buildRustPackage rec {
inherit pname version;
src = fetchFromGitHub {
owner = "wasmerio";
repo = "wasmer-python";
rev = version;
hash = "sha256-I1GfjLaPYMIHKh2m/5IQepUsJNiVUEJg49wyuuzUYtY=";
};
cargoHash = "sha256-txOOia1C4W+nsXuXp4EytEn82CFfSmiOYwRLC4WPImc=";
nativeBuildInputs = [ maturin python ];
preBuild = ''
cd packages/api
'';
buildPhase = ''
runHook preBuild
maturin build --release --manylinux off --strip
runHook postBuild
'';
postBuild = ''
cd ../..
'';
doCheck = false;
installPhase = ''
runHook preInstall
install -Dm644 -t $out target/wheels/*.whl
runHook postInstall
'';
};
in
buildPythonPackage rec {
in buildPythonPackage rec {
inherit pname version;
format = "wheel";
src = wheel;
src = fetchFromGitHub {
owner = "wasmerio";
repo = "wasmer-python";
rev = version;
hash = "sha256-I1GfjLaPYMIHKh2m/5IQepUsJNiVUEJg49wyuuzUYtY=";
};
unpackPhase = ''
mkdir -p dist
cp $src/*.whl dist
'';
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-txOOia1C4W+nsXuXp4EytEn82CFfSmiOYwRLC4WPImc=";
};
format = "pyproject";
nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ];
buildAndTestSubdir = "packages/api";
doCheck = false;
pythonImportsCheck = [ "wasmer" ];