Merge pull request #95542 from aaronjanse/aj-rust-custom-target

buildRustPackage: support custom targets
This commit is contained in:
John Ericson 2020-11-28 17:20:31 -05:00 committed by GitHub
commit 304913841b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 308 additions and 20 deletions

View file

@ -63,9 +63,52 @@ The fetcher will verify that the `Cargo.lock` file is in sync with the `src`
attribute, and fail the build if not. It will also will compress the vendor
directory into a tar.gz archive.
### Building a crate for a different target
### Cross compilation
To build your crate with a different cargo `--target` simply specify the `target` attribute:
By default, Rust packages are compiled for the host platform, just like any
other package is. The `--target` passed to rust tools is computed from this.
By default, it takes the `stdenv.hostPlatform.config` and replaces components
where they are known to differ. But there are ways to customize the argument:
- To choose a different target by name, define
`stdenv.hostPlatform.rustc.config` as that name (a string), and that
name will be used instead.
For example:
```nix
import <nixpkgs> {
crossSystem = (import <nixpkgs/lib>).systems.examples.armhf-embedded // {
rustc.config = "thumbv7em-none-eabi";
};
}
```
will result in:
```shell
--target thumbv7em-none-eabi
```
- To pass a completely custom target, define
`stdenv.hostPlatform.rustc.config` with its name, and
`stdenv.hostPlatform.rustc.platform` with the value. The value will be
serialized to JSON in a file called
`${stdenv.hostPlatform.rustc.config}.json`, and the path of that file
will be used instead.
For example:
```nix
import <nixpkgs> {
crossSystem = (import <nixpkgs/lib>).systems.examples.armhf-embedded // {
rustc.config = "thumb-crazy";
rustc.platform = { foo = ""; bar = ""; };
};
}
will result in:
```shell
--target /nix/store/asdfasdfsadf-thumb-crazy.json # contains {"foo":"","bar":""}
```
Finally, as an ad-hoc escape hatch, a computed target (string or JSON file
path) can be passed directly to `buildRustPackage`:
```nix
pkgs.rustPlatform.buildRustPackage {
@ -74,6 +117,15 @@ pkgs.rustPlatform.buildRustPackage {
}
```
This is useful to avoid rebuilding Rust tools, since they are actually target
agnostic and don't need to be rebuilt. But in the future, we should always
build the Rust tools and standard library crates separately so there is no
reason not to take the `stdenv.hostPlatform.rustc`-modifying approach, and the
ad-hoc escape hatch to `buildRustPackage` can be removed.
Note that currently custom targets aren't compiled with `std`, so `cargo test`
will fail. This can be ignored by adding `doCheck = false;` to your derivation.
### Running package tests
When using `buildRustPackage`, the `checkPhase` is enabled by default and runs

View file

@ -15,7 +15,7 @@
++ [(mkRustcDepArgs dependencies crateRenames)]
++ [(mkRustcFeatureArgs crateFeatures)]
++ extraRustcOpts
++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc"
++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTargetSpec stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc"
# since rustc 1.42 the "proc_macro" crate is part of the default crate prelude
# https://github.com/rust-lang/cargo/commit/4d64eb99a4#diff-7f98585dbf9d30aa100c8318e2c77e79R1021-R1022
++ lib.optional (lib.elem "proc-macro" crateType) "--extern proc_macro"

View file

@ -135,8 +135,8 @@ in ''
export CARGO_MANIFEST_DIR=$(pwd)
export DEBUG="${toString (!release)}"
export OPT_LEVEL="${toString optLevel}"
export TARGET="${rust.toRustTarget stdenv.hostPlatform}"
export HOST="${rust.toRustTarget stdenv.buildPlatform}"
export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}"
export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}"
export PROFILE=${if release then "release" else "debug"}
export OUT_DIR=$(pwd)/target/build/${crateName}.out
export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0}

View file

@ -4,6 +4,10 @@
, cargo
, diffutils
, fetchCargoTarball
, runCommandNoCC
, rustPlatform
, callPackage
, remarshal
, git
, rust
, rustc
@ -26,12 +30,15 @@
, cargoBuildFlags ? []
, buildType ? "release"
, meta ? {}
, target ? null
, target ? rust.toRustTargetSpec stdenv.hostPlatform
, cargoVendorDir ? null
, checkType ? buildType
, depsExtraArgs ? {}
, cargoParallelTestThreads ? true
# Toggles whether a custom sysroot is created when the target is a .json file.
, __internal_dontAddSysroot ? false
# Needed to `pushd`/`popd` into a subdir of a tarball if this subdir
# contains a Cargo.toml, but isn't part of a workspace (which is e.g. the
# case for `rustfmt`/etc from the `rust-sources).
@ -69,13 +76,26 @@ let
cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
'';
rustTarget = if target == null then rust.toRustTarget stdenv.hostPlatform else target;
targetIsJSON = stdenv.lib.hasSuffix ".json" target;
useSysroot = targetIsJSON && !__internal_dontAddSysroot;
# see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
# the "${}" is needed to transform the path into a /nix/store path before baseNameOf
shortTarget = if targetIsJSON then
(stdenv.lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
else target;
sysroot = (callPackage ./sysroot {}) {
inherit target shortTarget;
RUSTFLAGS = args.RUSTFLAGS or "";
originalCargoToml = src + /Cargo.toml; # profile info is later extracted
};
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
releaseDir = "target/${rustTarget}/${buildType}";
releaseDir = "target/${shortTarget}/${buildType}";
tmpDir = "${releaseDir}-tmp";
# Specify the stdenv's `diff` by abspath to ensure that the user's build
@ -85,7 +105,13 @@ let
in
stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
# See https://os.phil-opp.com/testing/ for more information.
assert useSysroot -> !(args.doCheck or true);
stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // stdenv.lib.optionalAttrs useSysroot {
RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or "");
} // {
inherit cargoDeps;
patchRegistryDeps = ./patch-registry-deps;
@ -115,7 +141,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
[target."${rust.toRustTarget stdenv.buildPlatform}"]
"linker" = "${ccForBuild}"
${stdenv.lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
[target."${rustTarget}"]
[target."${shortTarget}"]
"linker" = "${ccForHost}"
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
stdenv.lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
@ -185,7 +211,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
"CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \
cargo build -j $NIX_BUILD_CORES \
${stdenv.lib.optionalString (buildType == "release") "--release"} \
--target ${rustTarget} \
--target ${target} \
--frozen ${concatStringsSep " " cargoBuildFlags}
)
@ -205,7 +231,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // {
'';
checkPhase = args.checkPhase or (let
argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${rustTarget} --frozen";
argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${target} --frozen";
threads = if cargoParallelTestThreads then "$NIX_BUILD_CORES" else "1";
in ''
${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"}

View file

@ -0,0 +1,29 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "alloc"
version = "0.0.0"
dependencies = [
"compiler_builtins",
"core",
]
[[package]]
name = "compiler_builtins"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369"
dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "core"
version = "0.0.0"
[[package]]
name = "rustc-std-workspace-core"
version = "1.99.0"
dependencies = [
"core",
]

View file

@ -0,0 +1,45 @@
import os
import toml
rust_src = os.environ['RUSTC_SRC']
orig_cargo = os.environ['ORIG_CARGO'] if 'ORIG_CARGO' in os.environ else None
base = {
'package': {
'name': 'alloc',
'version': '0.0.0',
'authors': ['The Rust Project Developers'],
'edition': '2018',
},
'dependencies': {
'compiler_builtins': {
'version': '0.1.0',
'features': ['rustc-dep-of-std', 'mem'],
},
'core': {
'path': os.path.join(rust_src, 'libcore'),
},
},
'lib': {
'name': 'alloc',
'path': os.path.join(rust_src, 'liballoc/lib.rs'),
},
'patch': {
'crates-io': {
'rustc-std-workspace-core': {
'path': os.path.join(rust_src, 'tools/rustc-std-workspace-core'),
},
},
},
}
if orig_cargo is not None:
with open(orig_cargo, 'r') as f:
src = toml.loads(f.read())
if 'profile' in src:
base['profile'] = src['profile']
out = toml.dumps(base)
with open('Cargo.toml', 'x') as f:
f.write(out)

View file

@ -0,0 +1,41 @@
{ stdenv, rust, rustPlatform, buildPackages }:
{ shortTarget, originalCargoToml, target, RUSTFLAGS }:
let
cargoSrc = stdenv.mkDerivation {
name = "cargo-src";
preferLocalBuild = true;
phases = [ "installPhase" ];
installPhase = ''
RUSTC_SRC=${rustPlatform.rustcSrc.override { minimalContent = false; }} ORIG_CARGO=${originalCargoToml} \
${buildPackages.python3.withPackages (ps: with ps; [ toml ])}/bin/python3 ${./cargo.py}
mkdir -p $out
cp Cargo.toml $out/Cargo.toml
cp ${./Cargo.lock} $out/Cargo.lock
'';
};
in rustPlatform.buildRustPackage {
inherit target RUSTFLAGS;
name = "custom-sysroot";
src = cargoSrc;
RUSTC_BOOTSTRAP = 1;
__internal_dontAddSysroot = true;
cargoSha256 = "0y6dqfhsgk00y3fv5bnjzk0s7i30nwqc1rp0xlrk83hkh80x81mw";
doCheck = false;
installPhase = ''
export LIBS_DIR=$out/lib/rustlib/${shortTarget}/lib
mkdir -p $LIBS_DIR
for f in target/${shortTarget}/release/deps/*.{rlib,rmeta}; do
cp $f $LIBS_DIR
done
export RUST_SYSROOT=$(rustc --print=sysroot)
host=${rust.toRustTarget stdenv.buildPlatform}
cp -r $RUST_SYSROOT/lib/rustlib/$host $out
'';
}

View file

@ -0,0 +1,21 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p python3 python3.pkgs.toml cargo
set -e
HERE=$(dirname "${BASH_SOURCE[0]}")
NIXPKGS_ROOT="$HERE/../../../.."
# https://unix.stackexchange.com/a/84980/390173
tempdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-lockfile')
cd "$tempdir"
nix-build -E "with import (/. + \"${NIXPKGS_ROOT}\") {}; pkgs.rustPlatform.rustcSrc.override { minimalContent = false; }"
RUSTC_SRC="$(pwd)/result" python3 "$HERE/cargo.py"
RUSTC_BOOTSTRAP=1 cargo build || echo "Build failure is expected. All that's needed is the lockfile."
cp Cargo.lock "$HERE"
rm -rf "$tempdir"

View file

@ -24,9 +24,10 @@
if platform.isDarwin then "macos"
else platform.parsed.kernel.name;
# Target triple. Rust has slightly different naming conventions than we use.
# Returns the name of the rust target, even if it is custom. Adjustments are
# because rust has slightly different naming conventions than we do.
toRustTarget = platform: with platform.parsed; let
cpu_ = platform.rustc.arch or {
cpu_ = platform.rustc.platform.arch or {
"armv7a" = "armv7";
"armv7l" = "armv7";
"armv6l" = "arm";
@ -34,6 +35,13 @@
in platform.rustc.config
or "${cpu_}-${vendor.name}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}";
# Returns the name of the rust target if it is standard, or the json file
# containing the custom target spec.
toRustTargetSpec = platform:
if (platform.rustc or {}) ? platform
then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform)
else toRustTarget platform;
# This just contains tools for now. But it would conceivably contain
# libraries too, say if we picked some default/recommended versions from
# `cratesIO` to build by Hydra and/or try to prefer/bias in Cargo.lock for

View file

@ -1,4 +1,4 @@
{ stdenv, rustc }:
{ stdenv, rustc, minimalContent ? true }:
stdenv.mkDerivation {
name = "rust-src";
@ -6,6 +6,9 @@ stdenv.mkDerivation {
phases = [ "unpackPhase" "installPhase" ];
installPhase = ''
mv src $out
rm -rf $out/{ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,tools,vendor,stdarch}
rm -rf $out/{${if minimalContent
then "ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,tools,vendor,stdarch"
else "ci,doc,etc,grammar,llvm-project,llvm-emscripten,rtstartup,rustllvm,test,vendor"
}}
'';
}

View file

@ -70,9 +70,9 @@ in stdenv.mkDerivation rec {
"--set=build.cargo=${rustPlatform.rust.cargo}/bin/cargo"
"--enable-rpath"
"--enable-vendor"
"--build=${rust.toRustTarget stdenv.buildPlatform}"
"--host=${rust.toRustTarget stdenv.hostPlatform}"
"--target=${rust.toRustTarget stdenv.targetPlatform}"
"--build=${rust.toRustTargetSpec stdenv.buildPlatform}"
"--host=${rust.toRustTargetSpec stdenv.hostPlatform}"
"--target=${rust.toRustTargetSpec stdenv.targetPlatform}"
"${setBuild}.cc=${ccForBuild}"
"${setHost}.cc=${ccForHost}"

View file

@ -63,7 +63,8 @@ redoxRustPlatform.buildRustPackage rec {
DESTDIR=$out make install
'';
TARGET = buildPackages.rust.toRustTarget stdenvNoCC.targetPlatform;
# TODO: should be hostPlatform
TARGET = buildPackages.rust.toRustTargetSpec stdenvNoCC.targetPlatform;
cargoSha256 = "1fzz7ba3ga57x1cbdrcfrdwwjr70nh4skrpxp4j2gak2c3scj6rz";

View file

@ -38,6 +38,8 @@ with pkgs;
cross = callPackage ./cross {};
rustCustomSysroot = callPackage ./rust-sysroot {};
nixos-functions = callPackage ./nixos-functions {};
patch-shebangs = callPackage ./patch-shebangs {};

View file

@ -0,0 +1,60 @@
{ lib, rust, rustPlatform, fetchFromGitHub }:
let
mkBlogOsTest = target: rustPlatform.buildRustPackage rec {
name = "blog_os-sysroot-test";
src = fetchFromGitHub {
owner = "phil-opp";
repo = "blog_os";
rev = "4e38e7ddf8dd021c3cd7e4609dfa01afb827797b";
sha256 = "0k9ipm9ddm1bad7bs7368wzzp6xwrhyfzfpckdax54l4ffqwljcg";
};
cargoSha256 = "1cbcplgz28yxshyrp2krp1jphbrcqdw6wxx3rry91p7hiqyibd30";
inherit target;
RUSTFLAGS = "-C link-arg=-nostartfiles";
# Tests don't work for `no_std`. See https://os.phil-opp.com/testing/
doCheck = false;
meta = with lib; {
description = "Test for using custom sysroots with buildRustPackage";
maintainers = with maintainers; [ aaronjanse ];
platforms = lib.platforms.x86_64;
};
};
# The book uses rust-lld for linking, but rust-lld is not currently packaged for NixOS.
# The justification in the book for using rust-lld suggests that gcc can still be used for testing:
# > Instead of using the platform's default linker (which might not support Linux targets),
# > we use the cross platform LLD linker that is shipped with Rust for linking our kernel.
# https://github.com/phil-opp/blog_os/blame/7212ffaa8383122b1eb07fe1854814f99d2e1af4/blog/content/second-edition/posts/02-minimal-rust-kernel/index.md#L157
targetContents = {
"llvm-target" = "x86_64-unknown-none";
"data-layout" = "e-m:e-i64:64-f80:128-n8:16:32:64-S128";
"arch" = "x86_64";
"target-endian" = "little";
"target-pointer-width" = "64";
"target-c-int-width" = "32";
"os" = "none";
"executables" = true;
"linker-flavor" = "gcc";
"panic-strategy" = "abort";
"disable-redzone" = true;
"features" = "-mmx,-sse,+soft-float";
};
in {
blogOS-targetByFile = mkBlogOsTest (builtins.toFile "x86_64-blog_os.json" (builtins.toJSON targetContents));
blogOS-targetByNix = let
plat = lib.systems.elaborate { config = "x86_64-none"; } // {
rustc = {
config = "x86_64-blog_os";
platform = targetContents;
};
};
in mkBlogOsTest (rust.toRustTargetSpec plat);
}