diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index 3332dff1eb0..6f0ec7c0514 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -37,7 +37,7 @@ rustPlatform.buildRustPackage rec { }; cargoSha256 = "17ldqr3asrdcsh4l29m3b5r37r5d0b3npq1lrgjmxb6vlx6a36qh"; - verifyCargoDeps = true; + legacyCargoFetcher = false; meta = with stdenv.lib; { description = "A fast line-oriented regex search tool, similar to ag and ack"; @@ -59,12 +59,19 @@ When the `Cargo.lock`, provided by upstream, is not in sync with the added in `cargoPatches` will also be prepended to the patches in `patches` at build-time. -When `verifyCargoDeps` is set to `true`, the build will also verify that the -`cargoSha256` is not out of date by comparing the `Cargo.lock` file in both the -`cargoDeps` and `src`. Note that this option changes the value of `cargoSha256` -since it also copies the `Cargo.lock` in it. To avoid breaking -backward-compatibility this option is not enabled by default but hopefully will -be in the future. +Setting `legacyCargoFetcher` to `false` enables the following behavior: + +1. The `Cargo.lock` file is copied into the cargo vendor directory. +2. At buildtime, `buildRustPackage` will ensure that the `src` and `cargoSha256` + are consistent. This avoids errors where one but not the other is updated. +3. The builder will compress the vendored cargo src directory into a tar.gz file + for storage after vendoring, and decompress it before the build. This saves + disk space and enables hashed mirrors for Rust dependencies. + +Note that this option changes the value of `cargoSha256`, so it is currently +defaulted to `false`. When updating a Rust package, please set it to `true`; +eventually we will default this to true and update the remaining Rust packages, +then delete the option from all individual Rust package expressions. ### Building a crate for a different target diff --git a/pkgs/applications/editors/hexdino/default.nix b/pkgs/applications/editors/hexdino/default.nix index eee5a696555..91f048320a7 100644 --- a/pkgs/applications/editors/hexdino/default.nix +++ b/pkgs/applications/editors/hexdino/default.nix @@ -11,8 +11,8 @@ rustPlatform.buildRustPackage { sha256 = "11mz07735gxqfamjcjjmxya6swlvr1p77sgd377zjcmd6z54gwyf"; }; - cargoSha256 = "0qa8ypp5a7sf1gic482zh3i6s94w6k6bgmk5ynfvwi7g49ql7c4z"; - verifyCargoDeps = true; + cargoSha256 = "06ghcd4j751mdkzwb88nqwk8la4zdb137y0iqrkpykkfx0as43x3"; + legacyCargoFetcher = false; buildInputs = [ ncurses ]; diff --git a/pkgs/applications/version-management/git-and-tools/git-workspace/default.nix b/pkgs/applications/version-management/git-and-tools/git-workspace/default.nix index 2ceac55bf5c..61e3e1e3eef 100644 --- a/pkgs/applications/version-management/git-and-tools/git-workspace/default.nix +++ b/pkgs/applications/version-management/git-and-tools/git-workspace/default.nix @@ -15,9 +15,9 @@ rustPlatform.buildRustPackage rec { sha256 = "0pl5z0gx2ypkrgq7vj1cxj5iwj06vcd06x3b3nh0g7w7q7nl8pr4"; }; - cargoSha256 = "0jbsz7r9n3jcgb9sd6pdjwzjf1b35qpfqw8ba8fjjmzfvs9qn6ld"; + cargoSha256 = "1z4cb7rcb7ldj16xxynrjh4hg872rj39rbbp0vy15kdp3ifyi466"; - verifyCargoDeps = true; + legacyCargoFetcher = false; buildInputs = with stdenv; lib.optional isDarwin Security; diff --git a/pkgs/build-support/rust/README.md b/pkgs/build-support/rust/README.md new file mode 100644 index 00000000000..0e0ddb9648d --- /dev/null +++ b/pkgs/build-support/rust/README.md @@ -0,0 +1,45 @@ +# Updated fetchCargo behavior + +Changes to the `fetchcargo.nix` behavior that cause changes to the `cargoSha256` +are somewhat disruptive, so historically we've added conditionals to provide +backwards compatibility. We've now accumulated enough of these that it makes +sense to do a clean sweep updating hashes, and delete the conditionals in the +fetcher to simplify maintenance and implementation complexity. These +conditionals are: + +1. When cargo vendors dependencies, it generates a config. Previously, we were + hard-coding our own config, but this fails if there are git dependencies. We + have conditional logic to sometimes copy the vendored cargo config in, and + sometimes not. + +2. When a user updates the src package, they may forget to update the + `cargoSha256`. We have an opt-in conditional flag to add the `Cargo.lock` + into the vendor dir for inspection and compare at build-time, but it defaults + to false. + +3. We were previously vendoring into a directory with a recursive hash, but + would like to vendor into a compressed tar.gz file instead, for the reasons + specified in the git commit message adding this feature. + + +## Migration plan + +1. (DONE in this PR) Implement `fetchCargoTarball` as a separate, clean fetcher + implementation along-side `fetchcargo`. Rename `verifyCargoDeps` (default + false) to `legacyCargoFetcher` (default true), which switches the fetcher + implementation used. Replace `verifyCargoDeps = true;` with + `legacyCargoFetcher = false;` in Rust applications. + +2. Send a treewide Rust PR that sets `legacyCargoFetcher = true;` in all Rust + applications not using this (which is ~200 of them), with a note to + maintainers to delete if updating the package. Change the default in + `buildRustPackage` to false. + +3. Go through all Rust src packages deleting the `legacyCargoFetcher = false;` + line and re-computing the `cargoSha256`, merging as we go. + +4. Delete the `fetchcargo.nix` implementation entirely and also remove: + - All overrides in application-level packages + - The `fetchcargo-default-config.toml` and conditionals around using it when + no `$CARGO_CONFIG` exists + - This README.md file diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 4089436c0e0..ac0a8d3ae46 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -1,4 +1,4 @@ -{ stdenv, cacert, git, rust, cargo, rustc, fetchcargo, buildPackages, windows }: +{ stdenv, cacert, git, rust, cargo, rustc, fetchcargo, fetchCargoTarball, buildPackages, windows }: { name ? "${args.pname}-${args.version}" , cargoSha256 ? "unset" @@ -14,13 +14,13 @@ , cargoUpdateHook ? "" , cargoDepsHook ? "" , cargoBuildFlags ? [] -, # Set to true to verify if the cargo dependencies are up to date. - # This will change the value of cargoSha256. - verifyCargoDeps ? false + # Please set to true on any Rust package updates. Once all packages set this + # to true, we will delete and make it the default. For details, see the Rust + # section on the manual and ./README.md. +, legacyCargoFetcher ? true , buildType ? "release" , meta ? {} , target ? null - , cargoVendorDir ? null , ... } @ args: @@ -28,20 +28,27 @@ assert cargoVendorDir == null -> cargoSha256 != "unset"; assert buildType == "release" || buildType == "debug"; let + + cargoFetcher = if legacyCargoFetcher + then fetchcargo + else fetchCargoTarball; + cargoDeps = if cargoVendorDir == null - then fetchcargo { + then cargoFetcher { inherit name src srcs sourceRoot unpackPhase cargoUpdateHook; - copyLockfile = verifyCargoDeps; patches = cargoPatches; sha256 = cargoSha256; } else null; + # If we're using the modern fetcher that always preserves the original Cargo.lock + # and have vendored deps, check them against the src attr for consistency. + validateCargoDeps = cargoSha256 != "unset" && !legacyCargoFetcher; + setupVendorDir = if cargoVendorDir == null then '' unpackFile "$cargoDeps" - cargoDepsCopy=$(stripHash $(basename $cargoDeps)) - chmod -R +w "$cargoDepsCopy" + cargoDepsCopy=$(stripHash $cargoDeps) '' else '' cargoDepsCopy="$sourceRoot/${cargoVendorDir}" @@ -54,9 +61,14 @@ let ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; releaseDir = "target/${rustTarget}/${buildType}"; + + # Fetcher implementation choice should not be part of the hash in final + # derivation; only the cargoSha256 input matters. + filteredArgs = builtins.removeAttrs args [ "legacyCargoFetcher" ]; + in -stdenv.mkDerivation (args // { +stdenv.mkDerivation (filteredArgs // { inherit cargoDeps; patchRegistryDeps = ./patch-registry-deps; @@ -95,14 +107,13 @@ stdenv.mkDerivation (args // { ''} EOF - unset cargoDepsCopy export RUST_LOG=${logLevel} - '' + stdenv.lib.optionalString verifyCargoDeps '' - if ! diff source/Cargo.lock $cargoDeps/Cargo.lock ; then + '' + stdenv.lib.optionalString validateCargoDeps '' + if ! diff source/Cargo.lock $cargoDepsCopy/Cargo.lock ; then echo echo "ERROR: cargoSha256 is out of date" echo - echo "Cargo.lock is not the same in $cargoDeps" + echo "Cargo.lock is not the same in $cargoDepsCopy" echo echo "To fix the issue:" echo '1. Use "1111111111111111111111111111111111111111111111111111" as the cargoSha256 value' @@ -112,6 +123,8 @@ stdenv.mkDerivation (args // { exit 1 fi + '' + '' + unset cargoDepsCopy '' + (args.postUnpack or ""); configurePhase = args.configurePhase or '' diff --git a/pkgs/build-support/rust/fetchCargoTarball.nix b/pkgs/build-support/rust/fetchCargoTarball.nix new file mode 100644 index 00000000000..dff5d99da9e --- /dev/null +++ b/pkgs/build-support/rust/fetchCargoTarball.nix @@ -0,0 +1,81 @@ +{ stdenv, cacert, git, cargo, python3 }: +let cargo-vendor-normalise = stdenv.mkDerivation { + name = "cargo-vendor-normalise"; + src = ./cargo-vendor-normalise.py; + nativeBuildInputs = [ python3.pkgs.wrapPython ]; + dontUnpack = true; + installPhase = "install -D $src $out/bin/cargo-vendor-normalise"; + pythonPath = [ python3.pkgs.toml ]; + postFixup = "wrapPythonPrograms"; + doInstallCheck = true; + installCheckPhase = '' + # check that ./fetchcargo-default-config.toml is a fix point + reference=${./fetchcargo-default-config.toml} + < $reference $out/bin/cargo-vendor-normalise > test; + cmp test $reference + ''; + preferLocalBuild = true; +}; +in +{ name ? "cargo-deps" +, src ? null +, srcs ? [] +, patches ? [] +, sourceRoot +, sha256 +, cargoUpdateHook ? "" +, ... +} @ args: +stdenv.mkDerivation ({ + name = "${name}-vendor.tar.gz"; + nativeBuildInputs = [ cacert git cargo-vendor-normalise cargo ]; + + phases = "unpackPhase patchPhase buildPhase installPhase"; + + buildPhase = '' + # Ensure deterministic Cargo vendor builds + export SOURCE_DATE_EPOCH=1 + + if [[ ! -f Cargo.lock ]]; then + echo + echo "ERROR: The Cargo.lock file doesn't exist" + echo + echo "Cargo.lock is needed to make sure that cargoSha256 doesn't change" + echo "when the registry is updated." + echo + + exit 1 + fi + + # Keep the original around for copyLockfile + cp Cargo.lock Cargo.lock.orig + + export CARGO_HOME=$(mktemp -d cargo-home.XXX) + CARGO_CONFIG=$(mktemp cargo-config.XXXX) + + ${cargoUpdateHook} + + cargo vendor $name | cargo-vendor-normalise > $CARGO_CONFIG + + # Add the Cargo.lock to allow hash invalidation + cp Cargo.lock.orig $name/Cargo.lock + + # Packages with git dependencies generate non-default cargo configs, so + # always install it rather than trying to write a standard default template. + install -D $CARGO_CONFIG $name/.cargo/config; + ''; + + # Build a reproducible tar, per instructions at https://reproducible-builds.org/docs/archives/ + installPhase = '' + tar --owner=0 --group=0 --numeric-owner --format=gnu \ + --sort=name --mtime="@$SOURCE_DATE_EPOCH" \ + -czf $out $name + ''; + + outputHashAlgo = "sha256"; + outputHash = sha256; + + impureEnvVars = stdenv.lib.fetchers.proxyImpureEnvVars; +} // (builtins.removeAttrs args [ + "name" "sha256" "cargoUpdateHook" +])) diff --git a/pkgs/development/compilers/rust/default.nix b/pkgs/development/compilers/rust/default.nix index da3421a987d..af7f63b6f8a 100644 --- a/pkgs/development/compilers/rust/default.nix +++ b/pkgs/development/compilers/rust/default.nix @@ -25,12 +25,18 @@ inherit rustc cargo; }; + fetchCargoTarball = buildPackages.callPackage ../../../build-support/rust/fetchCargoTarball.nix { + inherit cargo; + }; + + # N.B. This is a legacy fetcher implementation that is being phased out and deleted. + # See ../../../build-support/rust/README.md for details. fetchcargo = buildPackages.callPackage ../../../build-support/rust/fetchcargo.nix { inherit cargo; }; buildRustPackage = callPackage ../../../build-support/rust { - inherit rustc cargo fetchcargo; + inherit rustc cargo fetchcargo fetchCargoTarball; }; rustcSrc = callPackage ./rust-src.nix { diff --git a/pkgs/development/tools/documentation/mdsh/default.nix b/pkgs/development/tools/documentation/mdsh/default.nix index 2ca2ef29dd2..ddce5bc29a4 100644 --- a/pkgs/development/tools/documentation/mdsh/default.nix +++ b/pkgs/development/tools/documentation/mdsh/default.nix @@ -11,8 +11,8 @@ rustPlatform.buildRustPackage rec { sha256 = "1a9i6h8fzrrfzjyfxaps73lxgkz92k0bnmwbjbwdmiwci4qgi9ms"; }; - cargoSha256 = "0rarpzfigyxr6s0ba13z00kvnms29qkjfbfjkay72mb6xn7f1059"; - verifyCargoDeps = true; + cargoSha256 = "1fxajh1n0qvcdas6w7dy3g92wilhfldy90pyk3779mrnh57fa6n5"; + legacyCargoFetcher = false; meta = with stdenv.lib; { description = "Markdown shell pre-processor"; diff --git a/pkgs/tools/misc/broot/default.nix b/pkgs/tools/misc/broot/default.nix index 45b26b25055..df26423c633 100644 --- a/pkgs/tools/misc/broot/default.nix +++ b/pkgs/tools/misc/broot/default.nix @@ -11,8 +11,8 @@ rustPlatform.buildRustPackage rec { sha256 = "13b1w9g68aj3r70w9bmrmdc772y959n77ajbdm2cpjs5f4kgfpak"; }; - cargoSha256 = "0vzpyymylzxjm613lf5xr6hd21ijkl3vwq4y6h1q3as41phw2sqb"; - verifyCargoDeps = true; + cargoSha256 = "0zrwpmsrzwnjml0964zky8w222zmlargha3z0n6hf8cfshx23s4k"; + legacyCargoFetcher = false; nativeBuildInputs = [ installShellFiles ]; diff --git a/pkgs/tools/misc/wagyu/default.nix b/pkgs/tools/misc/wagyu/default.nix index d56d21b1516..53e09849522 100644 --- a/pkgs/tools/misc/wagyu/default.nix +++ b/pkgs/tools/misc/wagyu/default.nix @@ -11,8 +11,8 @@ rustPlatform.buildRustPackage rec { sha256 = "1646j0lgg3hhznifvbkvr672p3yqlcavswijawaxq7n33ll8vmcn"; }; - cargoSha256 = "10b96l0b32zxq0xrnhivv3gihmi5y31rllbizv67hrg1axz095vn"; - verifyCargoDeps = true; + cargoSha256 = "16d1b3pamkg29nq80n6cbzc4zl9z3cgfvdxjkr2z4xrnzmkn1ysi"; + legacyCargoFetcher = false; meta = with lib; { description = "Rust library for generating cryptocurrency wallets"; diff --git a/pkgs/tools/package-management/nix-du/default.nix b/pkgs/tools/package-management/nix-du/default.nix index 74543cef83b..908f31b93ab 100644 --- a/pkgs/tools/package-management/nix-du/default.nix +++ b/pkgs/tools/package-management/nix-du/default.nix @@ -9,8 +9,8 @@ rustPlatform.buildRustPackage rec { rev = "v${version}"; sha256 = "149d60mid29s5alv5m3d7jrhyzc6cj7b6hpiq399gsdwzgxr00wq"; }; - cargoSha256 = "18kb4car5nzch3vpl6z1499silhs3fyn8c6xj3rzk94mm2m9srg4"; - verifyCargoDeps = true; + cargoSha256 = "1a6svl89dcdb5fpvs2i32i6agyhl0sx7kkkw70rqr17fyzl5psai"; + legacyCargoFetcher = false; doCheck = true; checkInputs = [ graphviz ]; diff --git a/pkgs/tools/security/fido2luks/default.nix b/pkgs/tools/security/fido2luks/default.nix index 4682a09acf5..ea911e46734 100644 --- a/pkgs/tools/security/fido2luks/default.nix +++ b/pkgs/tools/security/fido2luks/default.nix @@ -19,8 +19,8 @@ rustPlatform.buildRustPackage rec { buildInputs = [ cryptsetup ]; nativeBuildInputs = [ pkg-config ]; - cargoSha256 = "1i37k4ih6118z3wip2qh4jqk7ja2z0v1w8dri1lwqwlciqw17zi9"; - verifyCargoDeps = true; + cargoSha256 = "0rp4f6xnwmvf3pv6h0qwsg01jrndf77yn67675ac39kxzmrzfy2f"; + legacyCargoFetcher = false; meta = with stdenv.lib; { description = "Decrypt your LUKS partition using a FIDO2 compatible authenticator"; diff --git a/pkgs/tools/system/tre-command/default.nix b/pkgs/tools/system/tre-command/default.nix index 6050b2a709e..b8925dcfa42 100644 --- a/pkgs/tools/system/tre-command/default.nix +++ b/pkgs/tools/system/tre-command/default.nix @@ -11,8 +11,8 @@ rustPlatform.buildRustPackage rec { sha256 = "1fazw2wn738iknbv54gv7qll7d4q2gy9bq1s3f3cv21cdv6bqral"; }; - cargoSha256 = "0m82zbi610zgvcza6n03xl80g31x6bfkjyrfxcxa6fyf2l5cj9pv"; - verifyCargoDeps = true; + cargoSha256 = "1m3ccp5ncafkifg8sxyxczsg3ja1gvq8wmgni68bgzm2lwxh2qgw"; + legacyCargoFetcher = false; meta = with stdenv.lib; { description = "Tree command, improved";