prefer-fetch-remote: an overlay to fetch on remote builders

This is useful when running tools like NixOps or nix-review
on workstations where the upload to the builder is significantly
slower then downloading the source on the builder itself.
This commit is contained in:
Jörg Thalheim 2018-12-31 08:10:28 +01:00
parent 487d2a7ccd
commit eac6797380
No known key found for this signature in database
GPG key ID: CA4106B8D7CC79FA
9 changed files with 72 additions and 13 deletions

View file

@ -14,4 +14,5 @@
<xi:include href="functions/fhs-environments.xml" />
<xi:include href="functions/shell.xml" />
<xi:include href="functions/dockertools.xml" />
<xi:include href="functions/prefer-remote-fetch.xml" />
</chapter>

View file

@ -0,0 +1,27 @@
<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/xinclude"
xml:id="sec-prefer-remote-fetch">
<title>prefer-remote-fetch overlay</title>
<para>
<function>prefer-remote-fetch</function> is an overlay that download sources
on remote builder. This is useful when the evaluating machine has a slow
upload while the builder can fetch faster directly from the source.
To use it, put the following snippet as a new overlay:
<programlisting>
self: super:
(super.prefer-remote-fetch self super)
</programlisting>
A full configuration example for that sets the overlay up for your own account,
could look like this
<programlisting>
$ mkdir ~/.config/nixpkgs/overlays/
$ cat &gt; ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix &lt;&lt;EOF
self: super: super.prefer-remote-fetch self super
EOF
</programlisting>
</para>
</section>

View file

@ -19,6 +19,7 @@ in
, # Shell code executed after the file has been fetched
# successfully. This can do things like check or transform the file.
postFetch ? ""
, preferLocalBuild ? true
}:
/* NOTE:
@ -66,5 +67,5 @@ stdenvNoCC.mkDerivation {
"GIT_PROXY_COMMAND" "SOCKS_SERVER"
];
preferLocalBuild = true;
inherit preferLocalBuild;
}

View file

@ -1,4 +1,11 @@
{stdenvNoCC, mercurial}: {name ? null, url, rev ? null, md5 ? null, sha256 ? null, fetchSubrepos ? false}:
{ stdenvNoCC, mercurial }:
{ name ? null
, url
, rev ? null
, md5 ? null
, sha256 ? null
, fetchSubrepos ? false
, preferLocalBuild ? true }:
if md5 != null then
throw "fetchhg does not support md5 anymore, please use sha256"
@ -18,5 +25,5 @@ stdenvNoCC.mkDerivation {
outputHash = sha256;
inherit url rev;
preferLocalBuild = true;
inherit preferLocalBuild;
}

View file

@ -14,6 +14,7 @@
, meta ? {}
, port ? "8080"
, postFetch ? ""
, preferLocalBuild ? true
}:
assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0;
@ -42,11 +43,10 @@ if (!hasHash) then throw "Specify sha for fetchipfs fixed-output derivation" els
postFetch
ipfs
url
port;
port
meta;
# Doing the download on a remote machine just duplicates network
# traffic, so don't do that.
preferLocalBuild = true;
inherit meta;
inherit preferLocalBuild;
}

View file

@ -1,6 +1,7 @@
{stdenvNoCC, subversion, glibcLocales, sshSupport ? false, openssh ? null}:
{url, rev ? "HEAD", md5 ? "", sha256 ? "",
ignoreExternals ? false, ignoreKeywords ? false, name ? null}:
{url, rev ? "HEAD", md5 ? "", sha256 ? ""
, ignoreExternals ? false, ignoreKeywords ? false, name ? null
, preferLocalBuild ? true }:
let
repoName = with stdenvNoCC.lib;
@ -40,5 +41,5 @@ stdenvNoCC.mkDerivation {
inherit url rev sshSupport openssh ignoreExternals ignoreKeywords;
impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars;
preferLocalBuild = true;
inherit preferLocalBuild;
}

View file

@ -87,6 +87,9 @@ in
# Passthru information, if any.
, passthru ? {}
# Doing the download on a remote machine just duplicates network
# traffic, so don't do that by default
, preferLocalBuild ? true
}:
assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0;
@ -135,9 +138,7 @@ stdenvNoCC.mkDerivation {
nixpkgsVersion = lib.trivial.release;
# Doing the download on a remote machine just duplicates network
# traffic, so don't do that.
preferLocalBuild = true;
inherit preferLocalBuild;
postHook = if netrcPhase == null then null else ''
${netrcPhase}

View file

@ -0,0 +1,19 @@
# An overlay that download sources on remote builder.
# This is useful when the evaluating machine has a slow
# upload while the builder can fetch faster directly from the source.
# Usage: Put the following snippet in your usual overlay definition:
#
# self: super:
# (super.prefer-remote-fetch self super)
# Full configuration example for your own account:
#
# $ mkdir ~/.config/nixpkgs/overlays/
# $ echo 'self: super: super.prefer-remote-fetch self super' > ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix
#
self: super: {
fetchurl = args: super.fetchurl (args // { preferLocalBuild = false; });
fetchgit = args: super.fetchgit (args // { preferLocalBuild = false; });
fetchhg = args: super.fetchhg (args // { preferLocalBuild = false; });
fetchsvn = args: super.fetchsvn (args // { preferLocalBuild = false; });
fetchipfs = args: super.fetchipfs (args // { preferLocalBuild = false; });
}

View file

@ -207,6 +207,8 @@ in
fetchMavenArtifact = callPackage ../build-support/fetchmavenartifact { };
prefer-remote-fetch = import ../build-support/prefer-remote-fetch;
global-platform-pro = callPackage ../development/tools/global-platform-pro/default.nix { };
graph-easy = callPackage ../tools/graphics/graph-easy { };