Merge pull request #45432 from xeji/texlive-2018

[staging]: texlive: 2017 -> 2018
This commit is contained in:
xeji 2018-08-30 20:42:03 +02:00 committed by GitHub
commit d3036afd6d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 17118 additions and 8354 deletions

View file

@ -5,13 +5,13 @@
perlPackages.buildPerlModule rec {
name = "biber-${version}";
version = "2.10";
version = "2.11";
src = fetchFromGitHub {
owner = "plk";
repo = "biber";
rev = "v${version}";
sha256 = "0f6bb1iprl92iamxqlr8fc99mxr9n3722frd1ak9pbzh3m6c2ny6";
sha256 = "0qgkc1k9n36yfmndwz879pak6mjphld0p85lzn9g2ng0vhxsifzz";
};
buildInputs = with perlPackages; [

View file

@ -0,0 +1,68 @@
# Notes on maintaining/upgrading
## Upgrading texlive.bin
texlive contains a few binaries, defined in bin.nix and released once a year.
In order to reduce closure size for users who just need a few of them, we split it into
packages such as core, core-big, xvdi, etc. This requires making assumptions
about dependencies between the projects that may change between releases; if
you upgrade you may have to do some work here.
## Updating the package set
texlive contains several thousand packages from CTAN, defined in pkgs.nix.
The CTAN mirrors are not version-controlled and continuously moving,
with more than 100 updates per month.
To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN
and generate nix expressions for all packages in texlive at that point.
We mirror CTAN sources of this snapshot on community-operated servers and on IPFS.
To upgrade the package snapshot, follow this process:
### Snapshot sources and texlive package database
Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`).
See <https://tug.org/texlive/acquire-mirror.html> for instructions.
### Upgrade package information from texlive package database
```
$ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
| xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
```
This will download a current snapshot of the CTAN package database `texlive.tlpdb.xz`
and regenerate all of the sha512 hashes for the current upstream distribution in `pkgs.nix`.
### Build packages locally and generate fix hashes
To save disk space and prevent unnecessary rebuilds, texlive packages are built
as fixed-output derivations whose hashes are contained in `fixedHashes.nix`.
Updating the list of fixed hashes requires a local build of *all* packages,
which is a resource-intensive process:
```
# move fixedHashes away, otherwise build will fail on updated packages
$ mv fixedHashes.nix fixedHashes-old.nix
# start with empty fixedHashes
$ echo '{}' > fixedHashes.nix
$ nix-build ../../../../.. -Q --no-out-link -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes-new.nix
# The script wrongly includes the nix store path to `biber`, which is a separate nixpkgs package
$ grep -v -F '/nix/store/' fixedHashes-new.nix > fixedHashes.nix
```
### Commit changes
Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository.

View file

@ -14,25 +14,23 @@
let
withSystemLibs = map (libname: "--with-system-${libname}");
year = "2017";
year = "2018";
version = year; # keep names simple for now
common = rec {
src = fetchurl {
url = # "ftp://tug.org/historic/systems/texlive/${year}/"
#"http://lipa.ms.mff.cuni.cz/~cunav5am/nix/texlive-2016"
# FIXME: a proper mirror, though tarballs.nixos.org saves this case ATM
# http://146.185.144.154/texlive-2016
# + "/texlive-${year}0523b-source.tar.xz";
"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${year}/texlive-${year}0524-source.tar.xz";
sha256 = "1amjrxyasplv4alfwcxwnw4nrx7dz2ydmddkq16k6hg90i9njq81";
urls = [
"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${year}/texlive-${year}0414-source.tar.xz"
"ftp://tug.ctan.org/pub/tex/historic/systems/texlive/${year}/texlive-${year}0414-source.tar.xz"
];
sha256 = "0khyi6h015r2zfqgg0a44a2j7vmr1cy42knw7jbss237yvakc07y";
};
patches = [
(fetchurl {
name = "texlive-poppler-0.59.patch";
url = https://git.archlinux.org/svntogit/packages.git/plain/trunk/texlive-poppler-0.59.patch?h=packages/texlive-bin&id=6308ec39bce2a4d735f6ff8a4e94473748d7b450;
sha256 = "1c4ikq4kxw48bi3i33bzpabrjvbk01fwjr2lz20gkc9kv8l0bg3n";
name = "texlive-poppler-0.64.patch";
url = https://git.archlinux.org/svntogit/packages.git/plain/trunk/texlive-poppler-0.64.patch?h=packages/texlive-bin;
sha256 = "0443d074zl3c5raba8jyhavish706arjcd80ibb84zwnwck4ai0w";
})
];
@ -83,6 +81,8 @@ core = stdenv.mkDerivation rec {
for i in texk/kpathsea/mktex*; do
sed -i '/^mydir=/d' "$i"
done
cp -pv texk/web2c/pdftexdir/pdftoepdf{-newpoppler.cc,.cc}
cp -pv texk/web2c/pdftexdir/pdftosrc{-newpoppler.cc,.cc}
'';
preConfigure = ''
@ -165,7 +165,7 @@ inherit (core-big) metafont metapost luatex xetex;
core-big = stdenv.mkDerivation { #TODO: upmendex
name = "texlive-core-big.bin-${version}";
inherit (common) src;
inherit (common) src patches;
hardeningDisable = [ "format" ];
@ -176,20 +176,23 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
++ withSystemLibs [ "kpathsea" "ptexenc" "cairo" "harfbuzz" "icu" "graphite2" ]
++ map (prog: "--disable-${prog}") # don't build things we already have
[ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "pdftex"
"web-progs" "synctex" "luajittex" "mfluajit" # luajittex is mostly not needed, see:
"web-progs" "synctex"
# build fails on Darwin with luatex53
"luatex53" # TODO probably can be removed when TexLive 2019 is out
# luajittex is mostly not needed, see:
# http://tex.stackexchange.com/questions/97999/when-to-use-luajittex-in-favour-of-luatex
"luajittex" "mfluajit"
];
patches = common.patches ++ [ ./luatex-gcc7.patch ];
configureScript = ":";
# we use static libtexlua, because it's only used by a single binary
postConfigure = ''
mkdir ./WorkDir && cd ./WorkDir
# TODO add lua53 here when luatex53 is enabled again
for path in libs/{teckit,lua52} texk/web2c; do
(
if [[ "$path" == "libs/lua52" ]]; then
if [[ "$path" =~ "libs/lua5" ]]; then
extraConfig="--enable-static --disable-shared"
else
extraConfig=""
@ -202,7 +205,6 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
'';
preBuild = "cd texk/web2c";
CXXFLAGS = "-std=c++11 -Wno-reserved-user-defined-literal"; # TODO: remove once texlive 2018 is out?
enableParallelBuilding = true;
doCheck = false; # fails

View file

@ -1,34 +1,11 @@
/* TeX Live user docs
- source: ../../../../../doc/languages-frameworks/texlive.xml
- current html: http://nixos.org/nixpkgs/manual/#sec-language-texlive
Note on upgrading: The texlive package contains a few binaries, defined in
bin.nix and released once a year, and several thousand packages from CTAN,
defined in pkgs.nix.
The CTAN mirrors are continuously moving, with more than 100 updates per
month. Due to the size of the distribution, we snapshot it and generate nix
expressions for all packages in texlive at that point.
To upgrade this snapshot, run the following:
$ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
| xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
This will regenerate all of the sha512 hashes for the current upstream
distribution. You may want to find a more stable mirror, put the distribution
on IPFS, or contact a maintainer to get the tarballs from that point in time
into a more stable location, so that nix users who are building from source
can reproduce your work.
Upgrading the bin: texlive itself is a large collection of binaries. In order
to reduce closure size for users who just need a few of them, we split it into
packages such as core, core-big, xvdi, etc. This requires making assumptions
about dependencies between the projects that may change between releases; if
you upgrade you may have to do some work here.
*/
{ stdenv, lib, fetchurl, runCommand, writeText, buildEnv
, callPackage, ghostscriptX, harfbuzz, poppler_min
, makeWrapper, python, ruby, perl
, useFixedHashes ? true
, recurseIntoAttrs
}:
let
@ -41,6 +18,10 @@ let
};
};
# map: name -> fixed-output hash
# sha1 in base32 was chosen as a compromise between security and length
fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
# function for creating a working environment from a set of TL packages
combine = import ./combine.nix {
inherit bin combinePkgs buildEnv fastUnique lib makeWrapper writeText
@ -116,23 +97,34 @@ let
# the basename used by upstream (without ".tar.xz" suffix)
urlName = pname + lib.optionalString (tlType != "run") ".${tlType}";
tlName = urlName + "-${version}";
fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
urls = args.urls or (if args ? url then [ args.url ] else
map (up: "${up}/${urlName}.tar.xz") urlPrefixes
);
# Upstream refuses to distribute stable tarballs, so we host snapshots on IPFS.
# Upstream refuses to distribute stable tarballs,
# so we host snapshots on IPFS or on our own servers.
# Common packages should get served from the binary cache anyway.
# See discussions, e.g. https://github.com/NixOS/nixpkgs/issues/24683
urlPrefixes = args.urlPrefixes or [
# Should be stable for historic, archived releases
http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2017/tlnet-final/archive
# A snapshot temporarily hosted by @xeji.
# TODO: remove when there is a reliable long-term solution
https://cat3.de/texlive-2018/tlnet/archive
# TODO: Add IPFS and see if @veprbl is willing to add a texlive-2017-final mirror,
# or if we should just dump it and go to 2018.
# TODO: Add second, faster and more reliable snapshot mirror,
# maybe on one of our project's servers
# IPFS seeded by the mirror above - this may be quite slow
https://ipfs.io/ipfs/QmT4Z67wXin1Z9DhvqwSSkSZSuu8hT6LgDyMu6CBm9Tb7t/tlnet/archive
# The canonical source moves quickly and will be broken almost immediately
# http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
# Should be stable for historic, archived releases
# http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2018/tlnet-final/archive
# TODO: use this later when 2018 is archived
];
src = fetchurl { inherit urls sha512; };
@ -146,11 +138,30 @@ let
-C "$out" --anchored --exclude=tlpkg --keep-old-files
'' + postUnpack;
in runCommand "texlive-${tlName}" {
# lots of derivations, not meant to be cached
preferLocalBuild = true; allowSubstitutes = false;
inherit passthru;
}
in if sha512 == "" then
# hash stripped from pkgs.nix to save space -> fetch&unpack in a single step
fetchurl {
inherit urls;
sha1 = if fixedHash == null then throw "TeX Live package ${tlName} is missing hash!"
else fixedHash;
name = tlName;
recursiveHash = true;
downloadToTemp = true;
postFetch = ''mkdir "$out";'' + unpackCmd "$downloadedFile";
# TODO: perhaps override preferHashedMirrors and allowSubstitutes
}
// passthru
else runCommand "texlive-${tlName}"
( { # lots of derivations, not meant to be cached
preferLocalBuild = true; allowSubstitutes = false;
inherit passthru;
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha1";
outputHashMode = "recursive";
}
)
( ''
mkdir "$out"
'' + unpackCmd "'${src}'"

View file

@ -0,0 +1,10 @@
#!/bin/sh
echo "{"
grep -v -F '.bin-' | while read path; do
hash=`nix-hash --type sha1 --base32 "$path"`
echo -n "$path" | sed -E 's/[^-]*-texlive-(.*)/"\1"/'
echo "=\"$hash\";"
done
echo "}"

File diff suppressed because it is too large Load diff

View file

@ -1,30 +0,0 @@
https://www.tug.org/pipermail/tex-live/2017-June/040192.html
---
texk/web2c/luatexdir/luaffi/ctype.c | 4 ++++
texk/web2c/luatexdir/luaffi/ffi.h | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
--- a/texk/web2c/luatexdir/luaffi/ctype.c
+++ b/texk/web2c/luatexdir/luaffi/ctype.c 2017-05-31 13:08:25.421741873 +0000
@@ -245,6 +245,10 @@ void* to_cdata(lua_State* L, int idx, st
lua_pop(L, 1); /* mt */
cd = (struct cdata*) lua_touserdata(L, idx);
+ if (!cd) {
+ lua_pushnil(L);
+ return NULL;
+ }
*ct = cd->type;
lua_getuservalue(L, idx);
--- a/texk/web2c/luatexdir/luaffi/ffi.h
+++ b/texk/web2c/luatexdir/luaffi/ffi.h 2017-06-01 09:12:45.128442092 +0000
@@ -370,7 +370,7 @@ __declspec(align(16))
#endif
struct cdata {
const struct ctype type
-#ifdef __GNUC__
+#if 0 /* def __GNUC__ */
__attribute__ ((aligned(16)))
#endif
;

File diff suppressed because it is too large Load diff

View file

@ -11514,10 +11514,8 @@ with pkgs;
introspectionSupport = true;
});
poppler_min = poppler_0_61.override { # TODO: maybe reduce even more
poppler_min = poppler.override { # TODO: maybe reduce even more
# this is currently only used by texlive.bin.
# pinned to 0.61 because texlive 2017 doesn't build w/poppler 0.64
# TODO: review poppler version for texlive 2018
minimal = true;
suffix = "min";
};