Merge pull request #45432 from xeji/texlive-2018

[staging]: texlive: 2017 -> 2018

pkgs/tools/typesetting/biber/default.nix

@@ -5,13 +5,13 @@
 
 perlPackages.buildPerlModule rec {
   name = "biber-${version}";
-  version = "2.10";
+  version = "2.11";
 
   src = fetchFromGitHub {
     owner = "plk";
     repo = "biber";
     rev = "v${version}";
-    sha256 = "0f6bb1iprl92iamxqlr8fc99mxr9n3722frd1ak9pbzh3m6c2ny6";
+    sha256 = "0qgkc1k9n36yfmndwz879pak6mjphld0p85lzn9g2ng0vhxsifzz";
   };
 
   buildInputs = with perlPackages; [

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -0,0 +1,68 @@
+# Notes on maintaining/upgrading
+
+## Upgrading texlive.bin
+
+texlive contains a few binaries, defined in bin.nix and released once a year.
+
+In order to reduce closure size for users who just need a few of them, we split it into
+packages such as core, core-big, xvdi, etc. This requires making assumptions
+about dependencies between the projects that may change between releases; if
+you upgrade you may have to do some work here.
+
+
+## Updating the package set
+
+texlive contains several thousand packages from CTAN, defined in pkgs.nix.
+
+The CTAN mirrors are not version-controlled and continuously moving,
+with more than 100 updates per month.
+
+To create a consistent and reproducible package set in nixpkgs, we snapshot CTAN
+and generate nix expressions for all packages in texlive at that point.
+
+We mirror CTAN sources of this snapshot on community-operated servers and on IPFS.
+
+To upgrade the package snapshot, follow this process:
+
+
+### Snapshot sources and texlive package database
+
+Mirror the current CTAN archive to our mirror(s) and IPFS (URLs in `default.nix`).
+See <https://tug.org/texlive/acquire-mirror.html> for instructions.
+
+
+### Upgrade package information from texlive package database
+
+
+```
+$ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
+           | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
+```
+
+This will download a current snapshot of the CTAN package database `texlive.tlpdb.xz`
+and regenerate all of the sha512 hashes for the current upstream distribution in `pkgs.nix`.
+
+
+### Build packages locally and generate fix hashes
+
+To save disk space and prevent unnecessary rebuilds, texlive packages are built
+as fixed-output derivations whose hashes are contained in `fixedHashes.nix`.
+
+Updating the list of fixed hashes requires a local build of *all* packages,
+which is a resource-intensive process:
+
+
+```
+# move fixedHashes away, otherwise build will fail on updated packages
+$ mv fixedHashes.nix fixedHashes-old.nix
+# start with empty fixedHashes
+$ echo '{}' > fixedHashes.nix
+$ nix-build ../../../../.. -Q --no-out-link -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes-new.nix
+# The script wrongly includes the nix store path to `biber`, which is a separate nixpkgs package
+$ grep -v -F '/nix/store/' fixedHashes-new.nix > fixedHashes.nix 
+```
+
+### Commit changes
+
+Commit the updated `pkgs.nix` and `fixedHashes.nix` to the repository.
+

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -14,25 +14,23 @@
 let
   withSystemLibs = map (libname: "--with-system-${libname}");
 
-  year = "2017";
+  year = "2018";
   version = year; # keep names simple for now
 
   common = rec {
     src = fetchurl {
-      url = # "ftp://tug.org/historic/systems/texlive/${year}/"
+      urls = [
-      #"http://lipa.ms.mff.cuni.cz/~cunav5am/nix/texlive-2016"
+        "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${year}/texlive-${year}0414-source.tar.xz"
-      # FIXME: a proper mirror, though tarballs.nixos.org saves this case ATM
+              "ftp://tug.ctan.org/pub/tex/historic/systems/texlive/${year}/texlive-${year}0414-source.tar.xz"
-      # http://146.185.144.154/texlive-2016
+      ];
-      # + "/texlive-${year}0523b-source.tar.xz";
+      sha256 = "0khyi6h015r2zfqgg0a44a2j7vmr1cy42knw7jbss237yvakc07y";
-        "http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${year}/texlive-${year}0524-source.tar.xz";
-      sha256 = "1amjrxyasplv4alfwcxwnw4nrx7dz2ydmddkq16k6hg90i9njq81";
     };
 
     patches = [
       (fetchurl {
-        name = "texlive-poppler-0.59.patch";
+        name = "texlive-poppler-0.64.patch";
-        url = https://git.archlinux.org/svntogit/packages.git/plain/trunk/texlive-poppler-0.59.patch?h=packages/texlive-bin&id=6308ec39bce2a4d735f6ff8a4e94473748d7b450;
+        url = https://git.archlinux.org/svntogit/packages.git/plain/trunk/texlive-poppler-0.64.patch?h=packages/texlive-bin;
-        sha256 = "1c4ikq4kxw48bi3i33bzpabrjvbk01fwjr2lz20gkc9kv8l0bg3n";
+        sha256 = "0443d074zl3c5raba8jyhavish706arjcd80ibb84zwnwck4ai0w";
       })
     ];
 
@@ -83,6 +81,8 @@ core = stdenv.mkDerivation rec {
     for i in texk/kpathsea/mktex*; do
       sed -i '/^mydir=/d' "$i"
     done
+    cp -pv texk/web2c/pdftexdir/pdftoepdf{-newpoppler.cc,.cc}
+    cp -pv texk/web2c/pdftexdir/pdftosrc{-newpoppler.cc,.cc}
   '';
 
   preConfigure = ''
@@ -165,7 +165,7 @@ inherit (core-big) metafont metapost luatex xetex;
 core-big = stdenv.mkDerivation { #TODO: upmendex
   name = "texlive-core-big.bin-${version}";
 
-  inherit (common) src;
+  inherit (common) src patches;
 
   hardeningDisable = [ "format" ];
 
@@ -176,20 +176,23 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
     ++ withSystemLibs [ "kpathsea" "ptexenc" "cairo" "harfbuzz" "icu" "graphite2" ]
     ++ map (prog: "--disable-${prog}") # don't build things we already have
       [ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "pdftex"
-        "web-progs" "synctex" "luajittex" "mfluajit" # luajittex is mostly not needed, see:
+        "web-progs" "synctex"
+        # build fails on Darwin with luatex53
+        "luatex53" # TODO probably can be removed when TexLive 2019 is out
+        # luajittex is mostly not needed, see:
         # http://tex.stackexchange.com/questions/97999/when-to-use-luajittex-in-favour-of-luatex
+        "luajittex" "mfluajit"
       ];
 
-  patches = common.patches ++ [ ./luatex-gcc7.patch ];
-
   configureScript = ":";
 
   # we use static libtexlua, because it's only used by a single binary
   postConfigure = ''
     mkdir ./WorkDir && cd ./WorkDir
+    # TODO add lua53 here when luatex53 is enabled again
     for path in libs/{teckit,lua52} texk/web2c; do
       (
-        if [[ "$path" == "libs/lua52" ]]; then
+        if [[ "$path" =~ "libs/lua5" ]]; then
           extraConfig="--enable-static --disable-shared"
         else
           extraConfig=""
@@ -202,7 +205,6 @@ core-big = stdenv.mkDerivation { #TODO: upmendex
   '';
 
   preBuild = "cd texk/web2c";
-  CXXFLAGS = "-std=c++11 -Wno-reserved-user-defined-literal"; # TODO: remove once texlive 2018 is out?
   enableParallelBuilding = true;
 
   doCheck = false; # fails

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -1,34 +1,11 @@
 /* TeX Live user docs
   - source: ../../../../../doc/languages-frameworks/texlive.xml
   - current html: http://nixos.org/nixpkgs/manual/#sec-language-texlive
-
-  Note on upgrading: The texlive package contains a few binaries, defined in
-  bin.nix and released once a year, and several thousand packages from CTAN,
-  defined in pkgs.nix.
-
-  The CTAN mirrors are continuously moving, with more than 100 updates per
-  month. Due to the size of the distribution, we snapshot it and generate nix
-  expressions for all packages in texlive at that point.
-
-  To upgrade this snapshot, run the following:
-  $ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
-             | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
-
-  This will regenerate all of the sha512 hashes for the current upstream
-  distribution. You may want to find a more stable mirror, put the distribution
-  on IPFS, or contact a maintainer to get the tarballs from that point in time
-  into a more stable location, so that nix users who are building from source
-  can reproduce your work.
-
-  Upgrading the bin: texlive itself is a large collection of binaries. In order
-  to reduce closure size for users who just need a few of them, we split it into
-  packages such as core, core-big, xvdi, etc. This requires making assumptions
-  about dependencies between the projects that may change between releases; if
-  you upgrade you may have to do some work here.
 */
 { stdenv, lib, fetchurl, runCommand, writeText, buildEnv
 , callPackage, ghostscriptX, harfbuzz, poppler_min
 , makeWrapper, python, ruby, perl
+, useFixedHashes ? true
 , recurseIntoAttrs
 }:
 let
@@ -41,6 +18,10 @@ let
     };
   };
 
+  # map: name -> fixed-output hash
+  # sha1 in base32 was chosen as a compromise between security and length
+  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
+
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv fastUnique lib makeWrapper writeText
@@ -116,23 +97,34 @@ let
       # the basename used by upstream (without ".tar.xz" suffix)
       urlName = pname + lib.optionalString (tlType != "run") ".${tlType}";
       tlName = urlName + "-${version}";
+      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
               map (up: "${up}/${urlName}.tar.xz") urlPrefixes
             );
 
-      # Upstream refuses to distribute stable tarballs, so we host snapshots on IPFS.
+      # Upstream refuses to distribute stable tarballs,
+      # so we host snapshots on IPFS or on our own servers.
       # Common packages should get served from the binary cache anyway.
       # See discussions, e.g. https://github.com/NixOS/nixpkgs/issues/24683
       urlPrefixes = args.urlPrefixes or [
-        # Should be stable for historic, archived releases
+        # A snapshot temporarily hosted by @xeji.
-        http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2017/tlnet-final/archive
+        # TODO: remove when there is a reliable long-term solution
+        https://cat3.de/texlive-2018/tlnet/archive
 
-        # TODO: Add IPFS and see if @veprbl is willing to add a texlive-2017-final mirror,
+        # TODO: Add second, faster and more reliable snapshot mirror,
-        # or if we should just dump it and go to 2018.
+        # maybe on one of our project's servers
+
+        # IPFS seeded by the mirror above - this may be quite slow
+        https://ipfs.io/ipfs/QmT4Z67wXin1Z9DhvqwSSkSZSuu8hT6LgDyMu6CBm9Tb7t/tlnet/archive
 
         # The canonical source moves quickly and will be broken almost immediately
-        # http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
+        http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
+
+        # Should be stable for historic, archived releases
+        # http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2018/tlnet-final/archive
+        # TODO: use this later when 2018 is archived
+
       ];
 
       src = fetchurl { inherit urls sha512; };
@@ -146,11 +138,30 @@ let
           -C "$out" --anchored --exclude=tlpkg --keep-old-files
       '' + postUnpack;
 
-    in runCommand "texlive-${tlName}" {
+    in if sha512 == "" then
-        # lots of derivations, not meant to be cached
+      # hash stripped from pkgs.nix to save space -> fetch&unpack in a single step
-        preferLocalBuild = true; allowSubstitutes = false;
+      fetchurl {
-        inherit passthru;
+        inherit urls;
-      }
+        sha1 = if fixedHash == null then throw "TeX Live package ${tlName} is missing hash!"
+          else fixedHash;
+        name = tlName;
+        recursiveHash = true;
+        downloadToTemp = true;
+        postFetch = ''mkdir "$out";'' + unpackCmd "$downloadedFile";
+        # TODO: perhaps override preferHashedMirrors and allowSubstitutes
+     }
+        // passthru
+
+    else runCommand "texlive-${tlName}"
+      ( { # lots of derivations, not meant to be cached
+          preferLocalBuild = true; allowSubstitutes = false;
+          inherit passthru;
+        } // lib.optionalAttrs (fixedHash != null) {
+          outputHash = fixedHash;
+          outputHashAlgo = "sha1";
+          outputHashMode = "recursive";
+        }
+      )
       ( ''
           mkdir "$out"
         '' + unpackCmd "'${src}'"

pkgs/tools/typesetting/tex/texlive/fixHashes.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo "{"
+grep -v -F '.bin-' | while read path; do
+    hash=`nix-hash --type sha1 --base32 "$path"`
+    echo -n "$path" | sed -E 's/[^-]*-texlive-(.*)/"\1"/'
+    echo "=\"$hash\";"
+done
+echo "}"
+

pkgs/tools/typesetting/tex/texlive/luatex-gcc7.patch

@@ -1,30 +0,0 @@
-https://www.tug.org/pipermail/tex-live/2017-June/040192.html
----
- texk/web2c/luatexdir/luaffi/ctype.c |    4 ++++
- texk/web2c/luatexdir/luaffi/ffi.h   |    2 +-
- 2 files changed, 5 insertions(+), 1 deletion(-)
-
---- a/texk/web2c/luatexdir/luaffi/ctype.c
-+++ b/texk/web2c/luatexdir/luaffi/ctype.c       2017-05-31 13:08:25.421741873 +0000
-@@ -245,6 +245,10 @@ void* to_cdata(lua_State* L, int idx, st
- 
-     lua_pop(L, 1); /* mt */
-     cd = (struct cdata*) lua_touserdata(L, idx);
-+    if (!cd) {
-+        lua_pushnil(L);
-+        return NULL;
-+    }
-     *ct = cd->type;
-     lua_getuservalue(L, idx);
- 
---- a/texk/web2c/luatexdir/luaffi/ffi.h
-+++ b/texk/web2c/luatexdir/luaffi/ffi.h 2017-06-01 09:12:45.128442092 +0000
-@@ -370,7 +370,7 @@ __declspec(align(16))
- #endif
- struct cdata {
-     const struct ctype type
--#ifdef __GNUC__
-+#if 0 /* def __GNUC__ */
-       __attribute__ ((aligned(16)))
- #endif
-       ;

pkgs/top-level/all-packages.nix

@@ -11514,10 +11514,8 @@ with pkgs;
     introspectionSupport = true;
   });
 
-  poppler_min = poppler_0_61.override { # TODO: maybe reduce even more
+  poppler_min = poppler.override { # TODO: maybe reduce even more
     # this is currently only used by texlive.bin.
-    # pinned to 0.61 because texlive 2017 doesn't build w/poppler 0.64
-    # TODO: review poppler version for texlive 2018
     minimal = true;
     suffix = "min";
   };