Merge pull request #40826 from bhipple/u/texlive-2017-final

texlive: 2017 -> 2017-final

pkgs/tools/typesetting/biber/default.nix

@@ -1,21 +1,24 @@
 { stdenv, fetchFromGitHub, perlPackages }:
 
+# builds but doesn't work with perl 5.24, see discussion in #40826
+# TODO: build with perl >=5.26 and try to enable tests
+
 perlPackages.buildPerlModule rec {
   name = "biber-${version}";
-  version = "2.7";
+  version = "2.10";
 
   src = fetchFromGitHub {
     owner = "plk";
     repo = "biber";
     rev = "v${version}";
-    sha256 = "04jmsh59g2s0b61rm25z0hwb6yliqyh5gjs4y74va93d2b9mrd17";
+    sha256 = "0f6bb1iprl92iamxqlr8fc99mxr9n3722frd1ak9pbzh3m6c2ny6";
   };
 
   buildInputs = with perlPackages; [
     autovivification BusinessISBN BusinessISMN BusinessISSN ConfigAutoConf
     DataCompare DataDump DateSimple EncodeEUCJPASCII EncodeHanExtra EncodeJIS2K
     DateTime DateTimeFormatBuilder DateTimeCalendarJulian
-    ExtUtilsLibBuilder FileSlurp FileWhich IPCRun3 LogLog4perl LWPProtocolHttps ListAllUtils
+    ExtUtilsLibBuilder FileSlurper FileWhich IPCRun3 LogLog4perl LWPProtocolHttps ListAllUtils
     ListMoreUtils MozillaCA ReadonlyXS RegexpCommon TextBibTeX
     UnicodeCollate UnicodeLineBreak URI XMLLibXMLSimple XMLLibXSLT XMLWriter
     ClassAccessor TextCSV TextCSV_XS TextRoman DataUniqid LinguaTranslit UnicodeNormalize SortKey

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -312,6 +312,10 @@ xindy = stdenv.mkDerivation {
   name = "texlive-xindy.bin-${version}";
 
   inherit (common) src;
+
+  # If unset, xindy will try to mkdir /homeless-shelter
+  HOME = ".";
+
   prePatch = "cd utils/xindy";
   # hardcode clisp location
   postPatch = ''
@@ -338,5 +342,3 @@ xindy = stdenv.mkDerivation {
 };
 
 }
-
-

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -243,4 +243,3 @@ in buildEnv {
 }
 # TODO: make TeX fonts visible by fontconfig: it should be enough to install an appropriate file
 #       similarly, deal with xe(la)tex font visibility?
-

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -1,11 +1,34 @@
 /* TeX Live user docs
   - source: ../../../../../doc/languages-frameworks/texlive.xml
   - current html: http://nixos.org/nixpkgs/manual/#sec-language-texlive
+
+  Note on upgrading: The texlive package contains a few binaries, defined in
+  bin.nix and released once a year, and several thousand packages from CTAN,
+  defined in pkgs.nix.
+
+  The CTAN mirrors are continuously moving, with more than 100 updates per
+  month. Due to the size of the distribution, we snapshot it and generate nix
+  expressions for all packages in texlive at that point.
+
+  To upgrade this snapshot, run the following:
+  $ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
+             | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
+
+  This will regenerate all of the sha512 hashes for the current upstream
+  distribution. You may want to find a more stable mirror, put the distribution
+  on IPFS, or contact a maintainer to get the tarballs from that point in time
+  into a more stable location, so that nix users who are building from source
+  can reproduce your work.
+
+  Upgrading the bin: texlive itself is a large collection of binaries. In order
+  to reduce closure size for users who just need a few of them, we split it into
+  packages such as core, core-big, xvdi, etc. This requires making assumptions
+  about dependencies between the projects that may change between releases; if
+  you upgrade you may have to do some work here.
 */
 { stdenv, lib, fetchurl, runCommand, writeText, buildEnv
 , callPackage, ghostscriptX, harfbuzz, poppler_min
 , makeWrapper, python, ruby, perl
-, useFixedHashes ? true
 , recurseIntoAttrs
 }:
 let
@@ -18,13 +41,6 @@ let
     };
   };
 
-  # map: name -> fixed-output hash
-  # sha1 in base32 was chosen as a compromise between security and length
-  # warning: the following generator command takes lots of resources
-  # nix-build ../../../../.. -Q -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes-new.nix
-  # mv ./fixedHashes{-new,}.nix
-  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
-
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv fastUnique lib makeWrapper writeText
@@ -34,9 +50,6 @@ let
 
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    /* # beware: the URL below changes contents continuously
-      curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
-        | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix */
     orig = import ./pkgs.nix tl;
     removeSelfDep = lib.mapAttrs
       (n: p: if p ? deps then p // { deps = lib.filterAttrs (dn: _: n != dn) p.deps; }
@@ -103,7 +116,6 @@ let
       # the basename used by upstream (without ".tar.xz" suffix)
       urlName = pname + lib.optionalString (tlType != "run") ".${tlType}";
       tlName = urlName + "-${version}";
-      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
               map (up: "${up}/${urlName}.tar.xz") urlPrefixes
@@ -113,9 +125,14 @@ let
       # Common packages should get served from the binary cache anyway.
       # See discussions, e.g. https://github.com/NixOS/nixpkgs/issues/24683
       urlPrefixes = args.urlPrefixes or [
-        http://146.185.144.154/texlive-2017
+        # Should be stable for historic, archived releases
-        # IPFS GW is second, as it doesn't have a good time-outing behavior
+        http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2017/tlnet-final/archive
-        http://gateway.ipfs.io/ipfs/QmRLK45EC828vGXv5YDaBsJBj2LjMjjA2ReLVrXsasRzy7/texlive-2017
+
+        # TODO: Add IPFS and see if @veprbl is willing to add a texlive-2017-final mirror,
+        # or if we should just dump it and go to 2018.
+
+        # The canonical source moves quickly and will be broken almost immediately
+        # http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
       ];
 
       src = fetchurl { inherit urls sha512; };
@@ -129,30 +146,11 @@ let
           -C "$out" --anchored --exclude=tlpkg --keep-old-files
       '' + postUnpack;
 
-    in if sha512 == "" then
+    in runCommand "texlive-${tlName}" {
-      # hash stripped from pkgs.nix to save space -> fetch&unpack in a single step
+        # lots of derivations, not meant to be cached
-      fetchurl {
+        preferLocalBuild = true; allowSubstitutes = false;
-        inherit urls;
+        inherit passthru;
-        sha1 = if fixedHash == null then throw "TeX Live package ${tlName} is missing hash!"
-          else fixedHash;
-        name = tlName;
-        recursiveHash = true;
-        downloadToTemp = true;
-        postFetch = ''mkdir "$out";'' + unpackCmd "$downloadedFile";
-        # TODO: perhaps override preferHashedMirrors and allowSubstitutes
       }
-        // passthru
-
-    else runCommand "texlive-${tlName}"
-      ( { # lots of derivations, not meant to be cached
-          preferLocalBuild = true; allowSubstitutes = false;
-          inherit passthru;
-        } // lib.optionalAttrs (fixedHash != null) {
-          outputHash = fixedHash;
-          outputHashAlgo = "sha1";
-          outputHashMode = "recursive";
-        }
-      )
       ( ''
           mkdir "$out"
         '' + unpackCmd "'${src}'"
@@ -190,9 +188,8 @@ in
           })
         )
         { inherit (tl)
-            scheme-basic scheme-context scheme-full scheme-gust scheme-infraonly
+            scheme-basic scheme-context scheme-full scheme-gust
             scheme-medium scheme-minimal scheme-small scheme-tetex;
         }
     );
   }
-

pkgs/tools/typesetting/tex/texlive/fixHashes.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-echo "{"
-grep -v -F '.bin-' | while read path; do
-    hash=`nix-hash --type sha1 --base32 "$path"`
-    echo -n "$path" | sed -E 's/[^-]*-texlive-(.*)/"\1"/'
-    echo "=\"$hash\";"
-done
-echo "}"
-