texlive: 2017 -> 2017-final

This commit rebuilds texlive 2017 with the final release of 2017. As described
in these issues [1][2][3], the upstream CTAN mirrors are a continuously moving
rolling release without historical archives.

This particular FTP server is also a rolling release folling CTAN for the latest
version, but it has snapshots of the final texlive releases; it appears that the
2017 distribution has been unmodified since texlive-2018 was released earlier
this year.

Along the way, we needed to fix several issues:
- xindy: if $HOME is unset, it will try to mkdir /homeless-shelter, which fails
  due to insufficient permissions.
- scheme-infraonly: this scheme had symlinks into other releases that were
  read-only, so it couldn't patch and modify the scripts. This commit removes it
  for now, but that's not a particularly satisfying solution. Ideas?

This also adds some documentation on the upgrade process to prepare for
texlive-2018 [4].

This commit also replaces the sha1 hashes with upstream's standard sha512 hashes.
It appears the motivation for the shorter hashes was to save disk space in the
derivations; in master, the size of this directory is 1012K; in this commit it
is 1600K. The difference is not particularly large, and the downsides to using
our own sha1 hashes are:

- More nix code to maintain
- Multi-step upgrade process for maintainers: the maintainer first has to
  download all upstream tarballs by sha512 hash, then run the fix script, then
  rebuild with sha1 hashes.
- Less transparent. If we use the upstream sha512 hashes, any user can
  immediately verify that the hashes we're providing match upstream, or match
  the snapshot in time.
- Easier to debug. Since upstream is rolling and packages may disappear or fail
  to build, it's useful to be able to determine if the sha mismatch is because
  of an update or not; if we have a sha1 mismatch and no tarball to pull, we
  can't figure out which sha512sum would have produced that sha1.
- Less trust required. Due to the above, users don't have to trust the
  content-addressed mirrors on IPFS and @veprbl's servers as much.
- Easier to cobble together a source distribution from a variety of sources. It
  seems some FTP servers have more/less than others, or older/newer packages. If
  we know what we're looking for beforehand and we're just missing a few
  packages whose hashes match the advertised hashes upstream, it's easier to find.

[1] https://github.com/NixOS/nixpkgs/issues/24683
[2] https://github.com/NixOS/nixpkgs/issues/10026
[3] https://github.com/NixOS/nixpkgs/issues/34490
[4] https://github.com/NixOS/nixpkgs/issues/40232

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -312,6 +312,10 @@ xindy = stdenv.mkDerivation {
   name = "texlive-xindy.bin-${version}";
 
   inherit (common) src;
+
+  # If unset, xindy will try to mkdir /homeless-shelter
+  HOME = "/tmp/homeless-shelter";
+
   prePatch = "cd utils/xindy";
   # hardcode clisp location
   postPatch = ''
@@ -338,5 +342,3 @@ xindy = stdenv.mkDerivation {
 };
 
 }
-
-

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -243,4 +243,3 @@ in buildEnv {
 }
 # TODO: make TeX fonts visible by fontconfig: it should be enough to install an appropriate file
 #       similarly, deal with xe(la)tex font visibility?
-

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -1,11 +1,34 @@
 /* TeX Live user docs
   - source: ../../../../../doc/languages-frameworks/texlive.xml
   - current html: http://nixos.org/nixpkgs/manual/#sec-language-texlive
+
+  Note on upgrading: The texlive package contains a few binaries, defined in
+  bin.nix and released once a year, and several thousand packages from CTAN,
+  defined in pkgs.nix.
+
+  The CTAN mirrors are continuously moving, with more than 100 updates per
+  month. Due to the size of the distribution, we snapshot it and generate nix
+  expressions for all packages in texlive at that point.
+
+  To upgrade this snapshot, run the following:
+  $ curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
+             | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix
+
+  This will regenerate all of the sha512 hashes for the current upstream
+  distribution. You may want to find a more stable mirror, put the distribution
+  on IPFS, or contact a maintainer to get the tarballs from that point in time
+  into a more stable location, so that nix users who are building from source
+  can reproduce your work.
+
+  Upgrading the bin: texlive itself is a large collection of binaries. In order
+  to reduce closure size for users who just need a few of them, we split it into
+  packages such as core, core-big, xvdi, etc. This requires making assumptions
+  about dependencies between the projects that may change between releases; if
+  you upgrade you may have to do some work here.
 */
 { stdenv, lib, fetchurl, runCommand, writeText, buildEnv
 , callPackage, ghostscriptX, harfbuzz, poppler_min
 , makeWrapper, python, ruby, perl
-, useFixedHashes ? true
 , recurseIntoAttrs
 }:
 let
@@ -18,13 +41,6 @@ let
     };
   };
 
-  # map: name -> fixed-output hash
-  # sha1 in base32 was chosen as a compromise between security and length
-  # warning: the following generator command takes lots of resources
-  # nix-build ../../../../.. -Q -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes-new.nix
-  # mv ./fixedHashes{-new,}.nix
-  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
-
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv fastUnique lib makeWrapper writeText
@@ -34,9 +50,6 @@ let
 
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    /* # beware: the URL below changes contents continuously
-      curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
-        | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix */
     orig = import ./pkgs.nix tl;
     removeSelfDep = lib.mapAttrs
       (n: p: if p ? deps then p // { deps = lib.filterAttrs (dn: _: n != dn) p.deps; }
@@ -103,7 +116,6 @@ let
       # the basename used by upstream (without ".tar.xz" suffix)
       urlName = pname + lib.optionalString (tlType != "run") ".${tlType}";
       tlName = urlName + "-${version}";
-      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
               map (up: "${up}/${urlName}.tar.xz") urlPrefixes
@@ -113,9 +125,14 @@ let
       # Common packages should get served from the binary cache anyway.
       # See discussions, e.g. https://github.com/NixOS/nixpkgs/issues/24683
       urlPrefixes = args.urlPrefixes or [
-        http://146.185.144.154/texlive-2017
-        # IPFS GW is second, as it doesn't have a good time-outing behavior
-        http://gateway.ipfs.io/ipfs/QmRLK45EC828vGXv5YDaBsJBj2LjMjjA2ReLVrXsasRzy7/texlive-2017
+        # Should be stable for historic, archived releases
+        http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2017/tlnet-final/archive
+
+        # TODO: Add IPFS and see if @veprbl is willing to add a texlive-2017-final mirror,
+        # or if we should just dump it and go to 2018.
+
+        # The canonical source moves quickly and will be broken almost immediately
+        # http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/archive
       ];
 
       src = fetchurl { inherit urls sha512; };
@@ -129,30 +146,11 @@ let
           -C "$out" --anchored --exclude=tlpkg --keep-old-files
       '' + postUnpack;
 
-    in if sha512 == "" then
-      # hash stripped from pkgs.nix to save space -> fetch&unpack in a single step
-      fetchurl {
-        inherit urls;
-        sha1 = if fixedHash == null then throw "TeX Live package ${tlName} is missing hash!"
-          else fixedHash;
-        name = tlName;
-        recursiveHash = true;
-        downloadToTemp = true;
-        postFetch = ''mkdir "$out";'' + unpackCmd "$downloadedFile";
-        # TODO: perhaps override preferHashedMirrors and allowSubstitutes
+    in runCommand "texlive-${tlName}" {
+        # lots of derivations, not meant to be cached
+        preferLocalBuild = true; allowSubstitutes = false;
+        inherit passthru;
       }
-        // passthru
-
-    else runCommand "texlive-${tlName}"
-      ( { # lots of derivations, not meant to be cached
-          preferLocalBuild = true; allowSubstitutes = false;
-          inherit passthru;
-        } // lib.optionalAttrs (fixedHash != null) {
-          outputHash = fixedHash;
-          outputHashAlgo = "sha1";
-          outputHashMode = "recursive";
-        }
-      )
       ( ''
           mkdir "$out"
         '' + unpackCmd "'${src}'"
@@ -190,9 +188,8 @@ in
           })
         )
         { inherit (tl)
-            scheme-basic scheme-context scheme-full scheme-gust scheme-infraonly
+            scheme-basic scheme-context scheme-full scheme-gust
             scheme-medium scheme-minimal scheme-small scheme-tetex;
         }
     );
   }
-

pkgs/tools/typesetting/tex/texlive/fixHashes.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-echo "{"
-grep -v -F '.bin-' | while read path; do
-    hash=`nix-hash --type sha1 --base32 "$path"`
-    echo -n "$path" | sed -E 's/[^-]*-texlive-(.*)/"\1"/'
-    echo "=\"$hash\";"
-done
-echo "}"
-