Merge pull request #50977 from jtojnar/parallel-update.nix

update.nix: Run update scripts in parallel

doc/stdenv.xml

@@ -671,6 +671,43 @@ passthru = {
      </para>
     </listitem>
    </varlistentry>
+   <varlistentry>
+    <term>
+     <varname>passthru.updateScript</varname>
+    </term>
+    <listitem>
+     <para>
+      A script to be run by <filename>maintainers/scripts/update.nix</filename> when
+      the package is matched. It needs to be an executable file, either on the file
+      system:
+<programlisting>
+passthru.updateScript = ./update.sh;
+</programlisting>
+      or inside the expression itself:
+<programlisting>
+passthru.updateScript = writeScript "update-zoom-us" ''
+  #!/usr/bin/env nix-shell
+  #!nix-shell -i bash -p curl pcre common-updater-scripts
+
+  set -eu -o pipefail
+
+  version="$(curl -sI https://zoom.us/client/latest/zoom_x86_64.tar.xz | grep -Fi 'Location:' | pcregrep -o1 '/(([0-9]\.?)+)/')"
+  update-source-version zoom-us "$version"
+'';
+</programlisting>
+      The attribute can also contain a list, a script followed by arguments to be passed to it:
+<programlisting>
+passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
+</programlisting>
+      Note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
+     </para>
+
+     <para>
+      For information about how to run the updates, execute
+      <cmdsynopsis><command>nix-shell</command> <arg>maintainers/scripts/update.nix</arg></cmdsynopsis>.
+     </para>
+    </listitem>
+   </varlistentry>
   </variablelist>
  </section>
  <section xml:id="sec-stdenv-phases">

maintainers/scripts/update.nix

@@ -1,6 +1,8 @@
 { package ? null
 , maintainer ? null
 , path ? null
+, max-workers ? null
+, keep-going ? null
 }:
 
 # TODO: add assert statements
@@ -105,26 +107,23 @@ let
         % nix-shell maintainers/scripts/update.nix --argstr path gnome3
 
     to run update script for all package under an attribute path.
+
+    You can also add
+
+        --argstr max-workers 8
+
+    to increase the number of jobs in parallel, or
+
+        --argstr keep-going true
+
+    to continue running when a single update fails.
   '';
 
-  runUpdateScript = package: ''
+  packageData = package: {
-    echo -ne " - ${package.name}: UPDATING ..."\\r
+    name = package.name;
-    ${package.updateScript} &> ${(builtins.parseDrvName package.name).name}.log
+    pname = (builtins.parseDrvName package.name).name;
-    CODE=$?
+    updateScript = pkgs.lib.toList package.updateScript;
-    if [ "$CODE" != "0" ]; then
+  };
-      echo " - ${package.name}: ERROR       "
-      echo ""
-      echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
-      echo ""
-      cat ${(builtins.parseDrvName package.name).name}.log
-      echo ""
-      echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
-      exit $CODE
-    else
-      rm ${(builtins.parseDrvName package.name).name}.log
-    fi
-    echo " - ${package.name}: DONE.       "
-  '';
 
 in pkgs.stdenv.mkDerivation {
   name = "nixpkgs-update-script";
@@ -139,21 +138,7 @@ in pkgs.stdenv.mkDerivation {
     exit 1
   '';
   shellHook = ''
-    echo ""
+    unset shellHook # do not contaminate nested shells
-    echo "Going to be running update for following packages:"
+    exec ${pkgs.python3.interpreter} ${./update.py} ${pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages))}${pkgs.lib.optionalString (max-workers != null) " --max-workers=${max-workers}"}${pkgs.lib.optionalString (keep-going == "true") " --keep-going"}
-    echo "${builtins.concatStringsSep "\n" (map (x: " - ${x.name}") packages)}"
-    echo ""
-    read -n1 -r -p "Press space to continue..." confirm
-    if [ "$confirm" = "" ]; then
-      echo ""
-      echo "Running update for:"
-      ${builtins.concatStringsSep "\n" (map runUpdateScript packages)}
-      echo ""
-      echo "Packages updated!"
-      exit 0
-    else
-      echo "Aborting!"
-      exit 1
-    fi
   '';
 }

maintainers/scripts/update.py

@@ -0,0 +1,79 @@
+import argparse
+import concurrent.futures
+import json
+import os
+import subprocess
+import sys
+
+updates = {}
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def run_update_script(package):
+    eprint(f" - {package['name']}: UPDATING ...")
+
+    subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+
+
+def main(max_workers, keep_going, packages):
+    with open(sys.argv[1]) as f:
+        packages = json.load(f)
+
+    eprint()
+    eprint('Going to be running update for following packages:')
+    for package in packages:
+        eprint(f" - {package['name']}")
+    eprint()
+
+    confirm = input('Press Enter key to continue...')
+    if confirm == '':
+        eprint()
+        eprint('Running update for:')
+
+        with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+            for package in packages:
+                updates[executor.submit(run_update_script, package)] = package
+
+            for future in concurrent.futures.as_completed(updates):
+                package = updates[future]
+
+                try:
+                    future.result()
+                    eprint(f" - {package['name']}: DONE.")
+                except subprocess.CalledProcessError as e:
+                    eprint(f" - {package['name']}: ERROR")
+                    eprint()
+                    eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
+                    eprint()
+                    eprint(e.stdout.decode('utf-8'))
+                    with open(f"{package['pname']}.log", 'wb') as f:
+                        f.write(e.stdout)
+                    eprint()
+                    eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
+
+                    if not keep_going:
+                        sys.exit(1)
+
+        eprint()
+        eprint('Packages updated!')
+        sys.exit()
+    else:
+        eprint('Aborting!')
+        sys.exit(130)
+
+parser = argparse.ArgumentParser(description='Update packages')
+parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
+parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
+parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+
+    try:
+        main(args.max_workers, args.keep_going, args.packages)
+    except (KeyboardInterrupt, SystemExit) as e:
+        for update in updates:
+            update.cancel()
+
+        sys.exit(e.code if isinstance(e, SystemExit) else 130)

pkgs/applications/networking/browsers/firefox-bin/default.nix

@@ -191,7 +191,7 @@ stdenv.mkDerivation {
   # update with:
   # $ nix-shell maintainers/scripts/update.nix --argstr package firefox-bin-unwrapped
   passthru.updateScript = import ./update.nix {
-    inherit name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
+    inherit stdenv name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
     baseUrl =
       if channel == "devedition"
         then "http://archive.mozilla.org/pub/devedition/releases/"

pkgs/applications/networking/browsers/firefox-bin/update.nix

@@ -1,4 +1,5 @@
-{ name
+{ stdenv
+, name
 , channel
 , writeScript
 , xidel
@@ -17,6 +18,7 @@ let
     channel != "release";
 
 in writeScript "update-${name}" ''
+  #!${stdenv.shell}
   PATH=${coreutils}/bin:${gnused}/bin:${gnugrep}/bin:${xidel}/bin:${curl}/bin:${gnupg}/bin
   set -eux
   pushd ${basePath}

pkgs/applications/networking/browsers/firefox/update.nix

@@ -1,4 +1,5 @@
 { writeScript
+, stdenv
 , lib
 , xidel
 , common-updater-scripts
@@ -13,6 +14,7 @@
 }:
 
 writeScript "update-${attrPath}" ''
+  #!${stdenv.shell}
   PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep gnused xidel ]}
 
   url=${baseUrl}

pkgs/applications/office/mendeley/default.nix

@@ -131,7 +131,7 @@ stdenv.mkDerivation {
   dontStrip = true;
   dontPatchElf = true;
 
-  updateScript = import ./update.nix { inherit writeScript; };
+  updateScript = import ./update.nix { inherit stdenv writeScript; };
 
   meta = with stdenv.lib; {
     homepage = http://www.mendeley.com;

pkgs/applications/office/mendeley/update.nix

@@ -1,6 +1,7 @@
-{ writeScript }:
+{ stdenv, writeScript }:
 
 writeScript "update-mendeley" ''
+  #!${stdenv.shell}
   function follow() {
     local URL=$1
     while true; do

pkgs/desktops/deepin/update.nix

@@ -1,4 +1,4 @@
-{ lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
+{ stdenv, lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
 { name, ignored-versions ? "^2014\\.|^v[0-9]+" }:
 
 let
@@ -9,6 +9,7 @@ let
 in
 
 writeScript "update-${packageName}" ''
+  #!${stdenv.shell}
   set -o errexit
   set -x
 

pkgs/desktops/gnome-3/update.nix

@@ -1,11 +1,16 @@
-{ lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
+{ stdenv, lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
 { packageName, attrPath ? packageName, versionPolicy ? "odd-unstable" }:
 
 let
   python = python3.withPackages (p: [ p.requests ]);
-in writeScript "update-${packageName}" ''
+  updateScript = writeScript "gnome-update-script" ''
-  set -o errexit
+    #!${stdenv.shell}
-  PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
+    set -o errexit
-  latest_tag=$(python "${./find-latest-version.py}" "${packageName}" "${versionPolicy}" "stable")
+    package_name="$1"
-  update-source-version "${attrPath}" "$latest_tag"
+    attr_path="$2"
-''
+    version_policy="$3"
+    PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
+    latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable")
+    update-source-version "$attr_path" "$latest_tag"
+  '';
+in [ updateScript packageName attrPath versionPolicy ]

pkgs/development/libraries/safefile/default.nix

@@ -13,6 +13,7 @@ stdenv.mkDerivation rec {
 
   passthru = {
     updateScript = ''
+      #!${stdenv.shell}
       cd ${toString ./.}
       ${toString path}/pkgs/build-support/upstream-updater/update-walker.sh default.nix
     '';

pkgs/development/tools/build-managers/gup/default.nix

@@ -16,6 +16,7 @@ stdenv.mkDerivation rec {
     cp -r python/bin $out/bin
   '';
   passthru.updateScript = ''
+    #!${stdenv.shell}
     set -e
     echo
     cd ${toString ./.}

pkgs/development/web/nodejs/update.nix

@@ -1,4 +1,5 @@
-{ lib
+{ stdenv
+, lib
 , writeScript
 , coreutils
 , curl
@@ -11,6 +12,7 @@
 }:
 
 writeScript "update-nodejs" ''
+  #!${stdenv.shell}
   PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq gnupg nix ]}
 
   HOME=`mktemp -d`

pkgs/os-specific/linux/tp_smapi/default.nix

@@ -34,7 +34,7 @@ stdenv.mkDerivation rec {
   enableParallelBuilding = true;
 
   passthru.updateScript = import ./update.nix {
-    inherit lib writeScript coreutils gnugrep jq curl common-updater-scripts;
+    inherit stdenv lib writeScript coreutils gnugrep jq curl common-updater-scripts;
   };
 
   meta = {

pkgs/os-specific/linux/tp_smapi/update.nix

@@ -1,6 +1,7 @@
-{ lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
+{ stdenv, lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
 
 writeScript "update-tp_smapi" ''
+#!${stdenv.shell}
 PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq ]}
 
 tags=`curl -s https://api.github.com/repos/evgeni/tp_smapi/tags`

pkgs/tools/package-management/nix-pin/default.nix

@@ -26,6 +26,7 @@ let self = stdenv.mkDerivation rec {
         let impl = import "${self}/share/nix/api.nix" { inherit pkgs pinConfig; }; in
         { inherit (impl) augmentedPkgs pins callPackage; };
       updateScript = ''
+        #!${stdenv.shell}
         set -e
         echo
         cd ${toString ./.}

pkgs/tools/package-management/nix-update-source/default.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
+{ stdenv, lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
 python3Packages.buildPythonApplication rec {
   version = "0.6.3";
   name = "nix-update-source-${version}";
@@ -28,6 +28,7 @@ python3Packages.buildPythonApplication rec {
         overrideSrc = drv: lib.overrideDerivation drv (orig: { inherit src; });
       };
     updateScript = ''
+      #!${stdenv.shell}
       set -e
       echo
       cd ${toString ./.}