maintainers/scripts/update.nix: Add support for auto-commiting changes

Update scripts can now declare features using

	passthru.updateScript = {
	  command = [ ../../update.sh pname ];
	  supportedFeatures = [ "commit" ];
	};

A `commit` feature means that when the update script finishes successfully,
it will print a JSON list like the following:

	[
	  {
	    "attrPath": "volume_key",
	    "oldVersion": "0.3.11",
	    "newVersion": "0.3.12",
	    "files": [
	      "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
	    ]
	  }
	]

and data from that will be used when update.nix is run with --argstr commit true
to create commits.

We will create a new git worktree for each thread in the pool and run the update
script there. Then we will commit the change and cherry pick it in the main repo,
releasing the worktree for a next change.

maintainers/scripts/update.nix

@@ -4,6 +4,7 @@
 , max-workers ? null
 , include-overlays ? false
 , keep-going ? null
+, commit ? null
 }:
 
 # TODO: add assert statements
@@ -132,19 +133,26 @@ let
         --argstr keep-going true
 
     to continue running when a single update fails.
+
+    You can also make the updater automatically commit on your behalf from updateScripts
+    that support it by adding
+
+        --argstr commit true
   '';
 
   packageData = package: {
     name = package.name;
     pname = lib.getName package;
-    updateScript = map builtins.toString (lib.toList package.updateScript);
+    updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript));
+    supportedFeatures = package.updateScript.supportedFeatures or [];
   };
 
   packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages));
 
   optionalArgs =
     lib.optional (max-workers != null) "--max-workers=${max-workers}"
-    ++ lib.optional (keep-going == "true") "--keep-going";
+    ++ lib.optional (keep-going == "true") "--keep-going"
+    ++ lib.optional (commit == "true") "--commit";
 
   args = [ packagesJson ] ++ optionalArgs;
 

maintainers/scripts/update.py

@@ -1,22 +1,45 @@
 import argparse
+import contextlib
 import concurrent.futures
 import json
 import os
 import subprocess
 import sys
+import tempfile
+import threading
 
 updates = {}
 
+thread_name_prefix='UpdateScriptThread'
+
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
-def run_update_script(package):
+def run_update_script(package, commit):
+    if commit and 'commit' in package['supportedFeatures']:
+        thread_name = threading.current_thread().name
+        worktree, _branch, lock = temp_dirs[thread_name]
+        lock.acquire()
+        package['thread'] = thread_name
+    else:
+        worktree = None
+
     eprint(f" - {package['name']}: UPDATING ...")
 
-    subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree)
 
+@contextlib.contextmanager
+def make_worktree():
+    with tempfile.TemporaryDirectory() as wt:
+        branch_name = f'update-{os.path.basename(wt)}'
+        target_directory = f'{wt}/nixpkgs'
 
-def main(max_workers, keep_going, packages):
+        subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True)
+        yield (target_directory, branch_name)
+        subprocess.run(['git', 'worktree', 'remove', target_directory], check=True)
+        subprocess.run(['git', 'branch', '-D', branch_name], check=True)
+
+def main(max_workers, keep_going, commit, packages):
     with open(sys.argv[1]) as f:
         packages = json.load(f)
 
@@ -31,15 +54,29 @@ def main(max_workers, keep_going, packages):
         eprint()
         eprint('Running update for:')
 
-        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor:
+            global temp_dirs
+
+            if commit:
+                temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)}
+
             for package in packages:
-                updates[executor.submit(run_update_script, package)] = package
+                updates[executor.submit(run_update_script, package, commit)] = package
 
             for future in concurrent.futures.as_completed(updates):
                 package = updates[future]
 
                 try:
-                    future.result()
+                    p = future.result()
+                    if commit and 'commit' in package['supportedFeatures']:
+                        thread_name = package['thread']
+                        worktree, branch, lock = temp_dirs[thread_name]
+                        changes = json.loads(p.stdout)
+                        for change in changes:
+                            subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree)
+                            commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
+                            subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree)
+                            subprocess.run(['git', 'cherry-pick', branch], check=True)
                     eprint(f" - {package['name']}: DONE.")
                 except subprocess.CalledProcessError as e:
                     eprint(f" - {package['name']}: ERROR")
@@ -54,6 +91,9 @@ def main(max_workers, keep_going, packages):
 
                     if not keep_going:
                         sys.exit(1)
+                finally:
+                    if commit and 'commit' in package['supportedFeatures']:
+                        lock.release()
 
         eprint()
         eprint('Packages updated!')
@@ -65,13 +105,14 @@ def main(max_workers, keep_going, packages):
 parser = argparse.ArgumentParser(description='Update packages')
 parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
 parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
+parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
 parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
 
 if __name__ == '__main__':
     args = parser.parse_args()
 
     try:
-        main(args.max_workers, args.keep_going, args.packages)
+        main(args.max_workers, args.keep_going, args.commit, args.packages)
     except (KeyboardInterrupt, SystemExit) as e:
         for update in updates:
             update.cancel()