vimPlugins: Automate redirect updates in update.py

Many of the plugins in vim-plugin-names are out of date and redirect to
their new github repos. This commit adds a flag to automatically update
these and defines a process for committing these updates into nixpkgs.

doc/languages-frameworks/vim.section.md

@@ -263,7 +263,7 @@ Sometimes plugins require an override that must be changed when the plugin is up
 
 To add a new plugin:
 
-  1. run `./update.py` and create a commit named "vimPlugins: Update",
+  1. run `./update.py --update-redirects` and create a commit named "vimPlugins: Update",
   2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix),
   3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and
   4. create a pull request.

pkgs/misc/vim-plugins/update.py

@@ -9,13 +9,16 @@
 # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
 
 import argparse
+import fileinput
 import functools
+import http
 import json
 import os
 import subprocess
 import sys
 import traceback
 import urllib.error
+import urllib.parse
 import urllib.request
 import xml.etree.ElementTree as ET
 from datetime import datetime
@@ -71,9 +74,11 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa
 
 
 class Repo:
-    def __init__(self, owner: str, name: str) -> None:
+    def __init__(self, owner: str, name: str, alias: str) -> None:
         self.owner = owner
         self.name = name
+        self.alias = alias
+        self.redirect: Dict[str, str] = {}
 
     def url(self, path: str) -> str:
         return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
@@ -96,7 +101,9 @@ class Repo:
 
     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
     def latest_commit(self) -> Tuple[str, datetime]:
-        with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req:
+        commit_url = self.url("commits/master.atom")
+        with urllib.request.urlopen(commit_url, timeout=10) as req:
+            self.check_for_redirect(commit_url, req)
             xml = req.read()
             root = ET.fromstring(xml)
             latest_entry = root.find(ATOM_ENTRY)
@@ -111,6 +118,22 @@ class Repo:
             updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
             return Path(str(url.path)).name, updated
 
+    def check_for_redirect(self, url: str, req: http.client.HTTPResponse):
+        response_url = req.geturl()
+        if url != response_url:
+            new_owner, new_name = (
+                urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
+            )
+            end_line = f" as {self.alias}\n" if self.alias else "\n"
+            plugin_line = "{owner}/{name}" + end_line
+
+            old_plugin = plugin_line.format(owner=self.owner, name=self.name)
+            new_plugin = plugin_line.format(owner=new_owner, name=new_name)
+            self.redirect[old_plugin] = new_plugin
+
+            if new_name != self.name:
+                print(f"Plugin Name Changed: {self.name} -> {new_name}")
+
     def prefetch_git(self, ref: str) -> str:
         data = subprocess.check_output(
             ["nix-prefetch-git", "--fetch-submodules", self.url(""), ref]
@@ -197,15 +220,17 @@ def get_current_plugins() -> List[Plugin]:
     return plugins
 
 
-def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Plugin:
+def prefetch_plugin(
-    repo = Repo(user, repo_name)
+    user: str, repo_name: str, alias: str, cache: "Cache"
+) -> Tuple[Plugin, Dict[str, str]]:
+    repo = Repo(user, repo_name, alias)
     commit, date = repo.latest_commit()
     has_submodules = repo.has_submodules()
     cached_plugin = cache[commit]
     if cached_plugin is not None:
         cached_plugin.name = alias or repo_name
         cached_plugin.date = date
-        return cached_plugin
+        return cached_plugin, repo.redirect
 
     print(f"prefetch {user}/{repo_name}")
     if has_submodules:
@@ -213,7 +238,10 @@ def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Pl
     else:
         sha256 = repo.prefetch_github(commit)
 
-    return Plugin(alias or repo_name, commit, has_submodules, sha256, date=date)
+    return (
+        Plugin(alias or repo_name, commit, has_submodules, sha256, date=date),
+        repo.redirect,
+    )
 
 
 def print_download_error(plugin: str, ex: Exception):
@@ -227,20 +255,22 @@ def print_download_error(plugin: str, ex: Exception):
 
 
 def check_results(
-    results: List[Tuple[str, str, Union[Exception, Plugin]]]
+    results: List[Tuple[str, str, Union[Exception, Plugin], Dict[str, str]]]
-) -> List[Tuple[str, str, Plugin]]:
+) -> Tuple[List[Tuple[str, str, Plugin]], Dict[str, str]]:
     failures: List[Tuple[str, Exception]] = []
     plugins = []
-    for (owner, name, result) in results:
+    redirects: Dict[str, str] = {}
+    for (owner, name, result, redirect) in results:
         if isinstance(result, Exception):
             failures.append((name, result))
         else:
             plugins.append((owner, name, result))
+            redirects.update(redirect)
 
     print(f"{len(results) - len(failures)} plugins were checked", end="")
     if len(failures) == 0:
         print()
-        return plugins
+        return plugins, redirects
     else:
         print(f", {len(failures)} plugin(s) could not be downloaded:\n")
 
@@ -328,15 +358,15 @@ class Cache:
 
 def prefetch(
     args: Tuple[str, str, str], cache: Cache
-) -> Tuple[str, str, Union[Exception, Plugin]]:
+) -> Tuple[str, str, Union[Exception, Plugin], dict]:
     assert len(args) == 3
     owner, repo, alias = args
     try:
-        plugin = prefetch_plugin(owner, repo, alias, cache)
+        plugin, redirect = prefetch_plugin(owner, repo, alias, cache)
         cache[plugin.commit] = plugin
-        return (owner, repo, plugin)
+        return (owner, repo, plugin, redirect)
     except Exception as e:
-        return (owner, repo, e)
+        return (owner, repo, e, {})
 
 
 header = (
@@ -386,6 +416,31 @@ in lib.fix' (lib.extends overrides packages)
     print(f"updated {outfile}")
 
 
+def update_redirects(input_file: Path, output_file: Path, redirects: dict):
+    with fileinput.input(input_file, inplace=True) as f:
+        for line in f:
+            print(redirects.get(line, line), end="")
+    print(
+        f"""\
+Redirects have been detected and {input_file} has been updated. Please take the
+following steps:
+    1. Go ahead and commit just the updated expressions as you intended to do:
+            git add {output_file}
+            git commit -m "vimPlugins: Update"
+    2. If any of the plugin names were changed, add the old names as aliases in
+    aliases.nix
+    3. Make sure the updated {input_file} is still correctly sorted:
+            sort -udf ./vim-plugin-names > sorted && mv sorted vim-plugin-names
+    4. Run this script again so these changes will be reflected in the
+    generated expressions (no need to use the --update-redirects flag again):
+            ./update.py
+    5. Commit {input_file} along with aliases and generated expressions:
+            git add {output_file} {input_file} aliases.nix
+            git commit -m "vimPlugins: Update redirects"
+"""
+    )
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description=(
@@ -407,6 +462,12 @@ def parse_args():
         default=DEFAULT_OUT,
         help="Filename to save generated nix code",
     )
+    parser.add_argument(
+        "--update-redirects",
+        dest="update_redirects",
+        action="store_true",
+        help="Update input file if repos have been redirected.",
+    )
 
     return parser.parse_args()
 
@@ -428,10 +489,19 @@ def main() -> None:
     finally:
         cache.store()
 
-    plugins = check_results(results)
+    plugins, redirects = check_results(results)
 
     generate_nix(plugins, args.outfile)
 
+    if redirects:
+        if args.update_redirects:
+            update_redirects(args.input_file, args.outfile, redirects)
+        else:
+            print(
+                "Outdated vim-plugin-names found. Please run with "
+                "--update-redirects flag."
+            )
+
 
 if __name__ == "__main__":
     main()