From d4874a7ee62c817b90d80098f53ad205f9909ccb Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 21 Mar 2020 04:27:09 +0000 Subject: [PATCH] vimPlugins: Automate redirect updates in update.py Many of the plugins in vim-plugin-names are out of date and redirect to their new github repos. This commit adds a flag to automatically update these and defines a process for committing these updates into nixpkgs. --- doc/languages-frameworks/vim.section.md | 2 +- pkgs/misc/vim-plugins/update.py | 100 ++++++++++++++++++++---- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index 05a23d26cf2..cd53926defe 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -263,7 +263,7 @@ Sometimes plugins require an override that must be changed when the plugin is up To add a new plugin: - 1. run `./update.py` and create a commit named "vimPlugins: Update", + 1. run `./update.py --update-redirects` and create a commit named "vimPlugins: Update", 2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix), 3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and 4. create a pull request. diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index 0ef93ac569a..2a068892711 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -9,13 +9,16 @@ # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py import argparse +import fileinput import functools +import http import json import os import subprocess import sys import traceback import urllib.error +import urllib.parse import urllib.request import xml.etree.ElementTree as ET from datetime import datetime @@ -71,9 +74,11 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa class Repo: - def __init__(self, owner: str, name: str) -> None: + def __init__(self, owner: str, name: str, alias: str) -> None: self.owner = owner self.name = name + self.alias = alias + self.redirect: Dict[str, str] = {} def url(self, path: str) -> str: return urljoin(f"https://github.com/{self.owner}/{self.name}/", path) @@ -96,7 +101,9 @@ class Repo: @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: - with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req: + commit_url = self.url("commits/master.atom") + with urllib.request.urlopen(commit_url, timeout=10) as req: + self.check_for_redirect(commit_url, req) xml = req.read() root = ET.fromstring(xml) latest_entry = root.find(ATOM_ENTRY) @@ -111,6 +118,22 @@ class Repo: updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ") return Path(str(url.path)).name, updated + def check_for_redirect(self, url: str, req: http.client.HTTPResponse): + response_url = req.geturl() + if url != response_url: + new_owner, new_name = ( + urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2] + ) + end_line = f" as {self.alias}\n" if self.alias else "\n" + plugin_line = "{owner}/{name}" + end_line + + old_plugin = plugin_line.format(owner=self.owner, name=self.name) + new_plugin = plugin_line.format(owner=new_owner, name=new_name) + self.redirect[old_plugin] = new_plugin + + if new_name != self.name: + print(f"Plugin Name Changed: {self.name} -> {new_name}") + def prefetch_git(self, ref: str) -> str: data = subprocess.check_output( ["nix-prefetch-git", "--fetch-submodules", self.url(""), ref] @@ -197,15 +220,17 @@ def get_current_plugins() -> List[Plugin]: return plugins -def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Plugin: - repo = Repo(user, repo_name) +def prefetch_plugin( + user: str, repo_name: str, alias: str, cache: "Cache" +) -> Tuple[Plugin, Dict[str, str]]: + repo = Repo(user, repo_name, alias) commit, date = repo.latest_commit() has_submodules = repo.has_submodules() cached_plugin = cache[commit] if cached_plugin is not None: cached_plugin.name = alias or repo_name cached_plugin.date = date - return cached_plugin + return cached_plugin, repo.redirect print(f"prefetch {user}/{repo_name}") if has_submodules: @@ -213,7 +238,10 @@ def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Pl else: sha256 = repo.prefetch_github(commit) - return Plugin(alias or repo_name, commit, has_submodules, sha256, date=date) + return ( + Plugin(alias or repo_name, commit, has_submodules, sha256, date=date), + repo.redirect, + ) def print_download_error(plugin: str, ex: Exception): @@ -227,20 +255,22 @@ def print_download_error(plugin: str, ex: Exception): def check_results( - results: List[Tuple[str, str, Union[Exception, Plugin]]] -) -> List[Tuple[str, str, Plugin]]: + results: List[Tuple[str, str, Union[Exception, Plugin], Dict[str, str]]] +) -> Tuple[List[Tuple[str, str, Plugin]], Dict[str, str]]: failures: List[Tuple[str, Exception]] = [] plugins = [] - for (owner, name, result) in results: + redirects: Dict[str, str] = {} + for (owner, name, result, redirect) in results: if isinstance(result, Exception): failures.append((name, result)) else: plugins.append((owner, name, result)) + redirects.update(redirect) print(f"{len(results) - len(failures)} plugins were checked", end="") if len(failures) == 0: print() - return plugins + return plugins, redirects else: print(f", {len(failures)} plugin(s) could not be downloaded:\n") @@ -328,15 +358,15 @@ class Cache: def prefetch( args: Tuple[str, str, str], cache: Cache -) -> Tuple[str, str, Union[Exception, Plugin]]: +) -> Tuple[str, str, Union[Exception, Plugin], dict]: assert len(args) == 3 owner, repo, alias = args try: - plugin = prefetch_plugin(owner, repo, alias, cache) + plugin, redirect = prefetch_plugin(owner, repo, alias, cache) cache[plugin.commit] = plugin - return (owner, repo, plugin) + return (owner, repo, plugin, redirect) except Exception as e: - return (owner, repo, e) + return (owner, repo, e, {}) header = ( @@ -386,6 +416,31 @@ in lib.fix' (lib.extends overrides packages) print(f"updated {outfile}") +def update_redirects(input_file: Path, output_file: Path, redirects: dict): + with fileinput.input(input_file, inplace=True) as f: + for line in f: + print(redirects.get(line, line), end="") + print( + f"""\ +Redirects have been detected and {input_file} has been updated. Please take the +following steps: + 1. Go ahead and commit just the updated expressions as you intended to do: + git add {output_file} + git commit -m "vimPlugins: Update" + 2. If any of the plugin names were changed, add the old names as aliases in + aliases.nix + 3. Make sure the updated {input_file} is still correctly sorted: + sort -udf ./vim-plugin-names > sorted && mv sorted vim-plugin-names + 4. Run this script again so these changes will be reflected in the + generated expressions (no need to use the --update-redirects flag again): + ./update.py + 5. Commit {input_file} along with aliases and generated expressions: + git add {output_file} {input_file} aliases.nix + git commit -m "vimPlugins: Update redirects" +""" + ) + + def parse_args(): parser = argparse.ArgumentParser( description=( @@ -407,6 +462,12 @@ def parse_args(): default=DEFAULT_OUT, help="Filename to save generated nix code", ) + parser.add_argument( + "--update-redirects", + dest="update_redirects", + action="store_true", + help="Update input file if repos have been redirected.", + ) return parser.parse_args() @@ -428,10 +489,19 @@ def main() -> None: finally: cache.store() - plugins = check_results(results) + plugins, redirects = check_results(results) generate_nix(plugins, args.outfile) + if redirects: + if args.update_redirects: + update_redirects(args.input_file, args.outfile, redirects) + else: + print( + "Outdated vim-plugin-names found. Please run with " + "--update-redirects flag." + ) + if __name__ == "__main__": main()