Merge pull request #83788 from ryneeverett/vim-plugins-update-commit

vimPlugins: Automate git commits when updating.

doc/languages-frameworks/vim.section.md

@@ -261,12 +261,7 @@ deoplete-fish = super.deoplete-fish.overrideAttrs(old: {
 
 Sometimes plugins require an override that must be changed when the plugin is updated. This can cause issues when Vim plugins are auto-updated but the associated override isn't updated. For these plugins, the override should be written so that it specifies all information required to install the plugin, and running `./update.py` doesn't change the derivation for the plugin. Manually updating the override is required to update these types of plugins. An example of such a plugin is `LanguageClient-neovim`.
 
-To add a new plugin:
+To add a new plugin, run `./update.py --add "[owner]/[name]"`. **NOTE**: This script automatically commits to your git repository. Be sure to check out a fresh branch before running.
-
-  1. run `./update.py` and create a commit named "vimPlugins: Update",
-  2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix),
-  3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and
-  4. create a pull request.
 
 ## Important repositories
 

pkgs/misc/vim-plugins/deprecated.json

@@ -9,10 +9,10 @@
     },
     "vundle": {
         "date": "2020-03-27",
-        "new": "Vundle.vim"
+        "new": "Vundle-vim"
     },
     "youcompleteme": {
         "date": "2020-03-27",
         "new": "YouCompleteMe"
     }
-}
+}

pkgs/misc/vim-plugins/update.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -p nix-prefetch-git -p python3 nix -i python3
+#!nix-shell -p nix-prefetch-git -p python3 -p python3Packages.GitPython nix -i python3
 
 # format:
 # $ nix run nixpkgs.python3Packages.black -c black update.py
@@ -27,6 +27,8 @@ from typing import Dict, List, Optional, Tuple, Union, Any, Callable
 from urllib.parse import urljoin, urlparse
 from tempfile import NamedTemporaryFile
 
+import git
+
 ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry"  # " vim gets confused here
 ATOM_LINK = "{http://www.w3.org/2005/Atom}link"  # "
 ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated"  # "
@@ -74,7 +76,7 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa
 
 
 class Repo:
-    def __init__(self, owner: str, name: str, alias: str) -> None:
+    def __init__(self, owner: str, name: str, alias: Optional[str]) -> None:
         self.owner = owner
         self.name = name
         self.alias = alias
@@ -218,12 +220,12 @@ def get_current_plugins() -> List[Plugin]:
 
 
 def prefetch_plugin(
-    user: str, repo_name: str, alias: str, cache: "Cache"
+    user: str, repo_name: str, alias: Optional[str], cache: "Optional[Cache]" = None
 ) -> Tuple[Plugin, Dict[str, str]]:
     repo = Repo(user, repo_name, alias)
     commit, date = repo.latest_commit()
     has_submodules = repo.has_submodules()
-    cached_plugin = cache[commit]
+    cached_plugin = cache[commit] if cache else None
     if cached_plugin is not None:
         cached_plugin.name = alias or repo_name
         cached_plugin.date = date
@@ -241,6 +243,11 @@ def prefetch_plugin(
     )
 
 
+def fetch_plugin_from_pluginline(plugin_line: str) -> Plugin:
+    plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line))
+    return plugin
+
+
 def print_download_error(plugin: str, ex: Exception):
     print(f"{plugin}: {ex}", file=sys.stderr)
     ex_traceback = ex.__traceback__
@@ -413,10 +420,14 @@ in lib.fix' (lib.extends overrides packages)
     print(f"updated {outfile}")
 
 
-def rewrite_input(input_file: Path, output_file: Path, redirects: dict):
+def rewrite_input(
+    input_file: Path, redirects: Dict[str, str] = None, append: Tuple = ()
+):
     with open(input_file, "r") as f:
         lines = f.readlines()
 
+    lines.extend(append)
+
     if redirects:
         lines = [redirects.get(line, line) for line in lines]
 
@@ -424,32 +435,16 @@ def rewrite_input(input_file: Path, output_file: Path, redirects: dict):
         with open(DEPRECATED, "r") as f:
             deprecations = json.load(f)
         for old, new in redirects.items():
-            old_name = old.split("/")[1].split(" ")[0].strip("\n")
+            old_plugin = fetch_plugin_from_pluginline(old)
-            new_name = new.split("/")[1].split(" ")[0].strip("\n")
+            new_plugin = fetch_plugin_from_pluginline(new)
-            if old_name != new_name:
+            if old_plugin.normalized_name != new_plugin.normalized_name:
-                deprecations[old_name] = {
+                deprecations[old_plugin.normalized_name] = {
-                    "new": new_name,
+                    "new": new_plugin.normalized_name,
                     "date": cur_date_iso,
                 }
         with open(DEPRECATED, "w") as f:
             json.dump(deprecations, f, indent=4, sort_keys=True)
 
-        print(
-            f"""\
-Redirects have been detected and {input_file} has been updated. Please take the
-following steps:
-    1. Go ahead and commit just the updated expressions as you intended to do:
-            git add {output_file}
-            git commit -m "vimPlugins: Update"
-    2. Run this script again so these changes will be reflected in the
-    generated expressions:
-            ./update.py
-    3. Commit {input_file} along with deprecations and generated expressions:
-            git add {output_file} {input_file} {DEPRECATED}
-            git commit -m "vimPlugins: Update redirects"
-        """
-        )
-
     lines = sorted(lines, key=str.casefold)
 
     with open(input_file, "w") as f:
@@ -463,6 +458,13 @@ def parse_args():
             f"By default from {DEFAULT_IN} to {DEFAULT_OUT}"
         )
     )
+    parser.add_argument(
+        "--add",
+        dest="add_plugins",
+        default=[],
+        action="append",
+        help="Plugin to add to vimPlugins from Github in the form owner/repo",
+    )
     parser.add_argument(
         "--input-names",
         "-i",
@@ -485,30 +487,69 @@ def parse_args():
         default=30,
         help="Number of concurrent processes to spawn.",
     )
-
     return parser.parse_args()
 
 
-def main() -> None:
+def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
+    files_staged = repo.index.add([str(f.resolve()) for f in files])
 
+    if files_staged:
+        print(f'committing to nixpkgs "{message}"')
+        repo.index.commit(message)
+    else:
+        print("no changes in working tree to commit")
+
+
+def get_update(input_file: str, outfile: str, proc: int):
+    cache: Cache = Cache(get_current_plugins())
+    _prefetch = functools.partial(prefetch, cache=cache)
+
+    def update() -> dict:
+        plugin_names = load_plugin_spec(input_file)
+
+        try:
+            pool = Pool(processes=proc)
+            results = pool.map(_prefetch, plugin_names)
+        finally:
+            cache.store()
+
+        plugins, redirects = check_results(results)
+
+        generate_nix(plugins, outfile)
+
+        return redirects
+
+    return update
+
+
+def main():
     args = parse_args()
-    plugin_names = load_plugin_spec(args.input_file)
+    nixpkgs_repo = git.Repo(ROOT, search_parent_directories=True)
-    current_plugins = get_current_plugins()
+    update = get_update(args.input_file, args.outfile, args.proc)
-    cache = Cache(current_plugins)
 
-    prefetch_with_cache = functools.partial(prefetch, cache=cache)
+    redirects = update()
+    rewrite_input(args.input_file, redirects)
+    commit(nixpkgs_repo, "vimPlugins: update", [args.outfile])
 
-    try:
+    if redirects:
-        pool = Pool(processes=args.proc)
+        update()
-        results = pool.map(prefetch_with_cache, plugin_names)
+        commit(
-    finally:
+            nixpkgs_repo,
-        cache.store()
+            "vimPlugins: resolve github repository redirects",
+            [args.outfile, args.input_file, DEPRECATED],
+        )
 
-    plugins, redirects = check_results(results)
+    for plugin_line in args.add_plugins:
-
+        rewrite_input(args.input_file, append=(plugin_line + "\n",))
-    generate_nix(plugins, args.outfile)
+        update()
-
+        plugin = fetch_plugin_from_pluginline(plugin_line)
-    rewrite_input(args.input_file, args.outfile, redirects)
+        commit(
+            nixpkgs_repo,
+            "vimPlugins.{name}: init at {version}".format(
+                name=plugin.normalized_name, version=plugin.version
+            ),
+            [args.outfile, args.input_file],
+        )
 
 
 if __name__ == "__main__":