From df0db1774050f3e7f585e8902148a312f07b364e Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Fri, 27 Mar 2020 21:10:48 +0000 Subject: [PATCH 1/6] vimPlugins: automatically commit update --- doc/languages-frameworks/vim.section.md | 2 +- pkgs/misc/vim-plugins/update.py | 36 ++++++++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index 05a23d26cf2..4272d1f5969 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -263,7 +263,7 @@ Sometimes plugins require an override that must be changed when the plugin is up To add a new plugin: - 1. run `./update.py` and create a commit named "vimPlugins: Update", + 1. run `./update.py --commit`, 2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix), 3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and 4. create a pull request. diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index 0055e8eccbc..dda96816bb8 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -p nix-prefetch-git -p python3 nix -i python3 +#!nix-shell -p nix-prefetch-git -p python3 -p python3Packages.GitPython nix -i python3 # format: # $ nix run nixpkgs.python3Packages.black -c black update.py @@ -27,11 +27,14 @@ from typing import Dict, List, Optional, Tuple, Union, Any, Callable from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile +import git + ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " ROOT = Path(__file__).parent +NIXPKGS_PATH = ROOT.cwd().parents[2] DEFAULT_IN = ROOT.joinpath("vim-plugin-names") DEFAULT_OUT = ROOT.joinpath("generated.nix") DEPRECATED = ROOT.joinpath("deprecated.json") @@ -413,6 +416,11 @@ in lib.fix' (lib.extends overrides packages) print(f"updated {outfile}") +def commit_changes(repo: git.Repo, *files: Path): + repo.index.add([str(f.resolve()) for f in files]) + repo.index.commit("vimPlugins: Update") + + def rewrite_input(input_file: Path, output_file: Path, redirects: dict): with open(input_file, "r") as f: lines = f.readlines() @@ -438,13 +446,10 @@ def rewrite_input(input_file: Path, output_file: Path, redirects: dict): f"""\ Redirects have been detected and {input_file} has been updated. Please take the following steps: - 1. Go ahead and commit just the updated expressions as you intended to do: - git add {output_file} - git commit -m "vimPlugins: Update" - 2. Run this script again so these changes will be reflected in the + 1. Run this script again so these changes will be reflected in the generated expressions: ./update.py - 3. Commit {input_file} along with deprecations and generated expressions: + 2. Commit {input_file} along with deprecations and generated expressions: git add {output_file} {input_file} {DEPRECATED} git commit -m "vimPlugins: Update redirects" """ @@ -485,13 +490,27 @@ def parse_args(): default=30, help="Number of concurrent processes to spawn.", ) + parser.add_argument( + "--commit", + dest="commit", + action="store_true", + help="Automatically commit updates", + ) return parser.parse_args() -def main() -> None: +def get_nixpkgs_repo() -> git.Repo: + repo = git.Repo(NIXPKGS_PATH) + if repo.is_dirty(): + raise Exception("Please stash your changes before updating.") + return repo + +def main() -> None: args = parse_args() + if args.commit: + nixpkgs_repo = get_nixpkgs_repo() plugin_names = load_plugin_spec(args.input_file) current_plugins = get_current_plugins() cache = Cache(current_plugins) @@ -510,6 +529,9 @@ def main() -> None: rewrite_input(args.input_file, args.outfile, redirects) + if args.commit: + commit_changes(nixpkgs_repo, args.outfile) + if __name__ == "__main__": main() From 042c3dc356f24af8b6b87db1dbda335b4b0c535c Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 28 Mar 2020 06:29:53 +0000 Subject: [PATCH 2/6] vimPlugins: automatically commit redirect updates - When redirections are detected, rather than printing instructions, update.py now creates two commits -- one with the plugin updates and another with redirected plugin names updated to their new repos. - Added a --allow-dirty flag so that one can run ./update.py --commit with a dirty nixpkgs repository, which is necessary for development. I wouldn't mind removing this before merging if it's not in our flag budget but it's necessary scaffolding for now. --- pkgs/misc/vim-plugins/update.py | 94 +++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index dda96816bb8..ef0b640dd4e 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -416,12 +416,7 @@ in lib.fix' (lib.extends overrides packages) print(f"updated {outfile}") -def commit_changes(repo: git.Repo, *files: Path): - repo.index.add([str(f.resolve()) for f in files]) - repo.index.commit("vimPlugins: Update") - - -def rewrite_input(input_file: Path, output_file: Path, redirects: dict): +def rewrite_input(input_file: Path, redirects: dict): with open(input_file, "r") as f: lines = f.readlines() @@ -442,19 +437,6 @@ def rewrite_input(input_file: Path, output_file: Path, redirects: dict): with open(DEPRECATED, "w") as f: json.dump(deprecations, f, indent=4, sort_keys=True) - print( - f"""\ -Redirects have been detected and {input_file} has been updated. Please take the -following steps: - 1. Run this script again so these changes will be reflected in the - generated expressions: - ./update.py - 2. Commit {input_file} along with deprecations and generated expressions: - git add {output_file} {input_file} {DEPRECATED} - git commit -m "vimPlugins: Update redirects" - """ - ) - lines = sorted(lines, key=str.casefold) with open(input_file, "w") as f: @@ -496,25 +478,49 @@ def parse_args(): action="store_true", help="Automatically commit updates", ) + parser.add_argument( + "--allow-dirty", + dest="allow_dirty", + action="store_true", + help=( + "Allow commit to continue even if state is unexpectedly dirty. " + "This is only helpful when developing vimPlugins infrastructure." + ), + ) return parser.parse_args() -def get_nixpkgs_repo() -> git.Repo: - repo = git.Repo(NIXPKGS_PATH) - if repo.is_dirty(): - raise Exception("Please stash your changes before updating.") - return repo +class NixpkgsRepo: + def __init__(self, allow_dirty: bool): + self.allow_dirty: bool = allow_dirty + self.repo: git.Repo = git.Repo(NIXPKGS_PATH) + + if self.is_unexpectedly_dirty(): + raise Exception("Please stash changes before updating.") + + def commit(self, message: str, files: List[Path]) -> None: + file_paths = [str(f.resolve()) for f in files] + files_changed = False + for f in self.repo.index.diff(None): + if str(f) in file_paths: + files_changed = True + break + + if files_changed: + print(f'committing to nixpkgs "vimPlugins: {message}"') + self.repo.index.add(file_paths) + self.repo.index.commit(f"vimPlugins: {message}") + assert self.is_unexpectedly_dirty() is False + else: + print("no changes in working tree to commit") + + def is_unexpectedly_dirty(self) -> bool: + return self.repo.is_dirty() and not self.allow_dirty -def main() -> None: - args = parse_args() - if args.commit: - nixpkgs_repo = get_nixpkgs_repo() - plugin_names = load_plugin_spec(args.input_file) - current_plugins = get_current_plugins() - cache = Cache(current_plugins) - +def update_plugins(input_file: str, outfile: str, cache: Cache) -> Dict: + plugin_names = load_plugin_spec(input_file) prefetch_with_cache = functools.partial(prefetch, cache=cache) try: @@ -525,12 +531,30 @@ def main() -> None: plugins, redirects = check_results(results) - generate_nix(plugins, args.outfile) + generate_nix(plugins, outfile) - rewrite_input(args.input_file, args.outfile, redirects) + return redirects + + +def main() -> None: + args = parse_args() + if args.commit: + nixpkgs_repo = NixpkgsRepo(args.allow_dirty) + current_plugins = get_current_plugins() + cache = Cache(current_plugins) + redirects = {} + + redirects = update_plugins(args.input_file, args.outfile, cache) + + rewrite_input(args.input_file, redirects) if args.commit: - commit_changes(nixpkgs_repo, args.outfile) + nixpkgs_repo.commit("Update", [args.outfile]) + if redirects: + update_plugins(args.input_file, args.outfile, cache) + nixpkgs_repo.commit( + "Update redirects", [args.outfile, args.input_file, DEPRECATED] + ) if __name__ == "__main__": From f1ae95f6d3cbc55cea72fe2765105a6b998ec7db Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 28 Mar 2020 17:42:31 +0000 Subject: [PATCH 3/6] vimPlugins: update.py --add argument - update.py's new --add argument replaces manual editing of vim-plugin-names for basic use cases. --- doc/languages-frameworks/vim.section.md | 7 +--- pkgs/misc/vim-plugins/update.py | 54 ++++++++++++++++--------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index 4272d1f5969..1d316cf350e 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -261,12 +261,7 @@ deoplete-fish = super.deoplete-fish.overrideAttrs(old: { Sometimes plugins require an override that must be changed when the plugin is updated. This can cause issues when Vim plugins are auto-updated but the associated override isn't updated. For these plugins, the override should be written so that it specifies all information required to install the plugin, and running `./update.py` doesn't change the derivation for the plugin. Manually updating the override is required to update these types of plugins. An example of such a plugin is `LanguageClient-neovim`. -To add a new plugin: - - 1. run `./update.py --commit`, - 2. add the new plugin to [vim-plugin-names](/pkgs/misc/vim-plugins/vim-plugin-names) and add overrides if required to [overrides.nix](/pkgs/misc/vim-plugins/overrides.nix), - 3. run `./update.py` again and create a commit named "vimPlugins.[name]: init at [version]" (where `name` and `version` can be found in [generated.nix](/pkgs/misc/vim-plugins/generated.nix)), and - 4. create a pull request. +To add a new plugin, run `./update.py --commit --add "[owner]/[name]"`. ## Important repositories diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index ef0b640dd4e..0a08e4dac16 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -38,6 +38,7 @@ NIXPKGS_PATH = ROOT.cwd().parents[2] DEFAULT_IN = ROOT.joinpath("vim-plugin-names") DEFAULT_OUT = ROOT.joinpath("generated.nix") DEPRECATED = ROOT.joinpath("deprecated.json") +OVERRIDES = ROOT.joinpath("overrides.nix") import time from functools import wraps @@ -127,7 +128,7 @@ class Repo: new_owner, new_name = ( urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2] ) - end_line = "\n" if self.alias is None else f" as {self.alias}\n" + end_line = "\n" if self.alias == "" else f" as {self.alias}\n" plugin_line = "{owner}/{name}" + end_line old_plugin = plugin_line.format(owner=self.owner, name=self.name) @@ -280,17 +281,17 @@ def check_results( sys.exit(1) -def parse_plugin_line(line: str) -> Tuple[str, str, Optional[str]]: +def parse_plugin_line(line: str) -> Tuple[str, str, str]: name, repo = line.split("/") try: repo, alias = repo.split(" as ") return (name, repo, alias.strip()) except ValueError: # no alias defined - return (name, repo.strip(), None) + return (name, repo.strip(), "") -def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, Optional[str]]]: +def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, str]]: plugins = [] with open(plugin_file) as f: for line in f: @@ -416,10 +417,12 @@ in lib.fix' (lib.extends overrides packages) print(f"updated {outfile}") -def rewrite_input(input_file: Path, redirects: dict): +def rewrite_input(input_file: Path, redirects: dict = None, append: Tuple = ()): with open(input_file, "r") as f: lines = f.readlines() + lines.extend(append) + if redirects: lines = [redirects.get(line, line) for line in lines] @@ -450,6 +453,14 @@ def parse_args(): f"By default from {DEFAULT_IN} to {DEFAULT_OUT}" ) ) + parser.add_argument( + "--add", + "-a", + dest="add_plugins", + default=[], + action="append", + help="Plugin to add to vimPlugins in the form owner/repo", + ) parser.add_argument( "--input-names", "-i", @@ -500,17 +511,11 @@ class NixpkgsRepo: raise Exception("Please stash changes before updating.") def commit(self, message: str, files: List[Path]) -> None: - file_paths = [str(f.resolve()) for f in files] - files_changed = False - for f in self.repo.index.diff(None): - if str(f) in file_paths: - files_changed = True - break + files_staged = self.repo.index.add([str(f.resolve()) for f in files]) - if files_changed: - print(f'committing to nixpkgs "vimPlugins: {message}"') - self.repo.index.add(file_paths) - self.repo.index.commit(f"vimPlugins: {message}") + if files_staged: + print(f'committing to nixpkgs "{message}"') + self.repo.index.commit(message) assert self.is_unexpectedly_dirty() is False else: print("no changes in working tree to commit") @@ -538,6 +543,8 @@ def update_plugins(input_file: str, outfile: str, cache: Cache) -> Dict: def main() -> None: args = parse_args() + if args.add_plugins and not args.commit: + raise Exception("The --add argument requires setting the --commit flag.") if args.commit: nixpkgs_repo = NixpkgsRepo(args.allow_dirty) current_plugins = get_current_plugins() @@ -545,15 +552,26 @@ def main() -> None: redirects = {} redirects = update_plugins(args.input_file, args.outfile, cache) - rewrite_input(args.input_file, redirects) if args.commit: - nixpkgs_repo.commit("Update", [args.outfile]) + nixpkgs_repo.commit("vimPlugins: Update", [args.outfile]) if redirects: update_plugins(args.input_file, args.outfile, cache) nixpkgs_repo.commit( - "Update redirects", [args.outfile, args.input_file, DEPRECATED] + "vimPlugins: Update redirects", + [args.outfile, args.input_file, DEPRECATED], + ) + for plugin_line in args.add_plugins: + rewrite_input(args.input_file, append=(plugin_line + "\n",)) + update_plugins(args.input_file, args.outfile, cache) + + plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line), cache) + nixpkgs_repo.commit( + "vimPlugins.{name}: init at {version}".format( + name=plugin.normalized_name, version=plugin.version + ), + [args.outfile, args.input_file, OVERRIDES], ) From 1f32eb7987b44348b8c20bec0aea60bec836637c Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 28 Mar 2020 18:11:51 +0000 Subject: [PATCH 4/6] vimPlugins: remove repetition from update.main Adding some abstraction makes main() more readable which is important since it's the main control flow of the script. --- pkgs/misc/vim-plugins/update.py | 56 +++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index 0a08e4dac16..cea1ae21527 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -222,16 +222,17 @@ def get_current_plugins() -> List[Plugin]: def prefetch_plugin( - user: str, repo_name: str, alias: str, cache: "Cache" + user: str, repo_name: str, alias: str, cache: "Cache" = None ) -> Tuple[Plugin, Dict[str, str]]: repo = Repo(user, repo_name, alias) commit, date = repo.latest_commit() has_submodules = repo.has_submodules() - cached_plugin = cache[commit] - if cached_plugin is not None: - cached_plugin.name = alias or repo_name - cached_plugin.date = date - return cached_plugin, repo.redirect + if cache is not None: + cached_plugin = cache[commit] + if cached_plugin is not None: + cached_plugin.name = alias or repo_name + cached_plugin.date = date + return cached_plugin, repo.redirect print(f"prefetch {user}/{repo_name}") if has_submodules: @@ -524,21 +525,30 @@ class NixpkgsRepo: return self.repo.is_dirty() and not self.allow_dirty -def update_plugins(input_file: str, outfile: str, cache: Cache) -> Dict: - plugin_names = load_plugin_spec(input_file) - prefetch_with_cache = functools.partial(prefetch, cache=cache) +class PluginUpdater: + def __init__(self, input_file: str, outfile: str, proc: int): + self.input_file: str = input_file + self.outfile: str = outfile + self.proc = proc + self.cache: Cache = Cache(get_current_plugins()) + self.prefetch = functools.partial(prefetch, cache=self.cache) - try: - pool = Pool(processes=args.proc) - results = pool.map(prefetch_with_cache, plugin_names) - finally: - cache.store() + def __call__(self) -> Dict: + plugin_names = load_plugin_spec(self.input_file) - plugins, redirects = check_results(results) + try: + # synchronous variant for debugging + # results = list(map(self.prefetch, plugin_names)) + pool = Pool(processes=self.proc) + results = pool.map(self.prefetch, plugin_names) + finally: + self.cache.store() - generate_nix(plugins, outfile) + plugins, redirects = check_results(results) - return redirects + generate_nix(plugins, self.outfile) + + return redirects def main() -> None: @@ -547,26 +557,24 @@ def main() -> None: raise Exception("The --add argument requires setting the --commit flag.") if args.commit: nixpkgs_repo = NixpkgsRepo(args.allow_dirty) - current_plugins = get_current_plugins() - cache = Cache(current_plugins) - redirects = {} + updater = PluginUpdater(args.input_file, args.outfile, args.proc) - redirects = update_plugins(args.input_file, args.outfile, cache) + redirects = updater() rewrite_input(args.input_file, redirects) if args.commit: nixpkgs_repo.commit("vimPlugins: Update", [args.outfile]) if redirects: - update_plugins(args.input_file, args.outfile, cache) + updater() nixpkgs_repo.commit( "vimPlugins: Update redirects", [args.outfile, args.input_file, DEPRECATED], ) for plugin_line in args.add_plugins: rewrite_input(args.input_file, append=(plugin_line + "\n",)) - update_plugins(args.input_file, args.outfile, cache) + updater() - plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line), cache) + plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line)) nixpkgs_repo.commit( "vimPlugins.{name}: init at {version}".format( name=plugin.normalized_name, version=plugin.version From 1c0f0de46c231d6c392b9727327d9635fc4f60ec Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sun, 29 Mar 2020 16:09:59 +0000 Subject: [PATCH 5/6] vimPlugins: normalize deprecation names Parsing plugin lines in multiple places is hazardous and already manifested a bug! --- pkgs/misc/vim-plugins/deprecated.json | 4 ++-- pkgs/misc/vim-plugins/update.py | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pkgs/misc/vim-plugins/deprecated.json b/pkgs/misc/vim-plugins/deprecated.json index 2c02982f6c6..2af94f5df3d 100644 --- a/pkgs/misc/vim-plugins/deprecated.json +++ b/pkgs/misc/vim-plugins/deprecated.json @@ -9,10 +9,10 @@ }, "vundle": { "date": "2020-03-27", - "new": "Vundle.vim" + "new": "Vundle-vim" }, "youcompleteme": { "date": "2020-03-27", "new": "YouCompleteMe" } -} \ No newline at end of file +} diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index cea1ae21527..a01bd93a3cc 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -246,6 +246,11 @@ def prefetch_plugin( ) +def fetch_plugin_from_pluginline(plugin_line: str) -> Plugin: + plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line)) + return plugin + + def print_download_error(plugin: str, ex: Exception): print(f"{plugin}: {ex}", file=sys.stderr) ex_traceback = ex.__traceback__ @@ -418,7 +423,9 @@ in lib.fix' (lib.extends overrides packages) print(f"updated {outfile}") -def rewrite_input(input_file: Path, redirects: dict = None, append: Tuple = ()): +def rewrite_input( + input_file: Path, redirects: Dict[str, str] = None, append: Tuple = () +): with open(input_file, "r") as f: lines = f.readlines() @@ -431,11 +438,11 @@ def rewrite_input(input_file: Path, redirects: dict = None, append: Tuple = ()): with open(DEPRECATED, "r") as f: deprecations = json.load(f) for old, new in redirects.items(): - old_name = old.split("/")[1].split(" ")[0].strip("\n") - new_name = new.split("/")[1].split(" ")[0].strip("\n") - if old_name != new_name: - deprecations[old_name] = { - "new": new_name, + old_plugin = fetch_plugin_from_pluginline(old) + new_plugin = fetch_plugin_from_pluginline(new) + if old_plugin.normalized_name != new_plugin.normalized_name: + deprecations[old_plugin.normalized_name] = { + "new": new_plugin.normalized_name, "date": cur_date_iso, } with open(DEPRECATED, "w") as f: @@ -574,7 +581,7 @@ def main() -> None: rewrite_input(args.input_file, append=(plugin_line + "\n",)) updater() - plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line)) + plugin = fetch_plugin_from_pluginline(plugin_line) nixpkgs_repo.commit( "vimPlugins.{name}: init at {version}".format( name=plugin.normalized_name, version=plugin.version From 0a2759483a69d49a0580d3e9e994029f1d8f5bf0 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Tue, 31 Mar 2020 01:01:33 +0000 Subject: [PATCH 6/6] vimPlugins: update auto-commiting based on review - Use git.Repo(ROOT, search_parent_directories=True) to find nixpkgs repo. - Don't commit overrides.nix. - Remove "-a" short argument. - Remove "--commit" flag and commit by default. - Improve help/error messages. - Favor closure pattern over classes.Use a closure to wrap the update function with state rather than a callable class. - break NixpkgsRepo class into functions - Optional None-type arguments - Remove repo checks from update.py. Git is too flexible and permits too many workflows for my attempt to replace documentation with code to work. My goal would be to separate the `--add` functionality from the update functionality in the near term and then there will be no reason for this usage to create commits anyway. --- doc/languages-frameworks/vim.section.md | 2 +- pkgs/misc/vim-plugins/update.py | 144 +++++++++--------------- 2 files changed, 54 insertions(+), 92 deletions(-) diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index 1d316cf350e..4911509212e 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -261,7 +261,7 @@ deoplete-fish = super.deoplete-fish.overrideAttrs(old: { Sometimes plugins require an override that must be changed when the plugin is updated. This can cause issues when Vim plugins are auto-updated but the associated override isn't updated. For these plugins, the override should be written so that it specifies all information required to install the plugin, and running `./update.py` doesn't change the derivation for the plugin. Manually updating the override is required to update these types of plugins. An example of such a plugin is `LanguageClient-neovim`. -To add a new plugin, run `./update.py --commit --add "[owner]/[name]"`. +To add a new plugin, run `./update.py --add "[owner]/[name]"`. **NOTE**: This script automatically commits to your git repository. Be sure to check out a fresh branch before running. ## Important repositories diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index a01bd93a3cc..fe8cb90d103 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -34,11 +34,9 @@ ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " ROOT = Path(__file__).parent -NIXPKGS_PATH = ROOT.cwd().parents[2] DEFAULT_IN = ROOT.joinpath("vim-plugin-names") DEFAULT_OUT = ROOT.joinpath("generated.nix") DEPRECATED = ROOT.joinpath("deprecated.json") -OVERRIDES = ROOT.joinpath("overrides.nix") import time from functools import wraps @@ -78,7 +76,7 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa class Repo: - def __init__(self, owner: str, name: str, alias: str) -> None: + def __init__(self, owner: str, name: str, alias: Optional[str]) -> None: self.owner = owner self.name = name self.alias = alias @@ -128,7 +126,7 @@ class Repo: new_owner, new_name = ( urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2] ) - end_line = "\n" if self.alias == "" else f" as {self.alias}\n" + end_line = "\n" if self.alias is None else f" as {self.alias}\n" plugin_line = "{owner}/{name}" + end_line old_plugin = plugin_line.format(owner=self.owner, name=self.name) @@ -222,17 +220,16 @@ def get_current_plugins() -> List[Plugin]: def prefetch_plugin( - user: str, repo_name: str, alias: str, cache: "Cache" = None + user: str, repo_name: str, alias: Optional[str], cache: "Optional[Cache]" = None ) -> Tuple[Plugin, Dict[str, str]]: repo = Repo(user, repo_name, alias) commit, date = repo.latest_commit() has_submodules = repo.has_submodules() - if cache is not None: - cached_plugin = cache[commit] - if cached_plugin is not None: - cached_plugin.name = alias or repo_name - cached_plugin.date = date - return cached_plugin, repo.redirect + cached_plugin = cache[commit] if cache else None + if cached_plugin is not None: + cached_plugin.name = alias or repo_name + cached_plugin.date = date + return cached_plugin, repo.redirect print(f"prefetch {user}/{repo_name}") if has_submodules: @@ -287,17 +284,17 @@ def check_results( sys.exit(1) -def parse_plugin_line(line: str) -> Tuple[str, str, str]: +def parse_plugin_line(line: str) -> Tuple[str, str, Optional[str]]: name, repo = line.split("/") try: repo, alias = repo.split(" as ") return (name, repo, alias.strip()) except ValueError: # no alias defined - return (name, repo.strip(), "") + return (name, repo.strip(), None) -def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, str]]: +def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, Optional[str]]]: plugins = [] with open(plugin_file) as f: for line in f: @@ -463,11 +460,10 @@ def parse_args(): ) parser.add_argument( "--add", - "-a", dest="add_plugins", default=[], action="append", - help="Plugin to add to vimPlugins in the form owner/repo", + help="Plugin to add to vimPlugins from Github in the form owner/repo", ) parser.add_argument( "--input-names", @@ -491,103 +487,69 @@ def parse_args(): default=30, help="Number of concurrent processes to spawn.", ) - parser.add_argument( - "--commit", - dest="commit", - action="store_true", - help="Automatically commit updates", - ) - parser.add_argument( - "--allow-dirty", - dest="allow_dirty", - action="store_true", - help=( - "Allow commit to continue even if state is unexpectedly dirty. " - "This is only helpful when developing vimPlugins infrastructure." - ), - ) - return parser.parse_args() -class NixpkgsRepo: - def __init__(self, allow_dirty: bool): - self.allow_dirty: bool = allow_dirty - self.repo: git.Repo = git.Repo(NIXPKGS_PATH) +def commit(repo: git.Repo, message: str, files: List[Path]) -> None: + files_staged = repo.index.add([str(f.resolve()) for f in files]) - if self.is_unexpectedly_dirty(): - raise Exception("Please stash changes before updating.") - - def commit(self, message: str, files: List[Path]) -> None: - files_staged = self.repo.index.add([str(f.resolve()) for f in files]) - - if files_staged: - print(f'committing to nixpkgs "{message}"') - self.repo.index.commit(message) - assert self.is_unexpectedly_dirty() is False - else: - print("no changes in working tree to commit") - - def is_unexpectedly_dirty(self) -> bool: - return self.repo.is_dirty() and not self.allow_dirty + if files_staged: + print(f'committing to nixpkgs "{message}"') + repo.index.commit(message) + else: + print("no changes in working tree to commit") -class PluginUpdater: - def __init__(self, input_file: str, outfile: str, proc: int): - self.input_file: str = input_file - self.outfile: str = outfile - self.proc = proc - self.cache: Cache = Cache(get_current_plugins()) - self.prefetch = functools.partial(prefetch, cache=self.cache) +def get_update(input_file: str, outfile: str, proc: int): + cache: Cache = Cache(get_current_plugins()) + _prefetch = functools.partial(prefetch, cache=cache) - def __call__(self) -> Dict: - plugin_names = load_plugin_spec(self.input_file) + def update() -> dict: + plugin_names = load_plugin_spec(input_file) try: - # synchronous variant for debugging - # results = list(map(self.prefetch, plugin_names)) - pool = Pool(processes=self.proc) - results = pool.map(self.prefetch, plugin_names) + pool = Pool(processes=proc) + results = pool.map(_prefetch, plugin_names) finally: - self.cache.store() + cache.store() plugins, redirects = check_results(results) - generate_nix(plugins, self.outfile) + generate_nix(plugins, outfile) return redirects + return update -def main() -> None: + +def main(): args = parse_args() - if args.add_plugins and not args.commit: - raise Exception("The --add argument requires setting the --commit flag.") - if args.commit: - nixpkgs_repo = NixpkgsRepo(args.allow_dirty) - updater = PluginUpdater(args.input_file, args.outfile, args.proc) + nixpkgs_repo = git.Repo(ROOT, search_parent_directories=True) + update = get_update(args.input_file, args.outfile, args.proc) - redirects = updater() + redirects = update() rewrite_input(args.input_file, redirects) + commit(nixpkgs_repo, "vimPlugins: update", [args.outfile]) - if args.commit: - nixpkgs_repo.commit("vimPlugins: Update", [args.outfile]) - if redirects: - updater() - nixpkgs_repo.commit( - "vimPlugins: Update redirects", - [args.outfile, args.input_file, DEPRECATED], - ) - for plugin_line in args.add_plugins: - rewrite_input(args.input_file, append=(plugin_line + "\n",)) - updater() + if redirects: + update() + commit( + nixpkgs_repo, + "vimPlugins: resolve github repository redirects", + [args.outfile, args.input_file, DEPRECATED], + ) - plugin = fetch_plugin_from_pluginline(plugin_line) - nixpkgs_repo.commit( - "vimPlugins.{name}: init at {version}".format( - name=plugin.normalized_name, version=plugin.version - ), - [args.outfile, args.input_file, OVERRIDES], - ) + for plugin_line in args.add_plugins: + rewrite_input(args.input_file, append=(plugin_line + "\n",)) + update() + plugin = fetch_plugin_from_pluginline(plugin_line) + commit( + nixpkgs_repo, + "vimPlugins.{name}: init at {version}".format( + name=plugin.normalized_name, version=plugin.version + ), + [args.outfile, args.input_file], + ) if __name__ == "__main__":