update-luarocks-packages: use pluginupdate.py

Cleans up the common interface between the updaters.
Restores the ability to regen the lua packages in parallel.

2 files changed, 135 insertions, 125 deletions

diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py
index 0d656556557f..79f5b93be8af 100644
--- a/maintainers/scripts/pluginupdate.py
+++ b/maintainers/scripts/pluginupdate.py
@@ -42,8 +42,6 @@ LOG_LEVELS = {
 }
 
 log = logging.getLogger()
-log.addHandler(logging.StreamHandler())
-
 
 def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
     """Retry calling the decorated function using an exponential backoff.
@@ -203,7 +201,6 @@ class Editor:
         name: str,
         root: Path,
         get_plugins: str,
-        generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None],
         default_in: Optional[Path] = None,
         default_out: Optional[Path] = None,
         deprecated: Optional[Path] = None,
@@ -213,7 +210,6 @@ class Editor:
         self.name = name
         self.root = root
         self.get_plugins = get_plugins
-        self._generate_nix = generate_nix
         self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
         self.default_out = default_out or root.joinpath("generated.nix")
         self.deprecated = deprecated or root.joinpath("deprecated.json")
@@ -226,9 +222,9 @@ class Editor:
     def load_plugin_spec(self, plugin_file) -> List[PluginDesc]:
         return load_plugin_spec(plugin_file)
 
-    def generate_nix(self, plugins, outfile):
+    def generate_nix(self, plugins, outfile: str):
         '''Returns nothing for now, writes directly to outfile'''
-        self._generate_nix(plugins, outfile)
+        raise NotImplementedError()
 
     def get_update(self, input_file: str, outfile: str, proc: int):
         return get_update(input_file, outfile, proc, editor=self)
@@ -237,9 +233,58 @@ class Editor:
     def attr_path(self):
         return self.name + "Plugins"
 
+    def get_drv_name(self, name: str):
+        return self.attr_path + "." + name
+
     def rewrite_input(self, *args, **kwargs):
         return rewrite_input(*args, **kwargs)
 
+    def create_parser(self):
+        parser = argparse.ArgumentParser(
+            description=(
+                f"Updates nix derivations for {self.name} plugins"
+                f"By default from {self.default_in} to {self.default_out}"
+            )
+        )
+        parser.add_argument(
+            "--add",
+            dest="add_plugins",
+            default=[],
+            action="append",
+            help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
+        )
+        parser.add_argument(
+            "--input-names",
+            "-i",
+            dest="input_file",
+            default=self.default_in,
+            help="A list of plugins in the form owner/repo",
+        )
+        parser.add_argument(
+            "--out",
+            "-o",
+            dest="outfile",
+            default=self.default_out,
+            help="Filename to save generated nix code",
+        )
+        parser.add_argument(
+            "--proc",
+            "-p",
+            dest="proc",
+            type=int,
+            default=30,
+            help="Number of concurrent processes to spawn.",
+        )
+        parser.add_argument(
+            "--no-commit", "-n", action="store_true", default=False,
+            help="Whether to autocommit changes"
+        )
+        parser.add_argument(
+            "--debug", "-d", choices=LOG_LEVELS.keys(),
+            default=logging.getLevelName(logging.WARN),
+            help="Adjust log level"
+        )
+        return parser
 
 
 
@@ -466,54 +511,6 @@ def rewrite_input(
     with open(input_file, "w") as f:
         f.writelines(lines)
 
-# TODO move to Editor ?
-def parse_args(editor: Editor):
-    parser = argparse.ArgumentParser(
-        description=(
-            f"Updates nix derivations for {editor.name} plugins"
-            f"By default from {editor.default_in} to {editor.default_out}"
-        )
-    )
-    parser.add_argument(
-        "--add",
-        dest="add_plugins",
-        default=[],
-        action="append",
-        help=f"Plugin to add to {editor.attr_path} from Github in the form owner/repo",
-    )
-    parser.add_argument(
-        "--input-names",
-        "-i",
-        dest="input_file",
-        default=editor.default_in,
-        help="A list of plugins in the form owner/repo",
-    )
-    parser.add_argument(
-        "--out",
-        "-o",
-        dest="outfile",
-        default=editor.default_out,
-        help="Filename to save generated nix code",
-    )
-    parser.add_argument(
-        "--proc",
-        "-p",
-        dest="proc",
-        type=int,
-        default=30,
-        help="Number of concurrent processes to spawn.",
-    )
-    parser.add_argument(
-        "--no-commit", "-n", action="store_true", default=False,
-        help="Whether to autocommit changes"
-    )
-    parser.add_argument(
-        "--debug", "-d", choices=LOG_LEVELS.keys(),
-        default=logging.getLevelName(logging.WARN),
-        help="Adjust log level"
-    )
-    return parser.parse_args()
-
 
 def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
     repo.index.add([str(f.resolve()) for f in files])
@@ -547,12 +544,10 @@ def get_update(input_file: str, outfile: str, proc: int, editor: Editor):
     return update
 
 
-def update_plugins(editor: Editor):
+def update_plugins(editor: Editor, args):
     """The main entry function of this module. All input arguments are grouped in the `Editor`."""
 
-    args = parse_args(editor)
     log.setLevel(LOG_LEVELS[args.debug])
-
     log.info("Start updating plugins")
     nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
     update = editor.get_update(args.input_file, args.outfile, args.proc)
@@ -581,7 +576,7 @@ def update_plugins(editor: Editor):
         if autocommit:
             commit(
                 nixpkgs_repo,
-                "{editor.attr_path}.{name}: init at {version}".format(
+                "{editor.get_drv_name name}: init at {version}".format(
                     editor=editor.name, name=plugin.normalized_name, version=plugin.version
                 ),
                 [args.outfile, args.input_file],
diff --git a/maintainers/scripts/update-luarocks-packages b/maintainers/scripts/update-luarocks-packages
index 2524b4c9b89e..5123f86e93d0 100755
--- a/maintainers/scripts/update-luarocks-packages
+++ b/maintainers/scripts/update-luarocks-packages
@@ -16,20 +16,17 @@ from dataclasses import dataclass
 import subprocess
 import csv
 import logging
+import textwrap
+from multiprocessing.dummy import Pool
 
-from typing import List
+from typing import List, Tuple
 from pathlib import Path
 
-LOG_LEVELS = {
-    logging.getLevelName(level): level for level in [
-        logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
-}
-
 log = logging.getLogger()
 log.addHandler(logging.StreamHandler())
 
 ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent
-from pluginupdate import Editor, parse_args, update_plugins, PluginDesc, CleanEnvironment
+from pluginupdate import Editor, update_plugins, PluginDesc, CleanEnvironment, LOG_LEVELS, Cache
 
 PKG_LIST="maintainers/scripts/luarocks-packages.csv"
 TMP_FILE="$(mktemp)"
@@ -67,12 +64,11 @@ class LuaEditor(Editor):
     def get_current_plugins(self):
         return []
 
-    def load_plugin_spec(self, input_file) -> List[PluginDesc]:
+    def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
         luaPackages = []
         csvfilename=input_file
         log.info("Loading package descriptions from %s", csvfilename)
 
-
         with open(csvfilename, newline='') as csvfile:
             reader = csv.DictReader(csvfile,)
             for row in reader:
@@ -81,96 +77,115 @@ class LuaEditor(Editor):
                 luaPackages.append(plugin)
         return luaPackages
 
+    def generate_nix(
+        self,
+        results: List[Tuple[LuaPlugin, str]],
+        outfilename: str
+        ):
+
+        with tempfile.NamedTemporaryFile("w+") as f:
+            f.write(HEADER)
+            header2 = textwrap.dedent(
+            # header2 = inspect.cleandoc(
+            """
+                { self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
+                self: super:
+                with self;
+                {
+            """)
+            f.write(header2)
+            for (plugin, nix_expr) in results:
+                f.write(f"{plugin.normalized_name} = {nix_expr}")
+            f.write(FOOTER)
+            f.flush()
+
+            # if everything went fine, move the generated file to its destination
+            # using copy since move doesn't work across disks
+            shutil.copy(f.name, outfilename)
+
+        print(f"updated {outfilename}")
+
     @property
     def attr_path(self):
         return "luaPackages"
 
-    def get_update(self, input_file: str, outfile: str, _: int):
+    def get_update(self, input_file: str, outfile: str, proc: int):
+        cache: Cache = Cache(self.cache_file)
+        _prefetch = generate_pkg_nix
 
         def update() -> dict:
             plugin_specs = self.load_plugin_spec(input_file)
+            sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
+
+            try:
+                pool = Pool(processes=proc)
+                results = pool.map(_prefetch, sorted_plugin_specs)
+            finally:
+                pass
 
-            self.generate_nix(plugin_specs, outfile)
+            self.generate_nix(results, outfile)
 
             redirects = []
             return redirects
 
         return update
 
-    def rewrite_input(self, *args, **kwargs):
+    def rewrite_input(self, input_file: str, *args, **kwargs):
+        # vim plugin reads the file before update but that shouldn't be our case
         # not implemented yet
+        # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
+        # input_file = "toto.csv"
+        # with open(input_file, newline='') as csvfile:
+        #     writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        #     writer.writeheader()
+        #     for row in reader:
+        #         # name,server,version,luaversion,maintainers
+        #         plugin = LuaPlugin(**row)
+        #         luaPackages.append(plugin)
         pass
 
-def generate_nix(
-    plugins: List[LuaPlugin],
-    outfilename: str
-    ):
-    sorted_plugins = sorted(plugins, key=lambda v: v.name.lower())
-
-    # plug = {}
-    # selon le manifest luarocks.org/manifest
-    def _generate_pkg_nix(plug):
-        cmd = [ "luarocks", "nix", plug.name]
-        if plug.server:
-            cmd.append(f"--only-server={plug.server}")
-
-        if plug.maintainers:
-            cmd.append(f"--maintainers={plug.maintainers}")
-
-        if plug.version:
-            cmd.append(plug.version)
-
-        if plug.luaversion:
-            with CleanEnvironment():
-                local_pkgs = str(ROOT.resolve())
-                cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]
-
-                log.debug("running %s", cmd2)
-                lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
-                cmd.append(f"--lua-dir={lua_drv_path}/bin")
-
-        log.debug("running %s", cmd)
-        output = subprocess.check_output(cmd, text=True)
-        return output
-
-    with tempfile.NamedTemporaryFile("w+") as f:
-        f.write(HEADER)
-        f.write("""
-{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
-self: super:
-with self;
-{
-""")
+def generate_pkg_nix(plug: LuaPlugin):
+    '''
+    Generate nix expression for a luarocks package
+    Our cache key associates "p.name-p.version" to its rockspec
+    '''
+    log.debug("Generating nix expression for %s", plug.name)
+    cmd = [ "luarocks", "nix", plug.name]
 
-        for plugin in sorted_plugins:
+    if plug.server:
+        cmd.append(f"--only-server={plug.server}")
 
-            nix_expr = _generate_pkg_nix(plugin)
-            f.write(f"{plugin.normalized_name} = {nix_expr}"
-            )
-        f.write(FOOTER)
-        f.flush()
+    if plug.maintainers:
+        cmd.append(f"--maintainers={plug.maintainers}")
 
-        # if everything went fine, move the generated file to its destination
-        # using copy since move doesn't work across disks
-        shutil.copy(f.name, outfilename)
+    if plug.version:
+        cmd.append(plug.version)
 
-    print(f"updated {outfilename}")
+    if plug.luaversion:
+        with CleanEnvironment():
+            local_pkgs = str(ROOT.resolve())
+            cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]
 
-def load_plugin_spec():
-    pass
+            log.debug("running %s", ' '.join(cmd2))
+            lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
+            cmd.append(f"--lua-dir={lua_drv_path}/bin")
 
+    log.debug("running %s", cmd)
+    output = subprocess.check_output(cmd, text=True)
+    return (plug, output)
 
 def main():
 
-    editor = LuaEditor("lua", ROOT, '', generate_nix,
+    editor = LuaEditor("lua", ROOT, '',
         default_in = ROOT.joinpath(PKG_LIST),
         default_out = ROOT.joinpath(GENERATED_NIXFILE)
         )
 
-    args = parse_args(editor)
+    parser = editor.create_parser()
+    args = parser.parse_args()
     log.setLevel(LOG_LEVELS[args.debug])
 
-    update_plugins(editor)
+    update_plugins(editor, args)
 
 
 if __name__ == "__main__":