diff options
author | Jan Tojnar <jtojnar@gmail.com> | 2019-04-12 19:32:44 +0200 |
---|---|---|
committer | Jan Tojnar <jtojnar@gmail.com> | 2020-09-20 20:11:22 +0200 |
commit | 1efc042d92c0f20be6261be73e462789596fff09 (patch) | |
tree | 5eb9104f3710c720de30c294310f9a6f916debc8 /maintainers/scripts | |
parent | d351cea9f3a041c88d79fe9be5467bc7faabb4a4 (diff) | |
download | nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar.gz nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar.bz2 nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar.lz nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar.xz nixlib-1efc042d92c0f20be6261be73e462789596fff09.tar.zst nixlib-1efc042d92c0f20be6261be73e462789596fff09.zip |
maintainers/scripts/update.nix: Add support for auto-commiting changes
Update scripts can now declare features using passthru.updateScript = { command = [ ../../update.sh pname ]; supportedFeatures = [ "commit" ]; }; A `commit` feature means that when the update script finishes successfully, it will print a JSON list like the following: [ { "attrPath": "volume_key", "oldVersion": "0.3.11", "newVersion": "0.3.12", "files": [ "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix" ] } ] and data from that will be used when update.nix is run with --argstr commit true to create commits. We will create a new git worktree for each thread in the pool and run the update script there. Then we will commit the change and cherry pick it in the main repo, releasing the worktree for a next change.
Diffstat (limited to 'maintainers/scripts')
-rwxr-xr-x | maintainers/scripts/update.nix | 12 | ||||
-rw-r--r-- | maintainers/scripts/update.py | 55 |
2 files changed, 58 insertions, 9 deletions
diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 9568c6cbbccd..e11e2450bd0a 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -4,6 +4,7 @@ , max-workers ? null , include-overlays ? false , keep-going ? null +, commit ? null }: # TODO: add assert statements @@ -132,19 +133,26 @@ let --argstr keep-going true to continue running when a single update fails. + + You can also make the updater automatically commit on your behalf from updateScripts + that support it by adding + + --argstr commit true ''; packageData = package: { name = package.name; pname = lib.getName package; - updateScript = map builtins.toString (lib.toList package.updateScript); + updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript)); + supportedFeatures = package.updateScript.supportedFeatures or []; }; packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages)); optionalArgs = lib.optional (max-workers != null) "--max-workers=${max-workers}" - ++ lib.optional (keep-going == "true") "--keep-going"; + ++ lib.optional (keep-going == "true") "--keep-going" + ++ lib.optional (commit == "true") "--commit"; args = [ packagesJson ] ++ optionalArgs; diff --git a/maintainers/scripts/update.py b/maintainers/scripts/update.py index b9e17736bed3..b440f9defe43 100644 --- a/maintainers/scripts/update.py +++ b/maintainers/scripts/update.py @@ -1,22 +1,45 @@ import argparse +import contextlib import concurrent.futures import json import os import subprocess import sys +import tempfile +import threading updates = {} +thread_name_prefix='UpdateScriptThread' + def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def run_update_script(package): +def run_update_script(package, commit): + if commit and 'commit' in package['supportedFeatures']: + thread_name = threading.current_thread().name + worktree, _branch, lock = temp_dirs[thread_name] + lock.acquire() + package['thread'] = thread_name + else: + worktree = None + eprint(f" - {package['name']}: UPDATING ...") - subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree) +@contextlib.contextmanager +def make_worktree(): + with tempfile.TemporaryDirectory() as wt: + branch_name = f'update-{os.path.basename(wt)}' + target_directory = f'{wt}/nixpkgs' -def main(max_workers, keep_going, packages): + subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True) + yield (target_directory, branch_name) + subprocess.run(['git', 'worktree', 'remove', target_directory], check=True) + subprocess.run(['git', 'branch', '-D', branch_name], check=True) + +def main(max_workers, keep_going, commit, packages): with open(sys.argv[1]) as f: packages = json.load(f) @@ -31,15 +54,29 @@ def main(max_workers, keep_going, packages): eprint() eprint('Running update for:') - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor: + global temp_dirs + + if commit: + temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)} + for package in packages: - updates[executor.submit(run_update_script, package)] = package + updates[executor.submit(run_update_script, package, commit)] = package for future in concurrent.futures.as_completed(updates): package = updates[future] try: - future.result() + p = future.result() + if commit and 'commit' in package['supportedFeatures']: + thread_name = package['thread'] + worktree, branch, lock = temp_dirs[thread_name] + changes = json.loads(p.stdout) + for change in changes: + subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree) + commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change) + subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree) + subprocess.run(['git', 'cherry-pick', branch], check=True) eprint(f" - {package['name']}: DONE.") except subprocess.CalledProcessError as e: eprint(f" - {package['name']}: ERROR") @@ -54,6 +91,9 @@ def main(max_workers, keep_going, packages): if not keep_going: sys.exit(1) + finally: + if commit and 'commit' in package['supportedFeatures']: + lock.release() eprint() eprint('Packages updated!') @@ -65,13 +105,14 @@ def main(max_workers, keep_going, packages): parser = argparse.ArgumentParser(description='Update packages') parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4) parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure') +parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes') parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts') if __name__ == '__main__': args = parser.parse_args() try: - main(args.max_workers, args.keep_going, args.packages) + main(args.max_workers, args.keep_going, args.commit, args.packages) except (KeyboardInterrupt, SystemExit) as e: for update in updates: update.cancel() |