diff options
author | Martin Weinelt <hexa@darmstadt.ccc.de> | 2023-12-02 02:48:15 +0100 |
---|---|---|
committer | Martin Weinelt <hexa@darmstadt.ccc.de> | 2023-12-02 03:22:02 +0100 |
commit | 50311a8d071c7f741f9a4d12736a8025b962ec64 (patch) | |
tree | fbdb5bf5a0a0ad13014d60aea87332683b19194b /pkgs/development/interpreters/python | |
parent | 180ccef717a349e70479325dc56c7ee628b4ccf6 (diff) | |
download | nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar.gz nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar.bz2 nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar.lz nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar.xz nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.tar.zst nixlib-50311a8d071c7f741f9a4d12736a8025b962ec64.zip |
update-python-libraries: format with black/isort
Diffstat (limited to 'pkgs/development/interpreters/python')
-rwxr-xr-x | pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py | 289 |
1 files changed, 164 insertions, 125 deletions
diff --git a/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py b/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py index a7b6b58701fe..ff58c4438057 100755 --- a/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py +++ b/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py @@ -12,23 +12,24 @@ to update all non-pinned libraries in that folder. """ import argparse +import collections import json import logging import os import re -import requests +import subprocess from concurrent.futures import ThreadPoolExecutor as Pool -from packaging.version import Version as _Version -from packaging.version import InvalidVersion +from typing import Any, Optional + +import requests from packaging.specifiers import SpecifierSet -from typing import Optional, Any -import collections -import subprocess +from packaging.version import InvalidVersion +from packaging.version import Version as _Version INDEX = "https://pypi.io/pypi" """url of PyPI""" -EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] +EXTENSIONS = ["tar.gz", "tar.bz2", "tar", "zip", ".whl"] """Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" PRERELEASES = False @@ -37,13 +38,16 @@ BULK_UPDATE = False GIT = "git" -NIXPKGS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip() +NIXPKGS_ROOT = ( + subprocess.check_output(["git", "rev-parse", "--show-toplevel"]) + .decode("utf-8") + .strip() +) logging.basicConfig(level=logging.INFO) class Version(_Version, collections.abc.Sequence): - def __init__(self, version): super().__init__(version) # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21` @@ -65,7 +69,7 @@ def _get_values(attribute, text): :returns: List of matches. """ - regex = fr'{re.escape(attribute)}\s+=\s+"(.*)";' + regex = rf'{re.escape(attribute)}\s+=\s+"(.*)";' regex = re.compile(regex) values = regex.findall(text) return values @@ -73,14 +77,18 @@ def _get_values(attribute, text): def _get_attr_value(attr_path: str) -> Optional[Any]: try: - response = subprocess.check_output([ - "nix", - "--extra-experimental-features", "nix-command", - "eval", - "-f", f"{NIXPKGS_ROOT}/default.nix", - "--json", - f"{attr_path}" - ]) + response = subprocess.check_output( + [ + "nix", + "--extra-experimental-features", + "nix-command", + "eval", + "-f", + f"{NIXPKGS_ROOT}/default.nix", + "--json", + f"{attr_path}", + ] + ) return json.loads(response.decode()) except (subprocess.CalledProcessError, ValueError): return None @@ -100,12 +108,13 @@ def _get_unique_value(attribute, text): else: raise ValueError("no value found for {}".format(attribute)) + def _get_line_and_value(attribute, text, value=None): """Match attribute in text. Return the line and the value of the attribute.""" if value is None: - regex = rf'({re.escape(attribute)}\s+=\s+\"(.*)\";)' + regex = rf"({re.escape(attribute)}\s+=\s+\"(.*)\";)" else: - regex = rf'({re.escape(attribute)}\s+=\s+\"({re.escape(value)})\";)' + regex = rf"({re.escape(attribute)}\s+=\s+\"({re.escape(value)})\";)" regex = re.compile(regex) results = regex.findall(text) n = len(results) @@ -138,7 +147,7 @@ def _fetch_page(url): def _fetch_github(url): headers = {} - token = os.environ.get('GITHUB_API_TOKEN') + token = os.environ.get("GITHUB_API_TOKEN") if token: headers["Authorization"] = f"token {token}" r = requests.get(url, headers=headers) @@ -151,31 +160,26 @@ def _fetch_github(url): def _hash_to_sri(algorithm, value): """Convert a hash to its SRI representation""" - return subprocess.check_output([ - "nix", - "hash", - "to-sri", - "--type", algorithm, - value - ]).decode().strip() + return ( + subprocess.check_output(["nix", "hash", "to-sri", "--type", algorithm, value]) + .decode() + .strip() + ) def _skip_bulk_update(attr_name: str) -> bool: - return bool(_get_attr_value( - f"{attr_name}.skipBulkUpdate" - )) + return bool(_get_attr_value(f"{attr_name}.skipBulkUpdate")) SEMVER = { - 'major' : 0, - 'minor' : 1, - 'patch' : 2, + "major": 0, + "minor": 1, + "patch": 2, } def _determine_latest_version(current_version, target, versions): - """Determine latest version, given `target`. - """ + """Determine latest version, given `target`.""" current_version = Version(current_version) def _parse_versions(versions): @@ -193,7 +197,7 @@ def _determine_latest_version(current_version, target, versions): if len(ceiling) == 0: ceiling = None else: - ceiling[-1]+=1 + ceiling[-1] += 1 ceiling = Version(".".join(map(str, ceiling))) # We do not want prereleases @@ -210,17 +214,19 @@ def _get_latest_version_pypi(package, extension, current_version, target): url = "{}/{}/json".format(INDEX, package) json = _fetch_page(url) - versions = json['releases'].keys() + versions = json["releases"].keys() version = _determine_latest_version(current_version, target, versions) try: - releases = json['releases'][version] + releases = json["releases"][version] except KeyError as e: - raise KeyError('Could not find version {} for {}'.format(version, package)) from e + raise KeyError( + "Could not find version {} for {}".format(version, package) + ) from e for release in releases: - if release['filename'].endswith(extension): + if release["filename"].endswith(extension): # TODO: In case of wheel we need to do further checks! - sha256 = release['digests']['sha256'] + sha256 = release["digests"]["sha256"] break else: sha256 = None @@ -240,33 +246,40 @@ def _get_latest_version_github(package, extension, current_version, target): attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{package}") try: homepage = subprocess.check_output( - ["nix", "eval", "-f", f"{NIXPKGS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\ - .decode('utf-8') + [ + "nix", + "eval", + "-f", + f"{NIXPKGS_ROOT}/default.nix", + "--raw", + f"{attr_path}.src.meta.homepage", + ] + ).decode("utf-8") except Exception as e: raise ValueError(f"Unable to determine homepage: {e}") - owner_repo = homepage[len("https://github.com/"):] # remove prefix + owner_repo = homepage[len("https://github.com/") :] # remove prefix owner, repo = owner_repo.split("/") url = f"https://api.github.com/repos/{owner}/{repo}/releases" all_releases = _fetch_github(url) - releases = list(filter(lambda x: not x['prerelease'], all_releases)) + releases = list(filter(lambda x: not x["prerelease"], all_releases)) if len(releases) == 0: raise ValueError(f"{homepage} does not contain any stable releases") - versions = map(lambda x: strip_prefix(x['tag_name']), releases) + versions = map(lambda x: strip_prefix(x["tag_name"]), releases) version = _determine_latest_version(current_version, target, versions) - release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases)) - prefix = get_prefix(release['tag_name']) + release = next(filter(lambda x: strip_prefix(x["tag_name"]) == version, releases)) + prefix = get_prefix(release["tag_name"]) # some attributes require using the fetchgit git_fetcher_args = [] - if (_get_attr_value(f"{attr_path}.src.fetchSubmodules")): + if _get_attr_value(f"{attr_path}.src.fetchSubmodules"): git_fetcher_args.append("--fetch-submodules") - if (_get_attr_value(f"{attr_path}.src.fetchLFS")): + if _get_attr_value(f"{attr_path}.src.fetchLFS"): git_fetcher_args.append("--fetch-lfs") - if (_get_attr_value(f"{attr_path}.src.leaveDotGit")): + if _get_attr_value(f"{attr_path}.src.leaveDotGit"): git_fetcher_args.append("--leave-dotGit") if git_fetcher_args: @@ -274,8 +287,10 @@ def _get_latest_version_github(package, extension, current_version, target): cmd = [ "nix-prefetch-git", f"https://github.com/{owner}/{repo}.git", - "--hash", algorithm, - "--rev", f"refs/tags/{release['tag_name']}" + "--hash", + algorithm, + "--rev", + f"refs/tags/{release['tag_name']}", ] cmd.extend(git_fetcher_args) response = subprocess.check_output(cmd) @@ -283,45 +298,60 @@ def _get_latest_version_github(package, extension, current_version, target): hash = _hash_to_sri(algorithm, document[algorithm]) else: try: - hash = subprocess.check_output([ - "nix-prefetch-url", - "--type", "sha256", - "--unpack", - f"{release['tarball_url']}" - ], stderr=subprocess.DEVNULL).decode('utf-8').strip() + hash = ( + subprocess.check_output( + [ + "nix-prefetch-url", + "--type", + "sha256", + "--unpack", + f"{release['tarball_url']}", + ], + stderr=subprocess.DEVNULL, + ) + .decode("utf-8") + .strip() + ) except (subprocess.CalledProcessError, UnicodeError): # this may fail if they have both a branch and a tag of the same name, attempt tag name - tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags") - hash = subprocess.check_output([ - "nix-prefetch-url", - "--type", "sha256", - "--unpack", - tag_url - ], stderr=subprocess.DEVNULL).decode('utf-8').strip() + tag_url = str(release["tarball_url"]).replace( + "tarball", "tarball/refs/tags" + ) + hash = ( + subprocess.check_output( + ["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], + stderr=subprocess.DEVNULL, + ) + .decode("utf-8") + .strip() + ) return version, hash, prefix FETCHERS = { - 'fetchFromGitHub' : _get_latest_version_github, - 'fetchPypi' : _get_latest_version_pypi, - 'fetchurl' : _get_latest_version_pypi, + "fetchFromGitHub": _get_latest_version_github, + "fetchPypi": _get_latest_version_pypi, + "fetchurl": _get_latest_version_pypi, } -DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz' +DEFAULT_SETUPTOOLS_EXTENSION = "tar.gz" FORMATS = { - 'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION, - 'wheel' : 'whl', - 'pyproject' : 'tar.gz', - 'flit' : 'tar.gz' + "setuptools": DEFAULT_SETUPTOOLS_EXTENSION, + "wheel": "whl", + "pyproject": "tar.gz", + "flit": "tar.gz", } + def _determine_fetcher(text): # Count occurrences of fetchers. - nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys()) + nfetchers = sum( + text.count("src = {}".format(fetcher)) for fetcher in FETCHERS.keys() + ) if nfetchers == 0: raise ValueError("no fetcher.") elif nfetchers > 1: @@ -329,7 +359,7 @@ def _determine_fetcher(text): else: # Then we check which fetcher to use. for fetcher in FETCHERS.keys(): - if 'src = {}'.format(fetcher) in text: + if "src = {}".format(fetcher) in text: return fetcher @@ -341,47 +371,46 @@ def _determine_extension(text, fetcher): - fetchurl, we determine the extension from the url. - fetchFromGitHub we simply use `.tar.gz`. """ - if fetcher == 'fetchPypi': + if fetcher == "fetchPypi": try: - src_format = _get_unique_value('format', text) + src_format = _get_unique_value("format", text) except ValueError: - src_format = None # format was not given + src_format = None # format was not given try: - extension = _get_unique_value('extension', text) + extension = _get_unique_value("extension", text) except ValueError: - extension = None # extension was not given + extension = None # extension was not given if extension is None: if src_format is None: - src_format = 'setuptools' - elif src_format == 'other': + src_format = "setuptools" + elif src_format == "other": raise ValueError("Don't know how to update a format='other' package.") extension = FORMATS[src_format] - elif fetcher == 'fetchurl': - url = _get_unique_value('url', text) + elif fetcher == "fetchurl": + url = _get_unique_value("url", text) extension = os.path.splitext(url)[1] - if 'pypi' not in url: - raise ValueError('url does not point to PyPI.') + if "pypi" not in url: + raise ValueError("url does not point to PyPI.") - elif fetcher == 'fetchFromGitHub': + elif fetcher == "fetchFromGitHub": extension = "tar.gz" return extension def _update_package(path, target): - # Read the expression - with open(path, 'r') as f: + with open(path, "r") as f: text = f.read() # Determine pname. Many files have more than one pname - pnames = _get_values('pname', text) + pnames = _get_values("pname", text) # Determine version. - version = _get_unique_value('version', text) + version = _get_unique_value("version", text) # First we check how many fetchers are mentioned. fetcher = _determine_fetcher(text) @@ -396,7 +425,9 @@ def _update_package(path, target): elif _get_attr_value(f"python3Packages.{pname}.cargoDeps") is not None: raise ValueError(f"Cargo dependencies are unsupported, skipping {pname}") try: - new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target) + new_version, new_sha256, prefix = FETCHERS[fetcher]( + pname, extension, version, target + ) successful_fetch = True break except ValueError: @@ -413,7 +444,7 @@ def _update_package(path, target): if not new_sha256: raise ValueError("no file available for {}.".format(pname)) - text = _replace_value('version', new_version, text) + text = _replace_value("version", new_version, text) # hashes from pypi are 16-bit encoded sha256's, normalize it to sri to avoid merge conflicts # sri hashes have been the default format since nix 2.4+ @@ -423,16 +454,16 @@ def _update_package(path, target): if old_hash := _get_attr_value(f"python3Packages.{pname}.src.outputHash"): # fetchers can specify a sha256, or a sri hash try: - text = _replace_value('hash', sri_hash, text, old_hash) + text = _replace_value("hash", sri_hash, text, old_hash) except ValueError: - text = _replace_value('sha256', sri_hash, text, old_hash) + text = _replace_value("sha256", sri_hash, text, old_hash) else: raise ValueError(f"Unable to retrieve old hash for {pname}") - if fetcher == 'fetchFromGitHub': + if fetcher == "fetchFromGitHub": # in the case of fetchFromGitHub, it's common to see `rev = version;` or `rev = "v${version}";` # in which no string value is meant to be substituted. However, we can just overwrite the previous value. - regex = r'(rev\s+=\s+[^;]*;)' + regex = r"(rev\s+=\s+[^;]*;)" regex = re.compile(regex) matches = regex.findall(text) n = len(matches) @@ -444,30 +475,33 @@ def _update_package(path, target): match = matches[0] text = text.replace(match, f'rev = "refs/tags/{prefix}${{version}}";') # incase there's no prefix, just rewrite without interpolation - text = text.replace('"${version}";', 'version;') + text = text.replace('"${version}";', "version;") - with open(path, 'w') as f: + with open(path, "w") as f: f.write(text) - logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) + logging.info( + "Path {}: updated {} from {} to {}".format( + path, pname, version, new_version + ) + ) result = { - 'path' : path, - 'target': target, - 'pname': pname, - 'old_version' : version, - 'new_version' : new_version, + "path": path, + "target": target, + "pname": pname, + "old_version": version, + "new_version": new_version, #'fetcher' : fetcher, - } + } return result def _update(path, target): - # We need to read and modify a Nix expression. if os.path.isdir(path): - path = os.path.join(path, 'default.nix') + path = os.path.join(path, "default.nix") # If a default.nix does not exist, we quit. if not os.path.isfile(path): @@ -487,35 +521,41 @@ def _update(path, target): def _commit(path, pname, old_version, new_version, pkgs_prefix="python: ", **kwargs): - """Commit result. - """ + """Commit result.""" - msg = f'{pkgs_prefix}{pname}: {old_version} -> {new_version}' + msg = f"{pkgs_prefix}{pname}: {old_version} -> {new_version}" if changelog := _get_attr_value(f"{pkgs_prefix}{pname}.meta.changelog"): msg += f"\n\n{changelog}" try: - subprocess.check_call([GIT, 'add', path]) - subprocess.check_call([GIT, 'commit', '-m', msg]) + subprocess.check_call([GIT, "add", path]) + subprocess.check_call([GIT, "commit", "-m", msg]) except subprocess.CalledProcessError as e: - subprocess.check_call([GIT, 'checkout', path]) - raise subprocess.CalledProcessError(f'Could not commit {path}') from e + subprocess.check_call([GIT, "checkout", path]) + raise subprocess.CalledProcessError(f"Could not commit {path}") from e return True def main(): - epilog = """ environment variables: GITHUB_API_TOKEN\tGitHub API token used when updating github packages """ - parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog) - parser.add_argument('package', type=str, nargs='+') - parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major') - parser.add_argument('--commit', action='store_true', help='Create a commit for each package update') - parser.add_argument('--use-pkgs-prefix', action='store_true', help='Use python3Packages.${pname}: instead of python: ${pname}: when making commits') + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog + ) + parser.add_argument("package", type=str, nargs="+") + parser.add_argument("--target", type=str, choices=SEMVER.keys(), default="major") + parser.add_argument( + "--commit", action="store_true", help="Create a commit for each package update" + ) + parser.add_argument( + "--use-pkgs-prefix", + action="store_true", + help="Use python3Packages.${pname}: instead of python: ${pname}: when making commits", + ) args = parser.parse_args() target = args.target @@ -550,6 +590,5 @@ environment variables: logging.info("{} package(s) updated".format(count)) - -if __name__ == '__main__': +if __name__ == "__main__": main() |