diff options
Diffstat (limited to 'nixpkgs/maintainers/scripts')
| -rw-r--r-- | nixpkgs/maintainers/scripts/bootstrap-files/README.md | 2 | ||||
| -rwxr-xr-x | nixpkgs/maintainers/scripts/kde/collect-licenses.sh | 31 | ||||
| -rwxr-xr-x | nixpkgs/maintainers/scripts/kde/collect-logs.nu | 11 | ||||
| -rwxr-xr-x | nixpkgs/maintainers/scripts/kde/collect-metadata.py | 36 | ||||
| -rwxr-xr-x | nixpkgs/maintainers/scripts/kde/collect-missing-deps.py | 127 | ||||
| -rwxr-xr-x | nixpkgs/maintainers/scripts/kde/generate-sources.py | 113 | ||||
| -rw-r--r-- | nixpkgs/maintainers/scripts/kde/utils.py | 185 |
7 files changed, 504 insertions, 1 deletions
diff --git a/nixpkgs/maintainers/scripts/bootstrap-files/README.md b/nixpkgs/maintainers/scripts/bootstrap-files/README.md index ae385cbd6ce8..b55878f34192 100644 --- a/nixpkgs/maintainers/scripts/bootstrap-files/README.md +++ b/nixpkgs/maintainers/scripts/bootstrap-files/README.md @@ -39,7 +39,7 @@ target: ``` To validate cross-targets `binfmt` `NixOS` helper can be useful. - For `riscv64-unknown-linux-gnu` the `/etc/nixox/configuraqtion.nix` + For `riscv64-unknown-linux-gnu` the `/etc/nixos/configuration.nix` entry would be `boot.binfmt.emulatedSystems = [ "riscv64-linux" ]`. 3. Propose the commit as a PR to update bootstrap tarballs, tag people diff --git a/nixpkgs/maintainers/scripts/kde/collect-licenses.sh b/nixpkgs/maintainers/scripts/kde/collect-licenses.sh new file mode 100755 index 000000000000..87da901c255c --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/collect-licenses.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p gnutar jq reuse +set -eu +cd "$(dirname "$(readlink -f "$0")")"/../../.. + +TMPDIR=$(mktemp -d) +trap 'rm -rf $TMPDIR' EXIT + +echo "# Prebuilding sources..." +nix-build -A kdePackages.sources --no-link || true + +echo "# Evaluating sources..." +declare -A sources +eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)" + +echo "# Collecting licenses..." +for k in "${!sources[@]}"; do + echo "- Processing $k..." + + if [ ! -f "${sources[$k]}" ]; then + echo "Not found!" + continue + fi + + mkdir "$TMPDIR/$k" + tar -C "$TMPDIR/$k" -xf "${sources[$k]}" + + (cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json" +done + +jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json diff --git a/nixpkgs/maintainers/scripts/kde/collect-logs.nu b/nixpkgs/maintainers/scripts/kde/collect-logs.nu new file mode 100755 index 000000000000..1d07fa9d2caf --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/collect-logs.nu @@ -0,0 +1,11 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i nu -p nushell +cd $"($env.FILE_PWD)/../../.." + +mkdir logs +nix-env -qaP -f . -A kdePackages --json --out-path | from json | values | par-each { |it| + echo $"Processing ($it.pname)..." + if "outputs" in $it { + nix-store --read-log $it.outputs.out | save -f $"logs/($it.pname).log" + } +} diff --git a/nixpkgs/maintainers/scripts/kde/collect-metadata.py b/nixpkgs/maintainers/scripts/kde/collect-metadata.py new file mode 100755 index 000000000000..eaa619647136 --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/collect-metadata.py @@ -0,0 +1,36 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])" +import pathlib + +import click + +import utils + +@click.command +@click.argument( + "repo-metadata", + type=click.Path( + exists=True, + file_okay=False, + resolve_path=True, + path_type=pathlib.Path, + ), +) +@click.option( + "--nixpkgs", + type=click.Path( + exists=True, + file_okay=False, + resolve_path=True, + writable=True, + path_type=pathlib.Path, + ), + default=pathlib.Path(__file__).parent.parent.parent.parent +) +def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path): + metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata) + out_dir = nixpkgs / "pkgs/kde/generated" + metadata.write_json(out_dir) + +if __name__ == "__main__": + main() # type: ignore diff --git a/nixpkgs/maintainers/scripts/kde/collect-missing-deps.py b/nixpkgs/maintainers/scripts/kde/collect-missing-deps.py new file mode 100755 index 000000000000..f3943338b57f --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/collect-missing-deps.py @@ -0,0 +1,127 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 +import pathlib + +OK_MISSING = { + # we don't use precompiled QML + 'Qt6QuickCompiler', + 'Qt6QmlCompilerPlusPrivate', + # usually used for version numbers + 'Git', + # useless by itself, will warn if something else is not found + 'PkgConfig', + # license verification + 'ReuseTool', + # dev only + 'ClangFormat', + # doesn't exist + 'Qt6X11Extras', +} + +OK_MISSING_BY_PACKAGE = { + "angelfish": { + "Qt6Feedback", # we don't have it + }, + "attica": { + "Python3", # only used for license checks + }, + "discover": { + "rpm-ostree-1", # we don't have rpm-ostree (duh) + "Snapd", # we don't have snaps and probably never will + }, + "elisa": { + "UPNPQT", # upstream says it's broken + }, + "extra-cmake-modules": { + "Sphinx", # only used for docs, bloats closure size + "QCollectionGenerator" + }, + "kio-extras-kf5": { + "KDSoapWSDiscoveryClient", # actually vendored on KF5 version + }, + "kitinerary": { + "OsmTools", # used for map data updates, we use prebuilt + }, + "kosmindoormap": { + "OsmTools", # same + "Protobuf", + }, + "kpty": { + "UTEMPTER", # we don't have it and it probably wouldn't work anyway + }, + "kpublictransport": { + "OsmTools", # same + "PolyClipping", + "Protobuf", + }, + "krfb": { + "Qt6XkbCommonSupport", # not real + }, + "kuserfeedback": { + "Qt6Svg", # all used for backend console stuff we don't ship + "QmlLint", + "Qt6Charts", + "FLEX", + "BISON", + "Php", + "PhpUnit", + }, + "kwin": { + "display-info", # newer versions identify as libdisplay-info + }, + "mlt": { + "Qt5", # intentionally disabled + "SWIG", + }, + "plasma-desktop": { + "scim", # upstream is dead, not packaged in Nixpkgs + }, + "powerdevil": { + "DDCUtil", # cursed, intentionally disabled + }, + "pulseaudio-qt": { + "Qt6Qml", # tests only + "Qt6Quick", + }, + "syntax-highlighting": { + "XercesC", # only used for extra validation at build time + } +} + +def main(): + here = pathlib.Path(__file__).parent.parent.parent.parent + logs = (here / "logs").glob("*.log") + + for log in sorted(logs): + pname = log.stem + + missing = [] + is_in_block = False + with log.open(errors="replace") as fd: + for line in fd: + line = line.strip() + if line.startswith("-- No package '"): + package = line.removeprefix("-- No package '").removesuffix("' found") + missing.append(package) + if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:": + is_in_block = True + elif line.startswith("--") and is_in_block: + is_in_block = False + elif line.startswith("*") and is_in_block: + package = line.removeprefix("* ") + missing.append(package) + + missing = { + package + for package in missing + if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set())) + } + + if missing: + print(pname + ":") + for line in missing: + print(" -", line) + print() + +if __name__ == '__main__': + main() diff --git a/nixpkgs/maintainers/scripts/kde/generate-sources.py b/nixpkgs/maintainers/scripts/kde/generate-sources.py new file mode 100755 index 000000000000..e9f8c41ef4d7 --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/generate-sources.py @@ -0,0 +1,113 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.pyyaml ]) +import base64 +import binascii +import json +import pathlib +from urllib.parse import urlparse + +import bs4 +import click +import httpx +import jinja2 + +import utils + + +LEAF_TEMPLATE = jinja2.Template(''' +{mkKdeDerivation}: +mkKdeDerivation { + pname = "{{ pname }}"; +} +'''.strip()) + +ROOT_TEMPLATE = jinja2.Template(''' +{callPackage}: { + {%- for p in packages %} + {{ p }} = callPackage ./{{ p }} {}; + {%- endfor %} +} +'''.strip()); + +def to_sri(hash): + raw = binascii.unhexlify(hash) + b64 = base64.b64encode(raw).decode() + return f"sha256-{b64}" + + +@click.command +@click.argument( + "set", + type=click.Choice(["frameworks", "gear", "plasma"]), + required=True +) +@click.argument( + "version", + type=str, + required=True +) +@click.option( + "--nixpkgs", + type=click.Path( + exists=True, + file_okay=False, + resolve_path=True, + writable=True, + path_type=pathlib.Path, + ), + default=pathlib.Path(__file__).parent.parent.parent.parent +) +def main(set: str, version: str, nixpkgs: pathlib.Path): + root_dir = nixpkgs / "pkgs/kde" + set_dir = root_dir / set + generated_dir = root_dir / "generated" + metadata = utils.KDERepoMetadata.from_json(generated_dir) + + set_url = { + "frameworks": "kf", + "gear": "releases", + "plasma": "plasma", + }[set] + + sources = httpx.get(f"https://kde.org/info/sources/source-{set_url}-{version}.html") + sources.raise_for_status() + bs = bs4.BeautifulSoup(sources.text, features="html.parser") + + results = {} + for item in bs.select("tr")[1:]: + link = item.select_one("td:nth-child(1) a") + assert link + + hash = item.select_one("td:nth-child(3) tt") + assert hash + + project_name, version = link.text.rsplit("-", maxsplit=1) + if project_name not in metadata.projects_by_name: + print(f"Warning: unknown tarball: {project_name}") + + results[project_name] = { + "version": version, + "url": "mirror://kde" + urlparse(link.attrs["href"]).path, + "hash": to_sri(hash.text) + } + + pkg_dir = set_dir / project_name + pkg_file = pkg_dir / "default.nix" + if not pkg_file.exists(): + print(f"Generated new package: {set}/{project_name}") + pkg_dir.mkdir(parents=True, exist_ok=True) + with pkg_file.open("w") as fd: + fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n") + + set_dir.mkdir(parents=True, exist_ok=True) + with (set_dir / "default.nix").open("w") as fd: + fd.write(ROOT_TEMPLATE.render(packages=results.keys()) + "\n") + + sources_dir = generated_dir / "sources" + sources_dir.mkdir(parents=True, exist_ok=True) + with (sources_dir / f"{set}.json").open("w") as fd: + json.dump(results, fd, indent=2) + + +if __name__ == "__main__": + main() # type: ignore diff --git a/nixpkgs/maintainers/scripts/kde/utils.py b/nixpkgs/maintainers/scripts/kde/utils.py new file mode 100644 index 000000000000..7a82c4955c6b --- /dev/null +++ b/nixpkgs/maintainers/scripts/kde/utils.py @@ -0,0 +1,185 @@ +import collections +import dataclasses +import functools +import json +import pathlib +import subprocess + +import yaml + +class DataclassEncoder(json.JSONEncoder): + def default(self, it): + if dataclasses.is_dataclass(it): + return dataclasses.asdict(it) + return super().default(it) + + +@dataclasses.dataclass +class Project: + name: str + description: str | None + project_path: str + repo_path: str | None + + def __hash__(self) -> int: + return hash(self.name) + + @classmethod + def from_yaml(cls, path: pathlib.Path): + data = yaml.safe_load(path.open()) + return cls( + name=data["identifier"], + description=data["description"], + project_path=data["projectpath"], + repo_path=data["repopath"] + ) + + +def get_git_commit(path: pathlib.Path): + return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip() + + +def validate_unique(projects: list[Project], attr: str): + seen = set() + for item in projects: + attr_value = getattr(item, attr) + if attr_value in seen: + raise Exception(f"Duplicate {attr}: {attr_value}") + seen.add(attr_value) + + +THIRD_PARTY = { + "third-party/appstream": "appstream-qt", + "third-party/cmark": "cmark", + "third-party/gpgme": "gpgme", + "third-party/kdsoap": "kdsoap", + "third-party/libaccounts-qt": "accounts-qt", + "third-party/libgpg-error": "libgpg-error", + "third-party/libquotient": "libquotient", + "third-party/packagekit-qt": "packagekit-qt", + "third-party/poppler": "poppler", + "third-party/qcoro": "qcoro", + "third-party/qmltermwidget": "qmltermwidget", + "third-party/qtkeychain": "qtkeychain", + "third-party/signond": "signond", + "third-party/taglib": "taglib", + "third-party/wayland-protocols": "wayland-protocols", + "third-party/wayland": "wayland", + "third-party/zxing-cpp": "zxing-cpp", +} + +IGNORE = { + "kdesupport/phonon-directshow", + "kdesupport/phonon-mmf", + "kdesupport/phonon-mplayer", + "kdesupport/phonon-quicktime", + "kdesupport/phonon-waveout", + "kdesupport/phonon-xine" +} + +WARNED = set() + + +@dataclasses.dataclass +class KDERepoMetadata: + version: str + projects: list[Project] + dep_graph: dict[Project, set[Project]] + + @functools.cached_property + def projects_by_name(self): + return {p.name: p for p in self.projects} + + @functools.cached_property + def projects_by_path(self): + return {p.project_path: p for p in self.projects} + + def try_lookup_package(self, path): + if path in IGNORE: + return None + project = self.projects_by_path.get(path) + if project is None and path not in WARNED: + WARNED.add(path) + print(f"Warning: unknown project {path}") + return project + + @classmethod + def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path): + projects = [ + Project.from_yaml(metadata_file) + for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml") + ] + [ + Project(id, None, project_path, None) + for project_path, id in THIRD_PARTY.items() + ] + + validate_unique(projects, "name") + validate_unique(projects, "project_path") + + self = cls( + version=get_git_commit(repo_metadata), + projects=projects, + dep_graph={}, + ) + + dep_specs = [ + "dependency-data-common", + "dependency-data-kf6-qt6" + ] + dep_graph = collections.defaultdict(set) + + for spec in dep_specs: + spec_path = repo_metadata / "dependencies" / spec + for line in spec_path.open(): + line = line.strip() + if line.startswith("#"): + continue + if not line: + continue + + dependent, dependency = line.split(": ") + + dependent = self.try_lookup_package(dependent) + if dependent is None: + continue + + dependency = self.try_lookup_package(dependency) + if dependency is None: + continue + + dep_graph[dependent].add(dependency) + + self.dep_graph = dep_graph + + return self + + def write_json(self, root: pathlib.Path): + root.mkdir(parents=True, exist_ok=True) + + with (root / "projects.json").open("w") as fd: + json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2) + + with (root / "dependencies.json").open("w") as fd: + deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()} + json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2) + + @classmethod + def from_json(cls, root: pathlib.Path): + projects = [ + Project(**v) for v in json.load((root / "projects.json").open()).values() + ] + + deps = json.load((root / "dependencies.json").open()) + self = cls( + version=deps["version"], + projects=projects, + dep_graph={}, + ) + + dep_graph = collections.defaultdict(set) + for dependent, dependencies in deps["dependencies"].items(): + for dependency in dependencies: + dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency]) + + self.dep_graph = dep_graph + return self |