From 38593bc3c0ee89bbee1ee58519e7585a356850c6 Mon Sep 17 00:00:00 2001 From: Dominic Shelton Date: Sun, 19 Feb 2023 18:55:13 +1100 Subject: nixos/flipperzero: init --- nixos/modules/hardware/flipperzero.nix | 18 ++++++++++++++++++ nixos/modules/module-list.nix | 1 + 2 files changed, 19 insertions(+) create mode 100644 nixos/modules/hardware/flipperzero.nix (limited to 'nixos') diff --git a/nixos/modules/hardware/flipperzero.nix b/nixos/modules/hardware/flipperzero.nix new file mode 100644 index 000000000000..82f9b76fa3a7 --- /dev/null +++ b/nixos/modules/hardware/flipperzero.nix @@ -0,0 +1,18 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.hardware.flipperzero; + +in + +{ + options.hardware.flipperzero.enable = mkEnableOption (mdDoc "udev rules and software for Flipper Zero devices"); + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.qFlipper ]; + services.udev.packages = [ pkgs.qFlipper ]; + }; +} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 4a0e52f483c2..a19b0f7ea908 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -53,6 +53,7 @@ ./hardware/cpu/intel-sgx.nix ./hardware/device-tree.nix ./hardware/digitalbitbox.nix + ./hardware/flipperzero.nix ./hardware/flirc.nix ./hardware/gkraken.nix ./hardware/gpgsmartcards.nix -- cgit 1.4.1 From 4d3aef762f3c77f0e4040fcc66298b46694a7f6a Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 4 Feb 2023 23:16:30 +0100 Subject: nixos-render-docs: add options commonmark converter the old method of pasting parts of options.json into a markdown document and hoping for the best no longer works now that options.json contains more than just docbook. given the infrastructure we have now we can actually render options.md properly, so we may as well do that. --- nixos/lib/make-options-doc/default.nix | 11 +- nixos/lib/make-options-doc/generateDoc.py | 33 +-- .../src/nixos_render_docs/commonmark.py | 231 +++++++++++++++++++++ .../src/nixos_render_docs/options.py | 76 +++++++ .../nixos-render-docs/src/tests/test_commonmark.py | 92 ++++++++ 5 files changed, 408 insertions(+), 35 deletions(-) create mode 100644 pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py create mode 100644 pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py (limited to 'nixos') diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 50fb9ede08de..a0b9136ca7bc 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -98,11 +98,14 @@ in rec { > $out ''; - optionsCommonMark = pkgs.runCommand "options.md" {} '' - ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ - --format commonmark \ + optionsCommonMark = pkgs.runCommand "options.md" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options commonmark \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ ${optionsJSON}/share/doc/nixos/options.json \ - > $out + $out ''; optionsJSON = pkgs.runCommand "options.json" diff --git a/nixos/lib/make-options-doc/generateDoc.py b/nixos/lib/make-options-doc/generateDoc.py index 07884ed657e4..a41255067bfb 100644 --- a/nixos/lib/make-options-doc/generateDoc.py +++ b/nixos/lib/make-options-doc/generateDoc.py @@ -2,7 +2,7 @@ import argparse import json import sys -formats = ['commonmark', 'asciidoc'] +formats = ['asciidoc'] parser = argparse.ArgumentParser( description = 'Generate documentation for a set of JSON-formatted NixOS options' @@ -38,33 +38,6 @@ class OptionsEncoder(json.JSONEncoder): return super().encode(obj) -def generate_commonmark(options): - for (name, value) in options.items(): - print('##', name.replace('<', '<').replace('>', '>')) - print(value['description']) - print() - if 'type' in value: - print('*_Type_*') - print ('```') - print(value['type']) - print ('```') - print() - print() - if 'default' in value: - print('*_Default_*') - print('```') - print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('```') - print() - print() - if 'example' in value: - print('*_Example_*') - print('```') - print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('```') - print() - print() - # TODO: declarations: link to github def generate_asciidoc(options): for (name, value) in options.items(): @@ -103,9 +76,7 @@ def generate_asciidoc(options): with open(args.nix_options_path) as nix_options_json: options = json.load(nix_options_json) - if args.format == 'commonmark': - generate_commonmark(options) - elif args.format == 'asciidoc': + if args.format == 'asciidoc': generate_asciidoc(options) else: raise Exception(f'Unsupported documentation format `--format {args.format}`') diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py new file mode 100644 index 000000000000..4a708b1f92c6 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py @@ -0,0 +1,231 @@ +from collections.abc import Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from typing import Any, cast, Optional + +from .md import md_escape, md_make_code, Renderer + +import markdown_it +from markdown_it.token import Token +from markdown_it.utils import OptionsDict + +@dataclass(kw_only=True) +class List: + next_idx: Optional[int] = None + compact: bool + first_item_seen: bool = False + +@dataclass +class Par: + indent: str + continuing: bool = False + +class CommonMarkRenderer(Renderer): + __output__ = "commonmark" + + _parstack: list[Par] + _link_stack: list[str] + _list_stack: list[List] + + def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): + super().__init__(manpage_urls, parser) + self._parstack = [ Par("") ] + self._link_stack = [] + self._list_stack = [] + + def _enter_block(self, extra_indent: str) -> None: + self._parstack.append(Par(self._parstack[-1].indent + extra_indent)) + def _leave_block(self) -> None: + self._parstack.pop() + self._parstack[-1].continuing = True + def _break(self) -> str: + self._parstack[-1].continuing = True + return f"\n{self._parstack[-1].indent}" + def _maybe_parbreak(self) -> str: + result = f"\n{self._parstack[-1].indent}" * 2 if self._parstack[-1].continuing else "" + self._parstack[-1].continuing = True + return result + + def _admonition_open(self, kind: str) -> str: + pbreak = self._maybe_parbreak() + self._enter_block("") + return f"{pbreak}**{kind}:** " + def _admonition_close(self) -> str: + self._leave_block() + return "" + + def _indent_raw(self, s: str) -> str: + if '\n' not in s: + return s + return f"\n{self._parstack[-1].indent}".join(s.splitlines()) + + def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return self._indent_raw(md_escape(token.content)) + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._maybe_parbreak() + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f" {self._break()}" + def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return md_make_code(token.content) + def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self.fence(token, tokens, i, options, env) + def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + self._link_stack.append(cast(str, token.attrs['href'])) + return "[" + def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"]({md_escape(self._link_stack.pop())})" + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + lst = self._list_stack[-1] + lbreak = "" if not lst.first_item_seen else self._break() * (1 if lst.compact else 2) + lst.first_item_seen = True + head = " -" + if lst.next_idx is not None: + head = f" {lst.next_idx}." + lst.next_idx += 1 + self._enter_block(" " * (len(head) + 1)) + return f'{lbreak}{head} ' + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append(List(compact=bool(token.meta['compact']))) + return self._maybe_parbreak() + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" + def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "*" + def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "*" + def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + code = token.content + if code.endswith('\n'): + code = code[:-1] + pbreak = self._maybe_parbreak() + return pbreak + self._indent_raw(md_make_code(code, info=token.info, multiline=True)) + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._maybe_parbreak() + self._enter_block("> ") + return pbreak + "> " + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Note") + def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Caution") + def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Important") + def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Tip") + def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Warning") + def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append(List(compact=False)) + return "" + def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" + def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._maybe_parbreak() + self._enter_block(" ") + # add an opening zero-width non-joiner to separate *our* emphasis from possible + # emphasis in the provided term + return f'{pbreak} - *{chr(0x200C)}' + def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"{chr(0x200C)}*" + def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return "" + def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + content = md_make_code(token.content) + if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): + return f"[{content}]({url})" + return content # no roles in regular commonmark + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + # there's no way we can emit attrspans correctly in all cases. we could use inline + # html for ids, but that would not round-trip. same holds for classes. since this + # renderer is only used for approximate options export and all of these things are + # not allowed in options we can ignore them for now. + return "" + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return token.markup + " " + def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "\n" + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append( + List(next_idx = cast(int, token.attrs.get('start', 1)), + compact = bool(token.meta['compact']))) + return self._maybe_parbreak() + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 39ea8f8c627f..d8a24b885f87 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -13,6 +13,7 @@ from xml.sax.saxutils import escape, quoteattr import markdown_it from . import parallel +from .commonmark import CommonMarkRenderer from .docbook import DocBookRenderer, make_xml_id from .manpage import ManpageRenderer, man_escape from .md import Converter, md_escape, md_make_code @@ -422,6 +423,59 @@ class ManpageConverter(BaseConverter): return "\n".join(result) +class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer): + pass + +class CommonMarkConverter(BaseConverter): + __renderer__ = OptionsCommonMarkRenderer + __option_block_separator__ = "" + + def _parallel_render_prepare(self) -> Any: + return (self._manpage_urls, self._revision, self._markdown_by_default) + @classmethod + def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter: + return cls(*a) + + def _render_code(self, option: dict[str, Any], key: str) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if lit := option_is(option, key, 'literalDocBook'): + return [ f"*{key.capitalize()}:* {lit['text']}" ] + else: + return super()._render_code(option, key) + + def _render_description(self, desc: str | dict[str, Any]) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if isinstance(desc, str) and not self._markdown_by_default: + return [ desc ] + else: + return super()._render_description(desc) + + def _related_packages_header(self) -> list[str]: + return [ "*Related packages:*" ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ f"*{header}:*" ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + return [ f" - [{md_escape(name)}]({href})" ] + return [ f" - {md_escape(name)}" ] + + def _decl_def_footer(self) -> list[str]: + return [] + + def finalize(self) -> str: + result = [] + + for (name, opt) in self._sorted_options(): + result.append(f"## {md_escape(name)}\n") + result += opt.lines + result.append("\n\n") + + return "\n".join(result) + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) @@ -437,6 +491,13 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None: p.add_argument("infile") p.add_argument("outfile") +def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: + p.add_argument('--manpage-urls', required=True) + p.add_argument('--revision', required=True) + p.add_argument('--markdown-by-default', default=False, action='store_true') + p.add_argument("infile") + p.add_argument("outfile") + def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter( @@ -464,15 +525,30 @@ def _run_cli_manpage(args: argparse.Namespace) -> None: with open(args.outfile, 'w') as f: f.write(md.finalize()) +def _run_cli_commonmark(args: argparse.Namespace) -> None: + with open(args.manpage_urls, 'r') as manpage_urls: + md = CommonMarkConverter( + json.load(manpage_urls), + revision = args.revision, + markdown_by_default = args.markdown_by_default) + + with open(args.infile, 'r') as f: + md.add_options(json.load(f)) + with open(args.outfile, 'w') as f: + f.write(md.finalize()) + def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) _build_cli_db(formats.add_parser('docbook')) _build_cli_manpage(formats.add_parser('manpage')) + _build_cli_commonmark(formats.add_parser('commonmark')) def run_cli(args: argparse.Namespace) -> None: if args.format == 'docbook': _run_cli_db(args) elif args.format == 'manpage': _run_cli_manpage(args) + elif args.format == 'commonmark': + _run_cli_commonmark(args) else: raise RuntimeError('format not hooked up', args) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py new file mode 100644 index 000000000000..5e0d63eb6723 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py @@ -0,0 +1,92 @@ +import nixos_render_docs + +from sample_md import sample1 + +from typing import Mapping, Optional + +import markdown_it + +class Converter(nixos_render_docs.md.Converter): + __renderer__ = nixos_render_docs.commonmark.CommonMarkRenderer + +# NOTE: in these tests we represent trailing spaces by ` ` and replace them with real space later, +# since a number of editors will strip trailing whitespace on save and that would break the tests. + +def test_indented_fence() -> None: + c = Converter({}) + s = """\ +> - ```foo +> thing +>       +> rest +> ```\ +""".replace(' ', ' ') + assert c._render(s) == s + +def test_full() -> None: + c = Converter({ 'man(1)': 'http://example.org' }) + assert c._render(sample1) == f"""\ +**Warning:** foo + +**Note:** nested + +[ +multiline +](link) + +[` man(1) `](http://example.org) reference + +some nested anchors + +*emph* **strong** *nesting emph **and strong** and ` code `* + + - wide bullet + + - list + + 1. wide ordered + + 2. list + + - narrow bullet + - list + + 1. narrow ordered + 2. list + +> quotes +>  +> > with *nesting* +> >  +> > ``` +> > nested code block +> > ``` +>  +> - and lists +> - ``` +> containing code +> ``` +>  +> and more quote + + 100. list starting at 100 + 101. goes on + + - *‌deflist‌* +    + > with a quote + > and stuff +    + ``` + code block + ``` +    + ``` + fenced block + ``` +    + text + + - *‌more stuff in same deflist‌* +    + foo""".replace(' ', ' ') -- cgit 1.4.1 From 417dd2ad16040e43f14705d99298318708848b3e Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 5 Feb 2023 05:58:14 +0100 Subject: nixos-render-docs: add options asciidoc converter same reasoning as for the earlier commonmark converter. --- nixos/lib/make-options-doc/default.nix | 11 +- nixos/lib/make-options-doc/generateDoc.py | 83 ------- .../src/nixos_render_docs/asciidoc.py | 262 +++++++++++++++++++++ .../src/nixos_render_docs/options.py | 77 ++++++ .../nixos-render-docs/src/tests/test_asciidoc.py | 143 +++++++++++ 5 files changed, 489 insertions(+), 87 deletions(-) delete mode 100644 nixos/lib/make-options-doc/generateDoc.py create mode 100644 pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py create mode 100644 pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py (limited to 'nixos') diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index a0b9136ca7bc..a2385582a014 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -91,11 +91,14 @@ let in rec { inherit optionsNix; - optionsAsciiDoc = pkgs.runCommand "options.adoc" {} '' - ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ - --format asciidoc \ + optionsAsciiDoc = pkgs.runCommand "options.adoc" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ ${optionsJSON}/share/doc/nixos/options.json \ - > $out + $out ''; optionsCommonMark = pkgs.runCommand "options.md" { diff --git a/nixos/lib/make-options-doc/generateDoc.py b/nixos/lib/make-options-doc/generateDoc.py deleted file mode 100644 index a41255067bfb..000000000000 --- a/nixos/lib/make-options-doc/generateDoc.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import json -import sys - -formats = ['asciidoc'] - -parser = argparse.ArgumentParser( - description = 'Generate documentation for a set of JSON-formatted NixOS options' -) -parser.add_argument( - 'nix_options_path', - help = 'a path to a JSON file containing the NixOS options' -) -parser.add_argument( - '-f', - '--format', - choices = formats, - required = True, - help = f'the documentation format to generate' -) - -args = parser.parse_args() - -class OptionsEncoder(json.JSONEncoder): - def encode(self, obj): - # Unpack literal expressions and other Nix types. - # Don't escape the strings: they were escaped when initially serialized to JSON. - if isinstance(obj, dict): - _type = obj.get('_type') - if _type is not None: - if _type == 'literalExpression' or _type == 'literalDocBook': - return obj['text'] - - if _type == 'derivation': - return obj['name'] - - raise Exception(f'Unexpected type `{_type}` in {json.dumps(obj)}') - - return super().encode(obj) - -# TODO: declarations: link to github -def generate_asciidoc(options): - for (name, value) in options.items(): - print(f'== {name}') - print() - print(value['description']) - print() - print('[discrete]') - print('=== details') - print() - print(f'Type:: {value["type"]}') - if 'default' in value: - print('Default::') - print('+') - print('----') - print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('----') - print() - else: - print('No Default:: {blank}') - if value['readOnly']: - print('Read Only:: {blank}') - else: - print() - if 'example' in value: - print('Example::') - print('+') - print('----') - print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('----') - print() - else: - print('No Example:: {blank}') - print() - -with open(args.nix_options_path) as nix_options_json: - options = json.load(nix_options_json) - - if args.format == 'asciidoc': - generate_asciidoc(options) - else: - raise Exception(f'Unsupported documentation format `--format {args.format}`') - diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py new file mode 100644 index 000000000000..637185227e83 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py @@ -0,0 +1,262 @@ +from collections.abc import Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from typing import Any, cast, Optional +from urllib.parse import quote + +from .md import Renderer + +import markdown_it +from markdown_it.token import Token +from markdown_it.utils import OptionsDict + +_asciidoc_escapes = { + # escape all dots, just in case one is pasted at SOL + ord('.'): "{zwsp}.", + # may be replaced by typographic variants + ord("'"): "{apos}", + ord('"'): "{quot}", + # passthrough character + ord('+'): "{plus}", + # table marker + ord('|'): "{vbar}", + # xml entity reference + ord('&'): "{amp}", + # crossrefs. < needs extra escaping because links break in odd ways if they start with it + ord('<'): "{zwsp}+<+{zwsp}", + ord('>'): "{gt}", + # anchors, links, block attributes + ord('['): "{startsb}", + ord(']'): "{endsb}", + # superscript, subscript + ord('^'): "{caret}", + ord('~'): "{tilde}", + # bold + ord('*'): "{asterisk}", + # backslash + ord('\\'): "{backslash}", + # inline code + ord('`'): "{backtick}", +} +def asciidoc_escape(s: str) -> str: + s = s.translate(_asciidoc_escapes) + # :: is deflist item, ;; is has a replacement but no idea why + return s.replace("::", "{two-colons}").replace(";;", "{two-semicolons}") + +@dataclass(kw_only=True) +class List: + head: str + +@dataclass() +class Par: + sep: str + block_delim: str + continuing: bool = False + +class AsciiDocRenderer(Renderer): + __output__ = "asciidoc" + + _parstack: list[Par] + _list_stack: list[List] + _attrspans: list[str] + + def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): + super().__init__(manpage_urls, parser) + self._parstack = [ Par("\n\n", "====") ] + self._list_stack = [] + self._attrspans = [] + + def _enter_block(self, is_list: bool) -> None: + self._parstack.append(Par("\n+\n" if is_list else "\n\n", self._parstack[-1].block_delim + "=")) + def _leave_block(self) -> None: + self._parstack.pop() + def _break(self, force: bool = False) -> str: + result = self._parstack[-1].sep if force or self._parstack[-1].continuing else "" + self._parstack[-1].continuing = True + return result + + def _admonition_open(self, kind: str) -> str: + pbreak = self._break() + self._enter_block(False) + return f"{pbreak}[{kind}]\n{self._parstack[-2].block_delim}\n" + def _admonition_close(self) -> str: + self._leave_block() + return f"\n{self._parstack[-1].block_delim}\n" + + def _list_open(self, token: Token, head: str) -> str: + attrs = [] + if (idx := token.attrs.get('start')) is not None: + attrs.append(f"start={idx}") + if token.meta['compact']: + attrs.append('options="compact"') + if self._list_stack: + head *= len(self._list_stack[0].head) + 1 + self._list_stack.append(List(head=head)) + return f"{self._break()}[{','.join(attrs)}]" + def _list_close(self) -> str: + self._list_stack.pop() + return "" + + def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return asciidoc_escape(token.content) + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return " +\n" + def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f" " + def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"``{asciidoc_escape(token.content)}``" + def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self.fence(token, tokens, i, options, env) + def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}[" + def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "]" + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._enter_block(True) + # allow the next token to be a block or an inline. + return f'\n{self._list_stack[-1].head} {{empty}}' + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "\n" + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_open(token, '*') + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_close() + def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "__" + def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "__" + def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + attrs = f"[source,{token.info}]\n" if token.info else "" + code = token.content + if code.endswith('\n'): + code = code[:-1] + return f"{self._break(True)}{attrs}----\n{code}\n----" + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._break(True) + self._enter_block(False) + return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n" + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return f"\n{self._parstack[-1].block_delim}" + def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("NOTE") + def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("CAUTION") + def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("IMPORTANT") + def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("TIP") + def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("WARNING") + def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"{self._break()}[]" + def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._enter_block(True) + return ":: {empty}" + def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "\n" + def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + content = asciidoc_escape(token.content) + if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): + return f"link:{quote(url, safe='/:')}[{content}]" + return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``" + def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"[[{token.attrs['id']}]]" + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + (id_part, class_part) = ("", "") + if id := token.attrs.get('id'): + id_part = f"[[{id}]]" + if s := token.attrs.get('class'): + if s == 'keycap': + class_part = "kbd:[" + self._attrspans.append("]") + else: + return super().attr_span_begin(token, tokens, i, options, env) + else: + self._attrspans.append("") + return id_part + class_part + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._attrspans.pop() + def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return token.markup.replace("#", "=") + " " + def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "\n" + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_open(token, '.') + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_close() diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index d8a24b885f87..f29d8fdb8968 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -8,11 +8,13 @@ from collections.abc import Mapping, MutableMapping, Sequence from markdown_it.utils import OptionsDict from markdown_it.token import Token from typing import Any, Optional +from urllib.parse import quote from xml.sax.saxutils import escape, quoteattr import markdown_it from . import parallel +from .asciidoc import AsciiDocRenderer, asciidoc_escape from .commonmark import CommonMarkRenderer from .docbook import DocBookRenderer, make_xml_id from .manpage import ManpageRenderer, man_escape @@ -476,6 +478,59 @@ class CommonMarkConverter(BaseConverter): return "\n".join(result) +class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer): + pass + +class AsciiDocConverter(BaseConverter): + __renderer__ = AsciiDocRenderer + __option_block_separator__ = "" + + def _parallel_render_prepare(self) -> Any: + return (self._manpage_urls, self._revision, self._markdown_by_default) + @classmethod + def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter: + return cls(*a) + + def _render_code(self, option: dict[str, Any], key: str) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if lit := option_is(option, key, 'literalDocBook'): + return [ f"*{key.capitalize()}:* {lit['text']}" ] + else: + return super()._render_code(option, key) + + def _render_description(self, desc: str | dict[str, Any]) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if isinstance(desc, str) and not self._markdown_by_default: + return [ desc ] + else: + return super()._render_description(desc) + + def _related_packages_header(self) -> list[str]: + return [ "__Related packages:__" ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ f"__{header}:__\n" ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + return [ f"* link:{quote(href, safe='/:')}[{asciidoc_escape(name)}]" ] + return [ f"* {asciidoc_escape(name)}" ] + + def _decl_def_footer(self) -> list[str]: + return [] + + def finalize(self) -> str: + result = [] + + for (name, opt) in self._sorted_options(): + result.append(f"== {asciidoc_escape(name)}\n") + result += opt.lines + result.append("\n\n") + + return "\n".join(result) + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) @@ -498,6 +553,13 @@ def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: p.add_argument("infile") p.add_argument("outfile") +def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None: + p.add_argument('--manpage-urls', required=True) + p.add_argument('--revision', required=True) + p.add_argument('--markdown-by-default', default=False, action='store_true') + p.add_argument("infile") + p.add_argument("outfile") + def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter( @@ -537,11 +599,24 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None: with open(args.outfile, 'w') as f: f.write(md.finalize()) +def _run_cli_asciidoc(args: argparse.Namespace) -> None: + with open(args.manpage_urls, 'r') as manpage_urls: + md = AsciiDocConverter( + json.load(manpage_urls), + revision = args.revision, + markdown_by_default = args.markdown_by_default) + + with open(args.infile, 'r') as f: + md.add_options(json.load(f)) + with open(args.outfile, 'w') as f: + f.write(md.finalize()) + def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) _build_cli_db(formats.add_parser('docbook')) _build_cli_manpage(formats.add_parser('manpage')) _build_cli_commonmark(formats.add_parser('commonmark')) + _build_cli_asciidoc(formats.add_parser('asciidoc')) def run_cli(args: argparse.Namespace) -> None: if args.format == 'docbook': @@ -550,5 +625,7 @@ def run_cli(args: argparse.Namespace) -> None: _run_cli_manpage(args) elif args.format == 'commonmark': _run_cli_commonmark(args) + elif args.format == 'asciidoc': + _run_cli_asciidoc(args) else: raise RuntimeError('format not hooked up', args) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py new file mode 100644 index 000000000000..487506469954 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py @@ -0,0 +1,143 @@ +import nixos_render_docs + +from sample_md import sample1 + +class Converter(nixos_render_docs.md.Converter): + __renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer + +def test_lists() -> None: + c = Converter({}) + # attaching to the nth ancestor list requires n newlines before the + + assert c._render("""\ +- a + + b +- c + - d + - e + + 1 + + f +""") == """\ +[] +* {empty}a ++ +b + +* {empty}c ++ +[options="compact"] +** {empty}d ++ +[] +** {empty}e ++ +1 + + ++ +f +""" + +def test_full() -> None: + c = Converter({ 'man(1)': 'http://example.org' }) + assert c._render(sample1) == """\ +[WARNING] +==== +foo + +[NOTE] +===== +nested +===== + +==== + + +link:link[ multiline ] + +link:http://example.org[man(1)] reference + +[[b]]some [[a]]nested anchors + +__emph__ **strong** __nesting emph **and strong** and ``code``__ + +[] +* {empty}wide bullet + +* {empty}list + + +[] +. {empty}wide ordered + +. {empty}list + + +[options="compact"] +* {empty}narrow bullet + +* {empty}list + + +[options="compact"] +. {empty}narrow ordered + +. {empty}list + + +[quote] +==== +quotes + +[quote] +===== +with __nesting__ + +---- +nested code block +---- +===== + +[options="compact"] +* {empty}and lists + +* {empty} ++ +---- +containing code +---- + + +and more quote +==== + +[start=100,options="compact"] +. {empty}list starting at 100 + +. {empty}goes on + + +[] + +deflist:: {empty} ++ +[quote] +===== +with a quote and stuff +===== ++ +---- +code block +---- ++ +---- +fenced block +---- ++ +text + + +more stuff in same deflist:: {empty}foo +""" -- cgit 1.4.1