diff options
Diffstat (limited to 'nixpkgs/pkgs/pkgs-lib')
16 files changed, 1576 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/pkgs-lib/default.nix b/nixpkgs/pkgs/pkgs-lib/default.nix new file mode 100644 index 000000000000..113dcebf8c68 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/default.nix @@ -0,0 +1,11 @@ +# pkgs-lib is for functions and values that can't be in lib because +# they depend on some packages. This notably is *not* for supporting package +# building, instead pkgs/build-support is the place for that. +{ lib, pkgs }: { + # setting format types and generators. These do not fit in lib/types.nix, + # because they depend on pkgs for rendering some formats + formats = import ./formats.nix { + inherit lib pkgs; + }; +} + diff --git a/nixpkgs/pkgs/pkgs-lib/formats.nix b/nixpkgs/pkgs/pkgs-lib/formats.nix new file mode 100644 index 000000000000..3cbda3a7ebdd --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats.nix @@ -0,0 +1,462 @@ +{ lib, pkgs }: +rec { + + /* + + Every following entry represents a format for program configuration files + used for `settings`-style options (see https://github.com/NixOS/rfcs/pull/42). + Each entry should look as follows: + + <format> = <parameters>: { + # ^^ Parameters for controlling the format + + # The module system type most suitable for representing such a format + # The description needs to be overwritten for recursive types + type = ...; + + # Utility functions for convenience, or special interactions with the + # format (optional) + lib = { + exampleFunction = ... + # Types specific to the format (optional) + types = { ... }; + ... + }; + + # generate :: Name -> Value -> Path + # A function for generating a file with a value of such a type + generate = ...; + + }); + + Please note that `pkgs` may not always be available for use due to the split + options doc build introduced in fc614c37c653, so lazy evaluation of only the + 'type' field is required. + + */ + + + inherit (import ./formats/java-properties/default.nix { inherit lib pkgs; }) + javaProperties; + + libconfig = (import ./formats/libconfig/default.nix { inherit lib pkgs; }).format; + + json = {}: { + + type = with lib.types; let + valueType = nullOr (oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "JSON value"; + }; + in valueType; + + generate = name: value: pkgs.callPackage ({ runCommand, jq }: runCommand name { + nativeBuildInputs = [ jq ]; + value = builtins.toJSON value; + passAsFile = [ "value" ]; + } '' + jq . "$valuePath"> $out + '') {}; + + }; + + yaml = {}: { + + generate = name: value: pkgs.callPackage ({ runCommand, remarshal }: runCommand name { + nativeBuildInputs = [ remarshal ]; + value = builtins.toJSON value; + passAsFile = [ "value" ]; + } '' + json2yaml "$valuePath" "$out" + '') {}; + + type = with lib.types; let + valueType = nullOr (oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "YAML value"; + }; + in valueType; + + }; + + ini = { + # Represents lists as duplicate keys + listsAsDuplicateKeys ? false, + # Alternative to listsAsDuplicateKeys, converts list to non-list + # listToValue :: [IniAtom] -> IniAtom + listToValue ? null, + ... + }@args: + assert !listsAsDuplicateKeys || listToValue == null; + { + + type = with lib.types; let + + singleIniAtom = nullOr (oneOf [ + bool + int + float + str + ]) // { + description = "INI atom (null, bool, int, float or string)"; + }; + + iniAtom = + if listsAsDuplicateKeys then + coercedTo singleIniAtom lib.singleton (listOf singleIniAtom) // { + description = singleIniAtom.description + " or a list of them for duplicate keys"; + } + else if listToValue != null then + coercedTo singleIniAtom lib.singleton (nonEmptyListOf singleIniAtom) // { + description = singleIniAtom.description + " or a non-empty list of them"; + } + else + singleIniAtom; + + in attrsOf (attrsOf iniAtom); + + generate = name: value: + let + transformedValue = + if listToValue != null + then + lib.mapAttrs (section: lib.mapAttrs (key: val: + if lib.isList val then listToValue val else val + )) value + else value; + in pkgs.writeText name (lib.generators.toINI (removeAttrs args ["listToValue"]) transformedValue); + + }; + + keyValue = { + # Represents lists as duplicate keys + listsAsDuplicateKeys ? false, + # Alternative to listsAsDuplicateKeys, converts list to non-list + # listToValue :: [Atom] -> Atom + listToValue ? null, + ... + }@args: + assert !listsAsDuplicateKeys || listToValue == null; + { + + type = with lib.types; let + + singleAtom = nullOr (oneOf [ + bool + int + float + str + ]) // { + description = "atom (null, bool, int, float or string)"; + }; + + atom = + if listsAsDuplicateKeys then + coercedTo singleAtom lib.singleton (listOf singleAtom) // { + description = singleAtom.description + " or a list of them for duplicate keys"; + } + else if listToValue != null then + coercedTo singleAtom lib.singleton (nonEmptyListOf singleAtom) // { + description = singleAtom.description + " or a non-empty list of them"; + } + else + singleAtom; + + in attrsOf atom; + + generate = name: value: + let + transformedValue = + if listToValue != null + then + lib.mapAttrs (key: val: + if lib.isList val then listToValue val else val + ) value + else value; + in pkgs.writeText name (lib.generators.toKeyValue (removeAttrs args ["listToValue"]) transformedValue); + + }; + + gitIni = { listsAsDuplicateKeys ? false, ... }@args: { + + type = with lib.types; let + + iniAtom = (ini args).type/*attrsOf*/.functor.wrapped/*attrsOf*/.functor.wrapped; + + in attrsOf (attrsOf (either iniAtom (attrsOf iniAtom))); + + generate = name: value: pkgs.writeText name (lib.generators.toGitINI value); + }; + + toml = {}: json {} // { + type = with lib.types; let + valueType = oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ] // { + description = "TOML value"; + }; + in valueType; + + generate = name: value: pkgs.callPackage ({ runCommand, remarshal }: runCommand name { + nativeBuildInputs = [ remarshal ]; + value = builtins.toJSON value; + passAsFile = [ "value" ]; + } '' + json2toml "$valuePath" "$out" + '') {}; + + }; + + /* For configurations of Elixir project, like config.exs or runtime.exs + + Most Elixir project are configured using the [Config] Elixir DSL + + Since Elixir has more types than Nix, we need a way to map Nix types to + more than 1 Elixir type. To that end, this format provides its own library, + and its own set of types. + + To be more detailed, a Nix attribute set could correspond in Elixir to a + [Keyword list] (the more common type), or it could correspond to a [Map]. + + A Nix string could correspond in Elixir to a [String] (also called + "binary"), an [Atom], or a list of chars (usually discouraged). + + A Nix array could correspond in Elixir to a [List] or a [Tuple]. + + Some more types exists, like records, regexes, but since they are less used, + we can leave the `mkRaw` function as an escape hatch. + + For more information on how to use this format in modules, please refer to + the Elixir section of the Nixos documentation. + + TODO: special Elixir values doesn't show up nicely in the documentation + + [Config]: <https://hexdocs.pm/elixir/Config.html> + [Keyword list]: <https://hexdocs.pm/elixir/Keyword.html> + [Map]: <https://hexdocs.pm/elixir/Map.html> + [String]: <https://hexdocs.pm/elixir/String.html> + [Atom]: <https://hexdocs.pm/elixir/Atom.html> + [List]: <https://hexdocs.pm/elixir/List.html> + [Tuple]: <https://hexdocs.pm/elixir/Tuple.html> + */ + elixirConf = { elixir ? pkgs.elixir }: + with lib; let + toElixir = value: with builtins; + if value == null then "nil" else + if value == true then "true" else + if value == false then "false" else + if isInt value || isFloat value then toString value else + if isString value then string value else + if isAttrs value then attrs value else + if isList value then list value else + abort "formats.elixirConf: should never happen (value = ${value})"; + + escapeElixir = escape [ "\\" "#" "\"" ]; + string = value: "\"${escapeElixir value}\""; + + attrs = set: + if set ? _elixirType then specialType set + else + let + toKeyword = name: value: "${name}: ${toElixir value}"; + keywordList = concatStringsSep ", " (mapAttrsToList toKeyword set); + in + "[" + keywordList + "]"; + + listContent = values: concatStringsSep ", " (map toElixir values); + + list = values: "[" + (listContent values) + "]"; + + specialType = { value, _elixirType }: + if _elixirType == "raw" then value else + if _elixirType == "atom" then value else + if _elixirType == "map" then elixirMap value else + if _elixirType == "tuple" then tuple value else + abort "formats.elixirConf: should never happen (_elixirType = ${_elixirType})"; + + elixirMap = set: + let + toEntry = name: value: "${toElixir name} => ${toElixir value}"; + entries = concatStringsSep ", " (mapAttrsToList toEntry set); + in + "%{${entries}}"; + + tuple = values: "{${listContent values}}"; + + toConf = values: + let + keyConfig = rootKey: key: value: + "config ${rootKey}, ${key}, ${toElixir value}"; + keyConfigs = rootKey: values: mapAttrsToList (keyConfig rootKey) values; + rootConfigs = flatten (mapAttrsToList keyConfigs values); + in + '' + import Config + + ${concatStringsSep "\n" rootConfigs} + ''; + in + { + type = with lib.types; let + valueType = nullOr + (oneOf [ + bool + int + float + str + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "Elixir value"; + }; + in + attrsOf (attrsOf (valueType)); + + lib = + let + mkRaw = value: { + inherit value; + _elixirType = "raw"; + }; + + in + { + inherit mkRaw; + + /* Fetch an environment variable at runtime, with optional fallback + */ + mkGetEnv = { envVariable, fallback ? null }: + mkRaw "System.get_env(${toElixir envVariable}, ${toElixir fallback})"; + + /* Make an Elixir atom. + + Note: lowercase atoms still need to be prefixed by ':' + */ + mkAtom = value: { + inherit value; + _elixirType = "atom"; + }; + + /* Make an Elixir tuple out of a list. + */ + mkTuple = value: { + inherit value; + _elixirType = "tuple"; + }; + + /* Make an Elixir map out of an attribute set. + */ + mkMap = value: { + inherit value; + _elixirType = "map"; + }; + + /* Contains Elixir types. Every type it exports can also be replaced + by raw Elixir code (i.e. every type is `either type rawElixir`). + + It also reexports standard types, wrapping them so that they can + also be raw Elixir. + */ + types = with lib.types; let + isElixirType = type: x: (x._elixirType or "") == type; + + rawElixir = mkOptionType { + name = "rawElixir"; + description = "raw elixir"; + check = isElixirType "raw"; + }; + + elixirOr = other: either other rawElixir; + in + { + inherit rawElixir elixirOr; + + atom = elixirOr (mkOptionType { + name = "elixirAtom"; + description = "elixir atom"; + check = isElixirType "atom"; + }); + + tuple = elixirOr (mkOptionType { + name = "elixirTuple"; + description = "elixir tuple"; + check = isElixirType "tuple"; + }); + + map = elixirOr (mkOptionType { + name = "elixirMap"; + description = "elixir map"; + check = isElixirType "map"; + }); + # Wrap standard types, since anything in the Elixir configuration + # can be raw Elixir + } // lib.mapAttrs (_name: type: elixirOr type) lib.types; + }; + + generate = name: value: pkgs.runCommand name + { + value = toConf value; + passAsFile = [ "value" ]; + nativeBuildInputs = [ elixir ]; + } '' + cp "$valuePath" "$out" + mix format "$out" + ''; + }; + + # Outputs a succession of Python variable assignments + # Useful for many Django-based services + pythonVars = {}: { + type = with lib.types; let + valueType = nullOr(oneOf [ + bool + float + int + path + str + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "Python value"; + }; + in attrsOf valueType; + generate = name: value: pkgs.callPackage ({ runCommand, python3, black }: runCommand name { + nativeBuildInputs = [ python3 black ]; + value = builtins.toJSON value; + pythonGen = '' + import json + import os + + with open(os.environ["valuePath"], "r") as f: + for key, value in json.load(f).items(): + print(f"{key} = {repr(value)}") + ''; + passAsFile = [ "value" "pythonGen" ]; + } '' + cat "$valuePath" + python3 "$pythonGenPath" > $out + black $out + '') {}; + }; + +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/java-properties/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/default.nix new file mode 100644 index 000000000000..d3a4761f0f80 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/default.nix @@ -0,0 +1,132 @@ +{ lib, pkgs }: +let + inherit (lib) types; + inherit (types) attrsOf oneOf coercedTo str bool int float package; +in +{ + javaProperties = { comment ? "Generated with Nix", boolToString ? lib.boolToString }: { + + # Design note: + # A nested representation of inevitably leads to bad UX: + # 1. keys like "a.b" must be disallowed, or + # the addition of options in a freeformType module + # become breaking changes + # 2. adding a value for "a" after "a"."b" was already + # defined leads to a somewhat hard to understand + # Nix error, because that's not something you can + # do with attrset syntax. Workaround: "a"."", but + # that's too little too late. Another workaround: + # mkMerge [ { a = ...; } { a.b = ...; } ]. + # + # Choosing a non-nested representation does mean that + # we sacrifice the ability to override at the (conceptual) + # hierarchical levels, _if_ an application exhibits those. + # + # Some apps just use periods instead of spaces in an odd + # mix of attempted categorization and natural language, + # with no meaningful hierarchy. + # + # We _can_ choose to support hierarchical config files + # via nested attrsets, but the module author should + # make sure that problem (2) does not occur. + type = let + elemType = + oneOf ([ + # `package` isn't generalized to `path` because path values + # are ambiguous. Are they host path strings (toString /foo/bar) + # or should they be added to the store? ("${/foo/bar}") + # The user must decide. + (coercedTo package toString str) + + (coercedTo bool boolToString str) + (coercedTo int toString str) + (coercedTo float toString str) + ]) + // { description = "string, package, bool, int or float"; }; + in attrsOf elemType; + + generate = name: value: + pkgs.runCommandLocal name + { + # Requirements + # ============ + # + # 1. Strings in Nix carry over to the same + # strings in Java => need proper escapes + # 2. Generate files quickly + # - A JVM would have to match the app's + # JVM to avoid build closure bloat + # - Even then, JVM startup would slow + # down config generation. + # + # + # Implementation + # ============== + # + # Escaping has two steps + # + # 1. jq + # Escape known separators, in order not + # to break up the keys and values. + # This handles typical whitespace correctly, + # but may produce garbage for other control + # characters. + # + # 2. iconv + # Escape >ascii code points to java escapes, + # as .properties files are supposed to be + # encoded in ISO 8859-1. It's an old format. + # UTF-8 behavior may exist in some apps and + # libraries, but we can't rely on this in + # general. + + passAsFile = [ "value" ]; + value = builtins.toJSON value; + nativeBuildInputs = [ + pkgs.jq + pkgs.libiconvReal + ]; + + jqCode = + let + main = '' + to_entries + | .[] + | "\( + .key + | ${commonEscapes} + | gsub(" "; "\\ ") + | gsub("="; "\\=") + ) = \( + .value + | ${commonEscapes} + | gsub("^ "; "\\ ") + | gsub("\\n "; "\n\\ ") + )" + ''; + # Most escapes are equal for both keys and values. + commonEscapes = '' + gsub("\\\\"; "\\\\") + | gsub("\\n"; "\\n\\\n") + | gsub("#"; "\\#") + | gsub("!"; "\\!") + | gsub("\\t"; "\\t") + | gsub("\r"; "\\r") + ''; + in + main; + + inputEncoding = "UTF-8"; + + inherit comment; + + } '' + ( + echo "$comment" | while read -r ln; do echo "# $ln"; done + echo + jq -r --arg hash '#' "$jqCode" "$valuePath" \ + | iconv --from-code "$inputEncoding" --to-code JAVA \ + ) > "$out" + ''; + }; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/Main.java b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/Main.java new file mode 100644 index 000000000000..dc83944f24b0 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/Main.java @@ -0,0 +1,27 @@ +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.SortedSet; +import java.util.TreeSet; + +class Main { + public static void main (String args[]) { + try { + InputStream input = new FileInputStream(args[0]); + Properties prop = new Properties(); + prop.load(input); + SortedSet<String> keySet = new TreeSet(prop.keySet()); + for (String key : keySet) { + System.out.println("KEY"); + System.out.println(key); + System.out.println("VALUE"); + System.out.println(prop.get(key)); + System.out.println(""); + } + } catch (Exception e) { + e.printStackTrace(); + System.err.println(e.toString()); + System.exit(1); + } + } +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/default.nix new file mode 100644 index 000000000000..4a51179d1c86 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/java-properties/test/default.nix @@ -0,0 +1,92 @@ +{ fetchurl +, formats +, glibcLocales +, jdk +, lib +, stdenv +}: + +# This test primarily tests correct escaping. +# See also testJavaProperties in +# pkgs/pkgs-lib/tests/formats.nix, which tests +# type coercions and is a bit easier to read. + +let + inherit (lib) concatStrings attrValues mapAttrs; + + javaProperties = formats.javaProperties { }; + + input = { + foo = "bar"; + "empty value" = ""; + "typical.dot.syntax" = "com.sun.awt"; + "" = "empty key's value"; + "1" = "2 3"; + "#" = "not a comment # still not"; + "!" = "not a comment!"; + "!a" = "still not! a comment"; + "!b" = "still not ! a comment"; + "dos paths" = "C:\\Program Files\\Nix For Windows\\nix.exe"; + "a \t\nb" = " c"; + "angry \t\nkey" = '' + multi + ${"\tline\r"} + space- + indented + trailing-space${" "} + trailing-space${" "} + value + ''; + "this=not" = "bad"; + "nor = this" = "bad"; + "all stuff" = "foo = bar"; + "unicode big brain" = "e = mc□"; + "ütf-8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + +in +stdenv.mkDerivation { + name = "pkgs.formats.javaProperties-test-${jdk.name}"; + nativeBuildInputs = [ + jdk + glibcLocales + ]; + + # technically should go through the type.merge first, but that's tested + # in tests/formats.nix. + properties = javaProperties.generate "example.properties" input; + + # Expected output as printed by Main.java + passAsFile = [ "expected" ]; + expected = concatStrings (attrValues ( + mapAttrs + (key: value: + '' + KEY + ${key} + VALUE + ${value} + + '' + ) + input + )); + + src = lib.sourceByRegex ./. [ + ".*\.java" + ]; + # On Linux, this can be C.UTF-8, but darwin + zulu requires en_US.UTF-8 + LANG = "en_US.UTF-8"; + buildPhase = '' + javac Main.java + ''; + doCheck = true; + checkPhase = '' + cat -v $properties + java Main $properties >actual + diff -U3 $expectedPath actual + ''; + installPhase = "touch $out"; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/default.nix new file mode 100644 index 000000000000..7433a7285353 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/default.nix @@ -0,0 +1,121 @@ +{ lib +, pkgs +}: +let + inherit (pkgs) buildPackages callPackage; + # Implementation notes: + # Libconfig spec: https://hyperrealm.github.io/libconfig/libconfig_manual.html + # + # Since libconfig does not allow setting names to start with an underscore, + # this is used as a prefix for both special types and include directives. + # + # The difference between 32bit and 64bit values became optional in libconfig + # 1.5, so we assume 64bit values for all numbers. + + libconfig-generator = buildPackages.rustPlatform.buildRustPackage { + name = "libconfig-generator"; + version = "0.1.0"; + src = ./src; + + passthru.updateScript = ./update.sh; + + cargoLock.lockFile = ./src/Cargo.lock; + }; + + libconfig-validator = buildPackages.runCommandCC "libconfig-validator" + { + buildInputs = with buildPackages; [ libconfig ]; + } + '' + mkdir -p "$out/bin" + $CC -lconfig -x c - -o "$out/bin/libconfig-validator" ${./validator.c} + ''; +in +{ + format = { generator ? libconfig-generator, validator ? libconfig-validator }: { + inherit generator; + + type = with lib.types; + let + valueType = (oneOf [ + bool + int + float + str + path + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "libconfig value"; + }; + in + attrsOf valueType; + + lib = { + mkHex = value: { + _type = "hex"; + inherit value; + }; + mkOctal = value: { + _type = "octal"; + inherit value; + }; + mkFloat = value: { + _type = "float"; + inherit value; + }; + mkArray = value: { + _type = "array"; + inherit value; + }; + mkList = value: { + _type = "list"; + inherit value; + }; + }; + + generate = name: value: + callPackage + ({ + stdenvNoCC + , libconfig-generator + , libconfig-validator + , writeText + }: stdenvNoCC.mkDerivation rec { + inherit name; + + dontUnpack = true; + + json = builtins.toJSON value; + passAsFile = [ "json" ]; + + strictDeps = true; + nativeBuildInputs = [ libconfig-generator ]; + buildPhase = '' + runHook preBuild + libconfig-generator < $jsonPath > output.cfg + runHook postBuild + ''; + + doCheck = true; + nativeCheckInputs = [ libconfig-validator ]; + checkPhase = '' + runHook preCheck + libconfig-validator output.cfg + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mv output.cfg $out + runHook postInstall + ''; + + passthru.json = writeText "${name}.json" json; + }) + { + libconfig-generator = generator; + libconfig-validator = validator; + }; + }; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock new file mode 100644 index 000000000000..f8f921f996f9 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.lock @@ -0,0 +1,40 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "libconfig-generator" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" + +[[package]] +name = "serde_json" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +dependencies = [ + "itoa", + "ryu", + "serde", +] diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml new file mode 100644 index 000000000000..20ad44d22194 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "libconfig-generator" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1.0.178" +serde_json = "1.0.104" diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs new file mode 100644 index 000000000000..4da45f647d46 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/src/src/main.rs @@ -0,0 +1,271 @@ +use serde_json::Value; +use std::mem::discriminant; + +#[derive(Debug)] +enum LibConfigIntNumber { + Oct(i64), + Hex(i64), + Int(i64), +} + +#[derive(Debug)] +enum LibConfigValue { + Bool(bool), + Int(LibConfigIntNumber), + Float(f64), + String(String), + Array(Vec<LibConfigValue>), + List(Vec<LibConfigValue>), + Group(Vec<String>, Vec<(String, LibConfigValue)>), +} + +fn validate_setting_name(key: &str) -> bool { + let first_char = key.chars().next().expect("Empty setting name"); + (first_char.is_alphabetic() || first_char == '*') + && key[1..] + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '*') +} + +const SPECIAL_TYPES: [&str; 5] = ["octal", "hex", "float", "list", "array"]; + +fn object_is_special_type(o: &serde_json::Map<String, Value>) -> Option<&str> { + o.get("_type").and_then(|x| x.as_str()).and_then(|x| { + if SPECIAL_TYPES.contains(&x) { + Some(x) + } else { + None + } + }) +} + +fn vec_is_array(v: &Vec<LibConfigValue>) -> bool { + if v.is_empty() { + return true; + } + + let first_item = v.first().unwrap(); + + if match first_item { + LibConfigValue::Array(_) => true, + LibConfigValue::List(_) => true, + LibConfigValue::Group(_, _) => true, + _ => false, + } { + return false; + }; + + v[1..] + .iter() + .all(|item| discriminant(first_item) == discriminant(item)) +} + +fn json_to_libconfig(v: &Value) -> LibConfigValue { + match v { + Value::Null => panic!("Null value not allowed in libconfig"), + Value::Bool(b) => LibConfigValue::Bool(b.clone()), + Value::Number(n) => { + if n.is_i64() { + LibConfigValue::Int(LibConfigIntNumber::Int(n.as_i64().unwrap())) + } else if n.is_f64() { + LibConfigValue::Float(n.as_f64().unwrap()) + } else { + panic!("{} is not i64 or f64, cannot be represented as number in libconfig", n); + } + } + Value::String(s) => LibConfigValue::String(s.to_string()), + Value::Array(a) => { + let items = a + .iter() + .map(|item| json_to_libconfig(item)) + .collect::<Vec<LibConfigValue>>(); + LibConfigValue::List(items) + } + Value::Object(o) => { + if let Some(_type) = object_is_special_type(o) { + let value = o + .get("value") + .expect(format!("Missing value for special type: {}", &_type).as_str()); + + return match _type { + "octal" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Int(LibConfigIntNumber::Oct( + i64::from_str_radix(&str_value, 8) + .expect(format!("Invalid octal value: {}", value).as_str()), + )) + } + "hex" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Int(LibConfigIntNumber::Hex( + i64::from_str_radix(&str_value[2..], 16) + .expect(format!("Invalid hex value: {}", value).as_str()), + )) + } + "float" => { + let str_value = value + .as_str() + .expect( + format!("Value is not a string for special type: {}", &_type) + .as_str(), + ) + .to_owned(); + + LibConfigValue::Float( + str_value + .parse::<f64>() + .expect(format!("Invalid float value: {}", value).as_str()), + ) + } + "list" => { + let items = value + .as_array() + .expect( + format!("Value is not an array for special type: {}", &_type) + .as_str(), + ) + .to_owned() + .iter() + .map(|item| json_to_libconfig(item)) + .collect::<Vec<LibConfigValue>>(); + + LibConfigValue::List(items) + } + "array" => { + let items = value + .as_array() + .expect( + format!("Value is not an array for special type: {}", &_type) + .as_str(), + ) + .to_owned() + .iter() + .map(|item| json_to_libconfig(item)) + .collect::<Vec<LibConfigValue>>(); + + if !vec_is_array(&items) { + panic!( + "This can not be an array because of its contents: {:#?}", + items + ); + } + + LibConfigValue::Array(items) + } + _ => panic!("Invalid type: {}", _type), + }; + } + + let mut items = o + .iter() + .filter(|(key, _)| key.as_str() != "_includes") + .map(|(key, value)| (key.clone(), json_to_libconfig(value))) + .collect::<Vec<(String, LibConfigValue)>>(); + items.sort_by(|(a,_),(b,_)| a.partial_cmp(b).unwrap()); + + let includes = o + .get("_includes") + .map(|x| { + x.as_array() + .expect("_includes is not an array") + .iter() + .map(|x| { + x.as_str() + .expect("_includes item is not a string") + .to_owned() + }) + .collect::<Vec<String>>() + }) + .unwrap_or(vec![]); + + for (key,_) in items.iter() { + if !validate_setting_name(key) { + panic!("Invalid setting name: {}", key); + } + } + LibConfigValue::Group(includes, items) + } + } +} + +impl ToString for LibConfigValue { + fn to_string(&self) -> String { + match self { + LibConfigValue::Bool(b) => b.to_string(), + LibConfigValue::Int(i) => match i { + LibConfigIntNumber::Oct(n) => format!("0{:o}", n), + LibConfigIntNumber::Hex(n) => format!("0x{:x}", n), + LibConfigIntNumber::Int(n) => n.to_string(), + }, + LibConfigValue::Float(n) => format!("{:?}", n), + LibConfigValue::String(s) => { + format!("\"{}\"", s.replace("\\", "\\\\").replace("\"", "\\\"")) + } + LibConfigValue::Array(a) => { + let items = a + .iter() + .map(|item| item.to_string()) + .collect::<Vec<String>>() + .join(", "); + format!("[{}]", items) + } + LibConfigValue::List(a) => { + let items = a + .iter() + .map(|item| item.to_string()) + .collect::<Vec<String>>() + .join(", "); + format!("({})", items) + } + LibConfigValue::Group(i, o) => { + let includes = i + .iter() + .map(|x| x.replace("\\", "\\\\").replace("\"", "\\\"")) + .map(|x| format!("@include \"{}\"", x)) + .collect::<Vec<String>>() + .join("\n"); + let items = o + .iter() + .map(|(key, value)| format!("{}={};", key, value.to_string())) + .collect::<Vec<String>>() + .join(""); + if includes.is_empty() { + format!("{{{}}}", items) + } else { + format!("{{\n{}\n{}}}", includes, items) + } + } + } + } +} + +fn main() { + let stdin = std::io::stdin().lock(); + let json = serde_json::Deserializer::from_reader(stdin) + .into_iter::<Value>() + .next() + .expect("Could not read content from stdin") + .expect("Could not parse JSON from stdin"); + + for (key, value) in json + .as_object() + .expect("Top level of JSON file is not an object") + { + print!("{}={};", key, json_to_libconfig(value).to_string()); + } + print!("\n\n"); +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix new file mode 100644 index 000000000000..7b0df23ee663 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/default.nix @@ -0,0 +1,76 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + libconfig = formats.libconfig { }; + + include_expr = { + val = 1; + }; + + include_file = writeText "libconfig-test-include" '' + val=1; + ''; + + expression = { + simple_top_level_attr = "1.0"; + nested.attrset.has.a.integer.value = 100; + some_floaty = 29.95; + ## Same syntax here on these two, but they should get serialized differently: + # > A list may have zero or more elements, each of which can be a scalar value, an array, a group, or another list. + list1d = libconfig.lib.mkList [ 1 "mixed!" 5 2 ]; + # You might also omit the mkList, as a list will be a list (in contrast to an array) by default. + list2d = [ 1 [ 1 1.2 "foo" ] [ "bar" 1.2 1 ] ]; + # > An array may have zero or more elements, but the elements must all be scalar values of the same type. + array1d = libconfig.lib.mkArray [ 1 5 2 ]; + array2d = [ + (libconfig.lib.mkArray [ 1 2 ]) + (libconfig.lib.mkArray [ 2 1 ]) + ]; + nasty_string = "\"@\n\\\t^*\b\f\n\0\";'''$"; + + weirderTypes = { + _includes = [ include_file ]; + pi = 3.141592654; + bigint = 9223372036854775807; + hex = libconfig.lib.mkHex "0x1FC3"; + octal = libconfig.lib.mkOctal "0027"; + float = libconfig.lib.mkFloat "1.2E-3"; + array_of_ints = libconfig.lib.mkArray [ + (libconfig.lib.mkOctal "0732") + (libconfig.lib.mkHex "0xA3") + 1234 + ]; + list_of_weird_types = [ + 3.141592654 + 9223372036854775807 + (libconfig.lib.mkHex "0x1FC3") + (libconfig.lib.mkOctal "0027") + (libconfig.lib.mkFloat "1.2E-32") + (libconfig.lib.mkFloat "1") + ]; + }; + }; + + libconfig-test-cfg = libconfig.generate "libconfig-test.cfg" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.libconfig-test-comprehensive"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + cp ${./expected.txt} expected.txt + substituteInPlace expected.txt \ + --subst-var-by include_file "${include_file}" + diff -U3 ./expected.txt ${libconfig-test-cfg} + ''; + + installPhase = '' + mkdir $out + cp expected.txt $out + cp ${libconfig-test-cfg} $out/libconfig-test.cfg + cp ${libconfig-test-cfg.passthru.json} $out/libconfig-test.json + ''; + } + diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt new file mode 100644 index 000000000000..ef6e09f8111a --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/comprehensive/expected.txt @@ -0,0 +1,6 @@ +array1d=[1, 5, 2];array2d=([1, 2], [2, 1]);list1d=(1, "mixed!", 5, 2);list2d=(1, (1, 1.2, "foo"), ("bar", 1.2, 1));nasty_string="\"@ +\\ ^*bf +0\";'''$";nested={attrset={has={a={integer={value=100;};};};};};simple_top_level_attr="1.0";some_floaty=29.95;weirderTypes={ +@include "@include_file@" +array_of_ints=[0732, 0xa3, 1234];bigint=9223372036854775807;float=0.0012;hex=0x1fc3;list_of_weird_types=(3.141592654, 9223372036854775807, 0x1fc3, 027, 1.2e-32, 1.0);octal=027;pi=3.141592654;}; + diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/default.nix new file mode 100644 index 000000000000..6cd03fe4854f --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/test/default.nix @@ -0,0 +1,4 @@ +{ pkgs, ... }: +{ + comprehensive = pkgs.callPackage ./comprehensive { }; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/update.sh b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/update.sh new file mode 100755 index 000000000000..ffc5ad3917f7 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/update.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p cargo -i bash +cd "$(dirname "$0")" +cargo update diff --git a/nixpkgs/pkgs/pkgs-lib/formats/libconfig/validator.c b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/validator.c new file mode 100644 index 000000000000..738be0b774b5 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/libconfig/validator.c @@ -0,0 +1,21 @@ +// Copyright (C) 2005-2023 Mark A Lindner, ckie +// SPDX-License-Identifier: LGPL-2.1-or-later +#include <stdio.h> +#include <libconfig.h> +int main(int argc, char **argv) +{ + config_t cfg; + config_init(&cfg); + if (argc != 2) + { + fprintf(stderr, "USAGE: validator <path-to-validate>"); + } + if(! config_read_file(&cfg, argv[1])) + { + fprintf(stderr, "[libconfig] %s:%d - %s\n", config_error_file(&cfg), + config_error_line(&cfg), config_error_text(&cfg)); + config_destroy(&cfg); + return 1; + } + printf("[libconfig] validation ok\n"); +} \ No newline at end of file diff --git a/nixpkgs/pkgs/pkgs-lib/tests/default.nix b/nixpkgs/pkgs/pkgs-lib/tests/default.nix new file mode 100644 index 000000000000..289780f57650 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/tests/default.nix @@ -0,0 +1,46 @@ +# Call nix-build on this file to run all tests in this directory + +# This produces a link farm derivation with the original attrs +# merged on top of it. +# You can run parts of the "hierarchy" with for example: +# nix-build -A java-properties +# See `structured` below. + +{ pkgs ? import ../../.. { } }: +let + inherit (pkgs.lib) mapAttrs mapAttrsToList isDerivation mergeAttrs foldl' attrValues recurseIntoAttrs; + + structured = { + formats = import ./formats.nix { inherit pkgs; }; + java-properties = recurseIntoAttrs { + jdk8 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk8; }; + jdk11 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk11_headless; }; + jdk17 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk17_headless; }; + }; + libconfig = recurseIntoAttrs (import ../formats/libconfig/test { inherit pkgs; }); + }; + + flatten = prefix: as: + foldl' + mergeAttrs + { } + (attrValues + (mapAttrs + (k: v: + if isDerivation v + then { "${prefix}${k}" = v; } + else if v?recurseForDerivations + then flatten "${prefix}${k}-" (removeAttrs v [ "recurseForDerivations" ]) + else builtins.trace v throw "expected derivation or recurseIntoAttrs") + as + ) + ); +in + +# It has to be a link farm for inclusion in the hydra unstable jobset. +pkgs.linkFarm "pkgs-lib-formats-tests" + (mapAttrsToList + (k: v: { name = k; path = v; }) + (flatten "" structured) + ) +// structured diff --git a/nixpkgs/pkgs/pkgs-lib/tests/formats.nix b/nixpkgs/pkgs/pkgs-lib/tests/formats.nix new file mode 100644 index 000000000000..b7e100dd73bc --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/tests/formats.nix @@ -0,0 +1,253 @@ +{ pkgs }: +let + inherit (pkgs) lib formats; +in +with lib; +let + + evalFormat = format: args: def: + let + formatSet = format args; + config = formatSet.type.merge [] (imap1 (n: def: { + # We check the input values, so that + # - we don't write nonsensical tests that will impede progress + # - the test author has a slightly more realistic view of the + # final format during development. + value = lib.throwIfNot (formatSet.type.check def) (builtins.trace def "definition does not pass the type's check function") def; + file = "def${toString n}"; + }) [ def ]); + in formatSet.generate "test-format-file" config; + + runBuildTest = name: { drv, expected }: pkgs.runCommand name { + passAsFile = ["expected"]; + inherit expected drv; + } '' + if diff -u "$expectedPath" "$drv"; then + touch "$out" + else + echo + echo "Got different values than expected; diff above." + exit 1 + fi + ''; + + runBuildTests = tests: pkgs.linkFarm "nixpkgs-pkgs-lib-format-tests" (mapAttrsToList (name: value: { inherit name; path = runBuildTest name value; }) (filterAttrs (name: value: value != null) tests)); + +in runBuildTests { + + testJsonAtoms = { + drv = evalFormat formats.json {} { + null = null; + false = false; + true = true; + int = 10; + float = 3.141; + str = "foo"; + attrs.foo = null; + list = [ null null ]; + path = ./formats.nix; + }; + expected = '' + { + "attrs": { + "foo": null + }, + "false": false, + "float": 3.141, + "int": 10, + "list": [ + null, + null + ], + "null": null, + "path": "${./formats.nix}", + "str": "foo", + "true": true + } + ''; + }; + + testYamlAtoms = { + drv = evalFormat formats.yaml {} { + null = null; + false = false; + true = true; + float = 3.141; + str = "foo"; + attrs.foo = null; + list = [ null null ]; + path = ./formats.nix; + }; + expected = '' + attrs: + foo: null + 'false': false + float: 3.141 + list: + - null + - null + 'null': null + path: ${./formats.nix} + str: foo + 'true': true + ''; + }; + + testIniAtoms = { + drv = evalFormat formats.ini {} { + foo = { + bool = true; + int = 10; + float = 3.141; + str = "string"; + }; + }; + expected = '' + [foo] + bool=true + float=3.141000 + int=10 + str=string + ''; + }; + + testIniDuplicateKeys = { + drv = evalFormat formats.ini { listsAsDuplicateKeys = true; } { + foo = { + bar = [ null true "test" 1.2 10 ]; + baz = false; + qux = "qux"; + }; + }; + expected = '' + [foo] + bar=null + bar=true + bar=test + bar=1.200000 + bar=10 + baz=false + qux=qux + ''; + }; + + testIniListToValue = { + drv = evalFormat formats.ini { listToValue = concatMapStringsSep ", " (generators.mkValueStringDefault {}); } { + foo = { + bar = [ null true "test" 1.2 10 ]; + baz = false; + qux = "qux"; + }; + }; + expected = '' + [foo] + bar=null, true, test, 1.200000, 10 + baz=false + qux=qux + ''; + }; + + testKeyValueAtoms = { + drv = evalFormat formats.keyValue {} { + bool = true; + int = 10; + float = 3.141; + str = "string"; + }; + expected = '' + bool=true + float=3.141000 + int=10 + str=string + ''; + }; + + testKeyValueDuplicateKeys = { + drv = evalFormat formats.keyValue { listsAsDuplicateKeys = true; } { + bar = [ null true "test" 1.2 10 ]; + baz = false; + qux = "qux"; + }; + expected = '' + bar=null + bar=true + bar=test + bar=1.200000 + bar=10 + baz=false + qux=qux + ''; + }; + + testKeyValueListToValue = { + drv = evalFormat formats.keyValue { listToValue = concatMapStringsSep ", " (generators.mkValueStringDefault {}); } { + bar = [ null true "test" 1.2 10 ]; + baz = false; + qux = "qux"; + }; + expected = '' + bar=null, true, test, 1.200000, 10 + baz=false + qux=qux + ''; + }; + + testTomlAtoms = { + drv = evalFormat formats.toml {} { + false = false; + true = true; + int = 10; + float = 3.141; + str = "foo"; + attrs.foo = "foo"; + list = [ 1 2 ]; + level1.level2.level3.level4 = "deep"; + }; + expected = '' + false = false + float = 3.141 + int = 10 + list = [1, 2] + str = "foo" + true = true + [attrs] + foo = "foo" + + [level1.level2.level3] + level4 = "deep" + ''; + }; + + # This test is responsible for + # 1. testing type coercions + # 2. providing a more readable example test + # Whereas java-properties/default.nix tests the low level escaping, etc. + testJavaProperties = { + drv = evalFormat formats.javaProperties {} { + floaty = 3.1415; + tautologies = true; + contradictions = false; + foo = "bar"; + # # Disallowed at eval time, because it's ambiguous: + # # add to store or convert to string? + # root = /root; + "1" = 2; + package = pkgs.hello; + "ütf 8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + expected = '' + # Generated with Nix + + 1 = 2 + contradictions = false + floaty = 3.141500 + foo = bar + package = ${pkgs.hello} + tautologies = true + \u00fctf\ 8 = d\u00fbh + \u0627\u0644\u062c\u0628\u0631 = \u0623\u0643\u062b\u0631 \u0645\u0646 \u0645\u062c\u0631\u062f \u0623\u0631\u0642\u0627\u0645 + ''; + }; +} |