diff options
Diffstat (limited to 'pkgs/pkgs-lib')
| -rw-r--r-- | pkgs/pkgs-lib/formats.nix | 210 | ||||
| -rw-r--r-- | pkgs/pkgs-lib/formats/java-properties/default.nix | 132 | ||||
| -rw-r--r-- | pkgs/pkgs-lib/formats/java-properties/test/Main.java | 27 | ||||
| -rw-r--r-- | pkgs/pkgs-lib/formats/java-properties/test/default.nix | 92 | ||||
| -rw-r--r-- | pkgs/pkgs-lib/tests/default.nix | 48 | ||||
| -rw-r--r-- | pkgs/pkgs-lib/tests/formats.nix | 48 |
6 files changed, 547 insertions, 10 deletions
diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index e6e6a95c1f4a..cb46b63dd0c4 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -14,6 +14,15 @@ rec { # The description needs to be overwritten for recursive types type = ...; + # Utility functions for convenience, or special interactions with the + # format (optional) + lib = { + exampleFunction = ... + # Types specific to the format (optional) + types = { ... }; + ... + }; + # generate :: Name -> Value -> Path # A function for generating a file with a value of such a type generate = ...; @@ -22,6 +31,9 @@ rec { */ + inherit (import ./formats/java-properties/default.nix { inherit lib pkgs; }) + javaProperties; + json = {}: { type = with lib.types; let @@ -158,4 +170,202 @@ rec { ''; }; + + /* For configurations of Elixir project, like config.exs or runtime.exs + + Most Elixir project are configured using the [Config] Elixir DSL + + Since Elixir has more types than Nix, we need a way to map Nix types to + more than 1 Elixir type. To that end, this format provides its own library, + and its own set of types. + + To be more detailed, a Nix attribute set could correspond in Elixir to a + [Keyword list] (the more common type), or it could correspond to a [Map]. + + A Nix string could correspond in Elixir to a [String] (also called + "binary"), an [Atom], or a list of chars (usually discouraged). + + A Nix array could correspond in Elixir to a [List] or a [Tuple]. + + Some more types exists, like records, regexes, but since they are less used, + we can leave the `mkRaw` function as an escape hatch. + + For more information on how to use this format in modules, please refer to + the Elixir section of the Nixos documentation. + + TODO: special Elixir values doesn't show up nicely in the documentation + + [Config]: <https://hexdocs.pm/elixir/Config.html> + [Keyword list]: <https://hexdocs.pm/elixir/Keyword.html> + [Map]: <https://hexdocs.pm/elixir/Map.html> + [String]: <https://hexdocs.pm/elixir/String.html> + [Atom]: <https://hexdocs.pm/elixir/Atom.html> + [List]: <https://hexdocs.pm/elixir/List.html> + [Tuple]: <https://hexdocs.pm/elixir/Tuple.html> + */ + elixirConf = { elixir ? pkgs.elixir }: + with lib; let + toElixir = value: with builtins; + if value == null then "nil" else + if value == true then "true" else + if value == false then "false" else + if isInt value || isFloat value then toString value else + if isString value then string value else + if isAttrs value then attrs value else + if isList value then list value else + abort "formats.elixirConf: should never happen (value = ${value})"; + + escapeElixir = escape [ "\\" "#" "\"" ]; + string = value: "\"${escapeElixir value}\""; + + attrs = set: + if set ? _elixirType then specialType set + else + let + toKeyword = name: value: "${name}: ${toElixir value}"; + keywordList = concatStringsSep ", " (mapAttrsToList toKeyword set); + in + "[" + keywordList + "]"; + + listContent = values: concatStringsSep ", " (map toElixir values); + + list = values: "[" + (listContent values) + "]"; + + specialType = { value, _elixirType }: + if _elixirType == "raw" then value else + if _elixirType == "atom" then value else + if _elixirType == "map" then elixirMap value else + if _elixirType == "tuple" then tuple value else + abort "formats.elixirConf: should never happen (_elixirType = ${_elixirType})"; + + elixirMap = set: + let + toEntry = name: value: "${toElixir name} => ${toElixir value}"; + entries = concatStringsSep ", " (mapAttrsToList toEntry set); + in + "%{${entries}}"; + + tuple = values: "{${listContent values}}"; + + toConf = values: + let + keyConfig = rootKey: key: value: + "config ${rootKey}, ${key}, ${toElixir value}"; + keyConfigs = rootKey: values: mapAttrsToList (keyConfig rootKey) values; + rootConfigs = flatten (mapAttrsToList keyConfigs values); + in + '' + import Config + + ${concatStringsSep "\n" rootConfigs} + ''; + in + { + type = with lib.types; let + valueType = nullOr + (oneOf [ + bool + int + float + str + (attrsOf valueType) + (listOf valueType) + ]) // { + description = "Elixir value"; + }; + in + attrsOf (attrsOf (valueType)); + + lib = + let + mkRaw = value: { + inherit value; + _elixirType = "raw"; + }; + + in + { + inherit mkRaw; + + /* Fetch an environment variable at runtime, with optional fallback + */ + mkGetEnv = { envVariable, fallback ? null }: + mkRaw "System.get_env(${toElixir envVariable}, ${toElixir fallback})"; + + /* Make an Elixir atom. + + Note: lowercase atoms still need to be prefixed by ':' + */ + mkAtom = value: { + inherit value; + _elixirType = "atom"; + }; + + /* Make an Elixir tuple out of a list. + */ + mkTuple = value: { + inherit value; + _elixirType = "tuple"; + }; + + /* Make an Elixir map out of an attribute set. + */ + mkMap = value: { + inherit value; + _elixirType = "map"; + }; + + /* Contains Elixir types. Every type it exports can also be replaced + by raw Elixir code (i.e. every type is `either type rawElixir`). + + It also reexports standard types, wrapping them so that they can + also be raw Elixir. + */ + types = with lib.types; let + isElixirType = type: x: (x._elixirType or "") == type; + + rawElixir = mkOptionType { + name = "rawElixir"; + description = "raw elixir"; + check = isElixirType "raw"; + }; + + elixirOr = other: either other rawElixir; + in + { + inherit rawElixir elixirOr; + + atom = elixirOr (mkOptionType { + name = "elixirAtom"; + description = "elixir atom"; + check = isElixirType "atom"; + }); + + tuple = elixirOr (mkOptionType { + name = "elixirTuple"; + description = "elixir tuple"; + check = isElixirType "tuple"; + }); + + map = elixirOr (mkOptionType { + name = "elixirMap"; + description = "elixir map"; + check = isElixirType "map"; + }); + # Wrap standard types, since anything in the Elixir configuration + # can be raw Elixir + } // lib.mapAttrs (_name: type: elixirOr type) lib.types; + }; + + generate = name: value: pkgs.runCommandNoCC name + { + value = toConf value; + passAsFile = [ "value" ]; + nativeBuildInputs = [ elixir ]; + } '' + cp "$valuePath" "$out" + mix format "$out" + ''; + }; + } diff --git a/pkgs/pkgs-lib/formats/java-properties/default.nix b/pkgs/pkgs-lib/formats/java-properties/default.nix new file mode 100644 index 000000000000..d3a4761f0f80 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/default.nix @@ -0,0 +1,132 @@ +{ lib, pkgs }: +let + inherit (lib) types; + inherit (types) attrsOf oneOf coercedTo str bool int float package; +in +{ + javaProperties = { comment ? "Generated with Nix", boolToString ? lib.boolToString }: { + + # Design note: + # A nested representation of inevitably leads to bad UX: + # 1. keys like "a.b" must be disallowed, or + # the addition of options in a freeformType module + # become breaking changes + # 2. adding a value for "a" after "a"."b" was already + # defined leads to a somewhat hard to understand + # Nix error, because that's not something you can + # do with attrset syntax. Workaround: "a"."", but + # that's too little too late. Another workaround: + # mkMerge [ { a = ...; } { a.b = ...; } ]. + # + # Choosing a non-nested representation does mean that + # we sacrifice the ability to override at the (conceptual) + # hierarchical levels, _if_ an application exhibits those. + # + # Some apps just use periods instead of spaces in an odd + # mix of attempted categorization and natural language, + # with no meaningful hierarchy. + # + # We _can_ choose to support hierarchical config files + # via nested attrsets, but the module author should + # make sure that problem (2) does not occur. + type = let + elemType = + oneOf ([ + # `package` isn't generalized to `path` because path values + # are ambiguous. Are they host path strings (toString /foo/bar) + # or should they be added to the store? ("${/foo/bar}") + # The user must decide. + (coercedTo package toString str) + + (coercedTo bool boolToString str) + (coercedTo int toString str) + (coercedTo float toString str) + ]) + // { description = "string, package, bool, int or float"; }; + in attrsOf elemType; + + generate = name: value: + pkgs.runCommandLocal name + { + # Requirements + # ============ + # + # 1. Strings in Nix carry over to the same + # strings in Java => need proper escapes + # 2. Generate files quickly + # - A JVM would have to match the app's + # JVM to avoid build closure bloat + # - Even then, JVM startup would slow + # down config generation. + # + # + # Implementation + # ============== + # + # Escaping has two steps + # + # 1. jq + # Escape known separators, in order not + # to break up the keys and values. + # This handles typical whitespace correctly, + # but may produce garbage for other control + # characters. + # + # 2. iconv + # Escape >ascii code points to java escapes, + # as .properties files are supposed to be + # encoded in ISO 8859-1. It's an old format. + # UTF-8 behavior may exist in some apps and + # libraries, but we can't rely on this in + # general. + + passAsFile = [ "value" ]; + value = builtins.toJSON value; + nativeBuildInputs = [ + pkgs.jq + pkgs.libiconvReal + ]; + + jqCode = + let + main = '' + to_entries + | .[] + | "\( + .key + | ${commonEscapes} + | gsub(" "; "\\ ") + | gsub("="; "\\=") + ) = \( + .value + | ${commonEscapes} + | gsub("^ "; "\\ ") + | gsub("\\n "; "\n\\ ") + )" + ''; + # Most escapes are equal for both keys and values. + commonEscapes = '' + gsub("\\\\"; "\\\\") + | gsub("\\n"; "\\n\\\n") + | gsub("#"; "\\#") + | gsub("!"; "\\!") + | gsub("\\t"; "\\t") + | gsub("\r"; "\\r") + ''; + in + main; + + inputEncoding = "UTF-8"; + + inherit comment; + + } '' + ( + echo "$comment" | while read -r ln; do echo "# $ln"; done + echo + jq -r --arg hash '#' "$jqCode" "$valuePath" \ + | iconv --from-code "$inputEncoding" --to-code JAVA \ + ) > "$out" + ''; + }; +} diff --git a/pkgs/pkgs-lib/formats/java-properties/test/Main.java b/pkgs/pkgs-lib/formats/java-properties/test/Main.java new file mode 100644 index 000000000000..dc83944f24b0 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/test/Main.java @@ -0,0 +1,27 @@ +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.SortedSet; +import java.util.TreeSet; + +class Main { + public static void main (String args[]) { + try { + InputStream input = new FileInputStream(args[0]); + Properties prop = new Properties(); + prop.load(input); + SortedSet<String> keySet = new TreeSet(prop.keySet()); + for (String key : keySet) { + System.out.println("KEY"); + System.out.println(key); + System.out.println("VALUE"); + System.out.println(prop.get(key)); + System.out.println(""); + } + } catch (Exception e) { + e.printStackTrace(); + System.err.println(e.toString()); + System.exit(1); + } + } +} diff --git a/pkgs/pkgs-lib/formats/java-properties/test/default.nix b/pkgs/pkgs-lib/formats/java-properties/test/default.nix new file mode 100644 index 000000000000..4a51179d1c86 --- /dev/null +++ b/pkgs/pkgs-lib/formats/java-properties/test/default.nix @@ -0,0 +1,92 @@ +{ fetchurl +, formats +, glibcLocales +, jdk +, lib +, stdenv +}: + +# This test primarily tests correct escaping. +# See also testJavaProperties in +# pkgs/pkgs-lib/tests/formats.nix, which tests +# type coercions and is a bit easier to read. + +let + inherit (lib) concatStrings attrValues mapAttrs; + + javaProperties = formats.javaProperties { }; + + input = { + foo = "bar"; + "empty value" = ""; + "typical.dot.syntax" = "com.sun.awt"; + "" = "empty key's value"; + "1" = "2 3"; + "#" = "not a comment # still not"; + "!" = "not a comment!"; + "!a" = "still not! a comment"; + "!b" = "still not ! a comment"; + "dos paths" = "C:\\Program Files\\Nix For Windows\\nix.exe"; + "a \t\nb" = " c"; + "angry \t\nkey" = '' + multi + ${"\tline\r"} + space- + indented + trailing-space${" "} + trailing-space${" "} + value + ''; + "this=not" = "bad"; + "nor = this" = "bad"; + "all stuff" = "foo = bar"; + "unicode big brain" = "e = mc□"; + "ütf-8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + +in +stdenv.mkDerivation { + name = "pkgs.formats.javaProperties-test-${jdk.name}"; + nativeBuildInputs = [ + jdk + glibcLocales + ]; + + # technically should go through the type.merge first, but that's tested + # in tests/formats.nix. + properties = javaProperties.generate "example.properties" input; + + # Expected output as printed by Main.java + passAsFile = [ "expected" ]; + expected = concatStrings (attrValues ( + mapAttrs + (key: value: + '' + KEY + ${key} + VALUE + ${value} + + '' + ) + input + )); + + src = lib.sourceByRegex ./. [ + ".*\.java" + ]; + # On Linux, this can be C.UTF-8, but darwin + zulu requires en_US.UTF-8 + LANG = "en_US.UTF-8"; + buildPhase = '' + javac Main.java + ''; + doCheck = true; + checkPhase = '' + cat -v $properties + java Main $properties >actual + diff -U3 $expectedPath actual + ''; + installPhase = "touch $out"; +} diff --git a/pkgs/pkgs-lib/tests/default.nix b/pkgs/pkgs-lib/tests/default.nix index f3549ea9b0f2..ae91e15aa9ef 100644 --- a/pkgs/pkgs-lib/tests/default.nix +++ b/pkgs/pkgs-lib/tests/default.nix @@ -1,7 +1,45 @@ # Call nix-build on this file to run all tests in this directory -{ pkgs ? import ../../.. {} }: + +# This produces a link farm derivation with the original attrs +# merged on top of it. +# You can run parts of the "hierarchy" with for example: +# nix-build -A java-properties +# See `structured` below. + +{ pkgs ? import ../../.. { } }: let - formats = import ./formats.nix { inherit pkgs; }; -in pkgs.linkFarm "nixpkgs-pkgs-lib-tests" [ - { name = "formats"; path = import ./formats.nix { inherit pkgs; }; } -] + inherit (pkgs.lib) mapAttrs mapAttrsToList isDerivation mergeAttrs foldl' attrValues recurseIntoAttrs; + + structured = { + formats = import ./formats.nix { inherit pkgs; }; + java-properties = recurseIntoAttrs { + jdk8 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk8; }; + jdk11 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk11_headless; }; + jdk17 = pkgs.callPackage ../formats/java-properties/test { jdk = pkgs.jdk17_headless; }; + }; + }; + + flatten = prefix: as: + foldl' + mergeAttrs + { } + (attrValues + (mapAttrs + (k: v: + if isDerivation v + then { "${prefix}${k}" = v; } + else if v?recurseForDerivations + then flatten "${prefix}${k}-" (removeAttrs v [ "recurseForDerivations" ]) + else builtins.trace v throw "expected derivation or recurseIntoAttrs") + as + ) + ); +in + +# It has to be a link farm for inclusion in the hydra unstable jobset. +pkgs.linkFarm "pkgs-lib-formats-tests" + (mapAttrsToList + (k: v: { name = k; path = v; }) + (flatten "" structured) + ) +// structured diff --git a/pkgs/pkgs-lib/tests/formats.nix b/pkgs/pkgs-lib/tests/formats.nix index 2bc4e407fe75..dba7f981cbd9 100644 --- a/pkgs/pkgs-lib/tests/formats.nix +++ b/pkgs/pkgs-lib/tests/formats.nix @@ -9,13 +9,20 @@ let let formatSet = format args; config = formatSet.type.merge [] (imap1 (n: def: { - value = def; + # We check the input values, so that + # - we don't write nonsensical tests that will impede progress + # - the test author has a slightly more realistic view of the + # final format during development. + value = lib.throwIfNot (formatSet.type.check def) (builtins.trace def "definition does not pass the type's check function") def; file = "def${toString n}"; }) [ def ]); in formatSet.generate "test-format-file" config; - runBuildTest = name: { drv, expected }: pkgs.runCommand name {} '' - if diff -u '${builtins.toFile "expected" expected}' '${drv}'; then + runBuildTest = name: { drv, expected }: pkgs.runCommand name { + passAsFile = ["expected"]; + inherit expected drv; + } '' + if diff -u "$expectedPath" "$drv"; then touch "$out" else echo @@ -162,10 +169,41 @@ in runBuildTests { [attrs] foo = "foo" - [level1] - [level1.level2] [level1.level2.level3] level4 = "deep" ''; }; + + # This test is responsible for + # 1. testing type coercions + # 2. providing a more readable example test + # Whereas java-properties/default.nix tests the low level escaping, etc. + testJavaProperties = { + drv = evalFormat formats.javaProperties {} { + floaty = 3.1415; + tautologies = true; + contradictions = false; + foo = "bar"; + # # Disallowed at eval time, because it's ambiguous: + # # add to store or convert to string? + # root = /root; + "1" = 2; + package = pkgs.hello; + "ütf 8" = "dûh"; + # NB: Some editors (vscode) show this _whole_ line in right-to-left order + "الجبر" = "أكثر من مجرد أرقام"; + }; + expected = '' + # Generated with Nix + + 1 = 2 + contradictions = false + floaty = 3.141500 + foo = bar + package = ${pkgs.hello} + tautologies = true + \u00fctf\ 8 = d\u00fbh + \u0627\u0644\u062c\u0628\u0631 = \u0623\u0643\u062b\u0631 \u0645\u0646 \u0645\u062c\u0631\u062f \u0623\u0631\u0642\u0627\u0645 + ''; + }; } |