diff options
Diffstat (limited to 'nixpkgs/pkgs/pkgs-lib/formats/hocon')
11 files changed, 762 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/hocon/default.nix new file mode 100644 index 000000000000..318ee0143320 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/default.nix @@ -0,0 +1,189 @@ +{ lib +, pkgs +}: +let + inherit (pkgs) buildPackages callPackage; + + hocon-generator = buildPackages.rustPlatform.buildRustPackage { + name = "hocon-generator"; + version = "0.1.0"; + src = ./src; + + passthru.updateScript = ./update.sh; + + cargoLock.lockFile = ./src/Cargo.lock; + }; + + hocon-validator = pkgs.writers.writePython3Bin "hocon-validator" { + libraries = [ pkgs.python3Packages.pyhocon ]; + } '' + from sys import argv + from pyhocon import ConfigFactory + + if not len(argv) == 2: + print("USAGE: hocon-validator <file>") + + ConfigFactory.parse_file(argv[1]) + ''; +in +{ + # https://github.com/lightbend/config/blob/main/HOCON.md + format = { + generator ? hocon-generator + , validator ? hocon-validator + # `include classpath("")` is not implemented in pyhocon. + # In the case that you need this functionality, + # you will have to disable pyhocon validation. + , doCheck ? true + }: let + hoconLib = { + mkInclude = value: let + includeStatement = if lib.isAttrs value && !(lib.isDerivation value) then { + required = false; + type = null; + _type = "include"; + } // value else { + value = toString value; + required = false; + type = null; + _type = "include"; + }; + in + assert lib.assertMsg (lib.elem includeStatement.type [ "file" "url" "classpath" null ]) '' + Type of HOCON mkInclude is not of type 'file', 'url' or 'classpath': + ${(lib.generators.toPretty {}) includeStatement} + ''; + includeStatement; + + mkAppend = value: { + inherit value; + _type = "append"; + }; + + mkSubstitution = value: + if lib.isString value + then + { + inherit value; + optional = false; + _type = "substitution"; + } + else + assert lib.assertMsg (lib.isAttrs value) '' + Value of invalid type provided to `hocon.lib.mkSubstition`: ${lib.typeOf value} + ''; + assert lib.assertMsg (value ? "value") '' + Argument to `hocon.lib.mkSubstition` is missing a `value`: + ${builtins.toJSON value} + ''; + { + value = value.value; + optional = value.optional or false; + _type = "substitution"; + }; + }; + + in { + type = let + type' = with lib.types; let + atomType = nullOr (oneOf [ + bool + float + int + path + str + ]); + in (oneOf [ + atomType + (listOf atomType) + (attrsOf type') + ]) // { + description = "HOCON value"; + }; + in type'; + + lib = hoconLib; + + generate = name: value: + let + # TODO: remove in 24.11 + # Backwards compatability for generators in the following locations: + # - nixos/modules/services/networking/jibri/default.nix (__hocon_envvar) + # - nixos/modules/services/networking/jicofo.nix (__hocon_envvar, __hocon_unquoted_string) + # - nixos/modules/services/networking/jitsi-videobridge.nix (__hocon_envvar) + replaceOldIndicators = value: + if lib.isAttrs value then + (if value ? "__hocon_envvar" + then + lib.warn '' + Use of `__hocon_envvar` has been deprecated, and will + be removed in the future. + + Please use `(pkgs.formats.hocon {}).lib.mkSubstitution` instead. + '' + (hoconLib.mkSubstitution value.__hocon_envvar) + else if value ? "__hocon_unquoted_string" + then + lib.warn '' + Use of `__hocon_unquoted_string` has been deprecated, and will + be removed in the future. + + Please make use of the freeform options of + `(pkgs.formats.hocon {}).format` instead. + '' + { + value = value.__hocon_unquoted_string; + _type = "unquoted_string"; + } + else lib.mapAttrs (_: replaceOldIndicators) value) + else if lib.isList value + then map replaceOldIndicators value + else value; + + finalValue = replaceOldIndicators value; + in + callPackage + ({ + stdenvNoCC + , hocon-generator + , hocon-validator + , writeText + }: + stdenvNoCC.mkDerivation rec { + inherit name; + + dontUnpack = true; + + json = builtins.toJSON finalValue; + passAsFile = [ "json" ]; + + strictDeps = true; + nativeBuildInputs = [ hocon-generator ]; + buildPhase = '' + runHook preBuild + hocon-generator < $jsonPath > output.conf + runHook postBuild + ''; + + inherit doCheck; + nativeCheckInputs = [ hocon-validator ]; + checkPhase = '' + runHook preCheck + hocon-validator output.conf + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mv output.conf $out + runHook postInstall + ''; + + passthru.json = writeText "${name}.json" json; + }) + { + hocon-generator = generator; + hocon-validator = validator; + }; + }; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/.gitignore b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/.gitignore new file mode 100644 index 000000000000..eb5a316cbd19 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/.gitignore @@ -0,0 +1 @@ +target diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock new file mode 100644 index 000000000000..735461cd5f0e --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.lock @@ -0,0 +1,89 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "hocon-generator" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml new file mode 100644 index 000000000000..e39e636a9f50 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "hocon-generator" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1.0.178" +serde_json = "1.0.104" diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/src/main.rs b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/src/main.rs new file mode 100644 index 000000000000..2e53f3fd5659 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/src/src/main.rs @@ -0,0 +1,237 @@ +use serde_json::{value, Map, Value}; + +#[derive(Debug)] +enum HOCONValue { + Null, + Append(Box<HOCONValue>), + Bool(bool), + Number(value::Number), + String(String), + List(Vec<HOCONValue>), + Substitution(String, bool), + Object(Vec<HOCONInclude>, Vec<(String, HOCONValue)>), + Literal(String), +} + +#[derive(Debug)] +enum HOCONInclude { + Heuristic(String, bool), + Url(String, bool), + File(String, bool), + ClassPath(String, bool), +} + +impl HOCONInclude { + fn map_fst(&self, f: &dyn Fn(&String) -> String) -> HOCONInclude { + match self { + HOCONInclude::Heuristic(s, r) => HOCONInclude::Heuristic(f(s), *r), + HOCONInclude::Url(s, r) => HOCONInclude::Url(f(s), *r), + HOCONInclude::File(s, r) => HOCONInclude::File(f(s), *r), + HOCONInclude::ClassPath(s, r) => HOCONInclude::ClassPath(f(s), *r), + } + } +} + +fn parse_include(o: &Map<String, Value>) -> HOCONInclude { + let value = o + .get("value") + .expect("Missing field 'value' for include statement") + .as_str() + .expect("Field 'value' is not a string in include statement") + .to_string(); + let required = o + .get("required") + .expect("Missing field 'required' for include statement") + .as_bool() + .expect("Field 'required'is not a bool in include statement"); + let include_type = match o + .get("type") + .expect("Missing field 'type' for include statement") + { + Value::Null => None, + Value::String(s) => Some(s.as_str()), + t => panic!("Field 'type' is not a string in include statement: {:?}", t), + }; + + // Assert that this was an intentional include + debug_assert!(o.get("_type").and_then(|t| t.as_str()) == Some("include")); + + match include_type { + None => HOCONInclude::Heuristic(value, required), + Some("url") => HOCONInclude::Url(value, required), + Some("file") => HOCONInclude::File(value, required), + Some("classpath") => HOCONInclude::ClassPath(value, required), + _ => panic!( + "Could not recognize type for include statement: {}", + include_type.unwrap() + ), + } +} + +fn parse_special_types(o: &Map<String, Value>) -> Option<HOCONValue> { + o.get("_type") + .and_then(|r#type| r#type.as_str()) + .map(|r#type| match r#type { + "substitution" => { + let value = o + .get("value") + .expect("Missing value for substitution") + .as_str() + .unwrap_or_else(|| panic!("Substition value is not a string: {:?}", o)); + let required = o + .get("required") + .unwrap_or(&Value::Bool(false)) + .as_bool() + .unwrap_or_else(|| panic!("Substition value is not a string: {:?}", o)); + + debug_assert!(!value.contains('}')); + + HOCONValue::Substitution(value.to_string(), required) + } + "append" => { + let value = o.get("value").expect("Missing value for append"); + + HOCONValue::Append(Box::new(json_to_hocon(value))) + } + "unquoted_string" => { + let value = o + .get("value") + .expect("Missing value for unquoted_string") + .as_str() + .unwrap_or_else(|| panic!("Unquoted string value is not a string: {:?}", o)); + + HOCONValue::Literal(value.to_string()) + } + _ => panic!( + "\ + Attribute set contained special element '_type',\ + but its value is not recognized:\n{}", + r#type + ), + }) +} + +fn json_to_hocon(v: &Value) -> HOCONValue { + match v { + Value::Null => HOCONValue::Null, + Value::Bool(b) => HOCONValue::Bool(*b), + Value::Number(n) => HOCONValue::Number(n.clone()), + Value::String(s) => HOCONValue::String(s.clone()), + Value::Array(a) => { + let items = a.iter().map(json_to_hocon).collect::<Vec<HOCONValue>>(); + HOCONValue::List(items) + } + Value::Object(o) => { + if let Some(result) = parse_special_types(o) { + return result; + } + + let mut items = o + .iter() + .filter(|(key, _)| key.as_str() != "_includes") + .map(|(key, value)| (key.clone(), json_to_hocon(value))) + .collect::<Vec<(String, HOCONValue)>>(); + + items.sort_by(|(a, _), (b, _)| a.partial_cmp(b).unwrap()); + + let includes = o + .get("_includes") + .map(|x| { + x.as_array() + .expect("_includes is not an array") + .iter() + .map(|x| { + x.as_object() + .unwrap_or_else(|| panic!("Include is not an object: {}", x)) + }) + .map(parse_include) + .collect::<Vec<HOCONInclude>>() + }) + .unwrap_or(vec![]); + + HOCONValue::Object(includes, items) + } + } +} + +impl ToString for HOCONValue { + fn to_string(&self) -> String { + match self { + HOCONValue::Null => "null".to_string(), + HOCONValue::Bool(b) => b.to_string(), + HOCONValue::Number(n) => n.to_string(), + HOCONValue::String(s) => serde_json::to_string(&Value::String(s.clone())).unwrap(), + HOCONValue::Substitution(v, required) => { + format!("${{{}{}}}", if *required { "" } else { "?" }, v) + } + HOCONValue::List(l) => { + let items = l + .iter() + .map(|item| item.to_string()) + .collect::<Vec<String>>() + .join(",\n") + .split('\n') + .map(|s| " ".to_owned() + s) + .collect::<Vec<String>>() + .join("\n"); + format!("[\n{}\n]", items) + } + HOCONValue::Object(i, o) => { + let includes = i + .iter() + .map(|x| { + x.map_fst(&|s| serde_json::to_string(&Value::String(s.clone())).unwrap()) + }) + .map(|x| match x { + HOCONInclude::Heuristic(s, r) => (s.to_string(), r), + HOCONInclude::Url(s, r) => (format!("url({})", s), r), + HOCONInclude::File(s, r) => (format!("file({})", s), r), + HOCONInclude::ClassPath(s, r) => (format!("classpath({})", s), r), + }) + .map(|(i, r)| if r { format!("required({})", i) } else { i }) + .map(|s| format!("include {}", s)) + .collect::<Vec<String>>() + .join("\n"); + let items = o + .iter() + .map(|(key, value)| { + ( + serde_json::to_string(&Value::String(key.clone())).unwrap(), + value, + ) + }) + .map(|(key, value)| match value { + HOCONValue::Append(v) => format!("{} += {}", key, v.to_string()), + v => format!("{} = {}", key, v.to_string()), + }) + .collect::<Vec<String>>() + .join("\n"); + + let content = (if includes.is_empty() { + items + } else { + format!("{}{}", includes, items) + }) + .split('\n') + .map(|s| format!(" {}", s)) + .collect::<Vec<String>>() + .join("\n"); + + format!("{{\n{}\n}}", content) + } + HOCONValue::Append(_) => panic!("Append should not be present at this point"), + Self::Literal(s) => s.to_string(), + } + } +} + +fn main() { + let stdin = std::io::stdin().lock(); + let json = serde_json::Deserializer::from_reader(stdin) + .into_iter::<Value>() + .next() + .expect("Could not read content from stdin") + .expect("Could not parse JSON from stdin"); + + print!("{}\n\n", json_to_hocon(&json).to_string()); +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix new file mode 100644 index 000000000000..5f0b3d12a2d0 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/default.nix @@ -0,0 +1,65 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + hocon = formats.hocon { }; + + expression = { + substitution = { __hocon_envvar = "PATH"; }; + literal = { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + }; + + nested = { + substitution = { __hocon_envvar = "PATH"; }; + literal = { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + }; + }; + + nested_in_array = [ + { __hocon_envvar = "PATH"; } + { + __hocon_unquoted_string = '' + [ + 1, + "a", + ]''; + } + ]; + }; + + hocon-test-conf = hocon.generate "hocon-test.conf" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.hocon-test-backwards-compatibility"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + runHook preCheck + + diff -U3 ${./expected.txt} ${hocon-test-conf} + + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + cp ${./expected.txt} $out/expected.txt + cp ${hocon-test-conf} $out/hocon-test.conf + cp ${hocon-test-conf.passthru.json} $out/hocon-test.json + + runHook postInstall + ''; + } diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt new file mode 100644 index 000000000000..2835a3c6ca39 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/backwards-compatibility/expected.txt @@ -0,0 +1,22 @@ +{ + "literal" = [ + 1, + "a", + ] + "nested" = { + "literal" = [ + 1, + "a", + ] + "substitution" = ${?PATH} + } + "nested_in_array" = [ + ${?PATH}, + [ + 1, + "a", + ] + ] + "substitution" = ${?PATH} +} + diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix new file mode 100644 index 000000000000..ae4fae443d41 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/default.nix @@ -0,0 +1,83 @@ +{ lib, formats, stdenvNoCC, writeText, ... }: +let + hocon = formats.hocon { }; + + include_file = (writeText "hocon-test-include.conf" '' + "val" = 1 + '').overrideAttrs (_: _: { + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + outputHash = "sha256-UhkJLhT3bD6znq+IdDjs/ahP19mLzrLCy/R14pVrfew="; + }); + + expression = { + simple_top_level_attr = "1.0"; + nested.attrset.has.a.integer.value = 100; + some_floaty = 29.95; + + array2d = [ + [ 1 2 "a" ] + [ 2 1 "b" ] + ]; + nasty_string = "\"@\n\\\t^*\b\f\n\0\";'''$"; + + "misc attrs" = { + x = 1; + y = hocon.lib.mkAppend { a = 1; }; + }; + + "cursed \" .attrs \" " = { + "a" = 1; + "a b" = hocon.lib.mkSubstitution "a"; + "a b c" = hocon.lib.mkSubstitution { + value = "a b"; + required = false; + }; + }; + + to_include = { + _includes = [ + (hocon.lib.mkInclude include_file) + (hocon.lib.mkInclude "https://example.com") + (hocon.lib.mkInclude { + required = true; + type = "file"; + value = include_file; + }) + (hocon.lib.mkInclude { value = include_file; }) + (hocon.lib.mkInclude { + value = "https://example.com"; + type = "url"; + }) + ]; + }; + }; + + hocon-test-conf = hocon.generate "hocon-test.conf" expression; +in + stdenvNoCC.mkDerivation { + name = "pkgs.formats.hocon-test-comprehensive"; + + dontUnpack = true; + dontBuild = true; + + doCheck = true; + checkPhase = '' + runHook preCheck + + diff -U3 ${./expected.txt} ${hocon-test-conf} + + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + + mkdir $out + cp ${./expected.txt} $out/expected.txt + cp ${hocon-test-conf} $out/hocon-test.conf + cp ${hocon-test-conf.passthru.json} $out/hocon-test.json + + runHook postInstall + ''; + } diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt new file mode 100644 index 000000000000..ec196be4f686 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/comprehensive/expected.txt @@ -0,0 +1,47 @@ +{ + "array2d" = [ + [ + 1, + 2, + "a" + ], + [ + 2, + 1, + "b" + ] + ] + "cursed \" .attrs \" " = { + "a" = 1 + "a b" = ${?a} + "a b c" = ${?a b} + } + "misc attrs" = { + "x" = 1 + "y" += { + "a" = 1 + } + } + "nasty_string" = "\"@\n\\\t^*bf\n0\";'''$" + "nested" = { + "attrset" = { + "has" = { + "a" = { + "integer" = { + "value" = 100 + } + } + } + } + } + "simple_top_level_attr" = "1.0" + "some_floaty" = 29.95 + "to_include" = { + include "/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf" + include "https://example.com" + include required(file("/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf")) + include "/nix/store/ccnzr53dpipdacxgci3ii3bqacvb5hxm-hocon-test-include.conf" + include url("https://example.com") + } +} + diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/default.nix b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/default.nix new file mode 100644 index 000000000000..19928703b95e --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/test/default.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: +{ + comprehensive = pkgs.callPackage ./comprehensive { }; + backwards-compatibility = + let + pkgsNoWarn = pkgs.extend (final: prev: { + lib = prev.lib.extend (libFinal: libPrev: { + warn = msg: v: v; + trivial = libPrev.trivial // { + warn = msg: v: v; + }; + }); + }); + in pkgsNoWarn.callPackage ./backwards-compatibility { }; +} diff --git a/nixpkgs/pkgs/pkgs-lib/formats/hocon/update.sh b/nixpkgs/pkgs/pkgs-lib/formats/hocon/update.sh new file mode 100755 index 000000000000..ffc5ad3917f7 --- /dev/null +++ b/nixpkgs/pkgs/pkgs-lib/formats/hocon/update.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p cargo -i bash +cd "$(dirname "$0")" +cargo update |