diff options
| author | Alyssa Ross <hi@alyssa.is> | 2023-06-16 06:56:35 +0000 |
|---|---|---|
| committer | Alyssa Ross <hi@alyssa.is> | 2023-06-16 06:56:35 +0000 |
| commit | 99fcaeccb89621dd492203ce1f2d551c06f228ed (patch) | |
| tree | 41cb730ae07383004789779b0f6e11cb3f4642a3 /nixpkgs/lib | |
| parent | 59c5f5ac8682acc13bb22bc29c7cf02f7d75f01f (diff) | |
| parent | 75a5ebf473cd60148ba9aec0d219f72e5cf52519 (diff) | |
| download | nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar.gz nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar.bz2 nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar.lz nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar.xz nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.tar.zst nixlib-99fcaeccb89621dd492203ce1f2d551c06f228ed.zip | |
Merge branch 'nixos-unstable' of https://github.com/NixOS/nixpkgs
Conflicts: nixpkgs/nixos/modules/config/console.nix nixpkgs/nixos/modules/services/mail/mailman.nix nixpkgs/nixos/modules/services/mail/public-inbox.nix nixpkgs/nixos/modules/services/mail/rss2email.nix nixpkgs/nixos/modules/services/networking/ssh/sshd.nix nixpkgs/pkgs/applications/networking/instant-messengers/dino/default.nix nixpkgs/pkgs/applications/networking/irc/weechat/default.nix nixpkgs/pkgs/applications/window-managers/sway/default.nix nixpkgs/pkgs/build-support/go/module.nix nixpkgs/pkgs/build-support/rust/build-rust-package/default.nix nixpkgs/pkgs/development/interpreters/python/default.nix nixpkgs/pkgs/development/node-packages/overrides.nix nixpkgs/pkgs/development/tools/b4/default.nix nixpkgs/pkgs/servers/dict/dictd-db.nix nixpkgs/pkgs/servers/mail/public-inbox/default.nix nixpkgs/pkgs/tools/security/pinentry/default.nix nixpkgs/pkgs/tools/text/unoconv/default.nix nixpkgs/pkgs/top-level/all-packages.nix
Diffstat (limited to 'nixpkgs/lib')
65 files changed, 4560 insertions, 824 deletions
diff --git a/nixpkgs/lib/ascii-table.nix b/nixpkgs/lib/ascii-table.nix new file mode 100644 index 000000000000..74989936ea40 --- /dev/null +++ b/nixpkgs/lib/ascii-table.nix @@ -0,0 +1,99 @@ +{ "\t" = 9; + "\n" = 10; + "\r" = 13; + " " = 32; + "!" = 33; + "\"" = 34; + "#" = 35; + "$" = 36; + "%" = 37; + "&" = 38; + "'" = 39; + "(" = 40; + ")" = 41; + "*" = 42; + "+" = 43; + "," = 44; + "-" = 45; + "." = 46; + "/" = 47; + "0" = 48; + "1" = 49; + "2" = 50; + "3" = 51; + "4" = 52; + "5" = 53; + "6" = 54; + "7" = 55; + "8" = 56; + "9" = 57; + ":" = 58; + ";" = 59; + "<" = 60; + "=" = 61; + ">" = 62; + "?" = 63; + "@" = 64; + "A" = 65; + "B" = 66; + "C" = 67; + "D" = 68; + "E" = 69; + "F" = 70; + "G" = 71; + "H" = 72; + "I" = 73; + "J" = 74; + "K" = 75; + "L" = 76; + "M" = 77; + "N" = 78; + "O" = 79; + "P" = 80; + "Q" = 81; + "R" = 82; + "S" = 83; + "T" = 84; + "U" = 85; + "V" = 86; + "W" = 87; + "X" = 88; + "Y" = 89; + "Z" = 90; + "[" = 91; + "\\" = 92; + "]" = 93; + "^" = 94; + "_" = 95; + "`" = 96; + "a" = 97; + "b" = 98; + "c" = 99; + "d" = 100; + "e" = 101; + "f" = 102; + "g" = 103; + "h" = 104; + "i" = 105; + "j" = 106; + "k" = 107; + "l" = 108; + "m" = 109; + "n" = 110; + "o" = 111; + "p" = 112; + "q" = 113; + "r" = 114; + "s" = 115; + "t" = 116; + "u" = 117; + "v" = 118; + "w" = 119; + "x" = 120; + "y" = 121; + "z" = 122; + "{" = 123; + "|" = 124; + "}" = 125; + "~" = 126; +} diff --git a/nixpkgs/lib/asserts.nix b/nixpkgs/lib/asserts.nix index 9ae357cbc935..98e0b490acf2 100644 --- a/nixpkgs/lib/asserts.nix +++ b/nixpkgs/lib/asserts.nix @@ -16,11 +16,15 @@ rec { assertMsg :: Bool -> String -> Bool */ # TODO(Profpatsch): add tests that check stderr - assertMsg = pred: msg: + assertMsg = + # Predicate that needs to succeed, otherwise `msg` is thrown + pred: + # Message to throw in case `pred` fails + msg: pred || builtins.throw msg; - /* Specialized `assertMsg` for checking if val is one of the elements - of a list. Useful for checking enums. + /* Specialized `assertMsg` for checking if `val` is one of the elements + of the list `xs`. Useful for checking enums. Example: let sslLibrary = "libressl"; @@ -33,7 +37,14 @@ rec { Type: assertOneOf :: String -> ComparableVal -> List ComparableVal -> Bool */ - assertOneOf = name: val: xs: assertMsg + assertOneOf = + # The name of the variable the user entered `val` into, for inclusion in the error message + name: + # The value of what the user provided, to be compared against the values in `xs` + val: + # The list of valid values + xs: + assertMsg (lib.elem val xs) "${name} must be one of ${ lib.generators.toPretty {} xs}, but is: ${ diff --git a/nixpkgs/lib/attrsets.nix b/nixpkgs/lib/attrsets.nix index 5575e9577029..73b71f68db79 100644 --- a/nixpkgs/lib/attrsets.nix +++ b/nixpkgs/lib/attrsets.nix @@ -3,30 +3,42 @@ let inherit (builtins) head tail length; - inherit (lib.trivial) id; + inherit (lib.trivial) flip id mergeAttrs pipe; inherit (lib.strings) concatStringsSep concatMapStringsSep escapeNixIdentifier sanitizeDerivationName; inherit (lib.lists) foldr foldl' concatMap concatLists elemAt all partition groupBy take foldl; in rec { - inherit (builtins) attrNames listToAttrs hasAttr isAttrs getAttr; + inherit (builtins) attrNames listToAttrs hasAttr isAttrs getAttr removeAttrs; /* Return an attribute from nested attribute sets. Example: x = { a = { b = 3; }; } + # ["a" "b"] is equivalent to x.a.b + # 6 is a default value to return if the path does not exist in attrset attrByPath ["a" "b"] 6 x => 3 attrByPath ["z" "z"] 6 x => 6 + + Type: + attrByPath :: [String] -> Any -> AttrSet -> Any + */ - attrByPath = attrPath: default: e: + attrByPath = + # A list of strings representing the attribute path to return from `set` + attrPath: + # Default value if `attrPath` does not resolve to an existing value + default: + # The nested attribute set to select values from + set: let attr = head attrPath; in - if attrPath == [] then e - else if e ? ${attr} - then attrByPath (tail attrPath) default e.${attr} + if attrPath == [] then set + else if set ? ${attr} + then attrByPath (tail attrPath) default set.${attr} else default; /* Return if an attribute from nested attribute set exists. @@ -38,8 +50,14 @@ rec { hasAttrByPath ["z" "z"] x => false + Type: + hasAttrByPath :: [String] -> AttrSet -> Bool */ - hasAttrByPath = attrPath: e: + hasAttrByPath = + # A list of strings representing the attribute path to check from `set` + attrPath: + # The nested attribute set to check + e: let attr = head attrPath; in if attrPath == [] then true @@ -48,13 +66,20 @@ rec { else false; - /* Return nested attribute set in which an attribute is set. + /* Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. Example: setAttrByPath ["a" "b"] 3 => { a = { b = 3; }; } + + Type: + setAttrByPath :: [String] -> Any -> AttrSet */ - setAttrByPath = attrPath: value: + setAttrByPath = + # A list of strings representing the attribute path to set + attrPath: + # The value to set at the location described by `attrPath` + value: let len = length attrPath; atDepth = n: @@ -63,8 +88,8 @@ rec { else { ${elemAt attrPath n} = atDepth (n + 1); }; in atDepth 0; - /* Like `attrByPath' without a default value. If it doesn't find the - path it will throw. + /* Like `attrByPath`, but without a default value. If it doesn't find the + path it will throw an error. Example: x = { a = { b = 3; }; } @@ -72,29 +97,60 @@ rec { => 3 getAttrFromPath ["z" "z"] x => error: cannot find attribute `z.z' + + Type: + getAttrFromPath :: [String] -> AttrSet -> Any */ - getAttrFromPath = attrPath: + getAttrFromPath = + # A list of strings representing the attribute path to get from `set` + attrPath: + # The nested attribute set to find the value in. + set: let errorMsg = "cannot find attribute `" + concatStringsSep "." attrPath + "'"; - in attrByPath attrPath (abort errorMsg); + in attrByPath attrPath (abort errorMsg) set; + + /* Map each attribute in the given set and merge them into a new attribute set. + + Type: + concatMapAttrs :: (String -> a -> AttrSet) -> AttrSet -> AttrSet + + Example: + concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { x = "a"; y = "b"; } + => { x = "a"; xa = "a"; y = "b"; yb = "b"; } + */ + concatMapAttrs = f: v: + foldl' mergeAttrs { } + (attrValues + (mapAttrs f v) + ); /* Update or set specific paths of an attribute set. Takes a list of updates to apply and an attribute set to apply them to, and returns the attribute set with the updates applied. Updates are - represented as { path = ...; update = ...; } values, where `path` is a + represented as `{ path = ...; update = ...; }` values, where `path` is a list of strings representing the attribute path that should be updated, and `update` is a function that takes the old value at that attribute path as an argument and returns the new value it should be. Properties: + - Updates to deeper attribute paths are applied before updates to more shallow attribute paths + - Multiple updates to the same attribute path are applied in the order they appear in the update list + - If any but the last `path` element leads into a value that is not an attribute set, an error is thrown + - If there is an update for an attribute path that doesn't exist, accessing the argument in the update function causes an error, but intermediate attribute sets are implicitly created as needed @@ -115,6 +171,8 @@ rec { } ] { a.b.c = 0; } => { a = { b = { d = 1; }; }; x = { y = "xy"; }; } + + Type: updateManyAttrsByPath :: [{ path :: [String]; update :: (Any -> Any); }] -> AttrSet -> AttrSet */ updateManyAttrsByPath = let # When recursing into attributes, instead of updating the `path` of each @@ -180,8 +238,15 @@ rec { Example: attrVals ["a" "b" "c"] as => [as.a as.b as.c] + + Type: + attrVals :: [String] -> AttrSet -> [Any] */ - attrVals = nameList: set: map (x: set.${x}) nameList; + attrVals = + # The list of attributes to fetch from `set`. Each attribute name must exist on the attrbitue set + nameList: + # The set to get attribute values from + set: map (x: set.${x}) nameList; /* Return the values of all attributes in the given set, sorted by @@ -190,6 +255,9 @@ rec { Example: attrValues {c = 3; a = 1; b = 2;} => [1 2 3] + + Type: + attrValues :: AttrSet -> [Any] */ attrValues = builtins.attrValues or (attrs: attrVals (attrNames attrs) attrs); @@ -200,15 +268,25 @@ rec { Example: getAttrs [ "a" "b" ] { a = 1; b = 2; c = 3; } => { a = 1; b = 2; } + + Type: + getAttrs :: [String] -> AttrSet -> AttrSet */ - getAttrs = names: attrs: genAttrs names (name: attrs.${name}); + getAttrs = + # A list of attribute names to get out of `set` + names: + # The set to get the named attributes from + attrs: genAttrs names (name: attrs.${name}); - /* Collect each attribute named `attr' from a list of attribute + /* Collect each attribute named `attr` from a list of attribute sets. Sets that don't contain the named attribute are ignored. Example: catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] => [1 2] + + Type: + catAttrs :: String -> [AttrSet] -> [Any] */ catAttrs = builtins.catAttrs or (attr: l: concatLists (map (s: if s ? ${attr} then [s.${attr}] else []) l)); @@ -220,8 +298,15 @@ rec { Example: filterAttrs (n: v: n == "foo") { foo = 1; bar = 2; } => { foo = 1; } + + Type: + filterAttrs :: (String -> Any -> Bool) -> AttrSet -> AttrSet */ - filterAttrs = pred: set: + filterAttrs = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); @@ -231,8 +316,15 @@ rec { Example: filterAttrsRecursive (n: v: v != null) { foo = { bar = null; }; } => { foo = {}; } + + Type: + filterAttrsRecursive :: (String -> Any -> Bool) -> AttrSet -> AttrSet */ - filterAttrsRecursive = pred: set: + filterAttrsRecursive = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: listToAttrs ( concatMap (name: let v = set.${name}; in @@ -245,28 +337,94 @@ rec { ) (attrNames set) ); + /* + Like builtins.foldl' but for attribute sets. + Iterates over every name-value pair in the given attribute set. + The result of the callback function is often called `acc` for accumulator. It is passed between callbacks from left to right and the final `acc` is the return value of `foldlAttrs`. + + Attention: + There is a completely different function + `lib.foldAttrs` + which has nothing to do with this function, despite the similar name. + + Example: + foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [name]; + }) + { sum = 0; names = []; } + { + foo = 1; + bar = 10; + } + -> + { + sum = 11; + names = ["bar" "foo"]; + } + + foldlAttrs + (throw "function not needed") + 123 + {}; + -> + 123 + + foldlAttrs + (_: _: v: v) + (throw "initial accumulator not needed") + { z = 3; a = 2; }; + -> + 3 + + The accumulator doesn't have to be an attrset. + It can be as simple as a number or string. + + foldlAttrs + (acc: _: v: acc * 10 + v) + 1 + { z = 1; a = 2; }; + -> + 121 + + Type: + foldlAttrs :: ( a -> String -> b -> a ) -> a -> { ... :: b } -> a + */ + foldlAttrs = f: init: set: + foldl' + (acc: name: f acc name set.${name}) + init + (attrNames set); + /* Apply fold functions to values grouped by key. Example: foldAttrs (item: acc: [item] ++ acc) [] [{ a = 2; } { a = 3; }] => { a = [ 2 3 ]; } + + Type: + foldAttrs :: (Any -> Any -> Any) -> Any -> [AttrSets] -> Any + */ - foldAttrs = op: nul: + foldAttrs = + # A function, given a value and a collector combines the two. + op: + # The starting value. + nul: + # A list of attribute sets to fold together by key. + list_of_attrs: foldr (n: a: foldr (name: o: o // { ${name} = op n.${name} (a.${name} or nul); } ) a (attrNames n) - ) {}; + ) {} list_of_attrs; - /* Recursively collect sets that verify a given predicate named `pred' - from the set `attrs'. The recursion is stopped when the predicate is + /* Recursively collect sets that verify a given predicate named `pred` + from the set `attrs`. The recursion is stopped when the predicate is verified. - Type: - collect :: - (AttrSet -> Bool) -> AttrSet -> [x] - Example: collect isList { a = { b = ["b"]; }; c = [1]; } => [["b"] [1]] @@ -274,8 +432,15 @@ rec { collect (x: x ? outPath) { a = { outPath = "a/"; }; b = { outPath = "b/"; }; } => [{ outPath = "a/"; } { outPath = "b/"; }] + + Type: + collect :: (AttrSet -> Bool) -> AttrSet -> [x] */ - collect = pred: attrs: + collect = + # Given an attribute's value, determine if recursion should stop. + pred: + # The attribute set to recursively collect. + attrs: if pred attrs then [ attrs ] else if isAttrs attrs then @@ -293,8 +458,12 @@ rec { { a = 2; b = 10; } { a = 2; b = 20; } ] + Type: + cartesianProductOfSets :: AttrSet -> [AttrSet] */ - cartesianProductOfSets = attrsOfLists: + cartesianProductOfSets = + # Attribute set with attributes that are lists of values + attrsOfLists: foldl' (listOfAttrs: attrName: concatMap (attrs: map (listValue: attrs // { ${attrName} = listValue; }) attrsOfLists.${attrName} @@ -302,86 +471,109 @@ rec { ) [{}] (attrNames attrsOfLists); - /* Utility function that creates a {name, value} pair as expected by - builtins.listToAttrs. + /* Utility function that creates a `{name, value}` pair as expected by `builtins.listToAttrs`. Example: nameValuePair "some" 6 => { name = "some"; value = 6; } + + Type: + nameValuePair :: String -> Any -> { name :: String; value :: Any; } */ - nameValuePair = name: value: { inherit name value; }; + nameValuePair = + # Attribute name + name: + # Attribute value + value: + { inherit name value; }; - /* Apply a function to each element in an attribute set. The - function takes two arguments --- the attribute name and its value - --- and returns the new value for the attribute. The result is a - new attribute set. + /* Apply a function to each element in an attribute set, creating a new attribute set. Example: mapAttrs (name: value: name + "-" + value) { x = "foo"; y = "bar"; } => { x = "x-foo"; y = "y-bar"; } + + Type: + mapAttrs :: (String -> Any -> Any) -> AttrSet -> AttrSet */ mapAttrs = builtins.mapAttrs or (f: set: listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))); - /* Like `mapAttrs', but allows the name of each attribute to be + /* Like `mapAttrs`, but allows the name of each attribute to be changed in addition to the value. The applied function should - return both the new name and value as a `nameValuePair'. + return both the new name and value as a `nameValuePair`. Example: mapAttrs' (name: value: nameValuePair ("foo_" + name) ("bar-" + value)) { x = "a"; y = "b"; } => { foo_x = "bar-a"; foo_y = "bar-b"; } + + Type: + mapAttrs' :: (String -> Any -> { name :: String; value :: Any; }) -> AttrSet -> AttrSet */ - mapAttrs' = f: set: + mapAttrs' = + # A function, given an attribute's name and value, returns a new `nameValuePair`. + f: + # Attribute set to map over. + set: listToAttrs (map (attr: f attr set.${attr}) (attrNames set)); /* Call a function for each attribute in the given set and return the result in a list. - Type: - mapAttrsToList :: - (String -> a -> b) -> AttrSet -> [b] - Example: mapAttrsToList (name: value: name + value) { x = "a"; y = "b"; } => [ "xa" "yb" ] + + Type: + mapAttrsToList :: (String -> a -> b) -> AttrSet -> [b] + */ - mapAttrsToList = f: attrs: + mapAttrsToList = + # A function, given an attribute's name and value, returns a new value. + f: + # Attribute set to map over. + attrs: map (name: f name attrs.${name}) (attrNames attrs); - /* Like `mapAttrs', except that it recursively applies itself to - attribute sets. Also, the first argument of the argument - function is a *list* of the names of the containing attributes. + /* Like `mapAttrs`, except that it recursively applies itself to + the *leaf* attributes of a potentially-nested attribute set: + the second argument of the function will never be an attrset. + Also, the first argument of the argument function is a *list* + of the attribute names that form the path to the leaf attribute. - Type: - mapAttrsRecursive :: - ([String] -> a -> b) -> AttrSet -> AttrSet + For a function that gives you control over what counts as a leaf, + see `mapAttrsRecursiveCond`. Example: mapAttrsRecursive (path: value: concatStringsSep "-" (path ++ [value])) { n = { a = "A"; m = { b = "B"; c = "C"; }; }; d = "D"; } => { n = { a = "n-a-A"; m = { b = "n-m-b-B"; c = "n-m-c-C"; }; }; d = "d-D"; } + + Type: + mapAttrsRecursive :: ([String] -> a -> b) -> AttrSet -> AttrSet */ - mapAttrsRecursive = mapAttrsRecursiveCond (as: true); + mapAttrsRecursive = + # A function, given a list of attribute names and a value, returns a new value. + f: + # Set to recursively map over. + set: + mapAttrsRecursiveCond (as: true) f set; - /* Like `mapAttrsRecursive', but it takes an additional predicate + /* Like `mapAttrsRecursive`, but it takes an additional predicate function that tells it whether to recurse into an attribute - set. If it returns false, `mapAttrsRecursiveCond' does not + set. If it returns false, `mapAttrsRecursiveCond` does not recurse, but does apply the map function. If it returns true, it does recurse, and does not apply the map function. - Type: - mapAttrsRecursiveCond :: - (AttrSet -> Bool) -> ([String] -> a -> b) -> AttrSet -> AttrSet - Example: # To prevent recursing into derivations (which are attribute # sets with the attribute "type" equal to "derivation"): @@ -389,8 +581,17 @@ rec { (as: !(as ? "type" && as.type == "derivation")) (x: ... do something ...) attrs + + Type: + mapAttrsRecursiveCond :: (AttrSet -> Bool) -> ([String] -> a -> b) -> AttrSet -> AttrSet */ - mapAttrsRecursiveCond = cond: f: set: + mapAttrsRecursiveCond = + # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set. + cond: + # A function, given a list of attribute names and a value, returns a new value. + f: + # Attribute set to recursively map over. + set: let recurse = path: let @@ -409,13 +610,20 @@ rec { Example: genAttrs [ "foo" "bar" ] (name: "x_" + name) => { foo = "x_foo"; bar = "x_bar"; } + + Type: + genAttrs :: [ String ] -> (String -> Any) -> AttrSet */ - genAttrs = names: f: + genAttrs = + # Names of values in the resulting attribute set. + names: + # A function, given the name of the attribute, returns the attribute's value. + f: listToAttrs (map (n: nameValuePair n (f n)) names); /* Check whether the argument is a derivation. Any set with - { type = "derivation"; } counts as a derivation. + `{ type = "derivation"; }` counts as a derivation. Example: nixpkgs = import <nixpkgs> {} @@ -423,25 +631,36 @@ rec { => true isDerivation "foobar" => false + + Type: + isDerivation :: Any -> Bool */ - isDerivation = x: x.type or null == "derivation"; + isDerivation = + # Value to check. + value: value.type or null == "derivation"; - /* Converts a store path to a fake derivation. */ - toDerivation = path: - let - path' = builtins.storePath path; - res = - { type = "derivation"; - name = sanitizeDerivationName (builtins.substring 33 (-1) (baseNameOf path')); - outPath = path'; - outputs = [ "out" ]; - out = res; - outputName = "out"; - }; + /* Converts a store path to a fake derivation. + + Type: + toDerivation :: Path -> Derivation + */ + toDerivation = + # A store path to convert to a derivation. + path: + let + path' = builtins.storePath path; + res = + { type = "derivation"; + name = sanitizeDerivationName (builtins.substring 33 (-1) (baseNameOf path')); + outPath = path'; + outputs = [ "out" ]; + out = res; + outputName = "out"; + }; in res; - /* If `cond' is true, return the attribute set `as', + /* If `cond` is true, return the attribute set `as`, otherwise an empty attribute set. Example: @@ -449,47 +668,82 @@ rec { => { my = "set"; } optionalAttrs (false) { my = "set"; } => { } + + Type: + optionalAttrs :: Bool -> AttrSet -> AttrSet */ - optionalAttrs = cond: as: if cond then as else {}; + optionalAttrs = + # Condition under which the `as` attribute set is returned. + cond: + # The attribute set to return if `cond` is `true`. + as: + if cond then as else {}; - /* Merge sets of attributes and use the function f to merge attributes + /* Merge sets of attributes and use the function `f` to merge attributes values. Example: zipAttrsWithNames ["a"] (name: vs: vs) [{a = "x";} {a = "y"; b = "z";}] => { a = ["x" "y"]; } + + Type: + zipAttrsWithNames :: [ String ] -> (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet */ - zipAttrsWithNames = names: f: sets: + zipAttrsWithNames = + # List of attribute names to zip. + names: + # A function, accepts an attribute name, all the values, and returns a combined value. + f: + # List of values from the list of attribute sets. + sets: listToAttrs (map (name: { inherit name; value = f name (catAttrs name sets); }) names); - /* Implementation note: Common names appear multiple times in the list of + + /* Merge sets of attributes and use the function f to merge attribute values. + Like `lib.attrsets.zipAttrsWithNames` with all key names are passed for `names`. + + Implementation note: Common names appear multiple times in the list of names, hopefully this does not affect the system because the maximal - laziness avoid computing twice the same expression and listToAttrs does + laziness avoid computing twice the same expression and `listToAttrs` does not care about duplicated attribute names. Example: zipAttrsWith (name: values: values) [{a = "x";} {a = "y"; b = "z";}] - => { a = ["x" "y"]; b = ["z"] } + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrsWith :: (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet */ zipAttrsWith = builtins.zipAttrsWith or (f: sets: zipAttrsWithNames (concatMap attrNames sets) f sets); - /* Like `zipAttrsWith' with `(name: values: values)' as the function. - Example: - zipAttrs [{a = "x";} {a = "y"; b = "z";}] - => { a = ["x" "y"]; b = ["z"] } + + /* Merge sets of attributes and combine each attribute value in to a list. + + Like `lib.attrsets.zipAttrsWith` with `(name: values: values)` as the function. + + Example: + zipAttrs [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrs :: [ AttrSet ] -> AttrSet */ - zipAttrs = zipAttrsWith (name: values: values); + zipAttrs = + # List of attribute sets to zip together. + sets: + zipAttrsWith (name: values: values) sets; + /* Does the same as the update operator '//' except that attributes are merged until the given predicate is verified. The predicate should accept 3 arguments which are the path to reach the attribute, a part of the first attribute set and a part of the second attribute set. When - the predicate is verified, the value of the first attribute set is + the predicate is satisfied, the value of the first attribute set is replaced by the value of the second attribute set. Example: @@ -505,15 +759,23 @@ rec { baz = 4; } - returns: { + => { foo.bar = 1; # 'foo.*' from the second set foo.quz = 2; # bar = 3; # 'bar' from the first set baz = 4; # 'baz' from the second set } - */ - recursiveUpdateUntil = pred: lhs: rhs: + Type: + recursiveUpdateUntil :: ( [ String ] -> AttrSet -> AttrSet -> Bool ) -> AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdateUntil = + # Predicate, taking the path to the current attribute as a list of strings for attribute names, and the two values at that path from the original arguments. + pred: + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: let f = attrPath: zipAttrsWith (n: values: let here = attrPath ++ [n]; in @@ -525,6 +787,7 @@ rec { ); in f [] [rhs lhs]; + /* A recursive variant of the update operator ‘//’. The recursion stops when one of the attribute values is not an attribute set, in which case the right hand side value takes precedence over the @@ -543,16 +806,32 @@ rec { boot.loader.grub.device = ""; } - */ - recursiveUpdate = recursiveUpdateUntil (path: lhs: rhs: !(isAttrs lhs && isAttrs rhs)); + Type: + recursiveUpdate :: AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdate = + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: + recursiveUpdateUntil (path: lhs: rhs: !(isAttrs lhs && isAttrs rhs)) lhs rhs; + /* Returns true if the pattern is contained in the set. False otherwise. Example: matchAttrs { cpu = {}; } { cpu = { bits = 64; }; } => true - */ - matchAttrs = pattern: attrs: assert isAttrs pattern; + + Type: + matchAttrs :: AttrSet -> AttrSet -> Bool + */ + matchAttrs = + # Attribute set structure to match + pattern: + # Attribute set to find patterns in + attrs: + assert isAttrs pattern; all id (attrValues (zipAttrsWithNames (attrNames pattern) (n: values: let pat = head values; val = elemAt values 1; in if length values == 1 then false @@ -560,6 +839,7 @@ rec { else pat == val ) [pattern attrs])); + /* Override only the attributes that are already present in the old set useful for deep-overriding. @@ -570,61 +850,169 @@ rec { => { b = 2; } overrideExisting { a = 3; b = 2; } { a = 1; } => { a = 1; b = 2; } + + Type: + overrideExisting :: AttrSet -> AttrSet -> AttrSet */ - overrideExisting = old: new: + overrideExisting = + # Original attribute set + old: + # Attribute set with attributes to override in `old`. + new: mapAttrs (name: value: new.${name} or value) old; + /* Turns a list of strings into a human-readable description of those strings represented as an attribute path. The result of this function is not intended to be machine-readable. + Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. Example: showAttrPath [ "foo" "10" "bar" ] => "foo.\"10\".bar" showAttrPath [] => "<root attribute path>" + + Type: + showAttrPath :: [String] -> String */ - showAttrPath = path: + showAttrPath = + # Attribute path to render to a string + path: if path == [] then "<root attribute path>" else concatMapStringsSep "." escapeNixIdentifier path; + /* Get a package output. If no output is found, fallback to `.out` and then to the default. Example: getOutput "dev" pkgs.openssl => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getOutput :: String -> Derivation -> String */ getOutput = output: pkg: if ! pkg ? outputSpecified || ! pkg.outputSpecified then pkg.${output} or pkg.out or pkg else pkg; + /* Get a package's `bin` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getBin pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r" + + Type: + getBin :: Derivation -> String + */ getBin = getOutput "bin"; + + + /* Get a package's `lib` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getLib pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-lib" + + Type: + getLib :: Derivation -> String + */ getLib = getOutput "lib"; + + + /* Get a package's `dev` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getDev pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getDev :: Derivation -> String + */ getDev = getOutput "dev"; + + + /* Get a package's `man` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getMan pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-man" + + Type: + getMan :: Derivation -> String + */ getMan = getOutput "man"; - /* Pick the outputs of packages to place in buildInputs */ - chooseDevOutputs = drvs: builtins.map getDev drvs; + /* Pick the outputs of packages to place in `buildInputs` + + Type: chooseDevOutputs :: [Derivation] -> [String] + + */ + chooseDevOutputs = + # List of packages to pick `dev` outputs from + drvs: + builtins.map getDev drvs; /* Make various Nix tools consider the contents of the resulting attribute set when looking for what to build, find, etc. This function only affects a single attribute set; it does not apply itself recursively for nested attribute sets. + + Example: + { pkgs ? import <nixpkgs> {} }: + { + myTools = pkgs.lib.recurseIntoAttrs { + inherit (pkgs) hello figlet; + }; + } + + Type: + recurseIntoAttrs :: AttrSet -> AttrSet + */ recurseIntoAttrs = - attrs: attrs // { recurseForDerivations = true; }; + # An attribute set to scan for derivations. + attrs: + attrs // { recurseForDerivations = true; }; /* Undo the effect of recurseIntoAttrs. + + Type: + dontRecurseIntoAttrs :: AttrSet -> AttrSet */ dontRecurseIntoAttrs = - attrs: attrs // { recurseForDerivations = false; }; + # An attribute set to not scan for derivations. + attrs: + attrs // { recurseForDerivations = false; }; - /*** deprecated stuff ***/ + /* `unionOfDisjoint x y` is equal to `x // y // z` where the + attrnames in `z` are the intersection of the attrnames in `x` and + `y`, and all values `assert` with an error message. This + operator is commutative, unlike (//). + Type: unionOfDisjoint :: AttrSet -> AttrSet -> AttrSet + */ + unionOfDisjoint = x: y: + let + intersection = builtins.intersectAttrs x y; + collisions = lib.concatStringsSep " " (builtins.attrNames intersection); + mask = builtins.mapAttrs (name: value: builtins.throw + "unionOfDisjoint: collision on ${name}; complete list: ${collisions}") + intersection; + in + (x // y) // mask; + + # DEPRECATED zipWithNames = zipAttrsWithNames; + + # DEPRECATED zip = builtins.trace "lib.zip is deprecated, use lib.zipAttrsWith instead" zipAttrsWith; } diff --git a/nixpkgs/lib/customisation.nix b/nixpkgs/lib/customisation.nix index cc9a9b1c55d0..fe32e890f357 100644 --- a/nixpkgs/lib/customisation.nix +++ b/nixpkgs/lib/customisation.nix @@ -3,13 +3,13 @@ rec { - /* `overrideDerivation drv f' takes a derivation (i.e., the result - of a call to the builtin function `derivation') and returns a new + /* `overrideDerivation drv f` takes a derivation (i.e., the result + of a call to the builtin function `derivation`) and returns a new derivation in which the attributes of the original are overridden - according to the function `f'. The function `f' is called with + according to the function `f`. The function `f` is called with the original derivation attributes. - `overrideDerivation' allows certain "ad-hoc" customisation + `overrideDerivation` allows certain "ad-hoc" customisation scenarios (e.g. in ~/.config/nixpkgs/config.nix). For instance, if you want to "patch" the derivation returned by a package function in Nixpkgs to build another version than what the @@ -27,23 +27,34 @@ rec { For another application, see build-support/vm, where this function is used to build arbitrary derivations inside a QEMU virtual machine. + + Note that in order to preserve evaluation errors, the new derivation's + outPath depends on the old one's, which means that this function cannot + be used in circular situations when the old derivation also depends on the + new one. + + You should in general prefer `drv.overrideAttrs` over this function; + see the nixpkgs manual for more information on overriding. */ overrideDerivation = drv: f: let newDrv = derivation (drv.drvAttrs // (f drv)); - in lib.flip (extendDerivation true) newDrv ( + in lib.flip (extendDerivation (builtins.seq drv.drvPath true)) newDrv ( { meta = drv.meta or {}; passthru = if drv ? passthru then drv.passthru else {}; } // (drv.passthru or {}) // - (if (drv ? crossDrv && drv ? nativeDrv) - then { - crossDrv = overrideDerivation drv.crossDrv f; - nativeDrv = overrideDerivation drv.nativeDrv f; - } - else { })); + # TODO(@Artturin): remove before release 23.05 and only have __spliced. + (lib.optionalAttrs (drv ? crossDrv && drv ? nativeDrv) { + crossDrv = overrideDerivation drv.crossDrv f; + nativeDrv = overrideDerivation drv.nativeDrv f; + }) + // + lib.optionalAttrs (drv ? __spliced) { + __spliced = {} // (lib.mapAttrs (_: sDrv: overrideDerivation sDrv f) drv.__spliced); + }); /* `makeOverridable` takes a function from attribute set to attribute set and @@ -93,10 +104,10 @@ rec { else result; - /* Call the package function in the file `fn' with the required + /* Call the package function in the file `fn` with the required arguments automatically. The function is called with the - arguments `args', but any missing arguments are obtained from - `autoArgs'. This function is intended to be partially + arguments `args`, but any missing arguments are obtained from + `autoArgs`. This function is intended to be partially parameterised, e.g., callPackage = callPackageWith pkgs; @@ -105,9 +116,9 @@ rec { libbar = callPackage ./bar.nix { }; }; - If the `libbar' function expects an argument named `libfoo', it is + If the `libbar` function expects an argument named `libfoo`, it is automatically passed as an argument. Overrides or missing - arguments can be supplied in `args', e.g. + arguments can be supplied in `args`, e.g. libbar = callPackage ./bar.nix { libfoo = null; @@ -165,7 +176,7 @@ rec { # Only show the error for the first missing argument error = errorForArg (lib.head missingArgs); - in if missingArgs == [] then makeOverridable f allArgs else throw error; + in if missingArgs == [] then makeOverridable f allArgs else abort error; /* Like callPackage, but for a function that returns an attribute @@ -202,7 +213,14 @@ rec { outputSpecified = true; drvPath = assert condition; drv.${outputName}.drvPath; outPath = assert condition; drv.${outputName}.outPath; - }; + } // + # TODO: give the derivation control over the outputs. + # `overrideAttrs` may not be the only attribute that needs + # updating when switching outputs. + lib.optionalAttrs (passthru?overrideAttrs) { + # TODO: also add overrideAttrs when overrideAttrs is not custom, e.g. when not splicing. + overrideAttrs = f: (passthru.overrideAttrs f).${outputName}; + }; }; outputsList = map outputToAttrListElement outputs; @@ -241,16 +259,17 @@ rec { outputsList = map makeOutput outputs; drv' = (lib.head outputsList).value; - in lib.deepSeq drv' drv'; + in if drv == null then null else + lib.deepSeq drv' drv'; /* Make a set of packages with a common scope. All packages called - with the provided `callPackage' will be evaluated with the same + with the provided `callPackage` will be evaluated with the same arguments. Any package in the set may depend on any other. The `overrideScope'` function allows subsequent modification of the package set in a consistent way, i.e. all packages in the set will be called with the overridden packages. The package sets may be hierarchical: the packages in the set are called with the scope - provided by `newScope' and the set provides a `newScope' attribute + provided by `newScope` and the set provides a `newScope` attribute which can form the parent scope for later package sets. */ makeScope = newScope: f: let self = f self // { diff --git a/nixpkgs/lib/debug.nix b/nixpkgs/lib/debug.nix index e3ca3352397e..a851cd74778c 100644 --- a/nixpkgs/lib/debug.nix +++ b/nixpkgs/lib/debug.nix @@ -109,6 +109,8 @@ rec { traceSeqN 2 { a.b.c = 3; } null trace: { a = { b = {…}; }; } => null + + Type: traceSeqN :: Int -> a -> b -> b */ traceSeqN = depth: x: y: let snip = v: if isList v then noQuotes "[…]" v @@ -173,17 +175,63 @@ rec { # -- TESTING -- - /* Evaluate a set of tests. A test is an attribute set `{expr, - expected}`, denoting an expression and its expected result. The - result is a list of failed tests, each represented as `{name, - expected, actual}`, denoting the attribute name of the failing - test and its expected and actual results. + /* Evaluates a set of tests. - Used for regression testing of the functions in lib; see - tests.nix for an example. Only tests having names starting with - "test" are run. + A test is an attribute set `{expr, expected}`, + denoting an expression and its expected result. + + The result is a `list` of __failed tests__, each represented as + `{name, expected, result}`, - Add attr { tests = ["testName"]; } to run these tests only. + - expected + - What was passed as `expected` + - result + - The actual `result` of the test + + Used for regression testing of the functions in lib; see + tests.nix for more examples. + + Important: Only attributes that start with `test` are executed. + + - If you want to run only a subset of the tests add the attribute `tests = ["testName"];` + + Example: + + runTests { + testAndOk = { + expr = lib.and true false; + expected = false; + }; + testAndFail = { + expr = lib.and true false; + expected = true; + }; + } + -> + [ + { + name = "testAndFail"; + expected = true; + result = false; + } + ] + + Type: + runTests :: { + tests = [ String ]; + ${testName} :: { + expr :: a; + expected :: a; + }; + } + -> + [ + { + name :: String; + expected :: a; + result :: a; + } + ] */ runTests = # Tests to run @@ -202,90 +250,4 @@ rec { { testX = allTrue [ true ]; } */ testAllTrue = expr: { inherit expr; expected = map (x: true) expr; }; - - - # -- DEPRECATED -- - - traceShowVal = x: trace (showVal x) x; - traceShowValMarked = str: x: trace (str + showVal x) x; - - attrNamesToStr = a: - trace ( "Warning: `attrNamesToStr` is deprecated " - + "and will be removed in the next release. " - + "Please use more specific concatenation " - + "for your uses (`lib.concat(Map)StringsSep`)." ) - (concatStringsSep "; " (map (x: "${x}=") (attrNames a))); - - showVal = - trace ( "Warning: `showVal` is deprecated " - + "and will be removed in the next release, " - + "please use `traceSeqN`" ) - (let - modify = v: - let pr = f: { __pretty = f; val = v; }; - in if isDerivation v then pr - (drv: "<δ:${drv.name}:${concatStringsSep "," - (attrNames drv)}>") - else if [] == v then pr (const "[]") - else if isList v then pr (l: "[ ${go (head l)}, … ]") - else if isAttrs v then pr - (a: "{ ${ concatStringsSep ", " (attrNames a)} }") - else v; - go = x: generators.toPretty - { allowPrettyValues = true; } - (modify x); - in go); - - traceXMLVal = x: - trace ( "Warning: `traceXMLVal` is deprecated " - + "and will be removed in the next release. " - + "Please use `traceValFn builtins.toXML`." ) - (trace (builtins.toXML x) x); - traceXMLValMarked = str: x: - trace ( "Warning: `traceXMLValMarked` is deprecated " - + "and will be removed in the next release. " - + "Please use `traceValFn (x: str + builtins.toXML x)`." ) - (trace (str + builtins.toXML x) x); - - # trace the arguments passed to function and its result - # maybe rewrite these functions in a traceCallXml like style. Then one function is enough - traceCall = n: f: a: let t = n2: x: traceShowValMarked "${n} ${n2}:" x; in t "result" (f (t "arg 1" a)); - traceCall2 = n: f: a: b: let t = n2: x: traceShowValMarked "${n} ${n2}:" x; in t "result" (f (t "arg 1" a) (t "arg 2" b)); - traceCall3 = n: f: a: b: c: let t = n2: x: traceShowValMarked "${n} ${n2}:" x; in t "result" (f (t "arg 1" a) (t "arg 2" b) (t "arg 3" c)); - - traceValIfNot = c: x: - trace ( "Warning: `traceValIfNot` is deprecated " - + "and will be removed in the next release. " - + "Please use `if/then/else` and `traceValSeq 1`.") - (if c x then true else traceSeq (showVal x) false); - - - addErrorContextToAttrs = attrs: - trace ( "Warning: `addErrorContextToAttrs` is deprecated " - + "and will be removed in the next release. " - + "Please use `builtins.addErrorContext` directly." ) - (mapAttrs (a: v: addErrorContext "while evaluating ${a}" v) attrs); - - # example: (traceCallXml "myfun" id 3) will output something like - # calling myfun arg 1: 3 result: 3 - # this forces deep evaluation of all arguments and the result! - # note: if result doesn't evaluate you'll get no trace at all (FIXME) - # args should be printed in any case - traceCallXml = a: - trace ( "Warning: `traceCallXml` is deprecated " - + "and will be removed in the next release. " - + "Please complain if you use the function regularly." ) - (if !isInt a then - traceCallXml 1 "calling ${a}\n" - else - let nr = a; - in (str: expr: - if isFunction expr then - (arg: - traceCallXml (builtins.add 1 nr) "${str}\n arg ${builtins.toString nr} is \n ${builtins.toXML (builtins.seq arg arg)}" (expr arg) - ) - else - let r = builtins.seq expr expr; - in trace "${str}\n result:\n${builtins.toXML r}" r - )); } diff --git a/nixpkgs/lib/default.nix b/nixpkgs/lib/default.nix index e2a93e63ac1f..8fea4b8ad637 100644 --- a/nixpkgs/lib/default.nix +++ b/nixpkgs/lib/default.nix @@ -1,7 +1,7 @@ /* Library of low-level helper functions for nix expressions. * * Please implement (mostly) exhaustive unit tests - * for new functions in `./tests.nix'. + * for new functions in `./tests.nix`. */ let @@ -23,10 +23,10 @@ let # packaging customisation = callLibs ./customisation.nix; + derivations = callLibs ./derivations.nix; maintainers = import ../maintainers/maintainer-list.nix; teams = callLibs ../maintainers/team-list.nix; meta = callLibs ./meta.nix; - sources = callLibs ./sources.nix; versions = callLibs ./versions.nix; # module system @@ -52,7 +52,9 @@ let fetchers = callLibs ./fetchers.nix; # Eval-time filesystem handling + path = callLibs ./path; filesystem = callLibs ./filesystem.nix; + sources = callLibs ./sources.nix; # back-compat aliases platforms = self.systems.doubles; @@ -62,7 +64,7 @@ let inherit (builtins) add addErrorContext attrNames concatLists deepSeq elem elemAt filter genericClosure genList getAttr - hasAttr head isAttrs isBool isInt isList isString length + hasAttr head isAttrs isBool isInt isList isPath isString length lessThan listToAttrs pathExists readFile replaceStrings seq stringLength sub substring tail trace; inherit (self.trivial) id const pipe concat or and bitAnd bitOr bitXor @@ -76,8 +78,8 @@ let composeManyExtensions makeExtensible makeExtensibleWithCustomName; inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath getAttrFromPath attrVals attrValues getAttrs catAttrs filterAttrs - filterAttrsRecursive foldAttrs collect nameValuePair mapAttrs - mapAttrs' mapAttrsToList mapAttrsRecursive mapAttrsRecursiveCond + filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs + mapAttrs' mapAttrsToList concatMapAttrs mapAttrsRecursive mapAttrsRecursiveCond genAttrs isDerivation toDerivation optionalAttrs zipAttrsWithNames zipAttrsWith zipAttrs recursiveUpdateUntil recursiveUpdate matchAttrs overrideExisting showAttrPath getOutput getBin @@ -86,35 +88,40 @@ let updateManyAttrsByPath; inherit (self.lists) singleton forEach foldr fold foldl foldl' imap0 imap1 concatMap flatten remove findSingle findFirst any all count - optional optionals toList range partition zipListsWith zipLists + optional optionals toList range replicate partition zipListsWith zipLists reverseList listDfs toposort sort naturalSort compareLists take drop sublist last init crossLists unique intersectLists subtractLists mutuallyExclusive groupBy groupBy'; inherit (self.strings) concatStrings concatMapStrings concatImapStrings intersperse concatStringsSep concatMapStringsSep - concatImapStringsSep makeSearchPath makeSearchPathOutput + concatImapStringsSep concatLines makeSearchPath makeSearchPathOutput makeLibraryPath makeBinPath optionalString hasInfix hasPrefix hasSuffix stringToCharacters stringAsChars escape - escapeShellArg escapeShellArgs isValidPosixName toShellVar toShellVars - escapeRegex escapeXML replaceChars lowerChars + escapeShellArg escapeShellArgs + isStorePath isStringLike + isValidPosixName toShellVar toShellVars + escapeRegex escapeURL escapeXML replaceChars lowerChars upperChars toLower toUpper addContextFrom splitString removePrefix removeSuffix versionOlder versionAtLeast getName getVersion + mesonOption mesonBool mesonEnable nameFromURL enableFeature enableFeatureAs withFeature - withFeatureAs fixedWidthString fixedWidthNumber isStorePath - toInt readPathsFromFile fileContents; + withFeatureAs fixedWidthString fixedWidthNumber + toInt toIntBase10 readPathsFromFile fileContents; inherit (self.stringsWithDeps) textClosureList textClosureMap noDepEntry fullDepEntry packEntry stringAfter; inherit (self.customisation) overrideDerivation makeOverridable callPackageWith callPackagesWith extendDerivation hydraJob makeScope makeScopeWithSplicing; + inherit (self.derivations) lazyDerivation; inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio hiPrioSet getLicenseFromSpdxId getExe; - inherit (self.sources) pathType pathIsDirectory cleanSourceFilter + inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile; + inherit (self.sources) cleanSourceFilter cleanSource sourceByRegex sourceFilesBySuffices commitIdFromGitRepo cleanSourceWith pathHasContext - canCleanSource pathIsRegularFile pathIsGitRepo; + canCleanSource pathIsGitRepo; inherit (self.modules) evalModules setDefaultModuleLocation unifyModuleSyntax applyModuleArgsIfFunction mergeModules mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions @@ -125,24 +132,24 @@ let mkAliasAndWrapDefinitions fixMergeModules mkRemovedOptionModule mkRenamedOptionModule mkRenamedOptionModuleWith mkMergedOptionModule mkChangedOptionModule - mkAliasOptionModule mkDerivedConfig doRename; + mkAliasOptionModule mkDerivedConfig doRename + mkAliasOptionModuleMD; inherit (self.options) isOption mkEnableOption mkSinkUndeclaredOptions mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption getValues getFiles optionAttrSetToDocList optionAttrSetToDocList' scrubOptionValue literalExpression literalExample literalDocBook showOption showOptionWithDefLocs showFiles - unknownModule mkOption mkPackageOption + unknownModule mkOption mkPackageOption mkPackageOptionMD mdDoc literalMD; inherit (self.types) isType setType defaultTypeMerge defaultFunctor isOptionType mkOptionType; inherit (self.asserts) assertMsg assertOneOf; - inherit (self.debug) addErrorContextToAttrs traceIf traceVal traceValFn - traceXMLVal traceXMLValMarked traceSeq traceSeqN traceValSeq - traceValSeqFn traceValSeqN traceValSeqNFn traceFnSeqN traceShowVal - traceShowValMarked showVal traceCall traceCall2 traceCall3 - traceValIfNot runTests testAllTrue traceCallXml attrNamesToStr; + inherit (self.debug) traceIf traceVal traceValFn + traceSeq traceSeqN traceValSeq + traceValSeqFn traceValSeqN traceValSeqNFn traceFnSeqN + runTests testAllTrue; inherit (self.misc) maybeEnv defaultMergeArg defaultMerge foldArgs maybeAttrNullable maybeAttr ifEnable checkFlag getValue checkReqs uniqList uniqListExt condConcat lazyGenericClosure diff --git a/nixpkgs/lib/deprecated.nix b/nixpkgs/lib/deprecated.nix index ddce69f160cc..ed14e04bbd68 100644 --- a/nixpkgs/lib/deprecated.nix +++ b/nixpkgs/lib/deprecated.nix @@ -157,7 +157,36 @@ rec { } ); - closePropagation = list: (uniqList {inputList = (innerClosePropagation [] list);}); + closePropagationSlow = list: (uniqList {inputList = (innerClosePropagation [] list);}); + + # This is an optimisation of lib.closePropagation which avoids the O(n^2) behavior + # Using a list of derivations, it generates the full closure of the propagatedXXXBuildInputs + # The ordering / sorting / comparison is done based on the `outPath` + # attribute of each derivation. + # On some benchmarks, it performs up to 15 times faster than lib.closePropagation. + # See https://github.com/NixOS/nixpkgs/pull/194391 for details. + closePropagationFast = list: + builtins.map (x: x.val) (builtins.genericClosure { + startSet = builtins.map (x: { + key = x.outPath; + val = x; + }) (builtins.filter (x: x != null) list); + operator = item: + if !builtins.isAttrs item.val then + [ ] + else + builtins.concatMap (x: + if x != null then [{ + key = x.outPath; + val = x; + }] else + [ ]) ((item.val.propagatedBuildInputs or [ ]) + ++ (item.val.propagatedNativeBuildInputs or [ ])); + }); + + closePropagation = if builtins ? genericClosure + then closePropagationFast + else closePropagationSlow; # calls a function (f attr value ) for each record item. returns a list mapAttrsFlatten = f: r: map (attr: f attr r.${attr}) (attrNames r); diff --git a/nixpkgs/lib/derivations.nix b/nixpkgs/lib/derivations.nix new file mode 100644 index 000000000000..5b7ed1868e86 --- /dev/null +++ b/nixpkgs/lib/derivations.nix @@ -0,0 +1,101 @@ +{ lib }: + +let + inherit (lib) throwIfNot; +in +{ + /* + Restrict a derivation to a predictable set of attribute names, so + that the returned attrset is not strict in the actual derivation, + saving a lot of computation when the derivation is non-trivial. + + This is useful in situations where a derivation might only be used for its + passthru attributes, improving evaluation performance. + + The returned attribute set is lazy in `derivation`. Specifically, this + means that the derivation will not be evaluated in at least the + situations below. + + For illustration and/or testing, we define derivation such that its + evaluation is very noticeable. + + let derivation = throw "This won't be evaluated."; + + In the following expressions, `derivation` will _not_ be evaluated: + + (lazyDerivation { inherit derivation; }).type + + attrNames (lazyDerivation { inherit derivation; }) + + (lazyDerivation { inherit derivation; } // { foo = true; }).foo + + (lazyDerivation { inherit derivation; meta.foo = true; }).meta + + In these expressions, `derivation` _will_ be evaluated: + + "${lazyDerivation { inherit derivation }}" + + (lazyDerivation { inherit derivation }).outPath + + (lazyDerivation { inherit derivation }).meta + + And the following expressions are not valid, because the refer to + implementation details and/or attributes that may not be present on + some derivations: + + (lazyDerivation { inherit derivation }).buildInputs + + (lazyDerivation { inherit derivation }).passthru + + (lazyDerivation { inherit derivation }).pythonPath + + */ + lazyDerivation = + args@{ + # The derivation to be wrapped. + derivation + , # Optional meta attribute. + # + # While this function is primarily about derivations, it can improve + # the `meta` package attribute, which is usually specified through + # `mkDerivation`. + meta ? null + , # Optional extra values to add to the returned attrset. + # + # This can be used for adding package attributes, such as `tests`. + passthru ? { } + }: + let + # These checks are strict in `drv` and some `drv` attributes, but the + # attrset spine returned by lazyDerivation does not depend on it. + # Instead, the individual derivation attributes do depend on it. + checked = + throwIfNot (derivation.type or null == "derivation") + "lazySimpleDerivation: input must be a derivation." + throwIfNot + (derivation.outputs == [ "out" ]) + # Supporting multiple outputs should be a matter of inheriting more attrs. + "The derivation ${derivation.name or "<unknown>"} has multiple outputs. This is not supported by lazySimpleDerivation yet. Support could be added, and be useful as long as the set of outputs is known in advance, without evaluating the actual derivation." + derivation; + in + { + # Hardcoded `type` + # + # `lazyDerivation` requires its `derivation` argument to be a derivation, + # so if it is not, that is a programming error by the caller and not + # something that `lazyDerivation` consumers should be able to correct + # for after the fact. + # So, to improve laziness, we assume correctness here and check it only + # when actual derivation values are accessed later. + type = "derivation"; + + # A fixed set of derivation values, so that `lazyDerivation` can return + # its attrset before evaluating `derivation`. + # This must only list attributes that are available on _all_ derivations. + inherit (checked) outputs out outPath outputName drvPath name system; + + # The meta attribute can either be taken from the derivation, or if the + # `lazyDerivation` caller knew a shortcut, be taken from there. + meta = args.meta or checked.meta; + } // passthru; +} diff --git a/nixpkgs/lib/filesystem.nix b/nixpkgs/lib/filesystem.nix index 0a1275e547cf..4860d4d02a77 100644 --- a/nixpkgs/lib/filesystem.nix +++ b/nixpkgs/lib/filesystem.nix @@ -1,9 +1,103 @@ +# Functions for querying information about the filesystem +# without copying any files to the Nix store. { lib }: -{ # haskellPathsInDir : Path -> Map String Path - # A map of all haskell packages defined in the given path, - # identified by having a cabal file with the same name as the - # directory itself. - haskellPathsInDir = root: + +# Tested in lib/tests/filesystem.sh +let + inherit (builtins) + readDir + pathExists + ; + + inherit (lib.strings) + hasPrefix + ; + + inherit (lib.filesystem) + pathType + ; +in + +{ + + /* + The type of a path. The path needs to exist and be accessible. + The result is either "directory" for a directory, "regular" for a regular file, "symlink" for a symlink, or "unknown" for anything else. + + Type: + pathType :: Path -> String + + Example: + pathType /. + => "directory" + + pathType /some/file.nix + => "regular" + */ + pathType = + builtins.readFileType or + # Nix <2.14 compatibility shim + (path: + if ! pathExists path + # Fail irrecoverably to mimic the historic behavior of this function and + # the new builtins.readFileType + then abort "lib.filesystem.pathType: Path ${toString path} does not exist." + # The filesystem root is the only path where `dirOf / == /` and + # `baseNameOf /` is not valid. We can detect this and directly return + # "directory", since we know the filesystem root can't be anything else. + else if dirOf path == path + then "directory" + else (readDir (dirOf path)).${baseNameOf path} + ); + + /* + Whether a path exists and is a directory. + + Type: + pathIsDirectory :: Path -> Bool + + Example: + pathIsDirectory /. + => true + + pathIsDirectory /this/does/not/exist + => false + + pathIsDirectory /some/file.nix + => false + */ + pathIsDirectory = path: + pathExists path && pathType path == "directory"; + + /* + Whether a path exists and is a regular file, meaning not a symlink or any other special file type. + + Type: + pathIsRegularFile :: Path -> Bool + + Example: + pathIsRegularFile /. + => false + + pathIsRegularFile /this/does/not/exist + => false + + pathIsRegularFile /some/file.nix + => true + */ + pathIsRegularFile = path: + pathExists path && pathType path == "regular"; + + /* + A map of all haskell packages defined in the given path, + identified by having a cabal file with the same name as the + directory itself. + + Type: Path -> Map String Path + */ + haskellPathsInDir = + # The directory within to search + root: let # Files in the root root-files = builtins.attrNames (builtins.readDir root); # Files with their full paths @@ -17,15 +111,18 @@ builtins.pathExists (value + "/${name}.cabal") ) root-files-with-paths; in builtins.listToAttrs cabal-subdirs; - # locateDominatingFile : RegExp - # -> Path - # -> Nullable { path : Path; - # matches : [ MatchResults ]; - # } - # Find the first directory containing a file matching 'pattern' - # upward from a given 'file'. - # Returns 'null' if no directories contain a file matching 'pattern'. - locateDominatingFile = pattern: file: + /* + Find the first directory containing a file matching 'pattern' + upward from a given 'file'. + Returns 'null' if no directories contain a file matching 'pattern'. + + Type: RegExp -> Path -> Nullable { path : Path; matches : [ MatchResults ]; } + */ + locateDominatingFile = + # The pattern to search for + pattern: + # The file to start searching upward from + file: let go = path: let files = builtins.attrNames (builtins.readDir path); matches = builtins.filter (match: match != null) @@ -44,10 +141,15 @@ in go (if isDir then file else parent); - # listFilesRecursive: Path -> [ Path ] - # - # Given a directory, return a flattened list of all files within it recursively. - listFilesRecursive = dir: lib.flatten (lib.mapAttrsToList (name: type: + /* + Given a directory, return a flattened list of all files within it recursively. + + Type: Path -> [ Path ] + */ + listFilesRecursive = + # The path to recursively list + dir: + lib.flatten (lib.mapAttrsToList (name: type: if type == "directory" then lib.filesystem.listFilesRecursive (dir + "/${name}") else diff --git a/nixpkgs/lib/fixed-points.nix b/nixpkgs/lib/fixed-points.nix index bf1567a22a66..926428293c1c 100644 --- a/nixpkgs/lib/fixed-points.nix +++ b/nixpkgs/lib/fixed-points.nix @@ -107,7 +107,7 @@ rec { # Same as `makeExtensible` but the name of the extending attribute is # customized. makeExtensibleWithCustomName = extenderName: rattrs: - fix' rattrs // { + fix' (self: (rattrs self) // { ${extenderName} = f: makeExtensibleWithCustomName extenderName (extends f rattrs); - }; + }); } diff --git a/nixpkgs/lib/generators.nix b/nixpkgs/lib/generators.nix index 431b93c4ebbc..496845fc9ae4 100644 --- a/nixpkgs/lib/generators.nix +++ b/nixpkgs/lib/generators.nix @@ -240,10 +240,10 @@ rec { * to implicit typing rules, so it should work with older * parsers as well. */ - toYAML = {}@args: toJSON args; + toYAML = toJSON; withRecursion = - args@{ + { /* If this option is not null, the given value will stop evaluating at a certain depth */ depthLimit /* If this option is true, an error will be thrown, if a certain given depth is exceeded */ @@ -278,36 +278,46 @@ rec { mapAny 0; /* Pretty print a value, akin to `builtins.trace`. - * Should probably be a builtin as well. - */ + * Should probably be a builtin as well. + * The pretty-printed string should be suitable for rendering default values + * in the NixOS manual. In particular, it should be as close to a valid Nix expression + * as possible. + */ toPretty = { /* If this option is true, attrsets like { __pretty = fn; val = …; } will use fn to convert val to a pretty printed representation. (This means fn is type Val -> String.) */ allowPrettyValues ? false, /* If this option is true, the output is indented with newlines for attribute sets and lists */ - multiline ? true - }@args: + multiline ? true, + /* Initial indentation level */ + indent ? "" + }: let go = indent: v: with builtins; let isPath = v: typeOf v == "path"; introSpace = if multiline then "\n${indent} " else " "; outroSpace = if multiline then "\n${indent}" else " "; in if isInt v then toString v - else if isFloat v then "~${toString v}" + # toString loses precision on floats, so we use toJSON instead. This isn't perfect + # as the resulting string may not parse back as a float (e.g. 42, 1e-06), but for + # pretty-printing purposes this is acceptable. + else if isFloat v then builtins.toJSON v else if isString v then let - # Separate a string into its lines - newlineSplits = filter (v: ! isList v) (builtins.split "\n" v); - # For a '' string terminated by a \n, which happens when the closing '' is on a new line - multilineResult = "''" + introSpace + concatStringsSep introSpace (lib.init newlineSplits) + outroSpace + "''"; - # For a '' string not terminated by a \n, which happens when the closing '' is not on a new line - multilineResult' = "''" + introSpace + concatStringsSep introSpace newlineSplits + "''"; - # For single lines, replace all newlines with their escaped representation - singlelineResult = "\"" + libStr.escape [ "\"" ] (concatStringsSep "\\n" newlineSplits) + "\""; - in if multiline && length newlineSplits > 1 then - if lib.last newlineSplits == "" then multilineResult else multilineResult' - else singlelineResult + lines = filter (v: ! isList v) (builtins.split "\n" v); + escapeSingleline = libStr.escape [ "\\" "\"" "\${" ]; + escapeMultiline = libStr.replaceStrings [ "\${" "''" ] [ "''\${" "'''" ]; + singlelineResult = "\"" + concatStringsSep "\\n" (map escapeSingleline lines) + "\""; + multilineResult = let + escapedLines = map escapeMultiline lines; + # The last line gets a special treatment: if it's empty, '' is on its own line at the "outer" + # indentation level. Otherwise, '' is appended to the last line. + lastLine = lib.last escapedLines; + in "''" + introSpace + concatStringsSep introSpace (lib.init escapedLines) + + (if lastLine == "" then outroSpace else introSpace + lastLine) + "''"; + in + if multiline && length lines > 1 then multilineResult else singlelineResult else if true == v then "true" else if false == v then "false" else if null == v then "null" @@ -326,22 +336,26 @@ rec { else "<function, args: {${showFnas}}>" else if isAttrs v then # apply pretty values if allowed - if attrNames v == [ "__pretty" "val" ] && allowPrettyValues + if allowPrettyValues && v ? __pretty && v ? val then v.__pretty v.val else if v == {} then "{ }" else if v ? type && v.type == "derivation" then - "<derivation ${v.drvPath or "???"}>" + "<derivation ${v.name or "???"}>" else "{" + introSpace + libStr.concatStringsSep introSpace (libAttr.mapAttrsToList (name: value: - "${libStr.escapeNixIdentifier name} = ${go (indent + " ") value};") v) + "${libStr.escapeNixIdentifier name} = ${ + builtins.addErrorContext "while evaluating an attribute `${name}`" + (go (indent + " ") value) + };") v) + outroSpace + "}" else abort "generators.toPretty: should never happen (v = ${v})"; - in go ""; + in go indent; # PLIST handling toPlist = {}: v: let isFloat = builtins.isFloat or (x: false); + isPath = x: builtins.typeOf x == "path"; expr = ind: x: with builtins; if x == null then "" else if isBool x then bool ind x else @@ -349,6 +363,7 @@ rec { if isString x then str ind x else if isList x then list ind x else if isAttrs x then attrs ind x else + if isPath x then str ind (toString x) else if isFloat x then float ind x else abort "generators.toPlist: should never happen (v = ${v})"; @@ -378,7 +393,7 @@ rec { attr = let attrFilter = name: value: name != "_module" && value != null; in ind: x: libStr.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList - (name: value: lib.optional (attrFilter name value) [ + (name: value: lib.optionals (attrFilter name value) [ (key "\t${ind}" name) (expr "\t${ind}" value) ]) x)); @@ -409,8 +424,103 @@ ${expr "" v} (if v then "True" else "False") else if isFunction v then abort "generators.toDhall: cannot convert a function to Dhall" - else if isNull v then + else if v == null then abort "generators.toDhall: cannot convert a null to Dhall" else builtins.toJSON v; + + /* + Translate a simple Nix expression to Lua representation with occasional + Lua-inlines that can be constructed by mkLuaInline function. + + Configuration: + * multiline - by default is true which results in indented block-like view. + * indent - initial indent. + * asBindings - by default generate single value, but with this use attrset to set global vars. + + Attention: + Regardless of multiline parameter there is no trailing newline. + + Example: + generators.toLua {} + { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + } + -> + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + } + + Type: + toLua :: AttrSet -> Any -> String + */ + toLua = { + /* If this option is true, the output is indented with newlines for attribute sets and lists */ + multiline ? true, + /* Initial indentation level */ + indent ? "", + /* Interpret as variable bindings */ + asBindings ? false, + }@args: v: + with builtins; + let + innerIndent = "${indent} "; + introSpace = if multiline then "\n${innerIndent}" else " "; + outroSpace = if multiline then "\n${indent}" else " "; + innerArgs = args // { + indent = if asBindings then indent else innerIndent; + asBindings = false; + }; + concatItems = concatStringsSep ",${introSpace}"; + isLuaInline = { _type ? null, ... }: _type == "lua-inline"; + + generatedBindings = + assert lib.assertMsg (badVarNames == []) "Bad Lua var names: ${toPretty {} badVarNames}"; + libStr.concatStrings ( + lib.attrsets.mapAttrsToList (key: value: "${indent}${key} = ${toLua innerArgs value}\n") v + ); + + # https://en.wikibooks.org/wiki/Lua_Programming/variable#Variable_names + matchVarName = match "[[:alpha:]_][[:alnum:]_]*(\\.[[:alpha:]_][[:alnum:]_]*)*"; + badVarNames = filter (name: matchVarName name == null) (attrNames v); + in + if asBindings then + generatedBindings + else if v == null then + "nil" + else if isInt v || isFloat v || isString v || isBool v then + builtins.toJSON v + else if isList v then + (if v == [ ] then "{}" else + "{${introSpace}${concatItems (map (value: "${toLua innerArgs value}") v)}${outroSpace}}") + else if isAttrs v then + ( + if isLuaInline v then + "(${v.expr})" + else if v == { } then + "{}" + else + "{${introSpace}${concatItems ( + lib.attrsets.mapAttrsToList (key: value: "[${builtins.toJSON key}] = ${toLua innerArgs value}") v + )}${outroSpace}}" + ) + else + abort "generators.toLua: type ${typeOf v} is unsupported"; + + /* + Mark string as Lua expression to be inlined when processed by toLua. + + Type: + mkLuaInline :: String -> AttrSet + */ + mkLuaInline = expr: { _type = "lua-inline"; inherit expr; }; } diff --git a/nixpkgs/lib/kernel.nix b/nixpkgs/lib/kernel.nix index ffcbc268b76c..33da9663a8ed 100644 --- a/nixpkgs/lib/kernel.nix +++ b/nixpkgs/lib/kernel.nix @@ -8,9 +8,10 @@ with lib; option = x: x // { optional = true; }; - yes = { tristate = "y"; optional = false; }; - no = { tristate = "n"; optional = false; }; - module = { tristate = "m"; optional = false; }; + yes = { tristate = "y"; optional = false; }; + no = { tristate = "n"; optional = false; }; + module = { tristate = "m"; optional = false; }; + unset = { tristate = null; optional = false; }; freeform = x: { freeform = x; optional = false; }; /* diff --git a/nixpkgs/lib/licenses.nix b/nixpkgs/lib/licenses.nix index 56299612a0e6..1d2565fba6c0 100644 --- a/nixpkgs/lib/licenses.nix +++ b/nixpkgs/lib/licenses.nix @@ -78,6 +78,11 @@ in mkLicense lset) ({ url = "https://aomedia.org/license/patent-license/"; }; + apsl10 = { + spdxId = "APSL-1.0"; + fullName = "Apple Public Source License 1.0"; + }; + apsl20 = { spdxId = "APSL-2.0"; fullName = "Apple Public Source License 2.0"; @@ -103,6 +108,31 @@ in mkLicense lset) ({ fullName = "Apache License 2.0"; }; + asl20-llvm = { + spdxId = "Apache-2.0 WITH LLVM-exception"; + fullName = "Apache License 2.0 with LLVM Exceptions"; + }; + + bitstreamVera = { + spdxId = "Bitstream-Vera"; + fullName = "Bitstream Vera Font License"; + }; + + bitTorrent10 = { + spdxId = "BitTorrent-1.0"; + fullName = " BitTorrent Open Source License v1.0"; + }; + + bitTorrent11 = { + spdxId = "BitTorrent-1.1"; + fullName = " BitTorrent Open Source License v1.1"; + }; + + bola11 = { + url = "https://blitiri.com.ar/p/bola/"; + fullName = "Buena Onda License Agreement 1.1"; + }; + boost = { spdxId = "BSL-1.0"; fullName = "Boost Software License 1.0"; @@ -138,6 +168,11 @@ in mkLicense lset) ({ fullName = "BSD-2-Clause Plus Patent License"; }; + bsd2WithViews = { + spdxId = "BSD-2-Clause-Views"; + fullName = "BSD 2-Clause with views sentence"; + }; + bsd3 = { spdxId = "BSD-3-Clause"; fullName = ''BSD 3-clause "New" or "Revised" License''; @@ -148,6 +183,11 @@ in mkLicense lset) ({ fullName = ''BSD 4-clause "Original" or "Old" License''; }; + bsdOriginalShortened = { + spdxId = "BSD-4-Clause-Shortened"; + fullName = "BSD 4 Clause Shortened"; + }; + bsdOriginalUC = { spdxId = "BSD-4-Clause-UC"; fullName = "BSD 4-Clause University of California-Specific"; @@ -162,6 +202,23 @@ in mkLicense lset) ({ fullName = "Business Source License 1.1"; url = "https://mariadb.com/bsl11"; free = false; + redistributable = true; + }; + + caossl = { + fullName = "Computer Associates Open Source Licence Version 1.0"; + url = "http://jxplorer.org/licence.html"; + }; + + cal10 = { + fullName = "Cryptographic Autonomy License version 1.0 (CAL-1.0)"; + url = "https://opensource.org/licenses/CAL-1.0"; + }; + + caldera = { + spdxId = "Caldera"; + fullName = "Caldera License"; + url = "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf"; }; capec = { @@ -179,6 +236,18 @@ in mkLicense lset) ({ fullName = "Creative Commons Zero v1.0 Universal"; }; + cc-by-nc-nd-30 = { + spdxId = "CC-BY-NC-ND-3.0"; + fullName = "Creative Commons Attribution Non Commercial No Derivative Works 3.0 Unported"; + free = false; + }; + + cc-by-nc-nd-40 = { + spdxId = "CC-BY-NC-ND-4.0"; + fullName = "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International"; + free = false; + }; + cc-by-nc-sa-20 = { spdxId = "CC-BY-NC-SA-2.0"; fullName = "Creative Commons Attribution Non Commercial Share Alike 2.0"; @@ -302,6 +371,13 @@ in mkLicense lset) ({ free = false; }; + ecl20 = { + fullName = "Educational Community License, Version 2.0"; + url = "https://opensource.org/licenses/ECL-2.0"; + shortName = "ECL 2.0"; + spdxId = "ECL-2.0"; + }; + efl10 = { spdxId = "EFL-1.0"; fullName = "Eiffel Forum License v1.0"; @@ -486,6 +562,12 @@ in mkLicense lset) ({ fullName = "Imlib2 License"; }; + info-zip = { + spdxId = "Info-ZIP"; + fullName = "Info-ZIP License"; + url = "http://www.info-zip.org/pub/infozip/license.html"; + }; + inria-compcert = { fullName = "INRIA Non-Commercial License Agreement for the CompCert verified compiler"; url = "https://compcert.org/doc/LICENSE.txt"; @@ -527,12 +609,34 @@ in mkLicense lset) ({ redistributable = false; }; + fair = { + fullName = "Fair License"; + spdxId = "Fair"; + free = true; + }; + issl = { fullName = "Intel Simplified Software License"; url = "https://software.intel.com/en-us/license/intel-simplified-software-license"; free = false; }; + lal12 = { + spdxId = "LAL-1.2"; + fullName = "Licence Art Libre 1.2"; + }; + + lal13 = { + spdxId = "LAL-1.3"; + fullName = "Licence Art Libre 1.3"; + }; + + lens = { + fullName = "Lens Terms of Service Agreement"; + url = "https://k8slens.dev/licenses/tos"; + free = false; + }; + lgpl2Only = { spdxId = "LGPL-2.0-only"; fullName = "GNU Library General Public License v2 only"; @@ -578,6 +682,11 @@ in mkLicense lset) ({ fullName = "PNG Reference Library version 2"; }; + libssh2 = { + fullName = "libssh2 License"; + url = "https://www.libssh2.org/license.html"; + }; + libtiff = { spdxId = "libtiff"; fullName = "libtiff License"; @@ -588,11 +697,6 @@ in mkLicense lset) ({ url = "https://opensource.franz.com/preamble.html"; }; - llvm-exception = { - spdxId = "LLVM-exception"; - fullName = "LLVM Exception"; # LLVM exceptions to the Apache 2.0 License - }; - lppl12 = { spdxId = "LPPL-1.2"; fullName = "LaTeX Project Public License v1.2"; @@ -663,7 +767,12 @@ in mkLicense lset) ({ ncsa = { spdxId = "NCSA"; - fullName = "University of Illinois/NCSA Open Source License"; + fullName = "University of Illinois/NCSA Open Source License"; + }; + + nlpl = { + spdxId = "NLPL"; + fullName = "No Limit Public License"; }; nposl3 = { @@ -693,6 +802,11 @@ in mkLicense lset) ({ fullName = "SIL Open Font License 1.1"; }; + oml = { + spdxId = "OML"; + fullName = "Open Market License"; + }; + openldap = { spdxId = "OLDAP-2.8"; fullName = "Open LDAP Public License v2.8"; @@ -791,6 +905,12 @@ in mkLicense lset) ({ fullName = "SGI Free Software License B v2.0"; }; + # Gentoo seems to treat it as a license: + # https://gitweb.gentoo.org/repo/gentoo.git/tree/licenses/SGMLUG?id=7d999af4a47bf55e53e54713d98d145f935935c1 + sgmlug = { + fullName = "SGML UG SGML Parser Materials license"; + }; + sleepycat = { spdxId = "Sleepycat"; fullName = "Sleepycat License"; @@ -819,11 +939,23 @@ in mkLicense lset) ({ url = "https://github.com/thestk/stk/blob/master/LICENSE"; }; + tsl = { + shortName = "TSL"; + fullName = "Timescale License Agreegment"; + url = "https://github.com/timescale/timescaledb/blob/main/tsl/LICENSE-TIMESCALE"; + unfree = true; + }; + tcltk = { spdxId = "TCL"; fullName = "TCL/TK License"; }; + ucd = { + fullName = "Unicode Character Database License"; + url = "https://fedoraproject.org/wiki/Licensing:UCD"; + }; + ufl = { fullName = "Ubuntu Font License 1.0"; url = "https://ubuntu.com/legal/font-licence"; @@ -878,6 +1010,11 @@ in mkLicense lset) ({ free = false; }; + vol-sl = { + fullName = "Volatility Software License, Version 1.0"; + url = "https://www.volatilityfoundation.org/license/vsl-v1.0"; + }; + vsl10 = { spdxId = "VSL-1.0"; fullName = "Vovida Software License v1.0"; @@ -908,6 +1045,11 @@ in mkLicense lset) ({ fullName = "wxWindows Library Licence, Version 3.1"; }; + x11 = { + spdxId = "X11"; + fullName = "X11 License"; + }; + xfig = { fullName = "xfig"; url = "http://mcj.sourceforge.net/authors.html#xfig"; # https is broken @@ -934,26 +1076,6 @@ in mkLicense lset) ({ fullName = "GNU Affero General Public License v3.0"; deprecated = true; }; - fdl11 = { - spdxId = "GFDL-1.1"; - fullName = "GNU Free Documentation License v1.1"; - deprecated = true; - }; - fdl12 = { - spdxId = "GFDL-1.2"; - fullName = "GNU Free Documentation License v1.2"; - deprecated = true; - }; - fdl13 = { - spdxId = "GFDL-1.3"; - fullName = "GNU Free Documentation License v1.3"; - deprecated = true; - }; - gpl1 = { - spdxId = "GPL-1.0"; - fullName = "GNU General Public License v1.0"; - deprecated = true; - }; gpl2 = { spdxId = "GPL-2.0"; fullName = "GNU General Public License v2.0"; diff --git a/nixpkgs/lib/lists.nix b/nixpkgs/lib/lists.nix index 36056e18065e..5d9af0cf7114 100644 --- a/nixpkgs/lib/lists.nix +++ b/nixpkgs/lib/lists.nix @@ -36,7 +36,7 @@ rec { forEach = xs: f: map f xs; /* “right fold” a binary function `op` between successive elements of - `list` with `nul' as the starting value, i.e., + `list` with `nul` as the starting value, i.e., `foldr op nul [x_1 x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))`. Type: foldr :: (a -> b -> b) -> b -> [a] -> b @@ -198,8 +198,38 @@ rec { default: # Input list list: - let found = filter pred list; - in if found == [] then default else head found; + let + # A naive recursive implementation would be much simpler, but + # would also overflow the evaluator stack. We use `foldl'` as a workaround + # because it reuses the same stack space, evaluating the function for one + # element after another. We can't return early, so this means that we + # sacrifice early cutoff, but that appears to be an acceptable cost. A + # clever scheme with "exponential search" is possible, but appears over- + # engineered for now. See https://github.com/NixOS/nixpkgs/pull/235267 + + # Invariant: + # - if index < 0 then el == elemAt list (- index - 1) and all elements before el didn't satisfy pred + # - if index >= 0 then pred (elemAt list index) and all elements before (elemAt list index) didn't satisfy pred + # + # We start with index -1 and the 0'th element of the list, which satisfies the invariant + resultIndex = foldl' (index: el: + if index < 0 then + # No match yet before the current index, we need to check the element + if pred el then + # We have a match! Turn it into the actual index to prevent future iterations from modifying it + - index - 1 + else + # Still no match, update the index to the next element (we're counting down, so minus one) + index - 1 + else + # There's already a match, propagate the index without evaluating anything + index + ) (-1) list; + in + if resultIndex < 0 then + default + else + elemAt list resultIndex; /* Return true if function `pred` returns true for at least one element of `list`. @@ -242,7 +272,7 @@ rec { /* Return a singleton list or an empty list, depending on a boolean value. Useful when building lists with optional elements - (e.g. `++ optional (system == "i686-linux") firefox'). + (e.g. `++ optional (system == "i686-linux") firefox`). Type: optional :: bool -> a -> [a] @@ -283,7 +313,7 @@ rec { */ toList = x: if isList x then x else [x]; - /* Return a list of integers from `first' up to and including `last'. + /* Return a list of integers from `first` up to and including `last`. Type: range :: int -> int -> [int] @@ -303,10 +333,22 @@ rec { else genList (n: first + n) (last - first + 1); + /* Return a list with `n` copies of an element. + + Type: replicate :: int -> a -> [a] + + Example: + replicate 3 "a" + => [ "a" "a" "a" ] + replicate 2 true + => [ true true ] + */ + replicate = n: elem: genList (_: elem) n; + /* Splits the elements of a list in two lists, `right` and `wrong`, depending on the evaluation of a predicate. - Type: (a -> bool) -> [a] -> { right :: [a], wrong :: [a] } + Type: (a -> bool) -> [a] -> { right :: [a]; wrong :: [a]; } Example: partition (x: x > 2) [ 5 1 2 3 4 ] @@ -320,7 +362,7 @@ rec { ) { right = []; wrong = []; }); /* Splits the elements of a list into many lists, using the return value of a predicate. - Predicate should return a string which becomes keys of attrset `groupBy' returns. + Predicate should return a string which becomes keys of attrset `groupBy` returns. `groupBy'` allows to customise the combining function and initial value @@ -374,7 +416,7 @@ rec { /* Merges two lists of the same size together. If the sizes aren't the same the merging stops at the shortest. - Type: zipLists :: [a] -> [b] -> [{ fst :: a, snd :: b}] + Type: zipLists :: [a] -> [b] -> [{ fst :: a; snd :: b; }] Example: zipLists [ 1 2 ] [ "a" "b" ] diff --git a/nixpkgs/lib/meta.nix b/nixpkgs/lib/meta.nix index 74b94211552b..5fd55c4e90d6 100644 --- a/nixpkgs/lib/meta.nix +++ b/nixpkgs/lib/meta.nix @@ -27,7 +27,7 @@ rec { setName = name: drv: drv // {inherit name;}; - /* Like `setName', but takes the previous name as an argument. + /* Like `setName`, but takes the previous name as an argument. Example: updateName (oldName: oldName + "-experimental") somePkg @@ -76,7 +76,9 @@ rec { 1. (legacy) a system string. - 2. (modern) a pattern for the platform `parsed` field. + 2. (modern) a pattern for the entire platform structure (see `lib.systems.inspect.platformPatterns`). + + 3. (modern) a pattern for the platform `parsed` field (see `lib.systems.inspect.patterns`). We can inject these into a pattern for the whole of a structured platform, and then match that. @@ -85,6 +87,8 @@ rec { pattern = if builtins.isString elem then { system = elem; } + else if elem?parsed + then elem else { parsed = elem; }; in lib.matchAttrs pattern platform; @@ -92,12 +96,13 @@ rec { A package is available on a platform if both - 1. One of `meta.platforms` pattern matches the given platform. + 1. One of `meta.platforms` pattern matches the given + platform, or `meta.platforms` is not present. 2. None of `meta.badPlatforms` pattern matches the given platform. */ availableOn = platform: pkg: - lib.any (platformMatch platform) pkg.meta.platforms && + ((!pkg?meta.platforms) || lib.any (platformMatch platform) pkg.meta.platforms) && lib.all (elem: !platformMatch platform elem) (pkg.meta.badPlatforms or []); /* Get the corresponding attribute in lib.licenses diff --git a/nixpkgs/lib/minver.nix b/nixpkgs/lib/minver.nix index 86391bcd69e0..507d45bba4dc 100644 --- a/nixpkgs/lib/minver.nix +++ b/nixpkgs/lib/minver.nix @@ -1,2 +1,2 @@ # Expose the minimum required version for evaluating Nixpkgs -"2.2" +"2.3" diff --git a/nixpkgs/lib/modules.nix b/nixpkgs/lib/modules.nix index 7f1646e9b8bc..4dc8c663b2fe 100644 --- a/nixpkgs/lib/modules.nix +++ b/nixpkgs/lib/modules.nix @@ -12,7 +12,6 @@ let concatStringsSep elem filter - findFirst foldl' getAttrFromPath head @@ -22,6 +21,7 @@ let isBool isFunction isList + isPath isString length mapAttrs @@ -34,7 +34,6 @@ let recursiveUpdate reverseList sort setAttrByPath - toList types warnIf zipAttrsWith @@ -46,7 +45,9 @@ let showFiles showOption unknownModule - literalExpression + ; + inherit (lib.strings) + isConvertibleWithToString ; showDeclPrefix = loc: decl: prefix: @@ -62,39 +63,8 @@ let decls )); -in - -rec { - - /* - Evaluate a set of modules. The result is a set with the attributes: - - ‘options’: The nested set of all option declarations, - - ‘config’: The nested set of all option values. - - ‘type’: A module system type representing the module set as a submodule, - to be extended by configuration from the containing module set. - - This is also available as the module argument ‘moduleType’. - - ‘extendModules’: A function similar to ‘evalModules’ but building on top - of the module set. Its arguments, ‘modules’ and ‘specialArgs’ are - added to the existing values. - - Using ‘extendModules’ a few times has no performance impact as long - as you only reference the final ‘options’ and ‘config’. - If you do reference multiple ‘config’ (or ‘options’) from before and - after ‘extendModules’, performance is the same as with multiple - ‘evalModules’ invocations, because the new modules' ability to - override existing configuration fundamentally requires a new - fixpoint to be constructed. - - This is also available as a module argument. - - ‘_module’: A portion of the configuration tree which is elided from - ‘config’. It contains some values that are mostly internal to the - module system implementation. + /* See https://nixos.org/manual/nixpkgs/unstable/#module-system-lib-evalModules + or file://./../doc/module-system/module-system.chapter.md !!! Please think twice before adding to this argument list! The more that is specified here instead of in the modules themselves the harder @@ -109,8 +79,12 @@ rec { # there's _module.args. If specialArgs.modulesPath is defined it will be # used as the base path for disabledModules. specialArgs ? {} - , # This would be remove in the future, Prefer _module.args option instead. - args ? {} + , # `class`: + # A nominal type for modules. When set and non-null, this adds a check to + # make sure that only compatible modules are imported. + # This would be remove in the future, Prefer _module.args option instead. + class ? null + , args ? {} , # This would be remove in the future, Prefer _module.check option instead. check ? true }: @@ -160,87 +134,58 @@ rec { ${if prefix == [] then null # unset => visible else "internal"} = true; + # TODO: hidden during the markdown transition to not expose downstream + # users of the docs infra to markdown if they're not ready for it. + # we don't make this visible conditionally because it can impact + # performance (https://github.com/NixOS/nixpkgs/pull/208407#issuecomment-1368246192) + visible = false; # TODO: Change the type of this option to a submodule with a # freeformType, so that individual arguments can be documented # separately - description = '' + description = lib.mdDoc '' Additional arguments passed to each module in addition to ones - like <literal>lib</literal>, <literal>config</literal>, - and <literal>pkgs</literal>, <literal>modulesPath</literal>. - </para> - <para> + like `lib`, `config`, + and `pkgs`, `modulesPath`. + This option is also available to all submodules. Submodules do not inherit args from their parent module, nor do they provide args to their parent module or sibling submodules. The sole exception to - this is the argument <literal>name</literal> which is provided by + this is the argument `name` which is provided by parent modules to a submodule and contains the attribute name the submodule is bound to, or a unique generated name if it is not bound to an attribute. - </para> - <para> + Some arguments are already passed by default, of which the - following <emphasis>cannot</emphasis> be changed with this option: - <itemizedlist> - <listitem> - <para> - <varname>lib</varname>: The nixpkgs library. - </para> - </listitem> - <listitem> - <para> - <varname>config</varname>: The results of all options after merging the values from all modules together. - </para> - </listitem> - <listitem> - <para> - <varname>options</varname>: The options declared in all modules. - </para> - </listitem> - <listitem> - <para> - <varname>specialArgs</varname>: The <literal>specialArgs</literal> argument passed to <literal>evalModules</literal>. - </para> - </listitem> - <listitem> - <para> - All attributes of <varname>specialArgs</varname> - </para> - <para> - Whereas option values can generally depend on other option values - thanks to laziness, this does not apply to <literal>imports</literal>, which - must be computed statically before anything else. - </para> - <para> - For this reason, callers of the module system can provide <literal>specialArgs</literal> - which are available during import resolution. - </para> - <para> - For NixOS, <literal>specialArgs</literal> includes - <varname>modulesPath</varname>, which allows you to import - extra modules from the nixpkgs package tree without having to - somehow make the module aware of the location of the - <literal>nixpkgs</literal> or NixOS directories. - <programlisting> - { modulesPath, ... }: { - imports = [ - (modulesPath + "/profiles/minimal.nix") - ]; - } - </programlisting> - </para> - </listitem> - </itemizedlist> - </para> - <para> + following *cannot* be changed with this option: + - {var}`lib`: The nixpkgs library. + - {var}`config`: The results of all options after merging the values from all modules together. + - {var}`options`: The options declared in all modules. + - {var}`specialArgs`: The `specialArgs` argument passed to `evalModules`. + - All attributes of {var}`specialArgs` + + Whereas option values can generally depend on other option values + thanks to laziness, this does not apply to `imports`, which + must be computed statically before anything else. + + For this reason, callers of the module system can provide `specialArgs` + which are available during import resolution. + + For NixOS, `specialArgs` includes + {var}`modulesPath`, which allows you to import + extra modules from the nixpkgs package tree without having to + somehow make the module aware of the location of the + `nixpkgs` or NixOS directories. + ``` + { modulesPath, ... }: { + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + } + ``` + For NixOS, the default value for this option includes at least this argument: - <itemizedlist> - <listitem> - <para> - <varname>pkgs</varname>: The nixpkgs package set according to - the <option>nixpkgs.pkgs</option> option. - </para> - </listitem> - </itemizedlist> + - {var}`pkgs`: The nixpkgs package set according to + the {option}`nixpkgs.pkgs` option. ''; }; @@ -248,21 +193,21 @@ rec { type = types.bool; internal = true; default = true; - description = "Whether to check whether all option definitions have matching declarations."; + description = lib.mdDoc "Whether to check whether all option definitions have matching declarations."; }; _module.freeformType = mkOption { type = types.nullOr types.optionType; internal = true; default = null; - description = '' + description = lib.mdDoc '' If set, merge all definitions that don't have an associated option together using this type. The result then gets combined with the - values of all declared options to produce the final <literal> - config</literal> value. + values of all declared options to produce the final ` + config` value. - If this is <literal>null</literal>, definitions without an option - will throw an error unless <option>_module.check</option> is + If this is `null`, definitions without an option + will throw an error unless {option}`_module.check` is turned off. ''; }; @@ -270,7 +215,7 @@ rec { _module.specialArgs = mkOption { readOnly = true; internal = true; - description = '' + description = lib.mdDoc '' Externally provided module arguments that can't be modified from within a configuration, but can be used in module imports. ''; @@ -288,6 +233,7 @@ rec { merged = let collected = collectModules + class (specialArgs.modulesPath or "") (regularModules ++ [ internalModule ]) ({ inherit lib options config specialArgs; } // specialArgs); @@ -321,7 +267,18 @@ rec { if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then let firstDef = head merged.unmatchedDefns; - baseMsg = "The option `${showOption (prefix ++ firstDef.prefix)}' does not exist. Definition values:${showDefs [ firstDef ]}"; + baseMsg = + let + optText = showOption (prefix ++ firstDef.prefix); + defText = + builtins.addErrorContext + "while evaluating the error message for definitions for `${optText}', which is an option that does not exist" + (builtins.addErrorContext + "while evaluating a definition from `${firstDef.file}'" + ( showDefs [ firstDef ]) + ); + in + "The option `${optText}' does not exist. Definition values:${defText}"; in if attrNames options == [ "_module" ] then @@ -353,38 +310,64 @@ rec { prefix ? [], }: evalModules (evalModulesArgs // { + inherit class; modules = regularModules ++ modules; specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; prefix = extendArgs.prefix or evalModulesArgs.prefix or []; }); type = lib.types.submoduleWith { - inherit modules specialArgs; + inherit modules specialArgs class; }; result = withWarnings { + _type = "configuration"; options = checked options; config = checked (removeAttrs config [ "_module" ]); _module = checked (config._module); inherit extendModules type; + class = class; }; in result; - # collectModules :: (modulesPath: String) -> (modules: [ Module ]) -> (args: Attrs) -> [ Module ] + # collectModules :: (class: String) -> (modulesPath: String) -> (modules: [ Module ]) -> (args: Attrs) -> [ Module ] # # Collects all modules recursively through `import` statements, filtering out # all modules in disabledModules. - collectModules = let + collectModules = class: let # Like unifyModuleSyntax, but also imports paths and calls functions if necessary loadModule = args: fallbackFile: fallbackKey: m: - if isFunction m || isAttrs m then - unifyModuleSyntax fallbackFile fallbackKey (applyModuleArgsIfFunction fallbackKey m args) + if isFunction m then + unifyModuleSyntax fallbackFile fallbackKey (applyModuleArgs fallbackKey m args) + else if isAttrs m then + if m._type or "module" == "module" then + unifyModuleSyntax fallbackFile fallbackKey m + else if m._type == "if" || m._type == "override" then + loadModule args fallbackFile fallbackKey { config = m; } + else + throw ( + "Could not load a value as a module, because it is of type ${lib.strings.escapeNixString m._type}" + + lib.optionalString (fallbackFile != unknownModule) ", in file ${toString fallbackFile}." + + lib.optionalString (m._type == "configuration") " If you do intend to import this configuration, please only import the modules that make up the configuration. You may have to create a `let` binding, file or attribute to give yourself access to the relevant modules.\nWhile loading a configuration into the module system is a very sensible idea, it can not be done cleanly in practice." + # Extended explanation: That's because a finalized configuration is more than just a set of modules. For instance, it has its own `specialArgs` that, by the nature of `specialArgs` can't be loaded through `imports` or the the `modules` argument. So instead, we have to ask you to extract the relevant modules and use those instead. This way, we keep the module system comparatively simple, and hopefully avoid a bad surprise down the line. + ) else if isList m then let defs = [{ file = fallbackFile; value = m; }]; in throw "Module imports can't be nested lists. Perhaps you meant to remove one level of lists? Definitions: ${showDefs defs}" else unifyModuleSyntax (toString m) (toString m) (applyModuleArgsIfFunction (toString m) (import m) args); + checkModule = + if class != null + then + m: + if m._class != null -> m._class == class + then m + else + throw "The module ${m._file or m.key} was imported into ${class} instead of ${m._class}." + else + m: m; + /* Collects all modules recursively into the form @@ -418,13 +401,13 @@ rec { }; in parentFile: parentKey: initialModules: args: collectResults (imap1 (n: x: let - module = loadModule args parentFile "${parentKey}:anon-${toString n}" x; + module = checkModule (loadModule args parentFile "${parentKey}:anon-${toString n}" x); collectedImports = collectStructuredModules module._file module.key module.imports args; in { key = module.key; module = module; modules = collectedImports.modules; - disabled = module.disabledModules ++ collectedImports.disabled; + disabled = (if module.disabledModules != [] then [{ file = module._file; disabled = module.disabledModules; }] else []) ++ collectedImports.disabled; }) initialModules); # filterModules :: String -> { disabled, modules } -> [ Module ] @@ -433,8 +416,30 @@ rec { # modules recursively. It returns the final list of unique-by-key modules filterModules = modulesPath: { disabled, modules }: let - moduleKey = m: if isString m then toString modulesPath + "/" + m else toString m; - disabledKeys = map moduleKey disabled; + moduleKey = file: m: + if isString m + then + if builtins.substring 0 1 m == "/" + then m + else toString modulesPath + "/" + m + + else if isConvertibleWithToString m + then + if m?key && m.key != toString m + then + throw "Module `${file}` contains a disabledModules item that is an attribute set that can be converted to a string (${toString m}) but also has a `.key` attribute (${m.key}) with a different value. This makes it ambiguous which module should be disabled." + else + toString m + + else if m?key + then + m.key + + else if isAttrs m + then throw "Module `${file}` contains a disabledModules item that is an attribute set, presumably a module, that does not have a `key` attribute. This means that the module system doesn't have any means to identify the module that should be disabled. Make sure that you've put the correct value in disabledModules: a string path relative to modulesPath, a path value, or an attribute set with a `key` attribute." + else throw "Each disabledModules item must be a path, string, or a attribute set with a key attribute, or a value supported by toString. However, one of the disabledModules items in `${toString file}` is none of that, but is of type ${builtins.typeOf m}."; + + disabledKeys = concatMap ({ file, disabled }: map (moduleKey file) disabled) disabled; keyFilter = filter (attrs: ! elem attrs.key disabledKeys); in map (attrs: attrs.module) (builtins.genericClosure { startSet = keyFilter modules; @@ -460,11 +465,12 @@ rec { else config; in if m ? config || m ? options then - let badAttrs = removeAttrs m ["_file" "key" "disabledModules" "imports" "options" "config" "meta" "freeformType"]; in + let badAttrs = removeAttrs m ["_class" "_file" "key" "disabledModules" "imports" "options" "config" "meta" "freeformType"]; in if badAttrs != {} then throw "Module `${key}' has an unsupported attribute `${head (attrNames badAttrs)}'. This is caused by introducing a top-level `config' or `options' attribute. Add configuration attributes immediately on the top level instead, or move all of them (namely: ${toString (attrNames badAttrs)}) into the explicit `config' attribute." else { _file = toString m._file or file; + _class = m._class or null; key = toString m.key or key; disabledModules = m.disabledModules or []; imports = m.imports or []; @@ -472,16 +478,21 @@ rec { config = addFreeformType (addMeta (m.config or {})); } else + # shorthand syntax lib.throwIfNot (isAttrs m) "module ${file} (${key}) does not look like a module." { _file = toString m._file or file; + _class = m._class or null; key = toString m.key or key; disabledModules = m.disabledModules or []; imports = m.require or [] ++ m.imports or []; options = {}; - config = addFreeformType (addMeta (removeAttrs m ["_file" "key" "disabledModules" "require" "imports" "freeformType"])); + config = addFreeformType (removeAttrs m ["_class" "_file" "key" "disabledModules" "require" "imports" "freeformType"]); }; - applyModuleArgsIfFunction = key: f: args@{ config, options, lib, ... }: if isFunction f then + applyModuleArgsIfFunction = key: f: args@{ config, options, lib, ... }: + if isFunction f then applyModuleArgs key f args else f; + + applyModuleArgs = key: f: args@{ config, options, lib, ... }: let # Module arguments are resolved in a strict manner when attribute set # deconstruction is used. As the arguments are now defined with the @@ -502,12 +513,10 @@ rec { ) (lib.functionArgs f); # Note: we append in the opposite order such that we can add an error - # context on the explicited arguments of "args" too. This update + # context on the explicit arguments of "args" too. This update # operator is used to make the "args@{ ... }: with args.lib;" notation # works. - in f (args // extraArgs) - else - f; + in f (args // extraArgs); /* Merge a list of modules. This will recurse over the option declarations in all modules, combining them into a single set. @@ -561,15 +570,19 @@ rec { zipAttrsWith (n: concatLists) (map (module: let subtree = module.${attr}; in if !(builtins.isAttrs subtree) then - throw '' - You're trying to declare a value of type `${builtins.typeOf subtree}' - rather than an attribute-set for the option + throw (if attr == "config" then '' + You're trying to define a value of type `${builtins.typeOf subtree}' + rather than an attribute set for the option `${builtins.concatStringsSep "." prefix}'! This usually happens if `${builtins.concatStringsSep "." prefix}' has option definitions inside that are not matched. Please check how to properly define this option by e.g. referring to `man 5 configuration.nix'! - '' + '' else '' + An option declaration for `${builtins.concatStringsSep "." prefix}' has type + `${builtins.typeOf subtree}' rather than an attribute set. + Did you mean to define this outside of `options'? + '') else mapAttrs (n: f module) subtree ) modules); @@ -635,7 +648,6 @@ rec { } else let - firstNonOption = findFirst (m: !isOption m.options) "" decls; nonOptions = filter (m: !isOption m.options) decls; in throw "The option `${showOption loc}' in module `${(lib.head optionDecls)._file}' would be a parent of the following options, but its type `${(lib.head optionDecls).options.type.description or "<no description>"}' does not support nested options.\n${ @@ -683,11 +695,7 @@ rec { 'opts' is a list of modules. Each module has an options attribute which correspond to the definition of 'loc' in 'opt.file'. */ mergeOptionDecls = - let - coerceOption = file: opt: - if isFunction opt then setDefaultModuleLocation file opt - else setDefaultModuleLocation file { options = opt; }; - in loc: opts: + loc: opts: foldl' (res: opt: let t = res.type; t' = opt.options.type; @@ -752,6 +760,7 @@ rec { inherit (res.defsFinal') highestPrio; definitions = map (def: def.value) res.defsFinal; files = map (def: def.file) res.defsFinal; + definitionsWithLocations = res.defsFinal; inherit (res) isDefined; # This allows options to be correctly displayed using `${options.path.to.it}` __toString = _: showOption loc; @@ -871,7 +880,7 @@ rec { filterOverrides' = defs: let - getPrio = def: if def.value._type or "" == "override" then def.value.priority else defaultPriority; + getPrio = def: if def.value._type or "" == "override" then def.value.priority else defaultOverridePriority; highestPrio = foldl' (prio: def: min (getPrio def) prio) 9999 defs; strip = def: if def.value._type or "" == "override" then def // { value = def.value.content; } else def; in { @@ -880,7 +889,7 @@ rec { }; /* Sort a list of properties. The sort priority of a property is - 1000 by default, but can be overridden by wrapping the property + defaultOrderPriority by default, but can be overridden by wrapping the property using mkOrder. */ sortProperties = defs: let @@ -889,7 +898,7 @@ rec { then def // { value = def.value.content; inherit (def.value) priority; } else def; defs' = map strip defs; - compare = a: b: (a.priority or 1000) < (b.priority or 1000); + compare = a: b: (a.priority or defaultOrderPriority) < (b.priority or defaultOrderPriority); in sort compare defs'; # This calls substSubModules, whose entire purpose is only to ensure that @@ -925,10 +934,13 @@ rec { mkOptionDefault = mkOverride 1500; # priority of option defaults mkDefault = mkOverride 1000; # used in config sections of non-user modules to set a default + defaultOverridePriority = 100; mkImageMediaOverride = mkOverride 60; # image media profiles can be derived by inclusion into host config, hence needing to override host config, but do allow user to mkForce mkForce = mkOverride 50; mkVMOverride = mkOverride 10; # used by ‘nixos-rebuild build-vm’ + defaultPriority = lib.warnIf (lib.isInOldestRelease 2305) "lib.modules.defaultPriority is deprecated, please use lib.modules.defaultOverridePriority instead." defaultOverridePriority; + mkFixStrictness = lib.warn "lib.mkFixStrictness has no effect and will be removed. It returns its argument unmodified, so you can just remove any calls." id; mkOrder = priority: content: @@ -937,11 +949,9 @@ rec { }; mkBefore = mkOrder 500; + defaultOrderPriority = 1000; mkAfter = mkOrder 1500; - # The default priority for things that don't have a priority specified. - defaultPriority = 100; - # Convenient property used to transfer all definitions and their # properties from one option to another. This property is useful for # renaming options, and also for including properties from another module @@ -968,10 +978,10 @@ rec { # Similar to mkAliasAndWrapDefinitions but copies over the priority from the # option as well. # - # If a priority is not set, it assumes a priority of defaultPriority. + # If a priority is not set, it assumes a priority of defaultOverridePriority. mkAliasAndWrapDefsWithPriority = wrap: option: let - prio = option.highestPrio or defaultPriority; + prio = option.highestPrio or defaultOverridePriority; defsWithPrio = map (mkOverride prio) option.definitions; in mkAliasIfDef option (wrap (mkMerge defsWithPrio)); @@ -1136,6 +1146,15 @@ rec { use = id; }; + /* Transitional version of mkAliasOptionModule that uses MD docs. */ + mkAliasOptionModuleMD = from: to: doRename { + inherit from to; + visible = true; + warn = false; + use = id; + markdown = true; + }; + /* mkDerivedConfig : Option a -> (a -> Definition b) -> Definition b Create config definitions with the same priority as the definition of another option. @@ -1153,10 +1172,10 @@ rec { # to definitions. mkDerivedConfig = opt: f: mkOverride - (opt.highestPrio or defaultPriority) + (opt.highestPrio or defaultOverridePriority) (f opt.value); - doRename = { from, to, visible, warn, use, withPriority ? true }: + doRename = { from, to, visible, warn, use, withPriority ? true, markdown ? false }: { config, options, ... }: let fromOpt = getAttrFromPath from options; @@ -1167,16 +1186,18 @@ rec { { options = setAttrByPath from (mkOption { inherit visible; - description = "Alias of <option>${showOption to}</option>."; + description = if markdown + then lib.mdDoc "Alias of {option}`${showOption to}`." + else "Alias of <option>${showOption to}</option>."; apply = x: use (toOf config); } // optionalAttrs (toType != null) { type = toType; }); config = mkMerge [ - { + (optionalAttrs (options ? warnings) { warnings = optional (warn && fromOpt.isDefined) "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; - } + }) (if withPriority then mkAliasAndWrapDefsWithPriority (setAttrByPath to) fromOpt else mkAliasAndWrapDefinitions (setAttrByPath to) fromOpt) @@ -1200,4 +1221,67 @@ rec { _file = file; config = lib.importTOML file; }; + + private = lib.mapAttrs + (k: lib.warn "External use of `lib.modules.${k}` is deprecated. If your use case isn't covered by non-deprecated functions, we'd like to know more and perhaps support your use case well, instead of providing access to these low level functions. In this case please open an issue in https://github.com/nixos/nixpkgs/issues/.") + { + inherit + applyModuleArgsIfFunction + dischargeProperties + evalOptionValue + mergeModules + mergeModules' + pushDownProperties + unifyModuleSyntax + ; + collectModules = collectModules null; + }; + +in +private // +{ + # NOTE: not all of these functions are necessarily public interfaces; some + # are just needed by types.nix, but are not meant to be consumed + # externally. + inherit + defaultOrderPriority + defaultOverridePriority + defaultPriority + doRename + evalModules + filterOverrides + filterOverrides' + fixMergeModules + fixupOptionType # should be private? + importJSON + importTOML + mergeDefinitions + mergeOptionDecls # should be private? + mkAfter + mkAliasAndWrapDefinitions + mkAliasAndWrapDefsWithPriority + mkAliasDefinitions + mkAliasIfDef + mkAliasOptionModule + mkAliasOptionModuleMD + mkAssert + mkBefore + mkChangedOptionModule + mkDefault + mkDerivedConfig + mkFixStrictness + mkForce + mkIf + mkImageMediaOverride + mkMerge + mkMergedOptionModule + mkOptionDefault + mkOrder + mkOverride + mkRemovedOptionModule + mkRenamedOptionModule + mkRenamedOptionModuleWith + mkVMOverride + setDefaultModuleLocation + sortProperties; } diff --git a/nixpkgs/lib/options.nix b/nixpkgs/lib/options.nix index 84d9fd5e15de..af7914bb5137 100644 --- a/nixpkgs/lib/options.nix +++ b/nixpkgs/lib/options.nix @@ -8,7 +8,6 @@ let concatLists concatMap concatMapStringsSep - elemAt filter foldl' head @@ -37,6 +36,12 @@ let inherit (lib.types) mkOptionType ; + inherit (lib.lists) + last + ; + prioritySuggestion = '' + Use `lib.mkForce value` or `lib.mkDefault value` to change the priority on any of these definitions. + ''; in rec { @@ -95,24 +100,36 @@ rec { name: mkOption { default = false; example = true; - description = "Whether to enable ${name}."; + description = + if name ? _type && name._type == "mdDoc" + then lib.mdDoc "Whether to enable ${name.text}." + else "Whether to enable ${name}."; type = lib.types.bool; }; /* Creates an Option attribute set for an option that specifies the package a module should use for some purpose. - Type: mkPackageOption :: pkgs -> string -> { default :: [string], example :: null | string | [string] } -> option + The package is specified in the third argument under `default` as a list of strings + representing its attribute path in nixpkgs (or another package set). + Because of this, you need to pass nixpkgs itself (or a subset) as the first argument. - The package is specified as a list of strings representing its attribute path in nixpkgs. + The second argument may be either a string or a list of strings. + It provides the display name of the package in the description of the generated option + (using only the last element if the passed value is a list) + and serves as the fallback value for the `default` argument. - Because of this, you need to pass nixpkgs itself as the first argument. + To include extra information in the description, pass `extraDescription` to + append arbitrary text to the generated description. + You can also pass an `example` value, either a literal string or an attribute path. - The second argument is the name of the option, used in the description "The <name> package to use.". + The default argument can be omitted if the provided name is + an attribute of pkgs (if name is a string) or a + valid attribute path in pkgs (if name is a list). - You can also pass an example value, either a literal string or a package's attribute path. + If you wish to explicitly provide no default, pass `null` as `default`. - You can omit the default path if the name of the option is also attribute path in nixpkgs. + Type: mkPackageOption :: pkgs -> (string|[string]) -> { default? :: [string], example? :: null|string|[string], extraDescription? :: string } -> option Example: mkPackageOption pkgs "hello" { } @@ -121,26 +138,57 @@ rec { Example: mkPackageOption pkgs "GHC" { default = [ "ghc" ]; - example = "pkgs.haskell.packages.ghc924.ghc.withPackages (hkgs: [ hkgs.primes ])"; + example = "pkgs.haskell.packages.ghc92.ghc.withPackages (hkgs: [ hkgs.primes ])"; } => { _type = "option"; default = «derivation /nix/store/jxx55cxsjrf8kyh3fp2ya17q99w7541r-ghc-8.10.7.drv»; defaultText = { ... }; description = "The GHC package to use."; example = { ... }; type = { ... }; } + + Example: + mkPackageOption pkgs [ "python39Packages" "pytorch" ] { + extraDescription = "This is an example and doesn't actually do anything."; + } + => { _type = "option"; default = «derivation /nix/store/gvqgsnc4fif9whvwd9ppa568yxbkmvk8-python3.9-pytorch-1.10.2.drv»; defaultText = { ... }; description = "The pytorch package to use. This is an example and doesn't actually do anything."; type = { ... }; } + */ mkPackageOption = - # Package set (a specific version of nixpkgs) + # Package set (a specific version of nixpkgs or a subset) pkgs: # Name for the package, shown in option description name: - { default ? [ name ], example ? null }: - let default' = if !isList default then [ default ] else default; - in mkOption { - type = lib.types.package; - description = "The ${name} package to use."; - default = attrByPath default' - (throw "${concatStringsSep "." default'} cannot be found in pkgs") pkgs; - defaultText = literalExpression ("pkgs." + concatStringsSep "." default'); - ${if example != null then "example" else null} = literalExpression + { + # Whether the package can be null, for example to disable installing a package altogether. + nullable ? false, + # The attribute path where the default package is located (may be omitted) + default ? name, + # A string or an attribute path to use as an example (may be omitted) + example ? null, + # Additional text to include in the option description (may be omitted) + extraDescription ? "", + }: + let + name' = if isList name then last name else name; + in mkOption ({ + type = with lib.types; (if nullable then nullOr else lib.id) package; + description = "The ${name'} package to use." + + (if extraDescription == "" then "" else " ") + extraDescription; + } // (if default != null then let + default' = if isList default then default else [ default ]; + defaultPath = concatStringsSep "." default'; + defaultValue = attrByPath default' + (throw "${defaultPath} cannot be found in pkgs") pkgs; + in { + default = defaultValue; + defaultText = literalExpression ("pkgs." + defaultPath); + } else if nullable then { + default = null; + } else { }) // lib.optionalAttrs (example != null) { + example = literalExpression (if isList example then "pkgs." + concatStringsSep "." example else example); - }; + }); + + /* Like mkPackageOption, but emit an mdDoc description instead of DocBook. */ + mkPackageOptionMD = pkgs: name: extra: + let option = mkPackageOption pkgs name extra; + in option // { description = lib.mdDoc option.description; }; /* This option accepts anything, but it does not produce any result. @@ -177,7 +225,7 @@ rec { if length defs == 1 then (head defs).value else assert length defs > 1; - throw "The option `${showOption loc}' is defined multiple times.\n${message}\nDefinition values:${showDefs defs}"; + throw "The option `${showOption loc}' is defined multiple times while it's expected to be unique.\n${message}\nDefinition values:${showDefs defs}\n${prioritySuggestion}"; /* "Merge" option definitions by checking that they all have the same value. */ mergeEqualOption = loc: defs: @@ -188,13 +236,13 @@ rec { else if length defs == 1 then (head defs).value else (foldl' (first: def: if def.value != first.value then - throw "The option `${showOption loc}' has conflicting definition values:${showDefs [ first def ]}" + throw "The option `${showOption loc}' has conflicting definition values:${showDefs [ first def ]}\n${prioritySuggestion}" else first) (head defs) (tail defs)).value; /* Extracts values of all "value" keys of the given list. - Type: getValues :: [ { value :: a } ] -> [a] + Type: getValues :: [ { value :: a; } ] -> [a] Example: getValues [ { value = 1; } { value = 2; } ] // => [ 1 2 ] @@ -204,7 +252,7 @@ rec { /* Extracts values of all "file" keys of the given list - Type: getFiles :: [ { file :: a } ] -> [a] + Type: getFiles :: [ { file :: a; } ] -> [a] Example: getFiles [ { file = "file1"; } { file = "file2"; } ] // => [ "file1" "file2" ] @@ -216,12 +264,13 @@ rec { # the set generated with filterOptionSets. optionAttrSetToDocList = optionAttrSetToDocList' []; - optionAttrSetToDocList' = prefix: options: + optionAttrSetToDocList' = _: options: concatMap (opt: let - docOption = rec { + name = showOption opt.loc; + docOption = { loc = opt.loc; - name = showOption opt.loc; + inherit name; description = opt.description or null; declarations = filter (x: x != unknownModule) opt.declarations; internal = opt.internal or false; @@ -232,9 +281,18 @@ rec { readOnly = opt.readOnly or false; type = opt.type.description or "unspecified"; } - // optionalAttrs (opt ? example) { example = scrubOptionValue opt.example; } - // optionalAttrs (opt ? default) { default = scrubOptionValue opt.default; } - // optionalAttrs (opt ? defaultText) { default = opt.defaultText; } + // optionalAttrs (opt ? example) { + example = + builtins.addErrorContext "while evaluating the example of option `${name}`" ( + renderOptionValue opt.example + ); + } + // optionalAttrs (opt ? defaultText || opt ? default) { + default = + builtins.addErrorContext "while evaluating the ${if opt?defaultText then "defaultText" else "default value"} of option `${name}`" ( + renderOptionValue (opt.defaultText or opt.default) + ); + } // optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) { inherit (opt) relatedPackages; }; subOptions = @@ -254,6 +312,9 @@ rec { efficient: the XML representation of derivations is very large (on the order of megabytes) and is not actually used by the manual generator. + + This function was made obsolete by renderOptionValue and is kept for + compatibility with out-of-tree code. */ scrubOptionValue = x: if isDerivation x then @@ -263,6 +324,17 @@ rec { else x; + /* Ensures that the given option value (default or example) is a `_type`d string + by rendering Nix values to `literalExpression`s. + */ + renderOptionValue = v: + if v ? _type && v ? text then v + else literalExpression (lib.generators.toPretty { + multiline = true; + allowPrettyValues = true; + } v); + + /* For use in the `defaultText` and `example` option attributes. Causes the given string to be rendered verbatim in the documentation as Nix code. This is necessary for complex values, e.g. functions, or values that depend on @@ -281,7 +353,10 @@ rec { */ literalDocBook = text: if ! isString text then throw "literalDocBook expects a string." - else { _type = "literalDocBook"; inherit text; }; + else + lib.warnIf (lib.isInOldestRelease 2211) + "literalDocBook is deprecated, use literalMD instead" + { _type = "literalDocBook"; inherit text; }; /* Transition marker for documentation that's already migrated to markdown syntax. @@ -300,27 +375,31 @@ rec { # Helper functions. - /* Convert an option, described as a list of the option parts in to a - safe, human readable version. + /* Convert an option, described as a list of the option parts to a + human-readable version. Example: (showOption ["foo" "bar" "baz"]) == "foo.bar.baz" - (showOption ["foo" "bar.baz" "tux"]) == "foo.bar.baz.tux" + (showOption ["foo" "bar.baz" "tux"]) == "foo.\"bar.baz\".tux" + (showOption ["windowManager" "2bwm" "enable"]) == "windowManager.\"2bwm\".enable" Placeholders will not be quoted as they are not actual values: (showOption ["foo" "*" "bar"]) == "foo.*.bar" (showOption ["foo" "<name>" "bar"]) == "foo.<name>.bar" - - Unlike attributes, options can also start with numbers: - (showOption ["windowManager" "2bwm" "enable"]) == "windowManager.2bwm.enable" */ showOption = parts: let escapeOptionPart = part: let - escaped = lib.strings.escapeNixString part; - in if escaped == "\"${part}\"" + # We assume that these are "special values" and not real configuration data. + # If it is real configuration data, it is rendered incorrectly. + specialIdentifiers = [ + "<name>" # attrsOf (submodule {}) + "*" # listOf (submodule {}) + "<function body>" # functionTo + ]; + in if builtins.elem part specialIdentifiers then part - else escaped; + else lib.strings.escapeNixIdentifier part; in (concatStringsSep ".") (map escapeOptionPart parts); showFiles = files: concatStringsSep " and " (map (f: "`${f}'") files); diff --git a/nixpkgs/lib/path/README.md b/nixpkgs/lib/path/README.md new file mode 100644 index 000000000000..87e552d120d7 --- /dev/null +++ b/nixpkgs/lib/path/README.md @@ -0,0 +1,196 @@ +# Path library + +This document explains why the `lib.path` library is designed the way it is. + +The purpose of this library is to process [filesystem paths]. It does not read files from the filesystem. +It exists to support the native Nix [path value type] with extra functionality. + +[filesystem paths]: https://en.m.wikipedia.org/wiki/Path_(computing) +[path value type]: https://nixos.org/manual/nix/stable/language/values.html#type-path + +As an extension of the path value type, it inherits the same intended use cases and limitations: +- Only use paths to access files at evaluation time, such as the local project source. +- Paths cannot point to derivations, so they are unfit to represent dependencies. +- A path implicitly imports the referenced files into the Nix store when interpolated to a string. Therefore paths are not suitable to access files at build- or run-time, as you risk importing the path from the evaluation system instead. + +Overall, this library works with two types of paths: +- Absolute paths are represented with the Nix [path value type]. Nix automatically normalises these paths. +- Subpaths are represented with the [string value type] since path value types don't support relative paths. This library normalises these paths as safely as possible. Absolute paths in strings are not supported. + + A subpath refers to a specific file or directory within an absolute base directory. + It is a stricter form of a relative path, notably [without support for `..` components][parents] since those could escape the base directory. + +[string value type]: https://nixos.org/manual/nix/stable/language/values.html#type-string + +This library is designed to be as safe and intuitive as possible, throwing errors when operations are attempted that would produce surprising results, and giving the expected result otherwise. + +This library is designed to work well as a dependency for the `lib.filesystem` and `lib.sources` library components. Contrary to these library components, `lib.path` does not read any paths from the filesystem. + +This library makes only these assumptions about paths and no others: +- `dirOf path` returns the path to the parent directory of `path`, unless `path` is the filesystem root, in which case `path` is returned. + - There can be multiple filesystem roots: `p == dirOf p` and `q == dirOf q` does not imply `p == q`. + - While there's only a single filesystem root in stable Nix, the [lazy trees feature](https://github.com/NixOS/nix/pull/6530) introduces [additional filesystem roots](https://github.com/NixOS/nix/pull/6530#discussion_r1041442173). +- `path + ("/" + string)` returns the path to the `string` subdirectory in `path`. + - If `string` contains no `/` characters, then `dirOf (path + ("/" + string)) == path`. + - If `string` contains no `/` characters, then `baseNameOf (path + ("/" + string)) == string`. +- `path1 == path2` returns `true` only if `path1` points to the same filesystem path as `path2`. + +Notably we do not make the assumption that we can turn paths into strings using `toString path`. + +## Design decisions + +Each subsection here contains a decision along with arguments and counter-arguments for (+) and against (-) that decision. + +### Leading dots for relative paths +[leading-dots]: #leading-dots-for-relative-paths + +Observing: Since subpaths are a form of relative paths, they can have a leading `./` to indicate it being a relative path, this is generally not necessary for tools though. + +Considering: Paths should be as explicit, consistent and unambiguous as possible. + +Decision: Returned subpaths should always have a leading `./`. + +<details> +<summary>Arguments</summary> + +- (+) In shells, just running `foo` as a command wouldn't execute the file `foo`, whereas `./foo` would execute the file. In contrast, `foo/bar` does execute that file without the need for `./`. This can lead to confusion about when a `./` needs to be prefixed. If a `./` is always included, this becomes a non-issue. This effectively then means that paths don't overlap with command names. +- (+) Prepending with `./` makes the subpaths always valid as relative Nix path expressions. +- (+) Using paths in command line arguments could give problems if not escaped properly, e.g. if a path was `--version`. This is not a problem with `./--version`. This effectively then means that paths don't overlap with GNU-style command line options. +- (-) `./` is not required to resolve relative paths, resolution always has an implicit `./` as prefix. +- (-) It's less noisy without the `./`, e.g. in error messages. + - (+) But similarly, it could be confusing whether something was even a path. + e.g. `foo` could be anything, but `./foo` is more clearly a path. +- (+) Makes it more uniform with absolute paths (those always start with `/`). + - (-) That is not relevant for practical purposes. +- (+) `find` also outputs results with `./`. + - (-) But only if you give it an argument of `.`. If you give it the argument `some-directory`, it won't prefix that. +- (-) `realpath --relative-to` doesn't prefix relative paths with `./`. + - (+) There is no need to return the same result as `realpath`. + +</details> + +### Representation of the current directory +[curdir]: #representation-of-the-current-directory + +Observing: The subpath that produces the base directory can be represented with `.` or `./` or `./.`. + +Considering: Paths should be as consistent and unambiguous as possible. + +Decision: It should be `./.`. + +<details> +<summary>Arguments</summary> + +- (+) `./` would be inconsistent with [the decision to not persist trailing slashes][trailing-slashes]. +- (-) `.` is how `realpath` normalises paths. +- (+) `.` can be interpreted as a shell command (it's a builtin for sourcing files in `bash` and `zsh`). +- (+) `.` would be the only path without a `/`. It could not be used as a Nix path expression, since those require at least one `/` to be parsed as such. +- (-) `./.` is rather long. + - (-) We don't require users to type this though, as it's only output by the library. + As inputs all three variants are supported for subpaths (and we can't do anything about absolute paths) +- (-) `builtins.dirOf "foo" == "."`, so `.` would be consistent with that. +- (+) `./.` is consistent with the [decision to have leading `./`][leading-dots]. +- (+) `./.` is a valid Nix path expression, although this property does not hold for every relative path or subpath. + +</details> + +### Subpath representation +[relrepr]: #subpath-representation + +Observing: Subpaths such as `foo/bar` can be represented in various ways: +- string: `"foo/bar"` +- list with all the components: `[ "foo" "bar" ]` +- attribute set: `{ type = "relative-path"; components = [ "foo" "bar" ]; }` + +Considering: Paths should be as safe to use as possible. We should generate string outputs in the library and not encourage users to do that themselves. + +Decision: Paths are represented as strings. + +<details> +<summary>Arguments</summary> + +- (+) It's simpler for the users of the library. One doesn't have to convert a path a string before it can be used. + - (+) Naively converting the list representation to a string with `concatStringsSep "/"` would break for `[]`, requiring library users to be more careful. +- (+) It doesn't encourage people to do their own path processing and instead use the library. + With a list representation it would seem easy to just use `lib.lists.init` to get the parent directory, but then it breaks for `.`, which would be represented as `[ ]`. +- (+) `+` is convenient and doesn't work on lists and attribute sets. + - (-) Shouldn't use `+` anyways, we export safer functions for path manipulation. + +</details> + +### Parent directory +[parents]: #parent-directory + +Observing: Relative paths can have `..` components, which refer to the parent directory. + +Considering: Paths should be as safe and unambiguous as possible. + +Decision: `..` path components in string paths are not supported, neither as inputs nor as outputs. Hence, string paths are called subpaths, rather than relative paths. + +<details> +<summary>Arguments</summary> + +- (+) If we wanted relative paths to behave according to the "physical" interpretation (as a directory tree with relations between nodes), it would require resolving symlinks, since e.g. `foo/..` would not be the same as `.` if `foo` is a symlink. + - (-) The "logical" interpretation is also valid (treating paths as a sequence of names), and is used by some software. It is simpler, and not using symlinks at all is safer. + - (+) Mixing both models can lead to surprises. + - (+) We can't resolve symlinks without filesystem access. + - (+) Nix also doesn't support reading symlinks at evaluation time. + - (-) We could just not handle such cases, e.g. `equals "foo" "foo/bar/.. == false`. The paths are different, we don't need to check whether the paths point to the same thing. + - (+) Assume we said `relativeTo /foo /bar == "../bar"`. If this is used like `/bar/../foo` in the end, and `bar` turns out to be a symlink to somewhere else, this won't be accurate. + - (-) We could decide to not support such ambiguous operations, or mark them as such, e.g. the normal `relativeTo` will error on such a case, but there could be `extendedRelativeTo` supporting that. +- (-) `..` are a part of paths, a path library should therefore support it. + - (+) If we can convincingly argue that all such use cases are better done e.g. with runtime tools, the library not supporting it can nudge people towards using those. +- (-) We could allow "..", but only in the prefix. + - (+) Then we'd have to throw an error for doing `append /some/path "../foo"`, making it non-composable. + - (+) The same is for returning paths with `..`: `relativeTo /foo /bar => "../bar"` would produce a non-composable path. +- (+) We argue that `..` is not needed at the Nix evaluation level, since we'd always start evaluation from the project root and don't go up from there. + - (+) `..` is supported in Nix paths, turning them into absolute paths. + - (-) This is ambiguous in the presence of symlinks. +- (+) If you need `..` for building or runtime, you can use build-/run-time tooling to create those (e.g. `realpath` with `--relative-to`), or use absolute paths instead. + This also gives you the ability to correctly handle symlinks. + +</details> + +### Trailing slashes +[trailing-slashes]: #trailing-slashes + +Observing: Subpaths can contain trailing slashes, like `foo/`, indicating that the path points to a directory and not a file. + +Considering: Paths should be as consistent as possible, there should only be a single normalisation for the same path. + +Decision: All functions remove trailing slashes in their results. + +<details> +<summary>Arguments</summary> + +- (+) It allows normalisations to be unique, in that there's only a single normalisation for the same path. If trailing slashes were preserved, both `foo/bar` and `foo/bar/` would be valid but different normalisations for the same path. +- Comparison to other frameworks to figure out the least surprising behavior: + - (+) Nix itself doesn't support trailing slashes when parsing and doesn't preserve them when appending paths. + - (-) [Rust's std::path](https://doc.rust-lang.org/std/path/index.html) does preserve them during [construction](https://doc.rust-lang.org/std/path/struct.Path.html#method.new). + - (+) Doesn't preserve them when returning individual [components](https://doc.rust-lang.org/std/path/struct.Path.html#method.components). + - (+) Doesn't preserve them when [canonicalizing](https://doc.rust-lang.org/std/path/struct.Path.html#method.canonicalize). + - (+) [Python 3's pathlib](https://docs.python.org/3/library/pathlib.html#module-pathlib) doesn't preserve them during [construction](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath). + - Notably it represents the individual components as a list internally. + - (-) [Haskell's filepath](https://hackage.haskell.org/package/filepath-1.4.100.0) has [explicit support](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#g:6) for handling trailing slashes. + - (-) Does preserve them for [normalisation](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#v:normalise). + - (-) [NodeJS's Path library](https://nodejs.org/api/path.html) preserves trailing slashes for [normalisation](https://nodejs.org/api/path.html#pathnormalizepath). + - (+) For [parsing a path](https://nodejs.org/api/path.html#pathparsepath) into its significant elements, trailing slashes are not preserved. +- (+) Nix's builtin function `dirOf` gives an unexpected result for paths with trailing slashes: `dirOf "foo/bar/" == "foo/bar"`. + Inconsistently, `baseNameOf` works correctly though: `baseNameOf "foo/bar/" == "bar"`. + - (-) We are writing a path library to improve handling of paths though, so we shouldn't use these functions and discourage their use. +- (-) Unexpected result when normalising intermediate paths, like `relative.normalise ("foo" + "/") + "bar" == "foobar"`. + - (+) This is not a practical use case though. + - (+) Don't use `+` to append paths, this library has a `join` function for that. + - (-) Users might use `+` out of habit though. +- (+) The `realpath` command also removes trailing slashes. +- (+) Even with a trailing slash, the path is the same, it's only an indication that it's a directory. + +</details> + +## Other implementations and references + +- [Rust](https://doc.rust-lang.org/std/path/struct.Path.html) +- [Python](https://docs.python.org/3/library/pathlib.html) +- [Haskell](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html) +- [Nodejs](https://nodejs.org/api/path.html) +- [POSIX.1-2017](https://pubs.opengroup.org/onlinepubs/9699919799/nframe.html) diff --git a/nixpkgs/lib/path/default.nix b/nixpkgs/lib/path/default.nix new file mode 100644 index 000000000000..a4a08668ae62 --- /dev/null +++ b/nixpkgs/lib/path/default.nix @@ -0,0 +1,351 @@ +# Functions for working with paths, see ./path.md +{ lib }: +let + + inherit (builtins) + isString + isPath + split + match + ; + + inherit (lib.lists) + length + head + last + genList + elemAt + all + concatMap + foldl' + ; + + inherit (lib.strings) + concatStringsSep + substring + ; + + inherit (lib.asserts) + assertMsg + ; + + inherit (lib.path.subpath) + isValid + ; + + # Return the reason why a subpath is invalid, or `null` if it's valid + subpathInvalidReason = value: + if ! isString value then + "The given value is of type ${builtins.typeOf value}, but a string was expected" + else if value == "" then + "The given string is empty" + else if substring 0 1 value == "/" then + "The given string \"${value}\" starts with a `/`, representing an absolute path" + # We don't support ".." components, see ./path.md#parent-directory + else if match "(.*/)?\\.\\.(/.*)?" value != null then + "The given string \"${value}\" contains a `..` component, which is not allowed in subpaths" + else null; + + # Split and normalise a relative path string into its components. + # Error for ".." components and doesn't include "." components + splitRelPath = path: + let + # Split the string into its parts using regex for efficiency. This regex + # matches patterns like "/", "/./", "/././", with arbitrarily many "/"s + # together. These are the main special cases: + # - Leading "./" gets split into a leading "." part + # - Trailing "/." or "/" get split into a trailing "." or "" + # part respectively + # + # These are the only cases where "." and "" parts can occur + parts = split "/+(\\./+)*" path; + + # `split` creates a list of 2 * k + 1 elements, containing the k + + # 1 parts, interleaved with k matches where k is the number of + # (non-overlapping) matches. This calculation here gets the number of parts + # back from the list length + # floor( (2 * k + 1) / 2 ) + 1 == floor( k + 1/2 ) + 1 == k + 1 + partCount = length parts / 2 + 1; + + # To assemble the final list of components we want to: + # - Skip a potential leading ".", normalising "./foo" to "foo" + # - Skip a potential trailing "." or "", normalising "foo/" and "foo/." to + # "foo". See ./path.md#trailing-slashes + skipStart = if head parts == "." then 1 else 0; + skipEnd = if last parts == "." || last parts == "" then 1 else 0; + + # We can now know the length of the result by removing the number of + # skipped parts from the total number + componentCount = partCount - skipEnd - skipStart; + + in + # Special case of a single "." path component. Such a case leaves a + # componentCount of -1 due to the skipStart/skipEnd not verifying that + # they don't refer to the same character + if path == "." then [] + + # Generate the result list directly. This is more efficient than a + # combination of `filter`, `init` and `tail`, because here we don't + # allocate any intermediate lists + else genList (index: + # To get to the element we need to add the number of parts we skip and + # multiply by two due to the interleaved layout of `parts` + elemAt parts ((skipStart + index) * 2) + ) componentCount; + + # Join relative path components together + joinRelPath = components: + # Always return relative paths with `./` as a prefix (./path.md#leading-dots-for-relative-paths) + "./" + + # An empty string is not a valid relative path, so we need to return a `.` when we have no components + (if components == [] then "." else concatStringsSep "/" components); + +in /* No rec! Add dependencies on this file at the top. */ { + + /* Append a subpath string to a path. + + Like `path + ("/" + string)` but safer, because it errors instead of returning potentially surprising results. + More specifically, it checks that the first argument is a [path value type](https://nixos.org/manual/nix/stable/language/values.html#type-path"), + and that the second argument is a valid subpath string (see `lib.path.subpath.isValid`). + + Type: + append :: Path -> String -> Path + + Example: + append /foo "bar/baz" + => /foo/bar/baz + + # subpaths don't need to be normalised + append /foo "./bar//baz/./" + => /foo/bar/baz + + # can append to root directory + append /. "foo/bar" + => /foo/bar + + # first argument needs to be a path value type + append "/foo" "bar" + => <error> + + # second argument needs to be a valid subpath string + append /foo /bar + => <error> + append /foo "" + => <error> + append /foo "/bar" + => <error> + append /foo "../bar" + => <error> + */ + append = + # The absolute path to append to + path: + # The subpath string to append + subpath: + assert assertMsg (isPath path) '' + lib.path.append: The first argument is of type ${builtins.typeOf path}, but a path was expected''; + assert assertMsg (isValid subpath) '' + lib.path.append: Second argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + path + ("/" + subpath); + + /* Whether a value is a valid subpath string. + + - The value is a string + + - The string is not empty + + - The string doesn't start with a `/` + + - The string doesn't contain any `..` path components + + Type: + subpath.isValid :: String -> Bool + + Example: + # Not a string + subpath.isValid null + => false + + # Empty string + subpath.isValid "" + => false + + # Absolute path + subpath.isValid "/foo" + => false + + # Contains a `..` path component + subpath.isValid "../foo" + => false + + # Valid subpath + subpath.isValid "foo/bar" + => true + + # Doesn't need to be normalised + subpath.isValid "./foo//bar/" + => true + */ + subpath.isValid = + # The value to check + value: + subpathInvalidReason value == null; + + + /* Join subpath strings together using `/`, returning a normalised subpath string. + + Like `concatStringsSep "/"` but safer, specifically: + + - All elements must be valid subpath strings, see `lib.path.subpath.isValid` + + - The result gets normalised, see `lib.path.subpath.normalise` + + - The edge case of an empty list gets properly handled by returning the neutral subpath `"./."` + + Laws: + + - Associativity: + + subpath.join [ x (subpath.join [ y z ]) ] == subpath.join [ (subpath.join [ x y ]) z ] + + - Identity - `"./."` is the neutral element for normalised paths: + + subpath.join [ ] == "./." + subpath.join [ (subpath.normalise p) "./." ] == subpath.normalise p + subpath.join [ "./." (subpath.normalise p) ] == subpath.normalise p + + - Normalisation - the result is normalised according to `lib.path.subpath.normalise`: + + subpath.join ps == subpath.normalise (subpath.join ps) + + - For non-empty lists, the implementation is equivalent to normalising the result of `concatStringsSep "/"`. + Note that the above laws can be derived from this one. + + ps != [] -> subpath.join ps == subpath.normalise (concatStringsSep "/" ps) + + Type: + subpath.join :: [ String ] -> String + + Example: + subpath.join [ "foo" "bar/baz" ] + => "./foo/bar/baz" + + # normalise the result + subpath.join [ "./foo" "." "bar//./baz/" ] + => "./foo/bar/baz" + + # passing an empty list results in the current directory + subpath.join [ ] + => "./." + + # elements must be valid subpath strings + subpath.join [ /foo ] + => <error> + subpath.join [ "" ] + => <error> + subpath.join [ "/foo" ] + => <error> + subpath.join [ "../foo" ] + => <error> + */ + subpath.join = + # The list of subpaths to join together + subpaths: + # Fast in case all paths are valid + if all isValid subpaths + then joinRelPath (concatMap splitRelPath subpaths) + else + # Otherwise we take our time to gather more info for a better error message + # Strictly go through each path, throwing on the first invalid one + # Tracks the list index in the fold accumulator + foldl' (i: path: + if isValid path + then i + 1 + else throw '' + lib.path.subpath.join: Element at index ${toString i} is not a valid subpath string: + ${subpathInvalidReason path}'' + ) 0 subpaths; + + /* Normalise a subpath. Throw an error if the subpath isn't valid, see + `lib.path.subpath.isValid` + + - Limit repeating `/` to a single one + + - Remove redundant `.` components + + - Remove trailing `/` and `/.` + + - Add leading `./` + + Laws: + + - Idempotency - normalising multiple times gives the same result: + + subpath.normalise (subpath.normalise p) == subpath.normalise p + + - Uniqueness - there's only a single normalisation for the paths that lead to the same file system node: + + subpath.normalise p != subpath.normalise q -> $(realpath ${p}) != $(realpath ${q}) + + - Don't change the result when appended to a Nix path value: + + base + ("/" + p) == base + ("/" + subpath.normalise p) + + - Don't change the path according to `realpath`: + + $(realpath ${p}) == $(realpath ${subpath.normalise p}) + + - Only error on invalid subpaths: + + builtins.tryEval (subpath.normalise p)).success == subpath.isValid p + + Type: + subpath.normalise :: String -> String + + Example: + # limit repeating `/` to a single one + subpath.normalise "foo//bar" + => "./foo/bar" + + # remove redundant `.` components + subpath.normalise "foo/./bar" + => "./foo/bar" + + # add leading `./` + subpath.normalise "foo/bar" + => "./foo/bar" + + # remove trailing `/` + subpath.normalise "foo/bar/" + => "./foo/bar" + + # remove trailing `/.` + subpath.normalise "foo/bar/." + => "./foo/bar" + + # Return the current directory as `./.` + subpath.normalise "." + => "./." + + # error on `..` path components + subpath.normalise "foo/../bar" + => <error> + + # error on empty string + subpath.normalise "" + => <error> + + # error on absolute path + subpath.normalise "/foo" + => <error> + */ + subpath.normalise = + # The subpath string to normalise + subpath: + assert assertMsg (isValid subpath) '' + lib.path.subpath.normalise: Argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + joinRelPath (splitRelPath subpath); + +} diff --git a/nixpkgs/lib/path/tests/default.nix b/nixpkgs/lib/path/tests/default.nix new file mode 100644 index 000000000000..9a31e42828f4 --- /dev/null +++ b/nixpkgs/lib/path/tests/default.nix @@ -0,0 +1,34 @@ +{ + nixpkgs ? ../../.., + system ? builtins.currentSystem, + pkgs ? import nixpkgs { + config = {}; + overlays = []; + inherit system; + }, + libpath ? ../.., + # Random seed + seed ? null, +}: +pkgs.runCommand "lib-path-tests" { + nativeBuildInputs = with pkgs; [ + nix + jq + bc + ]; +} '' + # Needed to make Nix evaluation work + export NIX_STATE_DIR=$(mktemp -d) + + cp -r ${libpath} lib + export TEST_LIB=$PWD/lib + + echo "Running unit tests lib/path/tests/unit.nix" + nix-instantiate --eval lib/path/tests/unit.nix \ + --argstr libpath "$TEST_LIB" + + echo "Running property tests lib/path/tests/prop.sh" + bash lib/path/tests/prop.sh ${toString seed} + + touch $out +'' diff --git a/nixpkgs/lib/path/tests/generate.awk b/nixpkgs/lib/path/tests/generate.awk new file mode 100644 index 000000000000..811dd0c46d33 --- /dev/null +++ b/nixpkgs/lib/path/tests/generate.awk @@ -0,0 +1,64 @@ +# Generate random path-like strings, separated by null characters. +# +# Invocation: +# +# awk -f ./generate.awk -v <variable>=<value> | tr '\0' '\n' +# +# Customizable variables (all default to 0): +# - seed: Deterministic random seed to use for generation +# - count: Number of paths to generate +# - extradotweight: Give extra weight to dots being generated +# - extraslashweight: Give extra weight to slashes being generated +# - extranullweight: Give extra weight to null being generated, making paths shorter +BEGIN { + # Random seed, passed explicitly for reproducibility + srand(seed) + + # Don't include special characters below 32 + minascii = 32 + # Don't include DEL at 128 + maxascii = 127 + upperascii = maxascii - minascii + + # add extra weight for ., in addition to the one weight from the ascii range + upperdot = upperascii + extradotweight + + # add extra weight for /, in addition to the one weight from the ascii range + upperslash = upperdot + extraslashweight + + # add extra weight for null, indicating the end of the string + # Must be at least 1 to have strings end at all + total = upperslash + 1 + extranullweight + + # new=1 indicates that it's a new string + new=1 + while (count > 0) { + + # Random integer between [0, total) + value = int(rand() * total) + + if (value < upperascii) { + # Ascii range + printf("%c", value + minascii) + new=0 + + } else if (value < upperdot) { + # Dot range + printf "." + new=0 + + } else if (value < upperslash) { + # If it's the start of a new path, only generate a / in 10% of cases + # This is always an invalid subpath, which is not a very interesting case + if (new && rand() > 0.1) continue + printf "/" + + } else { + # Do not generate empty strings + if (new) continue + printf "\x00" + count-- + new=1 + } + } +} diff --git a/nixpkgs/lib/path/tests/prop.nix b/nixpkgs/lib/path/tests/prop.nix new file mode 100644 index 000000000000..67e5c1e9d61c --- /dev/null +++ b/nixpkgs/lib/path/tests/prop.nix @@ -0,0 +1,60 @@ +# Given a list of path-like strings, check some properties of the path library +# using those paths and return a list of attribute sets of the following form: +# +# { <string> = <lib.path.subpath.normalise string>; } +# +# If `normalise` fails to evaluate, the attribute value is set to `""`. +# If not, the resulting value is normalised again and an appropriate attribute set added to the output list. +{ + # The path to the nixpkgs lib to use + libpath, + # A flat directory containing files with randomly-generated + # path-like values + dir, +}: +let + lib = import libpath; + + # read each file into a string + strings = map (name: + builtins.readFile (dir + "/${name}") + ) (builtins.attrNames (builtins.readDir dir)); + + inherit (lib.path.subpath) normalise isValid; + inherit (lib.asserts) assertMsg; + + normaliseAndCheck = str: + let + originalValid = isValid str; + + tryOnce = builtins.tryEval (normalise str); + tryTwice = builtins.tryEval (normalise tryOnce.value); + + absConcatOrig = /. + ("/" + str); + absConcatNormalised = /. + ("/" + tryOnce.value); + in + # Check the lib.path.subpath.normalise property to only error on invalid subpaths + assert assertMsg + (originalValid -> tryOnce.success) + "Even though string \"${str}\" is valid as a subpath, the normalisation for it failed"; + assert assertMsg + (! originalValid -> ! tryOnce.success) + "Even though string \"${str}\" is invalid as a subpath, the normalisation for it succeeded"; + + # Check normalisation idempotency + assert assertMsg + (originalValid -> tryTwice.success) + "For valid subpath \"${str}\", the normalisation \"${tryOnce.value}\" was not a valid subpath"; + assert assertMsg + (originalValid -> tryOnce.value == tryTwice.value) + "For valid subpath \"${str}\", normalising it once gives \"${tryOnce.value}\" but normalising it twice gives a different result: \"${tryTwice.value}\""; + + # Check that normalisation doesn't change a string when appended to an absolute Nix path value + assert assertMsg + (originalValid -> absConcatOrig == absConcatNormalised) + "For valid subpath \"${str}\", appending to an absolute Nix path value gives \"${absConcatOrig}\", but appending the normalised result \"${tryOnce.value}\" gives a different value \"${absConcatNormalised}\""; + + # Return an empty string when failed + if tryOnce.success then tryOnce.value else ""; + +in lib.genAttrs strings normaliseAndCheck diff --git a/nixpkgs/lib/path/tests/prop.sh b/nixpkgs/lib/path/tests/prop.sh new file mode 100755 index 000000000000..e48c6667fa08 --- /dev/null +++ b/nixpkgs/lib/path/tests/prop.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +# Property tests for the `lib.path` library +# +# It generates random path-like strings and runs the functions on +# them, checking that the expected laws of the functions hold + +set -euo pipefail +shopt -s inherit_errexit + +# https://stackoverflow.com/a/246128 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if test -z "${TEST_LIB:-}"; then + TEST_LIB=$SCRIPT_DIR/../.. +fi + +tmp="$(mktemp -d)" +clean_up() { + rm -rf "$tmp" +} +trap clean_up EXIT +mkdir -p "$tmp/work" +cd "$tmp/work" + +# Defaulting to a random seed but the first argument can override this +seed=${1:-$RANDOM} +echo >&2 "Using seed $seed, use \`lib/path/tests/prop.sh $seed\` to reproduce this result" + +# The number of random paths to generate. This specific number was chosen to +# be fast enough while still generating enough variety to detect bugs. +count=500 + +debug=0 +# debug=1 # print some extra info +# debug=2 # print generated values + +# Fine tuning parameters to balance the number of generated invalid paths +# to the variance in generated paths. +extradotweight=64 # Larger value: more dots +extraslashweight=64 # Larger value: more slashes +extranullweight=16 # Larger value: shorter strings + +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Generating $count random path-like strings" +fi + +# Read stream of null-terminated strings entry-by-entry into bash, +# write it to a file and the `strings` array. +declare -a strings=() +mkdir -p "$tmp/strings" +while IFS= read -r -d $'\0' str; do + printf "%s" "$str" > "$tmp/strings/${#strings[@]}" + strings+=("$str") +done < <(awk \ + -f "$SCRIPT_DIR"/generate.awk \ + -v seed="$seed" \ + -v count="$count" \ + -v extradotweight="$extradotweight" \ + -v extraslashweight="$extraslashweight" \ + -v extranullweight="$extranullweight") + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Trying to normalise the generated path-like strings with Nix" +fi + +# Precalculate all normalisations with a single Nix call. Calling Nix for each +# string individually would take way too long +nix-instantiate --eval --strict --json \ + --argstr libpath "$TEST_LIB" \ + --argstr dir "$tmp/strings" \ + "$SCRIPT_DIR"/prop.nix \ + >"$tmp/result.json" + +# Uses some jq magic to turn the resulting attribute set into an associative +# bash array assignment +declare -A normalised_result="($(jq ' + to_entries + | map("[\(.key | @sh)]=\(.value | @sh)") + | join(" \n")' -r < "$tmp/result.json"))" + +# Looks up a normalisation result for a string +# Checks that the normalisation is only failing iff it's an invalid subpath +# For valid subpaths, returns 0 and prints the normalisation result +# For invalid subpaths, returns 1 +normalise() { + local str=$1 + # Uses the same check for validity as in the library implementation + if [[ "$str" == "" || "$str" == /* || "$str" =~ ^(.*/)?\.\.(/.*)?$ ]]; then + valid= + else + valid=1 + fi + + normalised=${normalised_result[$str]} + # An empty string indicates failure, this is encoded in ./prop.nix + if [[ -n "$normalised" ]]; then + if [[ -n "$valid" ]]; then + echo "$normalised" + else + die "For invalid subpath \"$str\", lib.path.subpath.normalise returned this result: \"$normalised\"" + fi + else + if [[ -n "$valid" ]]; then + die "For valid subpath \"$str\", lib.path.subpath.normalise failed" + else + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" is not a valid subpath" + fi + # Invalid and it correctly failed, we let the caller continue if they catch the exit code + return 1 + fi + fi +} + +# Intermediate result populated by test_idempotency_realpath +# and used in test_normalise_uniqueness +# +# Contains a mapping from a normalised subpath to the realpath result it represents +declare -A norm_to_real + +test_idempotency_realpath() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking idempotency of each result and making sure the realpath result isn't changed" + fi + + # Count invalid subpaths to display stats + invalid=0 + for str in "${strings[@]}"; do + if ! result=$(normalise "$str"); then + ((invalid++)) || true + continue + fi + + # Check the law that it doesn't change the result of a realpath + mkdir -p -- "$str" "$result" + real_orig=$(realpath -- "$str") + real_norm=$(realpath -- "$result") + + if [[ "$real_orig" != "$real_norm" ]]; then + die "realpath of the original string \"$str\" (\"$real_orig\") is not the same as realpath of the normalisation \"$result\" (\"$real_norm\")" + fi + + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" gets normalised to \"$result\" and file path \"$real_orig\"" + fi + norm_to_real["$result"]="$real_orig" + done + if [[ "$debug" -ge 1 ]]; then + echo >&2 "$(bc <<< "scale=1; 100 / $count * $invalid")% of the total $count generated strings were invalid subpath strings, and were therefore ignored" + fi +} + +test_normalise_uniqueness() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking for the uniqueness law" + fi + + for norm_p in "${!norm_to_real[@]}"; do + real_p=${norm_to_real["$norm_p"]} + for norm_q in "${!norm_to_real[@]}"; do + real_q=${norm_to_real["$norm_q"]} + # Checks normalisation uniqueness law for each pair of values + if [[ "$norm_p" != "$norm_q" && "$real_p" == "$real_q" ]]; then + die "Normalisations \"$norm_p\" and \"$norm_q\" are different, but the realpath of them is the same: \"$real_p\"" + fi + done + done +} + +test_idempotency_realpath +test_normalise_uniqueness + +echo >&2 tests ok diff --git a/nixpkgs/lib/path/tests/unit.nix b/nixpkgs/lib/path/tests/unit.nix new file mode 100644 index 000000000000..61c4ab4d6f2e --- /dev/null +++ b/nixpkgs/lib/path/tests/unit.nix @@ -0,0 +1,193 @@ +# Unit tests for lib.path functions. Use `nix-build` in this directory to +# run these +{ libpath }: +let + lib = import libpath; + inherit (lib.path) append subpath; + + cases = lib.runTests { + # Test examples from the lib.path.append documentation + testAppendExample1 = { + expr = append /foo "bar/baz"; + expected = /foo/bar/baz; + }; + testAppendExample2 = { + expr = append /foo "./bar//baz/./"; + expected = /foo/bar/baz; + }; + testAppendExample3 = { + expr = append /. "foo/bar"; + expected = /foo/bar; + }; + testAppendExample4 = { + expr = (builtins.tryEval (append "/foo" "bar")).success; + expected = false; + }; + testAppendExample5 = { + expr = (builtins.tryEval (append /foo /bar)).success; + expected = false; + }; + testAppendExample6 = { + expr = (builtins.tryEval (append /foo "")).success; + expected = false; + }; + testAppendExample7 = { + expr = (builtins.tryEval (append /foo "/bar")).success; + expected = false; + }; + testAppendExample8 = { + expr = (builtins.tryEval (append /foo "../bar")).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.isValid documentation + testSubpathIsValidExample1 = { + expr = subpath.isValid null; + expected = false; + }; + testSubpathIsValidExample2 = { + expr = subpath.isValid ""; + expected = false; + }; + testSubpathIsValidExample3 = { + expr = subpath.isValid "/foo"; + expected = false; + }; + testSubpathIsValidExample4 = { + expr = subpath.isValid "../foo"; + expected = false; + }; + testSubpathIsValidExample5 = { + expr = subpath.isValid "foo/bar"; + expected = true; + }; + testSubpathIsValidExample6 = { + expr = subpath.isValid "./foo//bar/"; + expected = true; + }; + # Some extra tests + testSubpathIsValidTwoDotsEnd = { + expr = subpath.isValid "foo/.."; + expected = false; + }; + testSubpathIsValidTwoDotsMiddle = { + expr = subpath.isValid "foo/../bar"; + expected = false; + }; + testSubpathIsValidTwoDotsPrefix = { + expr = subpath.isValid "..foo"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffix = { + expr = subpath.isValid "foo.."; + expected = true; + }; + testSubpathIsValidTwoDotsPrefixComponent = { + expr = subpath.isValid "foo/..bar/baz"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffixComponent = { + expr = subpath.isValid "foo/bar../baz"; + expected = true; + }; + testSubpathIsValidThreeDots = { + expr = subpath.isValid "..."; + expected = true; + }; + testSubpathIsValidFourDots = { + expr = subpath.isValid "...."; + expected = true; + }; + testSubpathIsValidThreeDotsComponent = { + expr = subpath.isValid "foo/.../bar"; + expected = true; + }; + testSubpathIsValidFourDotsComponent = { + expr = subpath.isValid "foo/..../bar"; + expected = true; + }; + + # Test examples from the lib.path.subpath.join documentation + testSubpathJoinExample1 = { + expr = subpath.join [ "foo" "bar/baz" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample2 = { + expr = subpath.join [ "./foo" "." "bar//./baz/" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample3 = { + expr = subpath.join [ ]; + expected = "./."; + }; + testSubpathJoinExample4 = { + expr = (builtins.tryEval (subpath.join [ /foo ])).success; + expected = false; + }; + testSubpathJoinExample5 = { + expr = (builtins.tryEval (subpath.join [ "" ])).success; + expected = false; + }; + testSubpathJoinExample6 = { + expr = (builtins.tryEval (subpath.join [ "/foo" ])).success; + expected = false; + }; + testSubpathJoinExample7 = { + expr = (builtins.tryEval (subpath.join [ "../foo" ])).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.normalise documentation + testSubpathNormaliseExample1 = { + expr = subpath.normalise "foo//bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample2 = { + expr = subpath.normalise "foo/./bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample3 = { + expr = subpath.normalise "foo/bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample4 = { + expr = subpath.normalise "foo/bar/"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample5 = { + expr = subpath.normalise "foo/bar/."; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample6 = { + expr = subpath.normalise "."; + expected = "./."; + }; + testSubpathNormaliseExample7 = { + expr = (builtins.tryEval (subpath.normalise "foo/../bar")).success; + expected = false; + }; + testSubpathNormaliseExample8 = { + expr = (builtins.tryEval (subpath.normalise "")).success; + expected = false; + }; + testSubpathNormaliseExample9 = { + expr = (builtins.tryEval (subpath.normalise "/foo")).success; + expected = false; + }; + # Some extra tests + testSubpathNormaliseIsValidDots = { + expr = subpath.normalise "./foo/.bar/.../baz...qux"; + expected = "./foo/.bar/.../baz...qux"; + }; + testSubpathNormaliseWrongType = { + expr = (builtins.tryEval (subpath.normalise null)).success; + expected = false; + }; + testSubpathNormaliseTwoDots = { + expr = (builtins.tryEval (subpath.normalise "..")).success; + expected = false; + }; + }; +in + if cases == [] then "Unit tests successful" + else throw "Path unit tests failed: ${lib.generators.toPretty {} cases}" diff --git a/nixpkgs/lib/sources.nix b/nixpkgs/lib/sources.nix index 343449d9a603..d990777c6fcc 100644 --- a/nixpkgs/lib/sources.nix +++ b/nixpkgs/lib/sources.nix @@ -4,7 +4,6 @@ # Tested in lib/tests/sources.sh let inherit (builtins) - hasContext match readDir split @@ -19,21 +18,11 @@ let pathExists readFile ; - - /* - Returns the type of a path: regular (for file), symlink, or directory. - */ - pathType = path: getAttr (baseNameOf path) (readDir (dirOf path)); - - /* - Returns true if the path exists and is a directory, false otherwise. - */ - pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false; - - /* - Returns true if the path exists and is a regular file, false otherwise. - */ - pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false; + inherit (lib.filesystem) + pathType + pathIsDirectory + pathIsRegularFile + ; /* A basic filter for `cleanSourceWith` that removes @@ -167,17 +156,27 @@ let in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; in cleanSourceWith { inherit filter src; }; - pathIsGitRepo = path: (tryEval (commitIdFromGitRepo path)).success; + pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value; /* Get the commit id of a git repo. Example: commitIdFromGitRepo <nixpkgs/.git> */ - commitIdFromGitRepo = + commitIdFromGitRepo = path: + let commitIdOrError = _commitIdFromGitRepoOrError path; + in commitIdOrError.value or (throw commitIdOrError.error); + + # Get the commit id of a git repo. + + # Returns `{ value = commitHash }` or `{ error = "... message ..." }`. + + # Example: commitIdFromGitRepo <nixpkgs/.git> + # not exported, used for commitIdFromGitRepo + _commitIdFromGitRepoOrError = let readCommitFromFile = file: path: - let fileName = toString path + "/" + file; - packedRefsName = toString path + "/packed-refs"; + let fileName = path + "/${file}"; + packedRefsName = path + "/packed-refs"; absolutePath = base: path: if lib.hasPrefix "/" path then path @@ -187,7 +186,7 @@ let then let m = match "^gitdir: (.*)$" (lib.fileContents path); in if m == null - then throw ("File contains no gitdir reference: " + path) + then { error = "File contains no gitdir reference: " + path; } else let gitDir = absolutePath (dirOf path) (lib.head m); commonDir'' = if pathIsRegularFile "${gitDir}/commondir" @@ -205,7 +204,7 @@ let let fileContent = lib.fileContents fileName; matchRef = match "^ref: (.*)$" fileContent; in if matchRef == null - then fileContent + then { value = fileContent; } else readCommitFromFile (lib.head matchRef) path else if pathIsRegularFile packedRefsName @@ -219,10 +218,10 @@ let # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795 refs = filter isRef (split "\n" fileContent); in if refs == [] - then throw ("Could not find " + file + " in " + packedRefsName) - else lib.head (matchRef (lib.head refs)) + then { error = "Could not find " + file + " in " + packedRefsName; } + else { value = lib.head (matchRef (lib.head refs)); } - else throw ("Not a .git directory: " + path); + else { error = "Not a .git directory: " + toString path; }; in readCommitFromFile "HEAD"; pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); @@ -262,11 +261,20 @@ let }; in { - inherit - pathType - pathIsDirectory - pathIsRegularFile + pathType = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathType has been moved to lib.filesystem.pathType." + lib.filesystem.pathType; + + pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory." + lib.filesystem.pathIsDirectory; + + pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile." + lib.filesystem.pathIsRegularFile; + + inherit pathIsGitRepo commitIdFromGitRepo diff --git a/nixpkgs/lib/strings.nix b/nixpkgs/lib/strings.nix index 295d98900e99..e875520c6858 100644 --- a/nixpkgs/lib/strings.nix +++ b/nixpkgs/lib/strings.nix @@ -2,7 +2,11 @@ { lib }: let -inherit (builtins) length; + inherit (builtins) length; + + inherit (lib.trivial) warnIf; + +asciiTable = import ./ascii-table.nix; in @@ -18,6 +22,7 @@ rec { isInt isList isAttrs + isPath isString match parseDrvName @@ -127,6 +132,17 @@ rec { # List of input strings list: concatStringsSep sep (lib.imap1 f list); + /* Concatenate a list of strings, adding a newline at the end of each one. + Defined as `concatMapStrings (s: s + "\n")`. + + Type: concatLines :: [string] -> string + + Example: + concatLines [ "foo" "bar" ] + => "foo\nbar\n" + */ + concatLines = concatMapStrings (s: s + "\n"); + /* Construct a Unix-style, colon-separated search path consisting of the given `subDir` appended to each of the given paths. @@ -185,6 +201,29 @@ rec { */ makeBinPath = makeSearchPathOutput "bin" "bin"; + /* Normalize path, removing extraneous /s + + Type: normalizePath :: string -> string + + Example: + normalizePath "/a//b///c/" + => "/a/b/c/" + */ + normalizePath = s: + warnIf + (isPath s) + '' + lib.strings.normalizePath: The argument (${toString s}) is a path value, but only strings are supported. + Path values are always normalised in Nix, so there's no need to call this function on them. + This function also copies the path to the Nix store and returns the store path, the same as "''${path}" will, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + builtins.foldl' + (x: y: if y == "/" && hasSuffix "/" x then x else x+y) + "" + (stringToCharacters s) + ); + /* Depending on the boolean `cond', return either the given string or the empty string. Useful to concatenate against a bigger string. @@ -216,7 +255,17 @@ rec { # Prefix to check for pref: # Input string - str: substring 0 (stringLength pref) str == pref; + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath pref) + '' + lib.strings.hasPrefix: The first argument (${toString pref}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (substring 0 (stringLength pref) str == pref); /* Determine whether a string has given suffix. @@ -236,8 +285,20 @@ rec { let lenContent = stringLength content; lenSuffix = stringLength suffix; - in lenContent >= lenSuffix && - substring (lenContent - lenSuffix) lenContent content == suffix; + in + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.hasSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + lenContent >= lenSuffix + && substring (lenContent - lenSuffix) lenContent content == suffix + ); /* Determine whether a string contains the given infix @@ -254,7 +315,16 @@ rec { => false */ hasInfix = infix: content: - builtins.match ".*${escapeRegex infix}.*" "${content}" != null; + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath infix) + '' + lib.strings.hasInfix: The first argument (${toString infix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (builtins.match ".*${escapeRegex infix}.*" "${content}" != null); /* Convert a string to a list of characters (i.e. singleton strings). This allows you to, e.g., map a function over each character. However, @@ -271,7 +341,7 @@ rec { => [ ] stringToCharacters "abc" => [ "a" "b" "c" ] - stringToCharacters "💩" + stringToCharacters "🦄" => [ "�" "�" "�" "�" ] */ stringToCharacters = s: @@ -294,6 +364,19 @@ rec { map f (stringToCharacters s) ); + /* Convert char to ascii value, must be in printable range + + Type: charToInt :: string -> int + + Example: + charToInt "A" + => 65 + charToInt "(" + => 40 + + */ + charToInt = c: builtins.getAttr c asciiTable; + /* Escape occurrence of the elements of `list` in `string` by prefixing it with a backslash. @@ -303,7 +386,35 @@ rec { escape ["(" ")"] "(foo)" => "\\(foo\\)" */ - escape = list: replaceChars list (map (c: "\\${c}") list); + escape = list: replaceStrings list (map (c: "\\${c}") list); + + /* Escape occurrence of the element of `list` in `string` by + converting to its ASCII value and prefixing it with \\x. + Only works for printable ascii characters. + + Type: escapeC = [string] -> string -> string + + Example: + escapeC [" "] "foo bar" + => "foo\\x20bar" + + */ + escapeC = list: replaceStrings list (map (c: "\\x${ toLower (lib.toHexString (charToInt c))}") list); + + /* Escape the string so it can be safely placed inside a URL + query. + + Type: escapeURL :: string -> string + + Example: + escapeURL "foo/bar baz" + => "foo%2Fbar%20baz" + */ + escapeURL = let + unreserved = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + toEscape = builtins.removeAttrs asciiTable unreserved; + in + replaceStrings (builtins.attrNames toEscape) (lib.mapAttrsToList (_: c: "%${fixedWidthString 2 "0" (lib.toHexString c)}") toEscape); /* Quote string to be used safely within the Bourne shell. @@ -357,7 +468,7 @@ rec { */ toShellVar = name: value: lib.throwIfNot (isValidPosixName name) "toShellVar: ${name} is not a valid shell variable name" ( - if isAttrs value && ! isCoercibleToString value then + if isAttrs value && ! isStringLike value then "declare -A ${name}=(${ concatStringsSep " " (lib.mapAttrsToList (n: v: "[${escapeShellArg n}]=${escapeShellArg v}" @@ -433,19 +544,8 @@ rec { ["\"" "'" "<" ">" "&"] [""" "'" "<" ">" "&"]; - # Obsolete - use replaceStrings instead. - replaceChars = builtins.replaceStrings or ( - del: new: s: - let - substList = lib.zipLists del new; - subst = c: - let found = lib.findFirst (sub: sub.fst == c) null substList; in - if found == null then - c - else - found.snd; - in - stringAsChars subst s); + # warning added 12-12-2022 + replaceChars = lib.warn "replaceChars is a deprecated alias of replaceStrings, replace usages of it with replaceStrings." builtins.replaceStrings; # Case conversion utilities. lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; @@ -459,7 +559,7 @@ rec { toLower "HOME" => "home" */ - toLower = replaceChars upperChars lowerChars; + toLower = replaceStrings upperChars lowerChars; /* Converts an ASCII string to upper-case. @@ -469,10 +569,10 @@ rec { toUpper "home" => "HOME" */ - toUpper = replaceChars lowerChars upperChars; + toUpper = replaceStrings lowerChars upperChars; /* Appends string context from another string. This is an implementation - detail of Nix. + detail of Nix and should be used carefully. Strings in Nix carry an invisible `context` which is a list of strings representing store paths. If the string is later used in a derivation @@ -495,13 +595,11 @@ rec { splitString "/" "/usr/local/bin" => [ "" "usr" "local" "bin" ] */ - splitString = _sep: _s: + splitString = sep: s: let - sep = builtins.unsafeDiscardStringContext _sep; - s = builtins.unsafeDiscardStringContext _s; - splits = builtins.filter builtins.isString (builtins.split (escapeRegex sep) s); + splits = builtins.filter builtins.isString (builtins.split (escapeRegex (toString sep)) (toString s)); in - map (v: addContextFrom _sep (addContextFrom _s v)) splits; + map (addContextFrom s) splits; /* Return a string without the specified prefix, if the prefix matches. @@ -518,14 +616,23 @@ rec { prefix: # Input string str: - let + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath prefix) + '' + lib.strings.removePrefix: The first argument (${toString prefix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any prefix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let preLen = stringLength prefix; sLen = stringLength str; in - if hasPrefix prefix str then + if substring 0 preLen str == prefix then substring preLen (sLen - preLen) str else - str; + str); /* Return a string without the specified suffix, if the suffix matches. @@ -542,14 +649,23 @@ rec { suffix: # Input string str: - let + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.removeSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any suffix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let sufLen = stringLength suffix; sLen = stringLength str; in if sufLen <= sLen && suffix == substring (sLen - sufLen) sufLen str then substring 0 (sLen - sufLen) str else - str; + str); /* Return true if string v1 denotes a version older than v2. @@ -623,6 +739,61 @@ rec { name = head (splitString sep filename); in assert name != filename; name; + /* Create a -D<feature>=<value> string that can be passed to typical Meson + invocations. + + Type: mesonOption :: string -> string -> string + + @param feature The feature to be set + @param value The desired value + + Example: + mesonOption "engine" "opengl" + => "-Dengine=opengl" + */ + mesonOption = feature: value: + assert (lib.isString feature); + assert (lib.isString value); + "-D${feature}=${value}"; + + /* Create a -D<condition>={true,false} string that can be passed to typical + Meson invocations. + + Type: mesonBool :: string -> bool -> string + + @param condition The condition to be made true or false + @param flag The controlling flag of the condition + + Example: + mesonBool "hardened" true + => "-Dhardened=true" + mesonBool "static" false + => "-Dstatic=false" + */ + mesonBool = condition: flag: + assert (lib.isString condition); + assert (lib.isBool flag); + mesonOption condition (lib.boolToString flag); + + /* Create a -D<feature>={enabled,disabled} string that can be passed to + typical Meson invocations. + + Type: mesonEnable :: string -> bool -> string + + @param feature The feature to be enabled or disabled + @param flag The controlling flag + + Example: + mesonEnable "docs" true + => "-Ddocs=enabled" + mesonEnable "savage" false + => "-Dsavage=disabled" + */ + mesonEnable = feature: flag: + assert (lib.isString feature); + assert (lib.isBool flag); + mesonOption feature (if flag then "enabled" else "disabled"); + /* Create an --{enable,disable}-<feat> string that can be passed to standard GNU Autoconf scripts. @@ -718,10 +889,31 @@ rec { in lib.warnIf (!precise) "Imprecise conversion from float to string ${result}" result; - /* Check whether a value can be coerced to a string */ - isCoercibleToString = x: - elem (typeOf x) [ "path" "string" "null" "int" "float" "bool" ] || - (isList x && lib.all isCoercibleToString x) || + /* Soft-deprecated function. While the original implementation is available as + isConvertibleWithToString, consider using isStringLike instead, if suitable. */ + isCoercibleToString = lib.warnIf (lib.isInOldestRelease 2305) + "lib.strings.isCoercibleToString is deprecated in favor of either isStringLike or isConvertibleWithToString. Only use the latter if it needs to return true for null, numbers, booleans and list of similarly coercibles." + isConvertibleWithToString; + + /* Check whether a list or other value can be passed to toString. + + Many types of value are coercible to string this way, including int, float, + null, bool, list of similarly coercible values. + */ + isConvertibleWithToString = x: + isStringLike x || + elem (typeOf x) [ "null" "int" "float" "bool" ] || + (isList x && lib.all isConvertibleWithToString x); + + /* Check whether a value can be coerced to a string. + The value must be a string, path, or attribute set. + + String-like values can be used without explicit conversion in + string interpolations and in most functions that expect a string. + */ + isStringLike = x: + isString x || + isPath x || x ? outPath || x ? __toString; @@ -738,31 +930,113 @@ rec { => false */ isStorePath = x: - if !(isList x) && isCoercibleToString x then + if isStringLike x then let str = toString x; in substring 0 1 str == "/" && dirOf str == storeDir else false; - /* Parse a string as an int. + /* Parse a string as an int. Does not support parsing of integers with preceding zero due to + ambiguity between zero-padded and octal numbers. See toIntBase10. Type: string -> int Example: + toInt "1337" => 1337 + toInt "-4" => -4 + + toInt " 123 " + => 123 + + toInt "00024" + => error: Ambiguity in interpretation of 00024 between octal and zero padded integer. + toInt "3.14" => error: floating point JSON numbers are not supported */ - # Obviously, it is a bit hacky to use fromJSON this way. toInt = str: - let may_be_int = fromJSON str; in - if isInt may_be_int - then may_be_int - else throw "Could not convert ${str} to int."; + let + # RegEx: Match any leading whitespace, possibly a '-', one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match a leading '0' then one or more digits. + isLeadingZero = match "0[[:digit:]]+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toInt: Could not convert ${escapeNixString str} to int."; + + octalAmbigError = "toInt: Ambiguity in interpretation of ${escapeNixString str}" + + " between octal and zero padded integer."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # Error on presence of leading zero/octal ambiguity. + else if isLeadingZero + then throw octalAmbigError + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; + + + /* Parse a string as a base 10 int. This supports parsing of zero-padded integers. + + Type: string -> int + + Example: + toIntBase10 "1337" + => 1337 + + toIntBase10 "-4" + => -4 + + toIntBase10 " 123 " + => 123 + + toIntBase10 "00024" + => 24 + + toIntBase10 "3.14" + => error: floating point JSON numbers are not supported + */ + toIntBase10 = str: + let + # RegEx: Match any leading whitespace, then match any zero padding, + # capture possibly a '-' followed by one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*0*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match at least one '0'. + isZero = match "0+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toIntBase10: Could not convert ${escapeNixString str} to int."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # In the special case zero-padded zero (00000), return early. + else if isZero + then 0 + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; /* Read a list of paths from `file`, relative to the `rootPath`. Lines beginning with `#` are treated as comments and ignored. diff --git a/nixpkgs/lib/systems/architectures.nix b/nixpkgs/lib/systems/architectures.nix index ddc320d24e0a..57b9184ca60c 100644 --- a/nixpkgs/lib/systems/architectures.nix +++ b/nixpkgs/lib/systems/architectures.nix @@ -40,14 +40,21 @@ rec { # a superior CPU has all the features of an inferior and is able to build and test code for it inferiors = { # x86_64 Intel + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html default = [ ]; westmere = [ ]; - sandybridge = [ "westmere" ] ++ inferiors.westmere; - ivybridge = [ "sandybridge" ] ++ inferiors.sandybridge; - haswell = [ "ivybridge" ] ++ inferiors.ivybridge; - broadwell = [ "haswell" ] ++ inferiors.haswell; - skylake = [ "broadwell" ] ++ inferiors.broadwell; - skylake-avx512 = [ "skylake" ] ++ inferiors.skylake; + sandybridge = [ "westmere" ] ++ inferiors.westmere; + ivybridge = [ "sandybridge" ] ++ inferiors.sandybridge; + haswell = [ "ivybridge" ] ++ inferiors.ivybridge; + broadwell = [ "haswell" ] ++ inferiors.haswell; + skylake = [ "broadwell" ] ++ inferiors.broadwell; + skylake-avx512 = [ "skylake" ] ++ inferiors.skylake; + cannonlake = [ "skylake-avx512" ] ++ inferiors.skylake-avx512; + icelake-client = [ "cannonlake" ] ++ inferiors.cannonlake; + icelake-server = [ "icelake-client" ] ++ inferiors.icelake-client; + cascadelake = [ "skylake-avx512" ] ++ inferiors.cannonlake; + cooperlake = [ "cascadelake" ] ++ inferiors.cascadelake; + tigerlake = [ "icelake-server" ] ++ inferiors.icelake-server; # x86_64 AMD # TODO: fill this (need testing) @@ -67,7 +74,7 @@ rec { # # Note: # - # - The succesors of `skylake` (`cannonlake`, `icelake`, etc) use `avx512` + # - The successors of `skylake` (`cannonlake`, `icelake`, etc) use `avx512` # which no current AMD Zen michroarch support. # - `znver1` uses `ABM`, `CLZERO`, `CX16`, `MWAITX`, and `SSE4A` which no # current Intel microarch support. diff --git a/nixpkgs/lib/systems/default.nix b/nixpkgs/lib/systems/default.nix index 2990afde3e9a..f4784c61c675 100644 --- a/nixpkgs/lib/systems/default.nix +++ b/nixpkgs/lib/systems/default.nix @@ -20,7 +20,7 @@ rec { # necessary. # # `parsed` is inferred from args, both because there are two options with one - # clearly prefered, and to prevent cycles. A simpler fixed point where the RHS + # clearly preferred, and to prevent cycles. A simpler fixed point where the RHS # always just used `final.*` would fail on both counts. elaborate = args': let args = if lib.isString args' then { system = args'; } @@ -47,9 +47,11 @@ rec { else if final.isUClibc then "uclibc" else if final.isAndroid then "bionic" else if final.isLinux /* default */ then "glibc" + else if final.isFreeBSD then "fblibc" + else if final.isNetBSD then "nblibc" else if final.isAvr then "avrlibc" + else if final.isGhcjs then null else if final.isNone then "newlib" - else if final.isNetBSD then "nblibc" # TODO(@Ericson2314) think more about other operating systems else "native/impure"; # Choose what linker we wish to use by default. Someday we might also @@ -61,15 +63,21 @@ rec { linker = /**/ if final.useLLVM or false then "lld" else if final.isDarwin then "cctools" - # "bfd" and "gold" both come from GNU binutils. The existance of Gold + # "bfd" and "gold" both come from GNU binutils. The existence of Gold # is why we use the more obscure "bfd" and not "binutils" for this # choice. else "bfd"; - extensions = { + extensions = rec { sharedLibrary = /**/ if final.isDarwin then ".dylib" else if final.isWindows then ".dll" else ".so"; + staticLibrary = + /**/ if final.isWindows then ".lib" + else ".a"; + library = + /**/ if final.isStatic then staticLibrary + else sharedLibrary; executable = /**/ if final.isWindows then ".exe" else ""; @@ -93,8 +101,15 @@ rec { genode = "Genode"; }.${final.parsed.kernel.name} or null; - # uname -p - processor = final.parsed.cpu.name; + # uname -m + processor = + if final.isPower64 + then "ppc64${lib.optionalString final.isLittleEndian "le"}" + else if final.isPower + then "ppc${lib.optionalString final.isLittleEndian "le"}" + else if final.isMips64 + then "mips64" # endianness is *not* included on mips64 + else final.parsed.cpu.name; # uname -r release = null; @@ -106,7 +121,7 @@ rec { ({ linux-kernel = args.linux-kernel or {}; gcc = args.gcc or {}; - rustc = args.rust or {}; + rustc = args.rustc or {}; } // platforms.select final) linux-kernel gcc rustc; @@ -115,23 +130,31 @@ rec { else if final.isAarch64 then "arm64" else if final.isx86_32 then "i386" else if final.isx86_64 then "x86_64" + # linux kernel does not distinguish microblaze/microblazeel + else if final.isMicroBlaze then "microblaze" else if final.isMips32 then "mips" else if final.isMips64 then "mips" # linux kernel does not distinguish mips32/mips64 else if final.isPower then "powerpc" else if final.isRiscV then "riscv" else if final.isS390 then "s390" + else if final.isLoongArch64 then "loongarch" else final.parsed.cpu.name; qemuArch = if final.isAarch32 then "arm" + else if final.isS390 && !final.isS390x then null else if final.isx86_64 then "x86_64" else if final.isx86 then "i386" - else { - powerpc = "ppc"; - powerpcle = "ppc"; - powerpc64 = "ppc64"; - powerpc64le = "ppc64le"; - }.${final.parsed.cpu.name} or final.parsed.cpu.name; + else if final.isMips64 then "mips64${lib.optionalString final.isLittleEndian "el"}" + else final.uname.processor; + + # Name used by UEFI for architectures. + efiArch = + if final.isx86_32 then "ia32" + else if final.isx86_64 then "x64" + else if final.isAarch32 then "arm" + else if final.isAarch64 then "aa64" + else final.parsed.cpu.name; darwinArch = { armv7a = "armv7"; @@ -150,38 +173,47 @@ rec { if final.isMacOS then "MACOSX_DEPLOYMENT_TARGET" else if final.isiOS then "IPHONEOS_DEPLOYMENT_TARGET" else null; + } // ( + let + selectEmulator = pkgs: + let + qemu-user = pkgs.qemu.override { + smartcardSupport = false; + spiceSupport = false; + openGLSupport = false; + virglSupport = false; + vncSupport = false; + gtkSupport = false; + sdlSupport = false; + pulseSupport = false; + smbdSupport = false; + seccompSupport = false; + enableDocs = false; + hostCpuTargets = [ "${final.qemuArch}-linux-user" ]; + }; + wine = (pkgs.winePackagesFor "wine${toString final.parsed.cpu.bits}").minimal; + in + if final.parsed.kernel.name == pkgs.stdenv.hostPlatform.parsed.kernel.name && + pkgs.stdenv.hostPlatform.canExecute final + then "${pkgs.runtimeShell} -c '\"$@\"' --" + else if final.isWindows + then "${wine}/bin/wine${lib.optionalString (final.parsed.cpu.bits == 64) "64"}" + else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux && final.qemuArch != null + then "${qemu-user}/bin/qemu-${final.qemuArch}" + else if final.isWasi + then "${pkgs.wasmtime}/bin/wasmtime" + else if final.isMmix + then "${pkgs.mmixware}/bin/mmix" + else null; + in { + emulatorAvailable = pkgs: (selectEmulator pkgs) != null; - emulator = pkgs: let - qemu-user = pkgs.qemu.override { - smartcardSupport = false; - spiceSupport = false; - openGLSupport = false; - virglSupport = false; - vncSupport = false; - gtkSupport = false; - sdlSupport = false; - pulseSupport = false; - smbdSupport = false; - seccompSupport = false; - hostCpuTargets = ["${final.qemuArch}-linux-user"]; - }; - wine-name = "wine${toString final.parsed.cpu.bits}"; - wine = (pkgs.winePackagesFor wine-name).minimal; - in - if final.parsed.kernel.name == pkgs.stdenv.hostPlatform.parsed.kernel.name && - pkgs.stdenv.hostPlatform.canExecute final - then "${pkgs.runtimeShell} -c '\"$@\"' --" - else if final.isWindows - then "${wine}/bin/${wine-name}" - else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux - then "${qemu-user}/bin/qemu-${final.qemuArch}" - else if final.isWasi - then "${pkgs.wasmtime}/bin/wasmtime" - else if final.isMmix - then "${pkgs.mmixware}/bin/mmix" - else throw "Don't know how to run ${final.config} executables."; + emulator = pkgs: + if (final.emulatorAvailable pkgs) + then selectEmulator pkgs + else throw "Don't know how to run ${final.config} executables."; - } // mapAttrs (n: v: v final.parsed) inspect.predicates + }) // mapAttrs (n: v: v final.parsed) inspect.predicates // mapAttrs (n: v: v final.gcc.arch or "default") architectures.predicates // args; in assert final.useAndroidPrebuilt -> final.isAndroid; diff --git a/nixpkgs/lib/systems/doubles.nix b/nixpkgs/lib/systems/doubles.nix index 90a6eb9f35c9..66bf3d872bee 100644 --- a/nixpkgs/lib/systems/doubles.nix +++ b/nixpkgs/lib/systems/doubles.nix @@ -13,7 +13,7 @@ let "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" # FreeBSD - "i686-freebsd" "x86_64-freebsd" + "i686-freebsd13" "x86_64-freebsd13" # Genode "aarch64-genode" "i686-genode" "x86_64-genode" @@ -22,12 +22,13 @@ let "x86_64-solaris" # JS - "js-ghcjs" + "javascript-ghcjs" # Linux "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" - "armv7l-linux" "i686-linux" "m68k-linux" "mipsel-linux" "mips64el-linux" - "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" + "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" + "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" + "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" # MMIXware @@ -40,9 +41,9 @@ let # none "aarch64_be-none" "aarch64-none" "arm-none" "armv6l-none" "avr-none" "i686-none" - "msp430-none" "or1k-none" "m68k-none" "powerpc-none" "powerpcle-none" - "riscv32-none" "riscv64-none" "rx-none" "s390-none" "s390x-none" "vc4-none" - "x86_64-none" + "microblaze-none" "microblazeel-none" "msp430-none" "or1k-none" "m68k-none" + "powerpc-none" "powerpcle-none" "riscv32-none" "riscv64-none" "rx-none" + "s390-none" "s390x-none" "vc4-none" "x86_64-none" # OpenBSD "i686-openbsd" "x86_64-openbsd" @@ -67,12 +68,15 @@ in { none = []; arm = filterDoubles predicates.isAarch32; + armv7 = filterDoubles predicates.isArmv7; aarch64 = filterDoubles predicates.isAarch64; x86 = filterDoubles predicates.isx86; i686 = filterDoubles predicates.isi686; x86_64 = filterDoubles predicates.isx86_64; + microblaze = filterDoubles predicates.isMicroBlaze; mips = filterDoubles predicates.isMips; mmix = filterDoubles predicates.isMmix; + power = filterDoubles predicates.isPower; riscv = filterDoubles predicates.isRiscV; riscv32 = filterDoubles predicates.isRiscV32; riscv64 = filterDoubles predicates.isRiscV64; @@ -81,6 +85,8 @@ in { or1k = filterDoubles predicates.isOr1k; m68k = filterDoubles predicates.isM68k; s390 = filterDoubles predicates.isS390; + s390x = filterDoubles predicates.isS390x; + loongarch64 = filterDoubles predicates.isLoongArch64; js = filterDoubles predicates.isJavaScript; bigEndian = filterDoubles predicates.isBigEndian; @@ -94,7 +100,9 @@ in { ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabi; }) ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabihf; }) ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabin32; }) - ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabi64; }); + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabi64; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv1; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv2; }); illumos = filterDoubles predicates.isSunOS; linux = filterDoubles predicates.isLinux; netbsd = filterDoubles predicates.isNetBSD; diff --git a/nixpkgs/lib/systems/examples.nix b/nixpkgs/lib/systems/examples.nix index 65dc9c07e346..45b95c150fce 100644 --- a/nixpkgs/lib/systems/examples.nix +++ b/nixpkgs/lib/systems/examples.nix @@ -22,12 +22,11 @@ rec { }; ppc64 = { - config = "powerpc64-unknown-linux-gnu"; - gcc = { abi = "elfv2"; }; # for gcc configuration + config = "powerpc64-unknown-linux-gnuabielfv2"; }; ppc64-musl = { config = "powerpc64-unknown-linux-musl"; - gcc = { abi = "elfv2"; }; # for gcc configuration + gcc = { abi = "elfv2"; }; }; sheevaplug = { @@ -92,22 +91,16 @@ rec { } // platforms.fuloong2f_n32; # can execute on 32bit chip - mips-linux-gnu = { config = "mips-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; - mipsel-linux-gnu = { config = "mipsel-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; - mipsisa32r6-linux-gnu = { config = "mipsisa32r6-unknown-linux-gnu"; } // platforms.gcc_mips32r6_o32; - mipsisa32r6el-linux-gnu = { config = "mipsisa32r6el-unknown-linux-gnu"; } // platforms.gcc_mips32r6_o32; + mips-linux-gnu = { config = "mips-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; + mipsel-linux-gnu = { config = "mipsel-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; # require 64bit chip (for more registers, 64-bit floating point, 64-bit "long long") but use 32bit pointers - mips64-linux-gnuabin32 = { config = "mips64-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; - mips64el-linux-gnuabin32 = { config = "mips64el-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; - mipsisa64r6-linux-gnuabin32 = { config = "mipsisa64r6-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r6_n32; - mipsisa64r6el-linux-gnuabin32 = { config = "mipsisa64r6el-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r6_n32; + mips64-linux-gnuabin32 = { config = "mips64-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; + mips64el-linux-gnuabin32 = { config = "mips64el-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; # 64bit pointers - mips64-linux-gnuabi64 = { config = "mips64-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; - mips64el-linux-gnuabi64 = { config = "mips64el-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; - mipsisa64r6-linux-gnuabi64 = { config = "mipsisa64r6-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r6_64; - mipsisa64r6el-linux-gnuabi64 = { config = "mipsisa64r6el-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r6_64; + mips64-linux-gnuabi64 = { config = "mips64-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; + mips64el-linux-gnuabi64 = { config = "mips64el-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; muslpi = raspberryPi // { config = "armv6l-unknown-linux-musleabihf"; @@ -136,6 +129,10 @@ rec { libc = "newlib"; }; + loongarch64-linux = { + config = "loongarch64-unknown-linux-gnu"; + }; + mmix = { config = "mmix-unknown-mmixware"; libc = "newlib"; @@ -304,15 +301,18 @@ rec { # BSDs + x86_64-freebsd = { + config = "x86_64-unknown-freebsd13"; + useLLVM = true; + }; + x86_64-netbsd = { config = "x86_64-unknown-netbsd"; - libc = "nblibc"; }; # this is broken and never worked fully x86_64-netbsd-llvm = { config = "x86_64-unknown-netbsd"; - libc = "nblibc"; useLLVM = true; }; @@ -327,6 +327,9 @@ rec { # Ghcjs ghcjs = { - config = "js-unknown-ghcjs"; + # This triple is special to GHC/Cabal/GHCJS and not recognized by autotools + # See: https://gitlab.haskell.org/ghc/ghc/-/commit/6636b670233522f01d002c9b97827d00289dbf5c + # https://github.com/ghcjs/ghcjs/issues/53 + config = "javascript-unknown-ghcjs"; }; } diff --git a/nixpkgs/lib/systems/flake-systems.nix b/nixpkgs/lib/systems/flake-systems.nix index 74124c32e836..b1988c6a4fbb 100644 --- a/nixpkgs/lib/systems/flake-systems.nix +++ b/nixpkgs/lib/systems/flake-systems.nix @@ -1,6 +1,6 @@ # See [RFC 46] for mandated platform support and ../../pkgs/stdenv for # implemented platform support. This list is mainly descriptive, i.e. all -# system doubles for platforms where nixpkgs can do native compiliation +# system doubles for platforms where nixpkgs can do native compilation # reasonably well are included. # # [RFC 46]: https://github.com/NixOS/rfcs/blob/master/rfcs/0046-platform-support-tiers.md diff --git a/nixpkgs/lib/systems/inspect.nix b/nixpkgs/lib/systems/inspect.nix index dbffca0300b5..89e9f4231d97 100644 --- a/nixpkgs/lib/systems/inspect.nix +++ b/nixpkgs/lib/systems/inspect.nix @@ -7,16 +7,36 @@ let abis_ = abis; in let abis = lib.mapAttrs (_: abi: builtins.removeAttrs abi [ "assertions" ]) abis_; in rec { + # these patterns are to be matched against {host,build,target}Platform.parsed patterns = rec { + # The patterns below are lists in sum-of-products form. + # + # Each attribute is list of product conditions; non-list values are treated + # as a singleton list. If *any* product condition in the list matches then + # the predicate matches. Each product condition is tested by + # `lib.attrsets.matchAttrs`, which requires a match on *all* attributes of + # the product. + isi686 = { cpu = cpuTypes.i686; }; isx86_32 = { cpu = { family = "x86"; bits = 32; }; }; isx86_64 = { cpu = { family = "x86"; bits = 64; }; }; isPower = { cpu = { family = "power"; }; }; isPower64 = { cpu = { family = "power"; bits = 64; }; }; + # This ABI is the default in NixOS PowerPC64 BE, but not on mainline GCC, + # so it sometimes causes issues in certain packages that makes the wrong + # assumption on the used ABI. + isAbiElfv2 = [ + { abi = { abi = "elfv2"; }; } + { abi = { name = "musl"; }; cpu = { family = "power"; bits = 64; }; } + ]; isx86 = { cpu = { family = "x86"; }; }; isAarch32 = { cpu = { family = "arm"; bits = 32; }; }; + isArmv7 = map ({ arch, ... }: { cpu = { inherit arch; }; }) + (lib.filter (cpu: lib.hasPrefix "armv7" cpu.arch or "") + (lib.attrValues cpuTypes)); isAarch64 = { cpu = { family = "arm"; bits = 64; }; }; isAarch = { cpu = { family = "arm"; }; }; + isMicroBlaze = { cpu = { family = "microblaze"; }; }; isMips = { cpu = { family = "mips"; }; }; isMips32 = { cpu = { family = "mips"; bits = 32; }; }; isMips64 = { cpu = { family = "mips"; bits = 64; }; }; @@ -36,10 +56,13 @@ rec { isOr1k = { cpu = { family = "or1k"; }; }; isM68k = { cpu = { family = "m68k"; }; }; isS390 = { cpu = { family = "s390"; }; }; - isJavaScript = { cpu = cpuTypes.js; }; + isS390x = { cpu = { family = "s390"; bits = 64; }; }; + isLoongArch64 = { cpu = { family = "loongarch"; bits = 64; }; }; + isJavaScript = { cpu = cpuTypes.javascript; }; is32bit = { cpu = { bits = 32; }; }; is64bit = { cpu = { bits = 64; }; }; + isILP32 = map (a: { abi = { abi = a; }; }) [ "n32" "ilp32" "x32" ]; isBigEndian = { cpu = { significantByte = significantBytes.bigEndian; }; }; isLittleEndian = { cpu = { significantByte = significantBytes.littleEndian; }; }; @@ -51,7 +74,7 @@ rec { isiOS = { kernel = kernels.ios; }; isLinux = { kernel = kernels.linux; }; isSunOS = { kernel = kernels.solaris; }; - isFreeBSD = { kernel = kernels.freebsd; }; + isFreeBSD = { kernel = { name = "freebsd"; }; }; isNetBSD = { kernel = kernels.netbsd; }; isOpenBSD = { kernel = kernels.openbsd; }; isWindows = { kernel = kernels.windows; }; @@ -64,12 +87,17 @@ rec { isNone = { kernel = kernels.none; }; isAndroid = [ { abi = abis.android; } { abi = abis.androideabi; } ]; - isGnu = with abis; map (a: { abi = a; }) [ gnuabi64 gnu gnueabi gnueabihf ]; + isGnu = with abis; map (a: { abi = a; }) [ gnuabi64 gnu gnueabi gnueabihf gnuabielfv1 gnuabielfv2 ]; isMusl = with abis; map (a: { abi = a; }) [ musl musleabi musleabihf muslabin32 muslabi64 ]; isUClibc = with abis; map (a: { abi = a; }) [ uclibc uclibceabi uclibceabihf ]; - isEfi = map (family: { cpu.family = family; }) - [ "x86" "arm" "aarch64" ]; + isEfi = [ + { cpu = { family = "arm"; version = "6"; }; } + { cpu = { family = "arm"; version = "7"; }; } + { cpu = { family = "arm"; version = "8"; }; } + { cpu = { family = "riscv"; }; } + { cpu = { family = "x86"; }; } + ]; }; matchAnyAttrs = patterns: @@ -77,4 +105,13 @@ rec { else matchAttrs patterns; predicates = mapAttrs (_: matchAnyAttrs) patterns; + + # these patterns are to be matched against the entire + # {host,build,target}Platform structure; they include a `parsed={}` marker so + # that `lib.meta.availableOn` can distinguish them from the patterns which + # apply only to the `parsed` field. + + platformPatterns = mapAttrs (_: p: { parsed = {}; } // p) { + isStatic = { isStatic = true; }; + }; } diff --git a/nixpkgs/lib/systems/parse.nix b/nixpkgs/lib/systems/parse.nix index 9d2571c993a9..6eb4f27cc519 100644 --- a/nixpkgs/lib/systems/parse.nix +++ b/nixpkgs/lib/systems/parse.nix @@ -88,6 +88,9 @@ rec { i686 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i686"; }; x86_64 = { bits = 64; significantByte = littleEndian; family = "x86"; arch = "x86-64"; }; + microblaze = { bits = 32; significantByte = bigEndian; family = "microblaze"; }; + microblazeel = { bits = 32; significantByte = littleEndian; family = "microblaze"; }; + mips = { bits = 32; significantByte = bigEndian; family = "mips"; }; mipsel = { bits = 32; significantByte = littleEndian; family = "mips"; }; mips64 = { bits = 64; significantByte = bigEndian; family = "mips"; }; @@ -124,7 +127,9 @@ rec { or1k = { bits = 32; significantByte = bigEndian; family = "or1k"; }; - js = { bits = 32; significantByte = littleEndian; family = "js"; }; + loongarch64 = { bits = 64; significantByte = littleEndian; family = "loongarch"; }; + + javascript = { bits = 32; significantByte = littleEndian; family = "javascript"; }; }; # GNU build systems assume that older NetBSD architectures are using a.out. @@ -175,23 +180,12 @@ rec { (b == armv7l && isCompatible a armv7a) (b == armv7l && isCompatible a armv7r) (b == armv7l && isCompatible a armv7m) - (b == armv7a && isCompatible a armv8a) - (b == armv7r && isCompatible a armv8a) - (b == armv7m && isCompatible a armv8a) - (b == armv7a && isCompatible a armv8r) - (b == armv7r && isCompatible a armv8r) - (b == armv7m && isCompatible a armv8r) - (b == armv7a && isCompatible a armv8m) - (b == armv7r && isCompatible a armv8m) - (b == armv7m && isCompatible a armv8m) # ARMv8 - (b == armv8r && isCompatible a armv8a) - (b == armv8m && isCompatible a armv8a) - - # XXX: not always true! Some arm64 cpus don’t support arm32 mode. (b == aarch64 && a == armv8a) (b == armv8a && isCompatible a aarch64) + (b == armv8r && isCompatible a armv8a) + (b == armv8m && isCompatible a armv8a) # PowerPC (b == powerpc && isCompatible a powerpc64) @@ -287,7 +281,11 @@ rec { # the normalized name for macOS. macos = { execFormat = macho; families = { inherit darwin; }; name = "darwin"; }; ios = { execFormat = macho; families = { inherit darwin; }; }; - freebsd = { execFormat = elf; families = { inherit bsd; }; }; + # A tricky thing about FreeBSD is that there is no stable ABI across + # versions. That means that putting in the version as part of the + # config string is paramount. + freebsd12 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 12; }; + freebsd13 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 13; }; linux = { execFormat = elf; families = { }; }; netbsd = { execFormat = elf; families = { inherit bsd; }; }; none = { execFormat = unknown; families = { }; }; @@ -350,6 +348,11 @@ rec { The "gnu" ABI is ambiguous on 32-bit ARM. Use "gnueabi" or "gnueabihf" instead. ''; } + { assertion = platform: with platform; !(isPower64 && isBigEndian); + message = '' + The "gnu" ABI is ambiguous on big-endian 64-bit PowerPC. Use "gnuabielfv2" or "gnuabielfv1" instead. + ''; + } ]; }; gnuabi64 = { abi = "64"; }; @@ -361,6 +364,9 @@ rec { gnuabin32 = { abi = "n32"; }; muslabin32 = { abi = "n32"; }; + gnuabielfv2 = { abi = "elfv2"; }; + gnuabielfv1 = { abi = "elfv1"; }; + musleabi = { float = "soft"; }; musleabihf = { float = "hard"; }; musl = {}; @@ -407,27 +413,29 @@ rec { else if (elemAt l 1) == "elf" then { cpu = elemAt l 0; vendor = "unknown"; kernel = "none"; abi = elemAt l 1; } else { cpu = elemAt l 0; kernel = elemAt l 1; }; - "3" = # Awkward hacks, beware! - if elemAt l 1 == "apple" - then { cpu = elemAt l 0; vendor = "apple"; kernel = elemAt l 2; } - else if (elemAt l 1 == "linux") || (elemAt l 2 == "gnu") - then { cpu = elemAt l 0; kernel = elemAt l 1; abi = elemAt l 2; } - else if (elemAt l 2 == "mingw32") # autotools breaks on -gnu for window - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "windows"; } - else if (elemAt l 2 == "wasi") - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "wasi"; } - else if (elemAt l 2 == "redox") - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "redox"; } - else if (elemAt l 2 == "mmixware") - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = "mmixware"; } - else if hasPrefix "netbsd" (elemAt l 2) - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; } - else if (elem (elemAt l 2) ["eabi" "eabihf" "elf"]) - then { cpu = elemAt l 0; vendor = "unknown"; kernel = elemAt l 1; abi = elemAt l 2; } - else if (elemAt l 2 == "ghcjs") - then { cpu = elemAt l 0; vendor = "unknown"; kernel = elemAt l 2; } - else if hasPrefix "genode" (elemAt l 2) - then { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; } + "3" = + # cpu-kernel-environment + if elemAt l 1 == "linux" || + elem (elemAt l 2) ["eabi" "eabihf" "elf" "gnu"] + then { + cpu = elemAt l 0; + kernel = elemAt l 1; + abi = elemAt l 2; + vendor = "unknown"; + } + # cpu-vendor-os + else if elemAt l 1 == "apple" || + elem (elemAt l 2) [ "wasi" "redox" "mmixware" "ghcjs" "mingw32" ] || + hasPrefix "freebsd" (elemAt l 2) || + hasPrefix "netbsd" (elemAt l 2) || + hasPrefix "genode" (elemAt l 2) + then { + cpu = elemAt l 0; + vendor = elemAt l 1; + kernel = if elemAt l 2 == "mingw32" + then "windows" # autotools breaks on -gnu for window + else elemAt l 2; + } else throw "Target specification with 3 components is ambiguous"; "4" = { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; abi = elemAt l 3; }; }.${toString (length l)} @@ -464,6 +472,8 @@ rec { if lib.versionAtLeast (parsed.cpu.version or "0") "6" then abis.gnueabihf else abis.gnueabi + # Default ppc64 BE to ELFv2 + else if isPower64 parsed && isBigEndian parsed then abis.gnuabielfv2 else abis.gnu else abis.unknown; }; @@ -472,10 +482,13 @@ rec { mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s)); + kernelName = kernel: + kernel.name + toString (kernel.version or ""); + doubleFromSystem = { cpu, kernel, abi, ... }: /**/ if abi == abis.cygnus then "${cpu.name}-cygwin" else if kernel.families ? darwin then "${cpu.name}-darwin" - else "${cpu.name}-${kernel.name}"; + else "${cpu.name}-${kernelName kernel}"; tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let optExecFormat = @@ -483,7 +496,7 @@ rec { gnuNetBSDDefaultExecFormat cpu != kernel.execFormat) kernel.execFormat.name; optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}"; - in "${cpu.name}-${vendor.name}-${kernel.name}${optExecFormat}${optAbi}"; + in "${cpu.name}-${vendor.name}-${kernelName kernel}${optExecFormat}${optAbi}"; ################################################################################ diff --git a/nixpkgs/lib/systems/platforms.nix b/nixpkgs/lib/systems/platforms.nix index 41c25484cea0..d574943e47df 100644 --- a/nixpkgs/lib/systems/platforms.nix +++ b/nixpkgs/lib/systems/platforms.nix @@ -557,7 +557,7 @@ rec { else if platform.isRiscV then riscv-multiplatform - else if platform.parsed.cpu == lib.systems.parse.cpuTypes.mipsel then fuloong2f_n32 + else if platform.parsed.cpu == lib.systems.parse.cpuTypes.mipsel then (import ./examples.nix { inherit lib; }).mipsel-linux-gnu else if platform.parsed.cpu == lib.systems.parse.cpuTypes.powerpc64le then powernv diff --git a/nixpkgs/lib/tests/filesystem.sh b/nixpkgs/lib/tests/filesystem.sh new file mode 100755 index 000000000000..cfd333d0001b --- /dev/null +++ b/nixpkgs/lib/tests/filesystem.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Tests lib/filesystem.nix +# Run: +# [nixpkgs]$ lib/tests/filesystem.sh +# or: +# [nixpkgs]$ nix-build lib/tests/release.nix + +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +mkdir directory +touch regular +ln -s target symlink +mkfifo fifo + +expectSuccess() { + local expr=$1 + local expectedResultRegex=$2 + if ! result=$(nix-instantiate --eval --strict --json \ + --expr "with (import <nixpkgs/lib>).filesystem; $expr"); then + die "$expr failed to evaluate, but it was expected to succeed" + fi + if [[ ! "$result" =~ $expectedResultRegex ]]; then + die "$expr == $result, but $expectedResultRegex was expected" + fi +} + +expectFailure() { + local expr=$1 + local expectedErrorRegex=$2 + if result=$(nix-instantiate --eval --strict --json 2>"$work/stderr" \ + --expr "with (import <nixpkgs/lib>).filesystem; $expr"); then + die "$expr evaluated successfully to $result, but it was expected to fail" + fi + if [[ ! "$(<"$work/stderr")" =~ $expectedErrorRegex ]]; then + die "Error was $(<"$work/stderr"), but $expectedErrorRegex was expected" + fi +} + +expectSuccess "pathType /." '"directory"' +expectSuccess "pathType $PWD/directory" '"directory"' +expectSuccess "pathType $PWD/regular" '"regular"' +expectSuccess "pathType $PWD/symlink" '"symlink"' +expectSuccess "pathType $PWD/fifo" '"unknown"' +# Different errors depending on whether the builtins.readFilePath primop is available or not +expectFailure "pathType $PWD/non-existent" "error: (evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'|getting status of '$PWD/non-existent': No such file or directory)" + +expectSuccess "pathIsDirectory /." "true" +expectSuccess "pathIsDirectory $PWD/directory" "true" +expectSuccess "pathIsDirectory $PWD/regular" "false" +expectSuccess "pathIsDirectory $PWD/symlink" "false" +expectSuccess "pathIsDirectory $PWD/fifo" "false" +expectSuccess "pathIsDirectory $PWD/non-existent" "false" + +expectSuccess "pathIsRegularFile /." "false" +expectSuccess "pathIsRegularFile $PWD/directory" "false" +expectSuccess "pathIsRegularFile $PWD/regular" "true" +expectSuccess "pathIsRegularFile $PWD/symlink" "false" +expectSuccess "pathIsRegularFile $PWD/fifo" "false" +expectSuccess "pathIsRegularFile $PWD/non-existent" "false" + +echo >&2 tests ok diff --git a/nixpkgs/lib/tests/maintainer-module.nix b/nixpkgs/lib/tests/maintainer-module.nix index 8cf8411b476a..afa12587a98d 100644 --- a/nixpkgs/lib/tests/maintainer-module.nix +++ b/nixpkgs/lib/tests/maintainer-module.nix @@ -7,7 +7,8 @@ in { type = types.str; }; email = lib.mkOption { - type = types.str; + type = types.nullOr types.str; + default = null; }; matrix = lib.mkOption { type = types.nullOr types.str; diff --git a/nixpkgs/lib/tests/maintainers.nix b/nixpkgs/lib/tests/maintainers.nix index 935d256d218d..be1c8aaa85c5 100644 --- a/nixpkgs/lib/tests/maintainers.nix +++ b/nixpkgs/lib/tests/maintainers.nix @@ -1,12 +1,12 @@ # to run these tests (and the others) # nix-build nixpkgs/lib/tests/release.nix +# These tests should stay in sync with the comment in maintainers/maintainers-list.nix { # The pkgs used for dependencies for the testing itself pkgs ? import ../.. {} , lib ? pkgs.lib }: let - inherit (lib) types; checkMaintainer = handle: uncheckedAttrs: let prefix = [ "lib" "maintainers" handle ]; @@ -21,7 +21,7 @@ let ]; }).config; - checkGithubId = lib.optional (checkedAttrs.github != null && checkedAttrs.githubId == null) '' + checks = lib.optional (checkedAttrs.github != null && checkedAttrs.githubId == null) '' echo ${lib.escapeShellArg (lib.showOption prefix)}': If `github` is specified, `githubId` must be too.' # Calling this too often would hit non-authenticated API limits, but this # shouldn't happen since such errors will get fixed rather quickly @@ -29,8 +29,12 @@ let id=$(jq -r '.id' <<< "$info") echo "The GitHub ID for GitHub user ${checkedAttrs.github} is $id:" echo -e " githubId = $id;\n" + '' ++ lib.optional (checkedAttrs.email == null && checkedAttrs.github == null && checkedAttrs.matrix == null) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': At least one of `email`, `github` or `matrix` must be specified, so that users know how to reach you.' + '' ++ lib.optional (checkedAttrs.email != null && lib.hasSuffix "noreply.github.com" checkedAttrs.email) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': If an email address is given, it should allow people to reach you. If you do not want that, you can just provide `github` or `matrix` instead.' ''; - in lib.deepSeq checkedAttrs checkGithubId; + in lib.deepSeq checkedAttrs checks; missingGithubIds = lib.concatLists (lib.mapAttrsToList checkMaintainer lib.maintainers); diff --git a/nixpkgs/lib/tests/misc.nix b/nixpkgs/lib/tests/misc.nix index 584a946e92cc..ce980436c1bc 100644 --- a/nixpkgs/lib/tests/misc.nix +++ b/nixpkgs/lib/tests/misc.nix @@ -4,6 +4,11 @@ with import ../default.nix; let + testingThrow = expr: { + expr = (builtins.tryEval (builtins.seq expr "didn't throw")); + expected = { success = false; value = false; }; + }; + testingDeepThrow = expr: testingThrow (builtins.deepSeq expr expr); testSanitizeDerivationName = { name, expected }: let @@ -153,6 +158,11 @@ runTests { expected = "a,b,c"; }; + testConcatLines = { + expr = concatLines ["a" "b" "c"]; + expected = "a\nb\nc\n"; + }; + testSplitStringsSimple = { expr = strings.splitString "." "a.b.c.d"; expected = [ "a" "b" "c" "d" ]; @@ -212,6 +222,21 @@ runTests { expected = [ "1" "2" "3" ]; }; + testPadVersionLess = { + expr = versions.pad 3 "1.2"; + expected = "1.2.0"; + }; + + testPadVersionLessExtra = { + expr = versions.pad 3 "1.3-rc1"; + expected = "1.3.0-rc1"; + }; + + testPadVersionMore = { + expr = versions.pad 3 "1.2.3.4"; + expected = "1.2.3"; + }; + testIsStorePath = { expr = let goodPath = @@ -312,6 +337,105 @@ runTests { expected = true; }; + testNormalizePath = { + expr = strings.normalizePath "//a/b//c////d/"; + expected = "/a/b/c/d/"; + }; + + testCharToInt = { + expr = strings.charToInt "A"; + expected = 65; + }; + + testEscapeC = { + expr = strings.escapeC [ " " ] "Hello World"; + expected = "Hello\\x20World"; + }; + + testEscapeURL = testAllTrue [ + ("" == strings.escapeURL "") + ("Hello" == strings.escapeURL "Hello") + ("Hello%20World" == strings.escapeURL "Hello World") + ("Hello%2FWorld" == strings.escapeURL "Hello/World") + ("42%25" == strings.escapeURL "42%") + ("%20%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%09%3A%2F%40%24%27%28%29%2A%2C%3B" == strings.escapeURL " ?&=#+%!<>#\"{}|\\^[]`\t:/@$'()*,;") + ]; + + testToInt = testAllTrue [ + # Naive + (123 == toInt "123") + (0 == toInt "0") + # Whitespace Padding + (123 == toInt " 123") + (123 == toInt "123 ") + (123 == toInt " 123 ") + (123 == toInt " 123 ") + (0 == toInt " 0") + (0 == toInt "0 ") + (0 == toInt " 0 ") + (-1 == toInt "-1") + (-1 == toInt " -1 ") + ]; + + testToIntFails = testAllTrue [ + ( builtins.tryEval (toInt "") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "00") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "01") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "002") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 002 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo123 ") == { success = false; value = false; } ) + ]; + + testToIntBase10 = testAllTrue [ + # Naive + (123 == toIntBase10 "123") + (0 == toIntBase10 "0") + # Whitespace Padding + (123 == toIntBase10 " 123") + (123 == toIntBase10 "123 ") + (123 == toIntBase10 " 123 ") + (123 == toIntBase10 " 123 ") + (0 == toIntBase10 " 0") + (0 == toIntBase10 "0 ") + (0 == toIntBase10 " 0 ") + # Zero Padding + (123 == toIntBase10 "0123") + (123 == toIntBase10 "0000123") + (0 == toIntBase10 "000000") + # Whitespace and Zero Padding + (123 == toIntBase10 " 0123") + (123 == toIntBase10 "0123 ") + (123 == toIntBase10 " 0123 ") + (123 == toIntBase10 " 0000123") + (123 == toIntBase10 "0000123 ") + (123 == toIntBase10 " 0000123 ") + (0 == toIntBase10 " 000000") + (0 == toIntBase10 "000000 ") + (0 == toIntBase10 " 000000 ") + (-1 == toIntBase10 "-1") + (-1 == toIntBase10 " -1 ") + ]; + + testToIntBase10Fails = testAllTrue [ + ( builtins.tryEval (toIntBase10 "") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo00123 ") == { success = false; value = false; } ) + ]; + # LISTS testFilter = { @@ -369,6 +493,11 @@ runTests { expected = [2 30 40 42]; }; + testReplicate = { + expr = replicate 3 "a"; + expected = ["a" "a" "a"]; + }; + testToIntShouldConvertStringToInt = { expr = toInt "27"; expected = 27; @@ -389,9 +518,97 @@ runTests { expected = false; }; + testFindFirstExample1 = { + expr = findFirst (x: x > 3) 7 [ 1 6 4 ]; + expected = 6; + }; + + testFindFirstExample2 = { + expr = findFirst (x: x > 9) 7 [ 1 6 4 ]; + expected = 7; + }; + + testFindFirstEmpty = { + expr = findFirst (abort "when the list is empty, the predicate is not needed") null []; + expected = null; + }; + + testFindFirstSingleMatch = { + expr = findFirst (x: x == 5) null [ 5 ]; + expected = 5; + }; + + testFindFirstSingleDefault = { + expr = findFirst (x: false) null [ (abort "if the predicate doesn't access the value, it must not be evaluated") ]; + expected = null; + }; + + testFindFirstNone = { + expr = builtins.tryEval (findFirst (x: x == 2) null [ 1 (throw "the last element must be evaluated when there's no match") ]); + expected = { success = false; value = false; }; + }; + + # Makes sure that the implementation doesn't cause a stack overflow + testFindFirstBig = { + expr = findFirst (x: x == 1000000) null (range 0 1000000); + expected = 1000000; + }; + + testFindFirstLazy = { + expr = findFirst (x: x == 1) 7 [ 1 (abort "list elements after the match must not be evaluated") ]; + expected = 1; + }; # ATTRSETS + testConcatMapAttrs = { + expr = concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { + foo = "bar"; + foobar = "baz"; + }; + expected = { + foo = "bar"; + foobar = "baz"; + foobarbaz = "baz"; + }; + }; + + # code from example + testFoldlAttrs = { + expr = { + example = foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [ name ]; + }) + { sum = 0; names = [ ]; } + { + foo = 1; + bar = 10; + }; + # should just return the initial value + emptySet = foldlAttrs (throw "function not needed") 123 { }; + # should just evaluate to the last value + accNotNeeded = foldlAttrs (_acc: _name: v: v) (throw "accumulator not needed") { z = 3; a = 2; }; + # the accumulator doesnt have to be an attrset it can be as trivial as being just a number or string + trivialAcc = foldlAttrs (acc: _name: v: acc * 10 + v) 1 { z = 1; a = 2; }; + }; + expected = { + example = { + sum = 11; + names = [ "bar" "foo" ]; + }; + emptySet = 123; + accNotNeeded = 3; + trivialAcc = 121; + }; + }; + # code from the example testRecursiveUpdateUntil = { expr = recursiveUpdateUntil (path: l: r: path == ["foo"]) { @@ -624,7 +841,7 @@ runTests { float = 0.1337; bool = true; emptystring = ""; - string = ''fno"rd''; + string = "fn\${o}\"r\\d"; newlinestring = "\n"; path = /. + "/foo"; null_ = null; @@ -632,16 +849,16 @@ runTests { functionArgs = { arg ? 4, foo }: arg; list = [ 3 4 function [ false ] ]; emptylist = []; - attrs = { foo = null; "foo bar" = "baz"; }; + attrs = { foo = null; "foo b/ar" = "baz"; }; emptyattrs = {}; drv = deriv; }; expected = rec { int = "42"; - float = "~0.133700"; + float = "0.1337"; bool = "true"; emptystring = ''""''; - string = ''"fno\"rd"''; + string = ''"fn\''${o}\"r\\d"''; newlinestring = "\"\\n\""; path = "/foo"; null_ = "null"; @@ -649,9 +866,9 @@ runTests { functionArgs = "<function, args: {arg?, foo}>"; list = "[ 3 4 ${function} [ false ] ]"; emptylist = "[ ]"; - attrs = "{ foo = null; \"foo bar\" = \"baz\"; }"; + attrs = "{ foo = null; \"foo b/ar\" = \"baz\"; }"; emptyattrs = "{ }"; - drv = "<derivation ${deriv.drvPath}>"; + drv = "<derivation ${deriv.name}>"; }; }; @@ -696,8 +913,8 @@ runTests { newlinestring = "\n"; multilinestring = '' hello - there - test + ''${there} + te'''st ''; multilinestring' = '' hello @@ -724,8 +941,8 @@ runTests { multilinestring = '' ''' hello - there - test + '''''${there} + te''''st '''''; multilinestring' = '' ''' @@ -742,6 +959,131 @@ runTests { expected = "«foo»"; }; + testToPlist = + let + deriv = derivation { name = "test"; builder = "/bin/sh"; system = "aarch64-linux"; }; + in { + expr = mapAttrs (const (generators.toPlist { })) { + value = { + nested.values = rec { + int = 42; + float = 0.1337; + bool = true; + emptystring = ""; + string = "fn\${o}\"r\\d"; + newlinestring = "\n"; + path = /. + "/foo"; + null_ = null; + list = [ 3 4 "test" ]; + emptylist = []; + attrs = { foo = null; "foo b/ar" = "baz"; }; + emptyattrs = {}; + }; + }; + }; + expected = { value = builtins.readFile ./test-to-plist-expected.plist; }; + }; + + testToLuaEmptyAttrSet = { + expr = generators.toLua {} {}; + expected = ''{}''; + }; + + testToLuaEmptyList = { + expr = generators.toLua {} []; + expected = ''{}''; + }; + + testToLuaListOfVariousTypes = { + expr = generators.toLua {} [ null 43 3.14159 true ]; + expected = '' + { + nil, + 43, + 3.14159, + true + }''; + }; + + testToLuaString = { + expr = generators.toLua {} ''double-quote (") and single quotes (')''; + expected = ''"double-quote (\") and single quotes (')"''; + }; + + testToLuaAttrsetWithLuaInline = { + expr = generators.toLua {} { x = generators.mkLuaInline ''"abc" .. "def"''; }; + expected = '' + { + ["x"] = ("abc" .. "def") + }''; + }; + + testToLuaAttrsetWithSpaceInKey = { + expr = generators.toLua {} { "some space and double-quote (\")" = 42; }; + expected = '' + { + ["some space and double-quote (\")"] = 42 + }''; + }; + + testToLuaWithoutMultiline = { + expr = generators.toLua { multiline = false; } [ 41 43 ]; + expected = ''{ 41, 43 }''; + }; + + testToLuaEmptyBindings = { + expr = generators.toLua { asBindings = true; } {}; + expected = ""; + }; + + testToLuaBindings = { + expr = generators.toLua { asBindings = true; } { x1 = 41; _y = { a = 43; }; }; + expected = '' + _y = { + ["a"] = 43 + } + x1 = 41 + ''; + }; + + testToLuaPartialTableBindings = { + expr = generators.toLua { asBindings = true; } { "x.y" = 42; }; + expected = '' + x.y = 42 + ''; + }; + + testToLuaIndentedBindings = { + expr = generators.toLua { asBindings = true; indent = " "; } { x = { y = 42; }; }; + expected = " x = {\n [\"y\"] = 42\n }\n"; + }; + + testToLuaBindingsWithSpace = testingThrow ( + generators.toLua { asBindings = true; } { "with space" = 42; } + ); + + testToLuaBindingsWithLeadingDigit = testingThrow ( + generators.toLua { asBindings = true; } { "11eleven" = 42; } + ); + + testToLuaBasicExample = { + expr = generators.toLua {} { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = generators.mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + }; + expected = '' + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + }''; + }; # CLI @@ -1206,4 +1548,110 @@ runTests { expr = strings.levenshteinAtMost 3 "hello" "Holla"; expected = true; }; + + # lazyDerivation + + testLazyDerivationIsLazyInDerivationForAttrNames = { + expr = attrNames (lazyDerivation { + derivation = throw "not lazy enough"; + }); + # It's ok to add attribute names here when lazyDerivation is improved + # in accordance with its inline comments. + expected = [ "drvPath" "meta" "name" "out" "outPath" "outputName" "outputs" "system" "type" ]; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.tests = "whatever is in tests"; + }).tests; + expected = "whatever is in tests"; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr2 = { + # passthru.tests is not a special case. It works for any attr. + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.foo = "whatever is in foo"; + }).foo; + expected = "whatever is in foo"; + }; + + testLazyDerivationIsLazyInDerivationForMeta = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + meta = "whatever is in meta"; + }).meta; + expected = "whatever is in meta"; + }; + + testLazyDerivationReturnsDerivationAttrs = let + derivation = { + type = "derivation"; + outputs = ["out"]; + out = "test out"; + outPath = "test outPath"; + outputName = "out"; + drvPath = "test drvPath"; + name = "test name"; + system = "test system"; + meta = "test meta"; + }; + in { + expr = lazyDerivation { inherit derivation; }; + expected = derivation; + }; + + testTypeDescriptionInt = { + expr = (with types; int).description; + expected = "signed integer"; + }; + testTypeDescriptionListOfInt = { + expr = (with types; listOf int).description; + expected = "list of signed integer"; + }; + testTypeDescriptionListOfListOfInt = { + expr = (with types; listOf (listOf int)).description; + expected = "list of list of signed integer"; + }; + testTypeDescriptionListOfEitherStrOrBool = { + expr = (with types; listOf (either str bool)).description; + expected = "list of (string or boolean)"; + }; + testTypeDescriptionEitherListOfStrOrBool = { + expr = (with types; either (listOf bool) str).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionEitherStrOrListOfBool = { + expr = (with types; either str (listOf bool)).description; + expected = "string or list of boolean"; + }; + testTypeDescriptionOneOfListOfStrOrBool = { + expr = (with types; oneOf [ (listOf bool) str ]).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionOneOfListOfStrOrBoolOrNumber = { + expr = (with types; oneOf [ (listOf bool) str number ]).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherListOfBoolOrEitherStringOrNumber = { + expr = (with types; either (listOf bool) (either str number)).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherEitherListOfBoolOrStringOrNumber = { + expr = (with types; either (either (listOf bool) str) number).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherNullOrBoolOrString = { + expr = (with types; either (nullOr bool) str).description; + expected = "null or boolean or string"; + }; + testTypeDescriptionEitherListOfEitherBoolOrStrOrInt = { + expr = (with types; either (listOf (either bool str)) int).description; + expected = "(list of (boolean or string)) or signed integer"; + }; + testTypeDescriptionEitherIntOrListOrEitherBoolOrStr = { + expr = (with types; either int (listOf (either bool str))).description; + expected = "signed integer or list of (boolean or string)"; + }; } diff --git a/nixpkgs/lib/tests/modules.sh b/nixpkgs/lib/tests/modules.sh index c92cc62023b5..7aebba6b589e 100755 --- a/nixpkgs/lib/tests/modules.sh +++ b/nixpkgs/lib/tests/modules.sh @@ -58,9 +58,15 @@ checkConfigError() { fi } +# Shorthand meta attribute does not duplicate the config +checkConfigOutput '^"one two"$' config.result ./shorthand-meta.nix + # Check boolean option. checkConfigOutput '^false$' config.enable ./declare-enable.nix checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*: true' config.enable ./define-enable.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' config.enable ./define-enable-throw.nix +checkConfigError 'while evaluating a definition from `.*/define-enable-abort.nix' config.enable ./define-enable-abort.nix +checkConfigError 'while evaluating the error message for definitions for .enable., which is an option that does not exist' config.enable ./define-enable-abort.nix checkConfigOutput '^1$' config.bare-submodule.nested ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix checkConfigOutput '^2$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix @@ -130,10 +136,19 @@ checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-enab set -- config.enable ./define-enable.nix ./declare-enable.nix checkConfigOutput '^true$' "$@" checkConfigOutput '^false$' "$@" ./disable-define-enable.nix +checkConfigOutput '^false$' "$@" ./disable-define-enable-string-path.nix checkConfigError "The option .*enable.* does not exist. Definition values:\n\s*- In .*: true" "$@" ./disable-declare-enable.nix checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-define-enable.nix ./disable-declare-enable.nix checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-enable-modules.nix +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-key.nix +checkConfigError 'Module ..*disable-module-bad-key.nix. contains a disabledModules item that is an attribute set, presumably a module, that does not have a .key. attribute. .*' 'config.enable' ./disable-module-bad-key.nix + +# Not sure if we want to keep supporting module keys that aren't strings, paths or v?key, but we shouldn't remove support accidentally. +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-toString-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-toString-key.nix + # Check _module.args. set -- config.enable ./declare-enable.nix ./define-enable-with-custom-arg.nix checkConfigError 'while evaluating the module argument .*custom.* in .*define-enable-with-custom-arg.nix.*:' "$@" @@ -151,6 +166,7 @@ checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' checkConfigOutput '^true$' "$@" ./define-module-check.nix # Check coerced value. +set -- checkConfigOutput '^"42"$' config.value ./declare-coerced-value.nix checkConfigOutput '^"24"$' config.value ./declare-coerced-value.nix ./define-value-string.nix checkConfigError 'A definition for option .* is not.*string or signed integer convertible to it.*. Definition values:\n\s*- In .*: \[ \]' config.value ./declare-coerced-value.nix ./define-value-list.nix @@ -158,7 +174,7 @@ checkConfigError 'A definition for option .* is not.*string or signed integer co # Check coerced value with unsound coercion checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix -checkConfigError 'json.exception.parse_error' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix +checkConfigError 'toInt: Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix # Check mkAliasOptionModule. checkConfigOutput '^true$' config.enable ./alias-with-priority.nix @@ -166,6 +182,11 @@ checkConfigOutput '^true$' config.enableAlias ./alias-with-priority.nix checkConfigOutput '^false$' config.enable ./alias-with-priority-can-override.nix checkConfigOutput '^false$' config.enableAlias ./alias-with-priority-can-override.nix +# Check mkPackageOption +checkConfigOutput '^"hello"$' config.package.pname ./declare-mkPackageOption.nix +checkConfigError 'The option .undefinedPackage. is used but not defined' config.undefinedPackage ./declare-mkPackageOption.nix +checkConfigOutput '^null$' config.nullablePackage ./declare-mkPackageOption.nix + # submoduleWith ## specialArgs should work @@ -174,7 +195,7 @@ checkConfigOutput '^"foo"$' config.submodule.foo ./declare-submoduleWith-special ## shorthandOnlyDefines config behaves as expected checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-shorthand.nix checkConfigError 'is not of type `boolean' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-noshorthand.nix -checkConfigError "You're trying to declare a value of type \`bool'\n\s*rather than an attribute-set for the option" config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-shorthand.nix +checkConfigError "You're trying to define a value of type \`bool'\n\s*rather than an attribute set for the option" config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-shorthand.nix checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-noshorthand.nix ## submoduleWith should merge all modules in one swoop @@ -239,7 +260,9 @@ checkConfigError 'A definition for option .* is not of type .*' \ ## Freeform modules # Assigning without a declared option should work checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-string.nix -# No freeform assigments shouldn't make it error +# Shorthand modules interpret `meta` and `class` as config items +checkConfigOutput '^true$' options._module.args.value.result ./freeform-attrsOf.nix ./define-freeform-keywords-shorthand.nix +# No freeform assignments shouldn't make it error checkConfigOutput '^{ }$' config ./freeform-attrsOf.nix # but only if the type matches checkConfigError 'A definition for option .* is not of type .*' config.value ./freeform-attrsOf.nix ./define-value-list.nix @@ -298,11 +321,11 @@ checkConfigOutput '^"baz"$' config.value.nested.bar.baz ./types-anything/mk-mods ## types.functionTo checkConfigOutput '^"input is input"$' config.result ./functionTo/trivial.nix checkConfigOutput '^"a b"$' config.result ./functionTo/merging-list.nix -checkConfigError 'A definition for option .fun.\[function body\]. is not of type .string.. Definition values:\n\s*- In .*wrong-type.nix' config.result ./functionTo/wrong-type.nix +checkConfigError 'A definition for option .fun.<function body>. is not of type .string.. Definition values:\n\s*- In .*wrong-type.nix' config.result ./functionTo/wrong-type.nix checkConfigOutput '^"b a"$' config.result ./functionTo/list-order.nix checkConfigOutput '^"a c"$' config.result ./functionTo/merging-attrs.nix checkConfigOutput '^"a bee"$' config.result ./functionTo/submodule-options.nix -checkConfigOutput '^"fun.\[function body\].a fun.\[function body\].b"$' config.optionsResult ./functionTo/submodule-options.nix +checkConfigOutput '^"fun.<function body>.a fun.<function body>.b"$' config.optionsResult ./functionTo/submodule-options.nix # moduleType checkConfigOutput '^"a b"$' config.resultFoo ./declare-variants.nix ./define-variant.nix @@ -344,6 +367,35 @@ checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survive # because of an `extendModules` bug, issue 168767. checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix +# Class checks, evalModules +checkConfigOutput '^{ }$' config.ok.config ./class-check.nix +checkConfigOutput '"nixos"' config.ok.class ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.fail.config ./class-check.nix +checkConfigError 'The module foo.nix#darwinModules.default was imported into nixos instead of darwin.' config.fail-anon.config ./class-check.nix + +# Class checks, submoduleWith +checkConfigOutput '^{ }$' config.sub.nixosOk ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.sub.nixosFail.config ./class-check.nix + +# submoduleWith type merge with different class +checkConfigError 'A submoduleWith option is declared multiple times with conflicting class values "darwin" and "nixos".' config.sub.mergeFail.config ./class-check.nix + +# _type check +checkConfigError 'Could not load a value as a module, because it is of type "flake", in file .*/module-imports-_type-check.nix' config.ok.config ./module-imports-_type-check.nix +checkConfigOutput '^true$' "$@" config.enable ./declare-enable.nix ./define-enable-with-top-level-mkIf.nix +checkConfigError 'Could not load a value as a module, because it is of type "configuration", in file .*/import-configuration.nix.*please only import the modules that make up the configuration.*' config ./import-configuration.nix + +# doRename works when `warnings` does not exist. +checkConfigOutput '^1234$' config.c.d.e ./doRename-basic.nix +# doRename adds a warning. +checkConfigOutput '^"The option `a\.b. defined in `.*/doRename-warnings\.nix. has been renamed to `c\.d\.e.\."$' \ + config.result \ + ./doRename-warnings.nix + +# Anonymous modules get deduplicated by key +checkConfigOutput '^"pear"$' config.once.raw ./merge-module-with-key.nix +checkConfigOutput '^"pear\\npear"$' config.twice.raw ./merge-module-with-key.nix + cat <<EOF ====== module tests ====== $pass Pass diff --git a/nixpkgs/lib/tests/modules/class-check.nix b/nixpkgs/lib/tests/modules/class-check.nix new file mode 100644 index 000000000000..293fd4abd628 --- /dev/null +++ b/nixpkgs/lib/tests/modules/class-check.nix @@ -0,0 +1,76 @@ +{ lib, ... }: { + options = { + sub = { + nixosOk = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + }; + # Same but will have bad definition + nixosFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + }; + + mergeFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + default = { }; + }; + }; + }; + imports = [ + { + options = { + sub = { + mergeFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "darwin"; + modules = [ ]; + }; + }; + }; + }; + } + ]; + config = { + _module.freeformType = lib.types.anything; + ok = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + ]; + }; + + fail = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + ./module-class-is-darwin.nix + ]; + }; + + fail-anon = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + { _file = "foo.nix#darwinModules.default"; + _class = "darwin"; + config = {}; + imports = []; + } + ]; + }; + + sub.nixosOk = { _class = "nixos"; }; + sub.nixosFail = { imports = [ ./module-class-is-darwin.nix ]; }; + }; +} diff --git a/nixpkgs/lib/tests/modules/declare-mkPackageOption.nix b/nixpkgs/lib/tests/modules/declare-mkPackageOption.nix new file mode 100644 index 000000000000..640b19a7bf22 --- /dev/null +++ b/nixpkgs/lib/tests/modules/declare-mkPackageOption.nix @@ -0,0 +1,19 @@ +{ lib, ... }: let + pkgs.hello = { + type = "derivation"; + pname = "hello"; + }; +in { + options = { + package = lib.mkPackageOption pkgs "hello" { }; + + undefinedPackage = lib.mkPackageOption pkgs "hello" { + default = null; + }; + + nullablePackage = lib.mkPackageOption pkgs "hello" { + nullable = true; + default = null; + }; + }; +} diff --git a/nixpkgs/lib/tests/modules/define-enable-abort.nix b/nixpkgs/lib/tests/modules/define-enable-abort.nix new file mode 100644 index 000000000000..85b58a567cad --- /dev/null +++ b/nixpkgs/lib/tests/modules/define-enable-abort.nix @@ -0,0 +1,3 @@ +{ + config.enable = abort "oops"; +} diff --git a/nixpkgs/lib/tests/modules/define-enable-throw.nix b/nixpkgs/lib/tests/modules/define-enable-throw.nix new file mode 100644 index 000000000000..16a59b781dc5 --- /dev/null +++ b/nixpkgs/lib/tests/modules/define-enable-throw.nix @@ -0,0 +1,3 @@ +{ + config.enable = throw "oops"; +} diff --git a/nixpkgs/lib/tests/modules/define-enable-with-top-level-mkIf.nix b/nixpkgs/lib/tests/modules/define-enable-with-top-level-mkIf.nix new file mode 100644 index 000000000000..4909c16d82b4 --- /dev/null +++ b/nixpkgs/lib/tests/modules/define-enable-with-top-level-mkIf.nix @@ -0,0 +1,5 @@ +{ lib, ... }: +# I think this might occur more realistically in a submodule +{ + imports = [ (lib.mkIf true { enable = true; }) ]; +} diff --git a/nixpkgs/lib/tests/modules/define-freeform-keywords-shorthand.nix b/nixpkgs/lib/tests/modules/define-freeform-keywords-shorthand.nix new file mode 100644 index 000000000000..8de1ec6a7475 --- /dev/null +++ b/nixpkgs/lib/tests/modules/define-freeform-keywords-shorthand.nix @@ -0,0 +1,15 @@ +{ config, ... }: { + class = { "just" = "data"; }; + a = "one"; + b = "two"; + meta = "meta"; + + _module.args.result = + let r = builtins.removeAttrs config [ "_module" ]; + in builtins.trace (builtins.deepSeq r r) (r == { + a = "one"; + b = "two"; + class = { "just" = "data"; }; + meta = "meta"; + }); +} diff --git a/nixpkgs/lib/tests/modules/disable-define-enable-string-path.nix b/nixpkgs/lib/tests/modules/disable-define-enable-string-path.nix new file mode 100644 index 000000000000..6429a6d6354a --- /dev/null +++ b/nixpkgs/lib/tests/modules/disable-define-enable-string-path.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ (toString ./define-enable.nix) ]; +} diff --git a/nixpkgs/lib/tests/modules/disable-module-bad-key.nix b/nixpkgs/lib/tests/modules/disable-module-bad-key.nix new file mode 100644 index 000000000000..f50d06f2f03c --- /dev/null +++ b/nixpkgs/lib/tests/modules/disable-module-bad-key.nix @@ -0,0 +1,16 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { config, ... }: { + config = { + enable = true; + }; + }; +in +{ + imports = [ + ./declare-enable.nix + ]; + disabledModules = [ { } ]; +} diff --git a/nixpkgs/lib/tests/modules/disable-module-with-key.nix b/nixpkgs/lib/tests/modules/disable-module-with-key.nix new file mode 100644 index 000000000000..ea2a60aa832d --- /dev/null +++ b/nixpkgs/lib/tests/modules/disable-module-with-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = "disable-module-with-key.nix#moduleWithKey"; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ moduleWithKey ]; + }; + default = {}; + }; + }; +} diff --git a/nixpkgs/lib/tests/modules/disable-module-with-toString-key.nix b/nixpkgs/lib/tests/modules/disable-module-with-toString-key.nix new file mode 100644 index 000000000000..3f8c81904ce6 --- /dev/null +++ b/nixpkgs/lib/tests/modules/disable-module-with-toString-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = 123; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ 123 ]; + }; + default = {}; + }; + }; +} diff --git a/nixpkgs/lib/tests/modules/doRename-basic.nix b/nixpkgs/lib/tests/modules/doRename-basic.nix new file mode 100644 index 000000000000..9d79fa4f26a3 --- /dev/null +++ b/nixpkgs/lib/tests/modules/doRename-basic.nix @@ -0,0 +1,11 @@ +{ lib, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + c.d.e = lib.mkOption {}; + }; + config = { + a.b = 1234; + }; +} diff --git a/nixpkgs/lib/tests/modules/doRename-warnings.nix b/nixpkgs/lib/tests/modules/doRename-warnings.nix new file mode 100644 index 000000000000..6f0f1e87e3aa --- /dev/null +++ b/nixpkgs/lib/tests/modules/doRename-warnings.nix @@ -0,0 +1,14 @@ +{ lib, config, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + warnings = lib.mkOption { type = lib.types.listOf lib.types.str; }; + c.d.e = lib.mkOption {}; + result = lib.mkOption {}; + }; + config = { + a.b = 1234; + result = lib.concatStringsSep "%" config.warnings; + }; +} diff --git a/nixpkgs/lib/tests/modules/import-configuration.nix b/nixpkgs/lib/tests/modules/import-configuration.nix new file mode 100644 index 000000000000..a2a32bbee4ca --- /dev/null +++ b/nixpkgs/lib/tests/modules/import-configuration.nix @@ -0,0 +1,12 @@ +{ lib, ... }: +let + myconf = lib.evalModules { modules = [ { } ]; }; +in +{ + imports = [ + # We can't do this. A configuration is not equal to its set of a modules. + # Equating those would lead to a mess, as specialArgs, anonymous modules + # that can't be deduplicated, and possibly more come into play. + myconf + ]; +} diff --git a/nixpkgs/lib/tests/modules/merge-module-with-key.nix b/nixpkgs/lib/tests/modules/merge-module-with-key.nix new file mode 100644 index 000000000000..21f00e6ef976 --- /dev/null +++ b/nixpkgs/lib/tests/modules/merge-module-with-key.nix @@ -0,0 +1,49 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithoutKey = { + config = { + raw = "pear"; + }; + }; + + moduleWithKey = { + key = __curPos.file + "#moduleWithKey"; + config = { + raw = "pear"; + }; + }; + + decl = { + options = { + raw = mkOption { + type = types.lines; + }; + }; + }; +in +{ + options = { + once = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithKey + moduleWithKey + ]; + }; + default = {}; + }; + twice = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithoutKey + moduleWithoutKey + ]; + }; + default = {}; + }; + }; +} diff --git a/nixpkgs/lib/tests/modules/module-class-is-darwin.nix b/nixpkgs/lib/tests/modules/module-class-is-darwin.nix new file mode 100644 index 000000000000..bacf45626d71 --- /dev/null +++ b/nixpkgs/lib/tests/modules/module-class-is-darwin.nix @@ -0,0 +1,4 @@ +{ + _class = "darwin"; + config = {}; +} diff --git a/nixpkgs/lib/tests/modules/module-class-is-nixos.nix b/nixpkgs/lib/tests/modules/module-class-is-nixos.nix new file mode 100644 index 000000000000..6d62feedae48 --- /dev/null +++ b/nixpkgs/lib/tests/modules/module-class-is-nixos.nix @@ -0,0 +1,4 @@ +{ + _class = "nixos"; + config = {}; +} diff --git a/nixpkgs/lib/tests/modules/module-imports-_type-check.nix b/nixpkgs/lib/tests/modules/module-imports-_type-check.nix new file mode 100644 index 000000000000..1e29c469daa5 --- /dev/null +++ b/nixpkgs/lib/tests/modules/module-imports-_type-check.nix @@ -0,0 +1,3 @@ +{ + imports = [ { _type = "flake"; } ]; +} diff --git a/nixpkgs/lib/tests/modules/shorthand-meta.nix b/nixpkgs/lib/tests/modules/shorthand-meta.nix new file mode 100644 index 000000000000..8c9619e18a2a --- /dev/null +++ b/nixpkgs/lib/tests/modules/shorthand-meta.nix @@ -0,0 +1,19 @@ +{ lib, ... }: +let + inherit (lib) types mkOption; +in +{ + imports = [ + ({ config, ... }: { + options = { + meta.foo = mkOption { + type = types.listOf types.str; + }; + result = mkOption { default = lib.concatStringsSep " " config.meta.foo; }; + }; + }) + { + meta.foo = [ "one" "two" ]; + } + ]; +} diff --git a/nixpkgs/lib/tests/release.nix b/nixpkgs/lib/tests/release.nix index b93a4236f91e..c3bf58db241f 100644 --- a/nixpkgs/lib/tests/release.nix +++ b/nixpkgs/lib/tests/release.nix @@ -1,44 +1,64 @@ { # The pkgs used for dependencies for the testing itself # Don't test properties of pkgs.lib, but rather the lib in the parent directory - pkgs ? import ../.. {} // { lib = throw "pkgs.lib accessed, but the lib tests should use nixpkgs' lib path directly!"; } + pkgs ? import ../.. {} // { lib = throw "pkgs.lib accessed, but the lib tests should use nixpkgs' lib path directly!"; }, + nix ? pkgs.nix, + nixVersions ? [ pkgs.nixVersions.minimum nix pkgs.nixVersions.unstable ], }: -pkgs.runCommand "nixpkgs-lib-tests" { - buildInputs = [ - pkgs.nix - (import ./check-eval.nix) - (import ./maintainers.nix { - inherit pkgs; - lib = import ../.; - }) - (import ./teams.nix { - inherit pkgs; - lib = import ../.; - }) - ]; -} '' - datadir="${pkgs.nix}/share" - export TEST_ROOT=$(pwd)/test-tmp - export NIX_BUILD_HOOK= - export NIX_CONF_DIR=$TEST_ROOT/etc - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_STORE_DIR=$TEST_ROOT/store - export PAGER=cat - cacheDir=$TEST_ROOT/binary-cache - - mkdir -p $NIX_CONF_DIR - echo "experimental-features = nix-command" >> $NIX_CONF_DIR/nix.conf - - nix-store --init - - cp -r ${../.} lib - echo "Running lib/tests/modules.sh" - bash lib/tests/modules.sh - - echo "Running lib/tests/sources.sh" - TEST_LIB=$PWD/lib bash lib/tests/sources.sh - - touch $out -'' +let + testWithNix = nix: + pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { + buildInputs = [ + (import ./check-eval.nix) + (import ./maintainers.nix { + inherit pkgs; + lib = import ../.; + }) + (import ./teams.nix { + inherit pkgs; + lib = import ../.; + }) + (import ../path/tests { + inherit pkgs; + }) + ]; + nativeBuildInputs = [ + nix + ]; + strictDeps = true; + } '' + datadir="${nix}/share" + export TEST_ROOT=$(pwd)/test-tmp + export NIX_BUILD_HOOK= + export NIX_CONF_DIR=$TEST_ROOT/etc + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_STORE_DIR=$TEST_ROOT/store + export PAGER=cat + cacheDir=$TEST_ROOT/binary-cache + + mkdir -p $NIX_CONF_DIR + echo "experimental-features = nix-command" >> $NIX_CONF_DIR/nix.conf + + nix-store --init + + cp -r ${../.} lib + echo "Running lib/tests/modules.sh" + bash lib/tests/modules.sh + + echo "Running lib/tests/filesystem.sh" + TEST_LIB=$PWD/lib bash lib/tests/filesystem.sh + + echo "Running lib/tests/sources.sh" + TEST_LIB=$PWD/lib bash lib/tests/sources.sh + + mkdir $out + echo success > $out/${nix.version} + ''; + +in + pkgs.symlinkJoin { + name = "nixpkgs-lib-tests"; + paths = map testWithNix nixVersions; + } diff --git a/nixpkgs/lib/tests/sources.sh b/nixpkgs/lib/tests/sources.sh index a7f490a79d74..cda77aa96b28 100755 --- a/nixpkgs/lib/tests/sources.sh +++ b/nixpkgs/lib/tests/sources.sh @@ -23,14 +23,19 @@ clean_up() { trap clean_up EXIT cd "$work" -touch {README.md,module.o,foo.bar} +# Crudely unquotes a JSON string by just taking everything between the first and the second quote. +# We're only using this for resulting /nix/store paths, which can't contain " anyways, +# nor can they contain any other characters that would need to be escaped specially in JSON +# This way we don't need to add a dependency on e.g. jq +crudeUnquoteJSON() { + cut -d \" -f2 +} -# nix-instantiate doesn't write out the source, only computing the hash, so -# this uses the experimental nix command instead. +touch {README.md,module.o,foo.bar} -dir="$(nix eval --impure --raw --expr '(with import <nixpkgs/lib>; "${ +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ cleanSource ./. -}")')" +}")' | crudeUnquoteJSON)" (cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF . ./foo.bar @@ -39,9 +44,9 @@ EOF ) || die "cleanSource 1" -dir="$(nix eval --impure --raw --expr '(with import <nixpkgs/lib>; "${ +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } -}")')" +}")' | crudeUnquoteJSON)" (cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF . ./module.o @@ -49,9 +54,9 @@ dir="$(nix eval --impure --raw --expr '(with import <nixpkgs/lib>; "${ EOF ) || die "cleanSourceWith 1" -dir="$(nix eval --impure --raw --expr '(with import <nixpkgs/lib>; "${ +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } -}")')" +}")' | crudeUnquoteJSON)" (cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF . ./README.md diff --git a/nixpkgs/lib/tests/systems.nix b/nixpkgs/lib/tests/systems.nix index 46e7bd992f1e..2afe128c4208 100644 --- a/nixpkgs/lib/tests/systems.nix +++ b/nixpkgs/lib/tests/systems.nix @@ -16,22 +16,25 @@ with lib.systems.doubles; lib.runTests { testall = mseteq all (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos ++ wasi ++ windows ++ embedded ++ mmix ++ js ++ genode ++ redox); testarm = mseteq arm [ "armv5tel-linux" "armv6l-linux" "armv6l-netbsd" "armv6l-none" "armv7a-linux" "armv7a-netbsd" "armv7l-linux" "armv7l-netbsd" "arm-none" "armv7a-darwin" ]; - testi686 = mseteq i686 [ "i686-linux" "i686-freebsd" "i686-genode" "i686-netbsd" "i686-openbsd" "i686-cygwin" "i686-windows" "i686-none" "i686-darwin" ]; - testmips = mseteq mips [ "mips64el-linux" "mipsel-linux" "mipsel-netbsd" ]; + testarmv7 = mseteq armv7 [ "armv7a-darwin" "armv7a-linux" "armv7l-linux" "armv7a-netbsd" "armv7l-netbsd" ]; + testi686 = mseteq i686 [ "i686-linux" "i686-freebsd13" "i686-genode" "i686-netbsd" "i686-openbsd" "i686-cygwin" "i686-windows" "i686-none" "i686-darwin" ]; + testmips = mseteq mips [ "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "mipsel-netbsd" ]; testmmix = mseteq mmix [ "mmix-mmixware" ]; + testpower = mseteq power [ "powerpc-netbsd" "powerpc-none" "powerpc64-linux" "powerpc64le-linux" "powerpcle-none" ]; testriscv = mseteq riscv [ "riscv32-linux" "riscv64-linux" "riscv32-netbsd" "riscv64-netbsd" "riscv32-none" "riscv64-none" ]; testriscv32 = mseteq riscv32 [ "riscv32-linux" "riscv32-netbsd" "riscv32-none" ]; testriscv64 = mseteq riscv64 [ "riscv64-linux" "riscv64-netbsd" "riscv64-none" ]; - testx86_64 = mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd" "x86_64-genode" "x86_64-redox" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" "x86_64-solaris" "x86_64-windows" "x86_64-none" ]; + tests390x = mseteq s390x [ "s390x-linux" "s390x-none" ]; + testx86_64 = mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd13" "x86_64-genode" "x86_64-redox" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" "x86_64-solaris" "x86_64-windows" "x86_64-none" ]; testcygwin = mseteq cygwin [ "i686-cygwin" "x86_64-cygwin" ]; testdarwin = mseteq darwin [ "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" ]; - testfreebsd = mseteq freebsd [ "i686-freebsd" "x86_64-freebsd" ]; + testfreebsd = mseteq freebsd [ "i686-freebsd13" "x86_64-freebsd13" ]; testgenode = mseteq genode [ "aarch64-genode" "i686-genode" "x86_64-genode" ]; testredox = mseteq redox [ "x86_64-redox" ]; testgnu = mseteq gnu (linux /* ++ kfreebsd ++ ... */); testillumos = mseteq illumos [ "x86_64-solaris" ]; - testlinux = mseteq linux [ "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" "armv7l-linux" "i686-linux" "mips64el-linux" "mipsel-linux" "riscv32-linux" "riscv64-linux" "x86_64-linux" "powerpc64-linux" "powerpc64le-linux" "m68k-linux" "s390-linux" "s390x-linux" ]; + testlinux = mseteq linux [ "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" ]; testnetbsd = mseteq netbsd [ "aarch64-netbsd" "armv6l-netbsd" "armv7a-netbsd" "armv7l-netbsd" "i686-netbsd" "m68k-netbsd" "mipsel-netbsd" "powerpc-netbsd" "riscv32-netbsd" "riscv64-netbsd" "x86_64-netbsd" ]; testopenbsd = mseteq openbsd [ "i686-openbsd" "x86_64-openbsd" ]; testwindows = mseteq windows [ "i686-cygwin" "x86_64-cygwin" "i686-windows" "x86_64-windows" ]; diff --git a/nixpkgs/lib/tests/test-to-plist-expected.plist b/nixpkgs/lib/tests/test-to-plist-expected.plist new file mode 100644 index 000000000000..df0528a60767 --- /dev/null +++ b/nixpkgs/lib/tests/test-to-plist-expected.plist @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>nested</key> + <dict> + <key>values</key> + <dict> + <key>attrs</key> + <dict> + <key>foo b/ar</key> + <string>baz</string> + </dict> + <key>bool</key> + <true/> + <key>emptyattrs</key> + <dict> + + </dict> + <key>emptylist</key> + <array> + + </array> + <key>emptystring</key> + <string></string> + <key>float</key> + <real>0.133700</real> + <key>int</key> + <integer>42</integer> + <key>list</key> + <array> + <integer>3</integer> + <integer>4</integer> + <string>test</string> + </array> + <key>newlinestring</key> + <string> +</string> + <key>path</key> + <string>/foo</string> + <key>string</key> + <string>fn${o}"r\d</string> + </dict> + </dict> +</dict> +</plist> \ No newline at end of file diff --git a/nixpkgs/lib/trivial.nix b/nixpkgs/lib/trivial.nix index 5d4fad8266bc..26e4b32400f2 100644 --- a/nixpkgs/lib/trivial.nix +++ b/nixpkgs/lib/trivial.nix @@ -179,7 +179,7 @@ rec { they take effect as soon as the oldest release reaches end of life. */ oldestSupportedRelease = # Update on master only. Do not backport. - 2205; + 2211; /* Whether a feature is supported in all supported releases (at the time of release branch-off, if applicable). See `oldestSupportedRelease`. */ @@ -195,7 +195,7 @@ rec { On each release the first letter is bumped and a new animal is chosen starting with that new letter. */ - codeName = "Raccoon"; + codeName = "Tapir"; /* Returns the current nixpkgs version suffix as string. */ versionSuffix = @@ -514,6 +514,8 @@ rec { in [r] ++ go q; in + assert (isInt base); + assert (isInt i); assert (base >= 2); assert (i >= 0); lib.reverseList (go i); diff --git a/nixpkgs/lib/types.nix b/nixpkgs/lib/types.nix index d7655bc1a6a2..9360d42f5850 100644 --- a/nixpkgs/lib/types.nix +++ b/nixpkgs/lib/types.nix @@ -6,7 +6,6 @@ let inherit (lib) elem flip - functionArgs isAttrs isBool isDerivation @@ -16,7 +15,6 @@ let isList isString isStorePath - setFunctionArgs toDerivation toList ; @@ -56,7 +54,7 @@ let concatStringsSep escapeNixString hasInfix - isCoercibleToString + isStringLike ; inherit (lib.trivial) boolToString @@ -113,8 +111,28 @@ rec { name , # Description of the type, defined recursively by embedding the wrapped type if any. description ? null - , # Function applied to each definition that should return true if - # its type-correct, false otherwise. + # A hint for whether or not this description needs parentheses. Possible values: + # - "noun": a simple noun phrase such as "positive integer" + # - "conjunction": a phrase with a potentially ambiguous "or" connective. + # - "composite": a phrase with an "of" connective + # See the `optionDescriptionPhrase` function. + , descriptionClass ? null + , # DO NOT USE WITHOUT KNOWING WHAT YOU ARE DOING! + # Function applied to each definition that must return false when a definition + # does not match the type. It should not check more than the root of the value, + # because checking nested values reduces laziness, leading to unnecessary + # infinite recursions in the module system. + # Further checks of nested values should be performed by throwing in + # the merge function. + # Strict and deep type checking can be performed by calling lib.deepSeq on + # the merged value. + # + # See https://github.com/NixOS/nixpkgs/pull/6794 that introduced this change, + # https://github.com/NixOS/nixpkgs/pull/173568 and + # https://github.com/NixOS/nixpkgs/pull/168295 that attempted to revert this, + # https://github.com/NixOS/nixpkgs/issues/191124 and + # https://github.com/NixOS/nixos-search/issues/391 for what happens if you ignore + # this disclaimer. check ? (x: true) , # Merge a list of definitions together into a single value. # This function is called with two arguments: the location of @@ -158,10 +176,36 @@ rec { nestedTypes ? {} }: { _type = "option-type"; - inherit name check merge emptyValue getSubOptions getSubModules substSubModules typeMerge functor deprecationMessage nestedTypes; + inherit + name check merge emptyValue getSubOptions getSubModules substSubModules + typeMerge functor deprecationMessage nestedTypes descriptionClass; description = if description == null then name else description; }; + # optionDescriptionPhrase :: (str -> bool) -> optionType -> str + # + # Helper function for producing unambiguous but readable natural language + # descriptions of types. + # + # Parameters + # + # optionDescriptionPhase unparenthesize optionType + # + # `unparenthesize`: A function from descriptionClass string to boolean. + # It must return true when the class of phrase will fit unambiguously into + # the description of the caller. + # + # `optionType`: The option type to parenthesize or not. + # The option whose description we're returning. + # + # Return value + # + # The description of the `optionType`, with parentheses if there may be an + # ambiguity. + optionDescriptionPhrase = unparenthesize: t: + if unparenthesize (t.descriptionClass or null) + then t.description + else "(${t.description})"; # When adding new types don't forget to document them in # nixos/doc/manual/development/option-types.xml! @@ -170,6 +214,7 @@ rec { raw = mkOptionType rec { name = "raw"; description = "raw value"; + descriptionClass = "noun"; check = value: true; merge = mergeOneOption; }; @@ -177,11 +222,12 @@ rec { anything = mkOptionType { name = "anything"; description = "anything"; + descriptionClass = "noun"; check = value: true; merge = loc: defs: let getType = value: - if isAttrs value && isCoercibleToString value + if isAttrs value && isStringLike value then "stringCoercibleSet" else builtins.typeOf value; @@ -217,21 +263,25 @@ rec { unspecified = mkOptionType { name = "unspecified"; + description = "unspecified value"; + descriptionClass = "noun"; }; bool = mkOptionType { name = "bool"; description = "boolean"; + descriptionClass = "noun"; check = isBool; merge = mergeEqualOption; }; int = mkOptionType { - name = "int"; - description = "signed integer"; - check = isInt; - merge = mergeEqualOption; - }; + name = "int"; + description = "signed integer"; + descriptionClass = "noun"; + check = isInt; + merge = mergeEqualOption; + }; # Specialized subdomains of int ints = @@ -292,15 +342,41 @@ rec { port = ints.u16; float = mkOptionType { - name = "float"; - description = "floating point number"; - check = isFloat; - merge = mergeEqualOption; + name = "float"; + description = "floating point number"; + descriptionClass = "noun"; + check = isFloat; + merge = mergeEqualOption; + }; + + number = either int float; + + numbers = let + betweenDesc = lowest: highest: + "${builtins.toJSON lowest} and ${builtins.toJSON highest} (both inclusive)"; + in { + between = lowest: highest: + assert lib.assertMsg (lowest <= highest) + "numbers.between: lowest must be smaller than highest"; + addCheck number (x: x >= lowest && x <= highest) // { + name = "numberBetween"; + description = "integer or floating point number between ${betweenDesc lowest highest}"; + }; + + nonnegative = addCheck number (x: x >= 0) // { + name = "numberNonnegative"; + description = "nonnegative integer or floating point number, meaning >=0"; + }; + positive = addCheck number (x: x > 0) // { + name = "numberPositive"; + description = "positive integer or floating point number, meaning >0"; + }; }; str = mkOptionType { name = "str"; description = "string"; + descriptionClass = "noun"; check = isString; merge = mergeEqualOption; }; @@ -308,6 +384,7 @@ rec { nonEmptyStr = mkOptionType { name = "nonEmptyStr"; description = "non-empty string"; + descriptionClass = "noun"; check = x: str.check x && builtins.match "[ \t\n]*" x == null; inherit (str) merge; }; @@ -320,6 +397,7 @@ rec { mkOptionType { name = "singleLineStr"; description = "(optionally newline-terminated) single-line string"; + descriptionClass = "noun"; inherit check; merge = loc: defs: lib.removeSuffix "\n" (merge loc defs); @@ -328,6 +406,7 @@ rec { strMatching = pattern: mkOptionType { name = "strMatching ${escapeNixString pattern}"; description = "string matching the pattern ${pattern}"; + descriptionClass = "noun"; check = x: str.check x && builtins.match pattern x != null; inherit (str) merge; }; @@ -340,6 +419,7 @@ rec { then "Concatenated string" # for types.string. else "strings concatenated with ${builtins.toJSON sep}" ; + descriptionClass = "noun"; check = isString; merge = loc: defs: concatStringsSep sep (getValues defs); functor = (defaultFunctor name) // { @@ -363,7 +443,7 @@ rec { passwdEntry = entryType: addCheck entryType (str: !(hasInfix ":" str || hasInfix "\n" str)) // { name = "passwdEntry ${entryType.name}"; - description = "${entryType.description}, not containing newlines or colons"; + description = "${optionDescriptionPhrase (class: class == "noun") entryType}, not containing newlines or colons"; }; attrs = mkOptionType { @@ -383,6 +463,7 @@ rec { # ("/nix/store/hash-foo"). These get a context added to them using builtins.storePath. package = mkOptionType { name = "package"; + descriptionClass = "noun"; check = x: isDerivation x || isStorePath x; merge = loc: defs: let res = mergeOneOption loc defs; @@ -395,15 +476,25 @@ rec { check = x: isDerivation x && hasAttr "shellPath" x; }; + pkgs = addCheck + (unique { message = "A Nixpkgs pkgs set can not be merged with another pkgs set."; } attrs // { + name = "pkgs"; + descriptionClass = "noun"; + description = "Nixpkgs package set"; + }) + (x: (x._type or null) == "pkgs"); + path = mkOptionType { name = "path"; - check = x: isCoercibleToString x && builtins.substring 0 1 (toString x) == "/"; + descriptionClass = "noun"; + check = x: isStringLike x && builtins.substring 0 1 (toString x) == "/"; merge = mergeEqualOption; }; listOf = elemType: mkOptionType rec { name = "listOf"; - description = "list of ${elemType.description}"; + description = "list of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; check = isList; merge = loc: defs: map (x: x.value) (filter (x: x ? value) (concatLists (imap1 (n: def: @@ -426,13 +517,14 @@ rec { nonEmptyListOf = elemType: let list = addCheck (types.listOf elemType) (l: l != []); in list // { - description = "non-empty " + list.description; + description = "non-empty ${optionDescriptionPhrase (class: class == "noun") list}"; emptyValue = { }; # no .value attr, meaning unset }; attrsOf = elemType: mkOptionType rec { name = "attrsOf"; - description = "attribute set of ${elemType.description}"; + description = "attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; check = isAttrs; merge = loc: defs: mapAttrs (n: v: v.value) (filterAttrs (n: v: v ? value) (zipAttrsWith (name: defs: @@ -455,7 +547,8 @@ rec { # error that it's not defined. Use only if conditional definitions don't make sense. lazyAttrsOf = elemType: mkOptionType rec { name = "lazyAttrsOf"; - description = "lazy attribute set of ${elemType.description}"; + description = "lazy attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; check = isAttrs; merge = loc: defs: zipAttrsWith (name: defs: @@ -473,7 +566,7 @@ rec { nestedTypes.elemType = elemType; }; - # TODO: drop this in the future: + # TODO: deprecate this in the future: loaOf = elemType: types.attrsOf elemType // { name = "loaOf"; deprecationMessage = "Mixing lists with attribute values is no longer" @@ -485,7 +578,7 @@ rec { # Value of given type but with no merging (i.e. `uniq list`s are not concatenated). uniq = elemType: mkOptionType rec { name = "uniq"; - inherit (elemType) description check; + inherit (elemType) description descriptionClass check; merge = mergeOneOption; emptyValue = elemType.emptyValue; getSubOptions = elemType.getSubOptions; @@ -497,7 +590,7 @@ rec { unique = { message }: type: mkOptionType rec { name = "unique"; - inherit (type) description check; + inherit (type) description descriptionClass check; merge = mergeUniqueOption { inherit message; }; emptyValue = type.emptyValue; getSubOptions = type.getSubOptions; @@ -510,7 +603,8 @@ rec { # Null or value of ... nullOr = elemType: mkOptionType rec { name = "nullOr"; - description = "null or ${elemType.description}"; + description = "null or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") elemType}"; + descriptionClass = "conjunction"; check = x: x == null || elemType.check x; merge = loc: defs: let nrNulls = count (def: def.value == null) defs; in @@ -528,11 +622,12 @@ rec { functionTo = elemType: mkOptionType { name = "functionTo"; - description = "function that evaluates to a(n) ${elemType.description}"; + description = "function that evaluates to a(n) ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; check = isFunction; merge = loc: defs: - fnArgs: (mergeDefinitions (loc ++ [ "[function body]" ]) elemType (map (fn: { inherit (fn) file; value = fn.value fnArgs; }) defs)).mergedValue; - getSubOptions = prefix: elemType.getSubOptions (prefix ++ [ "[function body]" ]); + fnArgs: (mergeDefinitions (loc ++ [ "<function body>" ]) elemType (map (fn: { inherit (fn) file; value = fn.value fnArgs; }) defs)).mergedValue; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ [ "<function body>" ]); getSubModules = elemType.getSubModules; substSubModules = m: functionTo (elemType.substSubModules m); functor = (defaultFunctor "functionTo") // { wrapped = elemType; }; @@ -554,6 +649,7 @@ rec { deferredModuleWith = attrs@{ staticModules ? [] }: mkOptionType { name = "deferredModule"; description = "module"; + descriptionClass = "noun"; check = x: isAttrs x || isFunction x || path.check x; merge = loc: defs: { imports = staticModules ++ map (def: lib.setDefaultModuleLocation "${def.file}, via option ${showOption loc}" def.value) defs; @@ -579,6 +675,7 @@ rec { optionType = mkOptionType { name = "optionType"; description = "optionType"; + descriptionClass = "noun"; check = value: value._type or null == "option-type"; merge = loc: defs: if length defs == 1 @@ -607,6 +704,7 @@ rec { , specialArgs ? {} , shorthandOnlyDefinesConfig ? false , description ? null + , class ? null }@attrs: let inherit (lib.modules) evalModules; @@ -618,7 +716,7 @@ rec { ) defs; base = evalModules { - inherit specialArgs; + inherit class specialArgs; modules = [{ # This is a work-around for the fact that some sub-modules, # such as the one included in an attribute set, expects an "args" @@ -673,9 +771,16 @@ rec { functor = defaultFunctor name // { type = types.submoduleWith; payload = { - inherit modules specialArgs shorthandOnlyDefinesConfig description; + inherit modules class specialArgs shorthandOnlyDefinesConfig description; }; binOp = lhs: rhs: { + class = + # `or null` was added for backwards compatibility only. `class` is + # always set in the current version of the module system. + if lhs.class or null == null then rhs.class or null + else if rhs.class or null == null then lhs.class or null + else if lhs.class or null == rhs.class then lhs.class or null + else throw "A submoduleWith option is declared multiple times with conflicting class values \"${toString lhs.class}\" and \"${toString rhs.class}\"."; modules = lhs.modules ++ rhs.modules; specialArgs = let intersecting = builtins.intersectAttrs lhs.specialArgs rhs.specialArgs; @@ -725,6 +830,10 @@ rec { "value ${show (builtins.head values)} (singular enum)" else "one of ${concatMapStringsSep ", " show values}"; + descriptionClass = + if builtins.length values < 2 + then "noun" + else "conjunction"; check = flip elem values; merge = mergeEqualOption; functor = (defaultFunctor name) // { payload = values; binOp = a: b: unique (a ++ b); }; @@ -733,7 +842,8 @@ rec { # Either value of type `t1` or `t2`. either = t1: t2: mkOptionType rec { name = "either"; - description = "${t1.description} or ${t2.description}"; + description = "${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t1} or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction" || class == "composite") t2}"; + descriptionClass = "conjunction"; check = x: t1.check x || t2.check x; merge = loc: defs: let @@ -771,7 +881,7 @@ rec { coercedType.description})"; mkOptionType rec { name = "coercedTo"; - description = "${finalType.description} or ${coercedType.description} convertible to it"; + description = "${optionDescriptionPhrase (class: class == "noun") finalType} or ${optionDescriptionPhrase (class: class == "noun") coercedType} convertible to it"; check = x: (coercedType.check x && finalType.check (coerceFunc x)) || finalType.check x; merge = loc: defs: let diff --git a/nixpkgs/lib/versions.nix b/nixpkgs/lib/versions.nix index 0e9d81ac78b1..986e7e5f9b37 100644 --- a/nixpkgs/lib/versions.nix +++ b/nixpkgs/lib/versions.nix @@ -46,4 +46,19 @@ rec { builtins.concatStringsSep "." (lib.take 2 (splitVersion v)); + /* Pad a version string with zeros to match the given number of components. + + Example: + pad 3 "1.2" + => "1.2.0" + pad 3 "1.3-rc1" + => "1.3.0-rc1" + pad 3 "1.2.3.4" + => "1.2.3" + */ + pad = n: version: let + numericVersion = lib.head (lib.splitString "-" version); + versionSuffix = lib.removePrefix numericVersion version; + in lib.concatStringsSep "." (lib.take n (lib.splitVersion numericVersion ++ lib.genList (_: "0") n)) + versionSuffix; + } |