diff options
Diffstat (limited to 'nixpkgs/lib/fileset')
| -rw-r--r-- | nixpkgs/lib/fileset/README.md | 16 | ||||
| -rw-r--r-- | nixpkgs/lib/fileset/default.nix | 484 | ||||
| -rw-r--r-- | nixpkgs/lib/fileset/internal.nix | 8 | ||||
| -rwxr-xr-x | nixpkgs/lib/fileset/tests.sh | 72 |
4 files changed, 370 insertions, 210 deletions
diff --git a/nixpkgs/lib/fileset/README.md b/nixpkgs/lib/fileset/README.md index 14b6877a9065..93e0199c32a4 100644 --- a/nixpkgs/lib/fileset/README.md +++ b/nixpkgs/lib/fileset/README.md @@ -253,7 +253,15 @@ The `fileFilter` function takes a path, and not a file set, as its second argume it would change the `subpath`/`components` value depending on which files are included. - (+) If necessary, this restriction can be relaxed later, the opposite wouldn't be possible -## To update in the future - -Here's a list of places in the library that need to be updated in the future: -- If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function +### Strict path existence checking + +Coercing paths that don't exist to file sets always gives an error. + +- (-) Sometimes you want to remove a file that may not always exist using `difference ./. ./does-not-exist`, + but this does not work because coercion of `./does-not-exist` fails, + even though its existence would have no influence on the result. + - (+) This is dangerous, because you wouldn't be protected against typos anymore. + E.g. when trying to prevent `./secret` from being imported, a typo like `difference ./. ./sercet` would import it regardless. + - (+) `difference ./. (maybeMissing ./does-not-exist)` can be used to do this more explicitly. + - (+) `difference ./. (difference ./foo ./foo/bar)` should report an error when `./foo/bar` does not exist ("double negation"). Unfortunately, the current internal representation does not lend itself to a behavior where both `difference x ./does-not-exists` and double negation are handled and checked correctly. + This could be fixed, but would require significant changes to the internal representation that are not worth the effort and the risk of introducing implicit behavior. diff --git a/nixpkgs/lib/fileset/default.nix b/nixpkgs/lib/fileset/default.nix index ca2ab9a35740..75e609a072e7 100644 --- a/nixpkgs/lib/fileset/default.nix +++ b/nixpkgs/lib/fileset/default.nix @@ -3,10 +3,56 @@ []{#sec-fileset} The [`lib.fileset`](#sec-functions-library-fileset) library allows you to work with _file sets_. - A file set is a mathematical set of local files that can be added to the Nix store for use in Nix derivations. + A file set is a (mathematical) set of local files that can be added to the Nix store for use in Nix derivations. File sets are easy and safe to use, providing obvious and composable semantics with good error messages to prevent mistakes. - See the [function reference](#sec-functions-library-fileset) for function-specific documentation. + ## Overview {#sec-fileset-overview} + + Basics: + - [Implicit coercion from paths to file sets](#sec-fileset-path-coercion) + + - [`lib.fileset.maybeMissing`](#function-library-lib.fileset.maybeMissing): + + Create a file set from a path that may be missing. + + - [`lib.fileset.trace`](#function-library-lib.fileset.trace)/[`lib.fileset.traceVal`](#function-library-lib.fileset.trace): + + Pretty-print file sets for debugging. + + - [`lib.fileset.toSource`](#function-library-lib.fileset.toSource): + + Add files in file sets to the store to use as derivation sources. + + Combinators: + - [`lib.fileset.union`](#function-library-lib.fileset.union)/[`lib.fileset.unions`](#function-library-lib.fileset.unions): + + Create a larger file set from all the files in multiple file sets. + + - [`lib.fileset.intersection`](#function-library-lib.fileset.intersection): + + Create a smaller file set from only the files in both file sets. + + - [`lib.fileset.difference`](#function-library-lib.fileset.difference): + + Create a smaller file set containing all files that are in one file set, but not another one. + + Filtering: + - [`lib.fileset.fileFilter`](#function-library-lib.fileset.fileFilter): + + Create a file set from all files that satisisfy a predicate in a directory. + + Utilities: + - [`lib.fileset.fromSource`](#function-library-lib.fileset.fromSource): + + Create a file set from a `lib.sources`-based value. + + - [`lib.fileset.gitTracked`](#function-library-lib.fileset.gitTracked)/[`lib.fileset.gitTrackedWith`](#function-library-lib.fileset.gitTrackedWith): + + Create a file set from all tracked files in a local Git repository. + + If you need more file set functions, + see [this issue](https://github.com/NixOS/nixpkgs/issues/266356) to request it. + ## Implicit coercion from paths to file sets {#sec-fileset-path-coercion} @@ -63,6 +109,7 @@ let _difference _mirrorStorePath _fetchGitSubmodulesMinver + _emptyWithoutBase ; inherit (builtins) @@ -107,6 +154,122 @@ let in { /* + Create a file set from a path that may or may not exist: + - If the path does exist, the path is [coerced to a file set](#sec-fileset-path-coercion). + - If the path does not exist, a file set containing no files is returned. + + Type: + maybeMissing :: Path -> FileSet + + Example: + # All files in the current directory, but excluding main.o if it exists + difference ./. (maybeMissing ./main.o) + */ + maybeMissing = + path: + if ! isPath path then + if isStringLike path then + throw '' + lib.fileset.maybeMissing: Argument ("${toString path}") is a string-like value, but it should be a path instead.'' + else + throw '' + lib.fileset.maybeMissing: Argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + _emptyWithoutBase + else + _singleton path; + + /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + This function takes a final argument to return. + In comparison, [`traceVal`](#function-library-lib.fileset.traceVal) returns + the given file set argument. + + This variant is useful for tracing file sets in the Nix repl. + + Type: + trace :: FileSet -> Any -> Any + + Example: + trace (unions [ ./Makefile ./src ./tests/run.sh ]) null + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + null + */ + trace = + /* + The file set to trace. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.trace: Argument" fileset; + in + seq + (_printFileset actualFileset) + (x: x); + + /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + This function returns the given file set. + In comparison, [`trace`](#function-library-lib.fileset.trace) takes another argument to return. + + This variant is useful for tracing file sets passed as arguments to other functions. + + Type: + traceVal :: FileSet -> FileSet + + Example: + toSource { + root = ./.; + fileset = traceVal (unions [ + ./Makefile + ./src + ./tests/run.sh + ]); + } + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + "/nix/store/...-source" + */ + traceVal = + /* + The file set to trace and return. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.traceVal: Argument" fileset; + in + seq + (_printFileset actualFileset) + # We could also return the original fileset argument here, + # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again + actualFileset; + + /* Add the local files contained in `fileset` to the store as a single [store path](https://nixos.org/manual/nix/stable/glossary#gloss-store-path) rooted at `root`. The result is the store path as a string-like value, making it usable e.g. as the `src` of a derivation, or in string interpolation: @@ -251,75 +414,6 @@ in { }; /* - Create a file set with the same files as a `lib.sources`-based value. - This does not import any of the files into the store. - - This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. - - A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). - - :::{.note} - File sets cannot represent empty directories. - Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. - ::: - - Type: - fromSource :: SourceLike -> FileSet - - Example: - # There's no cleanSource-like function for file sets yet, - # but we can just convert cleanSource to a file set and use it that way - toSource { - root = ./.; - fileset = fromSource (lib.sources.cleanSource ./.); - } - - # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), - # but removing a subdirectory using file set functions - difference - (fromSource (lib.sources.sourceByRegex ./. [ - "^README\.md$" - # This regex includes everything in ./doc - "^doc(/.*)?$" - ]) - ./doc/generated - - # Use cleanSource, but limit it to only include ./Makefile and files under ./src - intersection - (fromSource (lib.sources.cleanSource ./.)) - (unions [ - ./Makefile - ./src - ]); - */ - fromSource = source: - let - # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, - # which are technically internal to lib.sources, - # but we'll allow this since both libraries are in the same code base - # and this function is a bridge between them. - isFiltered = source ? _isLibCleanSourceWith; - path = if isFiltered then source.origSrc else source; - in - # We can only support sources created from paths - if ! isPath path then - if isStringLike path then - throw '' - lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. - Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' - else - throw '' - lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' - else if ! pathExists path then - throw '' - lib.fileset.fromSource: The source origin (${toString path}) of the argument is a path that does not exist.'' - else if isFiltered then - _fromSourceFilter path source.filter - else - # If there's no filter, no need to run the expensive conversion, all subpaths will be included - _singleton path; - - /* The file set containing all files that are in either of two given file sets. This is the same as [`unions`](#function-library-lib.fileset.unions), but takes just two file sets instead of a list. @@ -412,66 +506,6 @@ in { ]; /* - Filter a file set to only contain files matching some predicate. - - Type: - fileFilter :: - ({ - name :: String, - type :: String, - ... - } -> Bool) - -> Path - -> FileSet - - Example: - # Include all regular `default.nix` files in the current directory - fileFilter (file: file.name == "default.nix") ./. - - # Include all non-Nix files from the current directory - fileFilter (file: ! hasSuffix ".nix" file.name) ./. - - # Include all files that start with a "." in the current directory - fileFilter (file: hasPrefix "." file.name) ./. - - # Include all regular files (not symlinks or others) in the current directory - fileFilter (file: file.type == "regular") ./. - */ - fileFilter = - /* - The predicate function to call on all files contained in given file set. - A file is included in the resulting file set if this function returns true for it. - - This function is called with an attribute set containing these attributes: - - - `name` (String): The name of the file - - - `type` (String, one of `"regular"`, `"symlink"` or `"unknown"`): The type of the file. - This matches result of calling [`builtins.readFileType`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-readFileType) on the file's path. - - Other attributes may be added in the future. - */ - predicate: - # The path whose files to filter - path: - if ! isFunction predicate then - throw '' - lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' - else if ! isPath path then - if path._type or "" == "fileset" then - throw '' - lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. - If you need to filter files in a file set, use `intersection fileset (fileFilter pred ./.)` instead.'' - else - throw '' - lib.fileset.fileFilter: Second argument is of type ${typeOf path}, but it should be a path instead.'' - else if ! pathExists path then - throw '' - lib.fileset.fileFilter: Second argument (${toString path}) is a path that does not exist.'' - else - _fileFilter predicate path; - - /* The file set containing all files that are in both of two given file sets. See also [Intersection (set theory)](https://en.wikipedia.org/wiki/Intersection_(set_theory)). @@ -563,94 +597,140 @@ in { (elemAt filesets 1); /* - Incrementally evaluate and trace a file set in a pretty way. - This function is only intended for debugging purposes. - The exact tracing format is unspecified and may change. - - This function takes a final argument to return. - In comparison, [`traceVal`](#function-library-lib.fileset.traceVal) returns - the given file set argument. - - This variant is useful for tracing file sets in the Nix repl. + Filter a file set to only contain files matching some predicate. Type: - trace :: FileSet -> Any -> Any + fileFilter :: + ({ + name :: String, + type :: String, + hasExt :: String -> Bool, + ... + } -> Bool) + -> Path + -> FileSet Example: - trace (unions [ ./Makefile ./src ./tests/run.sh ]) null - => - trace: /home/user/src/myProject - trace: - Makefile (regular) - trace: - src (all files in directory) - trace: - tests - trace: - run.sh (regular) - null + # Include all regular `default.nix` files in the current directory + fileFilter (file: file.name == "default.nix") ./. + + # Include all non-Nix files from the current directory + fileFilter (file: ! file.hasExt "nix") ./. + + # Include all files that start with a "." in the current directory + fileFilter (file: hasPrefix "." file.name) ./. + + # Include all regular files (not symlinks or others) in the current directory + fileFilter (file: file.type == "regular") ./. */ - trace = + fileFilter = /* - The file set to trace. + The predicate function to call on all files contained in given file set. + A file is included in the resulting file set if this function returns true for it. - This argument can also be a path, - which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + This function is called with an attribute set containing these attributes: + + - `name` (String): The name of the file + + - `type` (String, one of `"regular"`, `"symlink"` or `"unknown"`): The type of the file. + This matches result of calling [`builtins.readFileType`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-readFileType) on the file's path. + + - `hasExt` (String -> Bool): Whether the file has a certain file extension. + `hasExt ext` is true only if `hasSuffix ".${ext}" name`. + + This also means that e.g. for a file with name `.gitignore`, + `hasExt "gitignore"` is true. + + Other attributes may be added in the future. */ - fileset: - let - # "fileset" would be a better name, but that would clash with the argument name, - # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.trace: Argument" fileset; - in - seq - (_printFileset actualFileset) - (x: x); + predicate: + # The path whose files to filter + path: + if ! isFunction predicate then + throw '' + lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' + else if ! isPath path then + if path._type or "" == "fileset" then + throw '' + lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. + If you need to filter files in a file set, use `intersection fileset (fileFilter pred ./.)` instead.'' + else + throw '' + lib.fileset.fileFilter: Second argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fileFilter: Second argument (${toString path}) is a path that does not exist.'' + else + _fileFilter predicate path; /* - Incrementally evaluate and trace a file set in a pretty way. - This function is only intended for debugging purposes. - The exact tracing format is unspecified and may change. + Create a file set with the same files as a `lib.sources`-based value. + This does not import any of the files into the store. - This function returns the given file set. - In comparison, [`trace`](#function-library-lib.fileset.trace) takes another argument to return. + This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. - This variant is useful for tracing file sets passed as arguments to other functions. + A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). - Type: - traceVal :: FileSet -> FileSet + :::{.note} + File sets cannot represent empty directories. + Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. + ::: - Example: - toSource { - root = ./.; - fileset = traceVal (unions [ - ./Makefile - ./src - ./tests/run.sh - ]); - } - => - trace: /home/user/src/myProject - trace: - Makefile (regular) - trace: - src (all files in directory) - trace: - tests - trace: - run.sh (regular) - "/nix/store/...-source" - */ - traceVal = - /* - The file set to trace and return. + Type: + fromSource :: SourceLike -> FileSet - This argument can also be a path, - which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). - */ - fileset: + Example: + # There's no cleanSource-like function for file sets yet, + # but we can just convert cleanSource to a file set and use it that way + toSource { + root = ./.; + fileset = fromSource (lib.sources.cleanSource ./.); + } + + # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), + # but removing a subdirectory using file set functions + difference + (fromSource (lib.sources.sourceByRegex ./. [ + "^README\.md$" + # This regex includes everything in ./doc + "^doc(/.*)?$" + ]) + ./doc/generated + + # Use cleanSource, but limit it to only include ./Makefile and files under ./src + intersection + (fromSource (lib.sources.cleanSource ./.)) + (unions [ + ./Makefile + ./src + ]); + */ + fromSource = source: let - # "fileset" would be a better name, but that would clash with the argument name, - # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.traceVal: Argument" fileset; + # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, + # which are technically internal to lib.sources, + # but we'll allow this since both libraries are in the same code base + # and this function is a bridge between them. + isFiltered = source ? _isLibCleanSourceWith; + path = if isFiltered then source.origSrc else source; in - seq - (_printFileset actualFileset) - # We could also return the original fileset argument here, - # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again - actualFileset; + # We can only support sources created from paths + if ! isPath path then + if isStringLike path then + throw '' + lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. + Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' + else + throw '' + lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fromSource: The source origin (${toString path}) of the argument is a path that does not exist.'' + else if isFiltered then + _fromSourceFilter path source.filter + else + # If there's no filter, no need to run the expensive conversion, all subpaths will be included + _singleton path; /* Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository. diff --git a/nixpkgs/lib/fileset/internal.nix b/nixpkgs/lib/fileset/internal.nix index 23d7b847204c..35d556e78391 100644 --- a/nixpkgs/lib/fileset/internal.nix +++ b/nixpkgs/lib/fileset/internal.nix @@ -52,6 +52,7 @@ let concatStringsSep substring stringLength + hasSuffix ; in @@ -181,7 +182,8 @@ rec { ${context} is of type ${typeOf value}, but it should be a file set or a path instead.'' else if ! pathExists value then throw '' - ${context} (${toString value}) is a path that does not exist.'' + ${context} (${toString value}) is a path that does not exist. + To create a file set from a path that may not exist, use `lib.fileset.maybeMissing`.'' else _singleton value; @@ -796,9 +798,11 @@ rec { if predicate { inherit name type; + hasExt = ext: hasSuffix ".${ext}" name; + # To ensure forwards compatibility with more arguments being added in the future, # adding an attribute which can't be deconstructed :) - "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file }:`, use `{ name, file, ... }:` instead." = null; + "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file, hasExt }:`, use `{ name, file, hasExt, ... }:` instead." = null; } then type diff --git a/nixpkgs/lib/fileset/tests.sh b/nixpkgs/lib/fileset/tests.sh index 06f92f297d88..077aefe371c3 100755 --- a/nixpkgs/lib/fileset/tests.sh +++ b/nixpkgs/lib/fileset/tests.sh @@ -413,7 +413,8 @@ expectFailure 'toSource { root = ./.; fileset = cleanSourceWith { src = ./.; }; \s*Note that this only works for sources created from paths.' # Path coercion errors for non-existent paths -expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) is a path that does not exist.' +expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) is a path that does not exist. +\s*To create a file set from a path that may not exist, use `lib.fileset.maybeMissing`.' # File sets cannot be evaluated directly expectFailure 'union ./. ./.' 'lib.fileset: Directly evaluating a file set is not supported. @@ -846,7 +847,7 @@ checkFileset 'fileFilter (file: abort "this is not needed") ./.' # The predicate must be able to handle extra attributes touch a -expectFailure 'toSource { root = ./.; fileset = fileFilter ({ name, type }: true) ./.; }' 'called with unexpected argument '\''"lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you'\''re using `\{ name, file \}:`, use `\{ name, file, ... \}:` instead."'\' +expectFailure 'toSource { root = ./.; fileset = fileFilter ({ name, type, hasExt }: true) ./.; }' 'called with unexpected argument '\''"lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you'\''re using `\{ name, file, hasExt \}:`, use `\{ name, file, hasExt, ... \}:` instead."'\' rm -rf -- * # .name is the name, and it works correctly, even recursively @@ -894,6 +895,39 @@ expectEqual \ 'toSource { root = ./.; fileset = union ./d/a ./d/b; }' rm -rf -- * +# Check that .hasExt checks for the file extension +# The empty extension is the same as a file ending with a . +tree=( + [a]=0 + [a.]=1 + [a.b]=0 + [a.b.]=1 + [a.b.c]=0 +) +checkFileset 'fileFilter (file: file.hasExt "") ./.' + +# It can check for the last extension +tree=( + [a]=0 + [.a]=1 + [.a.]=0 + [.b.a]=1 + [.b.a.]=0 +) +checkFileset 'fileFilter (file: file.hasExt "a") ./.' + +# It can check for any extension +tree=( + [a.b.c.d]=1 +) +checkFileset 'fileFilter (file: + all file.hasExt [ + "b.c.d" + "c.d" + "d" + ] +) ./.' + # It's lazy tree=( [b]=1 @@ -1450,6 +1484,40 @@ checkGitTracked rm -rf -- * +## lib.fileset.maybeMissing + +# Argument must be a path +expectFailure 'maybeMissing "someString"' 'lib.fileset.maybeMissing: Argument \("someString"\) is a string-like value, but it should be a path instead.' +expectFailure 'maybeMissing null' 'lib.fileset.maybeMissing: Argument is of type null, but it should be a path instead.' + +tree=( +) +checkFileset 'maybeMissing ./a' +checkFileset 'maybeMissing ./b' +checkFileset 'maybeMissing ./b/c' + +# Works on single files +tree=( + [a]=1 + [b/c]=0 + [b/d]=0 +) +checkFileset 'maybeMissing ./a' +tree=( + [a]=0 + [b/c]=1 + [b/d]=0 +) +checkFileset 'maybeMissing ./b/c' + +# Works on directories +tree=( + [a]=0 + [b/c]=1 + [b/d]=1 +) +checkFileset 'maybeMissing ./b' + # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets echo >&2 tests ok |