diff options
Diffstat (limited to 'nixpkgs/pkgs/development/misc/resholve')
| -rw-r--r-- | nixpkgs/pkgs/development/misc/resholve/README.md | 158 | ||||
| -rw-r--r-- | nixpkgs/pkgs/development/misc/resholve/default.nix | 9 | ||||
| -rw-r--r-- | nixpkgs/pkgs/development/misc/resholve/deps.nix | 120 | ||||
| -rw-r--r-- | nixpkgs/pkgs/development/misc/resholve/resholve-package.nix | 97 | ||||
| -rw-r--r-- | nixpkgs/pkgs/development/misc/resholve/resholve.nix | 80 |
5 files changed, 464 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/development/misc/resholve/README.md b/nixpkgs/pkgs/development/misc/resholve/README.md new file mode 100644 index 000000000000..6b99aebb5979 --- /dev/null +++ b/nixpkgs/pkgs/development/misc/resholve/README.md @@ -0,0 +1,158 @@ +# Using resholve's Nix API + +resholve converts bare executable references in shell scripts to absolute +paths. This will hopefully make its way into the Nixpkgs manual soon, but +until then I'll outline how to use the `resholvePackage` function. + +> Fair warning: resholve does *not* aspire to resolving all valid Shell +> scripts. It depends on the OSH/Oil parser, which aims to support most (but +> not all) Bash. resholve aims to be a ~90% sort of solution. + +## API Concepts + +The main difference between `resholvePackage` and other builder functions +is the `solutions` attrset, which describes which scripts to resolve and how. +Each "solution" (k=v pair) in this attrset describes one resholve invocation. + +> NOTE: For most shell packages, one invocation will probably be enough: +> - Packages with a single script will only need one solution. +> - Packages with multiple scripts can still use one solution if the scripts +> don't require conflicting directives. +> - Packages with scripts that require conflicting directives can use multiple +> solutions to resolve the scripts separately, but produce a single package. + +## Basic Example + +Here's a simple example from one of my own projects, with annotations: +<!-- +TODO: ideally this will use a nixpkgs example; but we don't have any IN yet +and the first package PR (bashup-events) is too complex for this context. +--> + +```nix +{ stdenv, lib, resholvePackage, fetchFromGitHub, bashup-events44, bashInteractive_5, doCheck ? true, shellcheck }: + +resholvePackage rec { + pname = "shellswain"; + version = "unreleased"; + + src = fetchFromGitHub { + # ... + }; + + solutions = { + # Give each solution a short name. This is what you'd use to + # override its settings, and it shows in (some) error messages. + profile = { + # the only *required* arguments are the 3 below + + # Specify 1 or more $out-relative script paths. Unlike many + # builders, resholvePackage modifies the output files during + # fixup (to correctly resolve in-package sourcing). + scripts = [ "bin/shellswain.bash" ]; + + # "none" for no shebang, "${bash}/bin/bash" for bash, etc. + interpreter = "none"; + + # packages resholve should resolve executables from + inputs = [ bashup-events44 ]; + }; + }; + + makeFlags = [ "prefix=${placeholder "out"}" ]; + + inherit doCheck; + checkInputs = [ shellcheck ]; + + # ... +} +``` + +## Options + +`resholvePackage` maps Nix types/idioms into the flags and environment variables +that the `resholve` CLI expects. Here's an overview: + +| Option | Type | Containing | +| ------------- | ------- | ----------------------------------------------------- | +| scripts | list | $out-relative string paths to resolve | +| inputs | list | packages to resolve executables from | +| interpreter | string | 'none' or abspath for shebang | +| prologue | file | text to insert before the first code-line | +| epilogue | file | text to isnert after the last code-line | +| flags | list | strings to pass as flags | +| fake | attrset | [directives](#controlling-resolution-with-directives) | +| fix | attrset | [directives](#controlling-resolution-with-directives) | +| keep | attrset | [directives](#controlling-resolution-with-directives) | + +## Controlling resolution with directives + +In order to resolve a script, resholve will make you disambiguate how it should +handle any potential problems it encounters with directives. There are currently +3 types: +1. `fake` directives tell resholve to pretend it knows about an identifier + such as a function, builtin, external command, etc. if there's a good reason + it doesn't already know about it. Common examples: + - builtins for a non-bash shell + - loadable builtins + - platform-specific external commands in cross-platform conditionals +2. `fix` directives give resholve permission to fix something that it can't + safely fix automatically. Common examples: + - resolving commands in aliases (this is appropriate for standalone scripts + that use aliases non-interactively--but it would prevent profile/rc + scripts from using the latest current-system symlinks.) + - resolve commands in a variable definition + - resolve an absolute command path from inputs as if it were a bare reference +3. `keep` directives tell resholve not to raise an error (i.e., ignore) + something it would usually object to. Common examples: + - variables used as/within the first word of a command + - pre-existing absolute or user-relative (~) command paths + - dynamic (variable) arguments to commands known to accept/run other commands + +> NOTE: resholve has a (growing) number of directives detailed in `man resholve` +> via `nixpkgs.resholve`. + +Each of these 3 types is represented by its own attrset, where you can think +of the key as a scope. The value should be: +- `true` for any directives that the resholve CLI accepts as a single word +- a list of strings for all other options +<!-- +TODO: these should be fully-documented here, but I'm already maintaining +more copies of their specification/behavior than I like, and continuing to +add more at this early date will only ensure that I spend more time updating +docs and less time filling in feature gaps. + +Full documentation may be greatly accellerated if someone can help me sort out +single-sourcing. See: https://github.com/abathur/resholve/issues/19 +--> + +This will hopefully make more sense when you see it. Here are CLI examples +from the manpage, and the Nix equivalents: + +```nix +# --fake 'f:setUp;tearDown builtin:setopt source:/etc/bashrc' +fake = { + # fake accepts the initial of valid identifier types as a CLI convienience. + # Use full names in the Nix API. + function = [ "setUp" "tearDown" ]; + builtin = [ "setopt" ]; + source = [ "/etc/bashrc" ]; +}; + +# --fix 'aliases xargs:ls $GIT:gix' +fix = { + # all single-word directives use `true` as value + aliases = true; + xargs = [ "ls" ]; + "$GIT" = [ "gix" ]; +}; + +# --keep 'which:git;ls .:$HOME $LS:exa /etc/bashrc ~/.bashrc' +keep = { + which = [ "git" "ls" ]; + "." = [ "$HOME" ]; + "$LS" = [ "exa" ]; + "/etc/bashrc" = true; + "~/.bashrc" = true; +}; +``` diff --git a/nixpkgs/pkgs/development/misc/resholve/default.nix b/nixpkgs/pkgs/development/misc/resholve/default.nix new file mode 100644 index 000000000000..7b5a79dd221a --- /dev/null +++ b/nixpkgs/pkgs/development/misc/resholve/default.nix @@ -0,0 +1,9 @@ +{ callPackage +, doCheck ? true +}: + +rec { + resholve = callPackage ./resholve.nix { inherit doCheck; }; + resholvePackage = + callPackage ./resholve-package.nix { inherit resholve; }; +} diff --git a/nixpkgs/pkgs/development/misc/resholve/deps.nix b/nixpkgs/pkgs/development/misc/resholve/deps.nix new file mode 100644 index 000000000000..86bcba570759 --- /dev/null +++ b/nixpkgs/pkgs/development/misc/resholve/deps.nix @@ -0,0 +1,120 @@ +{ lib, stdenv +, python27Packages +, fetchFromGitHub +, makeWrapper +, # re2c deps + autoreconfHook +, # py-yajl deps + git +, # oil deps + readline +, cmark +, file +, glibcLocales +, oilPatches ? [ ] +}: + +/* +Notes on specific dependencies: +- if/when python2.7 is removed from nixpkgs, this may need to figure + out how to build oil's vendored python2 +- I'm not sure if glibcLocales is worth the addition here. It's to fix + a libc test oil runs. My oil fork just disabled the libc tests, but + I haven't quite decided if that's the right long-term call, so I + didn't add a patch for it here yet. +*/ + +rec { + # had to add this as well; 1.3 causes a break here; sticking + # to oil's official 1.0.3 dep for now. + re2c = stdenv.mkDerivation rec { + pname = "re2c"; + version = "1.0.3"; + sourceRoot = "${src.name}/re2c"; + src = fetchFromGitHub { + owner = "skvadrik"; + repo = "re2c"; + rev = version; + sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi"; + }; + nativeBuildInputs = [ autoreconfHook ]; + preCheck = '' + patchShebangs run_tests.sh + ''; + }; + + py-yajl = python27Packages.buildPythonPackage rec { + pname = "oil-pyyajl-unstable"; + version = "2019-12-05"; + src = fetchFromGitHub { + owner = "oilshell"; + repo = "py-yajl"; + rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df"; + sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh"; + fetchSubmodules = true; + }; + # just for submodule IIRC + nativeBuildInputs = [ git ]; + }; + + # resholve's primary dependency is this developer build of the oil shell. + oildev = python27Packages.buildPythonPackage rec { + pname = "oildev-unstable"; + version = "2020-03-31"; + + src = fetchFromGitHub { + owner = "oilshell"; + repo = "oil"; + rev = "ea80cdad7ae1152a25bd2a30b87fe3c2ad32394a"; + sha256 = "0pxn0f8qbdman4gppx93zwml7s5byqfw560n079v68qjgzh2brq2"; + + /* + It's not critical to drop most of these; the primary target is + the vendored fork of Python-2.7.13, which is ~ 55M and over 3200 + files, dozens of which get interpreter script patches in fixup. + */ + extraPostFetch = '' + rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp + ''; + }; + + # TODO: not sure why I'm having to set this for nix-build... + # can anyone tell if I'm doing something wrong? + SOURCE_DATE_EPOCH = 315532800; + + # These aren't, strictly speaking, nix/nixpkgs specific, but I've + # had hell upstreaming them. Pulling from resholve source and + # passing in from resholve.nix + patches = oilPatches; + + buildInputs = [ readline cmark py-yajl ]; + + nativeBuildInputs = [ re2c file makeWrapper ]; + + propagatedBuildInputs = with python27Packages; [ six typing ]; + + doCheck = true; + + preBuild = '' + build/dev.sh all + ''; + + postPatch = '' + patchShebangs asdl build core doctools frontend native oil_lang + ''; + + _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}"; + + # See earlier note on glibcLocales + LOCALE_ARCHIVE = lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive"; + + meta = { + description = "A new unix shell"; + homepage = "https://www.oilshell.org/"; + license = with lib.licenses; [ + psfl # Includes a portion of the python interpreter and standard library + asl20 # Licence for Oil itself + ]; + }; + }; +} diff --git a/nixpkgs/pkgs/development/misc/resholve/resholve-package.nix b/nixpkgs/pkgs/development/misc/resholve/resholve-package.nix new file mode 100644 index 000000000000..cc971196a4f8 --- /dev/null +++ b/nixpkgs/pkgs/development/misc/resholve/resholve-package.nix @@ -0,0 +1,97 @@ +{ stdenv, lib, resholve }: + +{ pname +, src +, version +, passthru ? { } +, solutions +, ... +}@attrs: +let + inherit stdenv; + /* These functions break up the work of partially validating the + * 'solutions' attrset and massaging it into env/cli args. + * + * Note: some of the left-most args do not *have* to be passed as + * deep as they are, but I've done so to provide more error context + */ + + # for brevity / line length + spaces = l: builtins.concatStringsSep " " l; + semicolons = l: builtins.concatStringsSep ";" l; + + /* Throw a fit with dotted attr path context */ + nope = path: msg: + throw "${builtins.concatStringsSep "." path}: ${msg}"; + + /* Special-case directive value representations by type */ + makeDirective = solution: env: name: val: + if builtins.isInt val then builtins.toString val + else if builtins.isString val then name + else if true == val then name + else if false == val then "" # omit! + else if null == val then "" # omit! + else if builtins.isList val then "${name}:${semicolons val}" + else nope [ solution env name ] "unexpected type: ${builtins.typeOf val}"; + + /* Build fake/fix/keep directives from Nix types */ + makeDirectives = solution: env: val: + lib.mapAttrsToList (makeDirective solution env) val; + + /* Special-case value representation by type/name */ + makeEnvVal = solution: env: val: + if env == "inputs" then lib.makeBinPath val + else if builtins.isString val then val + else if builtins.isList val then spaces val + else if builtins.isAttrs val then spaces (makeDirectives solution env val) + else nope [ solution env ] "unexpected type: ${builtins.typeOf val}"; + + /* Shell-format each env value */ + shellEnv = solution: env: value: + lib.escapeShellArg (makeEnvVal solution env value); + + /* Build a single ENV=val pair */ + makeEnv = solution: env: value: + "RESHOLVE_${lib.toUpper env}=${shellEnv solution env value}"; + + /* Discard attrs claimed by makeArgs */ + removeCliArgs = value: + removeAttrs value [ "scripts" "flags" ]; + + /* Verify required arguments are present */ + validateSolution = { scripts, inputs, interpreter, ... }: true; + + /* Pull out specific solution keys to build ENV=val pairs */ + makeEnvs = solution: value: + spaces (lib.mapAttrsToList (makeEnv solution) (removeCliArgs value)); + + /* Pull out specific solution keys to build CLI argstring */ + makeArgs = { flags ? [ ], scripts, ... }: + spaces (flags ++ scripts); + + /* Build a single resholve invocation */ + makeInvocation = solution: value: + if validateSolution value then + "${makeEnvs solution value} resholve --overwrite ${makeArgs value}" + else throw "invalid solution"; # shouldn't trigger for now + + /* Build resholve invocation for each solution. */ + makeCommands = solutions: + lib.mapAttrsToList makeInvocation solutions; + + self = (stdenv.mkDerivation ((removeAttrs attrs [ "solutions" ]) + // { + inherit pname version src; + buildInputs = [ resholve ]; + + # enable below for verbose debug info if needed + # supports default python.logging levels + # LOGLEVEL="INFO"; + preFixup = '' + pushd "$out" + ${builtins.concatStringsSep "\n" (makeCommands solutions)} + popd + ''; + })); +in +lib.extendDerivation true passthru self diff --git a/nixpkgs/pkgs/development/misc/resholve/resholve.nix b/nixpkgs/pkgs/development/misc/resholve/resholve.nix new file mode 100644 index 000000000000..e8b4ed2cfb25 --- /dev/null +++ b/nixpkgs/pkgs/development/misc/resholve/resholve.nix @@ -0,0 +1,80 @@ +{ lib +, callPackage +, python27Packages +, installShellFiles +, fetchFromGitHub +, file +, findutils +, gettext +, bats +, bash +, doCheck ? true +}: +let + version = "0.4.2"; + rSrc = fetchFromGitHub { + owner = "abathur"; + repo = "resholve"; + rev = "v${version}"; + hash = "sha256-ArUQjqh4LRvFLzHiTIcae0q/VFxFF/X9eOFeRnYmTO0="; + }; + deps = callPackage ./deps.nix { + /* + resholve needs to patch Oil, but trying to avoid adding + them all *to* nixpkgs, since they aren't specific to + nix/nixpkgs. + */ + oilPatches = [ + "${rSrc}/0001-add_setup_py.patch" + "${rSrc}/0002-add_MANIFEST_in.patch" + "${rSrc}/0003-fix_codegen_shebang.patch" + "${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch" + ]; + }; +in +python27Packages.buildPythonApplication { + pname = "resholve"; + inherit version; + src = rSrc; + format = "other"; + + nativeBuildInputs = [ installShellFiles ]; + + propagatedBuildInputs = [ deps.oildev python27Packages.ConfigArgParse ]; + + patchPhase = '' + for file in resholve; do + substituteInPlace $file --subst-var-by version ${version} + done + ''; + + installPhase = '' + install -Dm755 resholve $out/bin/resholve + installManPage resholve.1 + ''; + + inherit doCheck; + checkInputs = [ bats ]; + RESHOLVE_PATH = "${lib.makeBinPath [ file findutils gettext ]}"; + + checkPhase = '' + # explicit interpreter for test suite + export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH" + patchShebangs . + ./test.sh + ''; + + # Do not propagate Python; may be obsoleted by nixos/nixpkgs#102613 + # for context on why, see abathur/resholve#20 + postFixup = '' + rm $out/nix-support/propagated-build-inputs + ''; + + meta = with lib; { + description = "Resolve external shell-script dependencies"; + homepage = "https://github.com/abathur/resholve"; + license = with licenses; [ mit ]; + maintainers = with maintainers; [ abathur ]; + platforms = platforms.all; + }; +} |