diff options
author | Ryan Trinkle <ryan.trinkle@gmail.com> | 2017-07-05 16:18:22 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-07-05 16:18:22 -0400 |
commit | 7004641566c6c1e11464cec360613341e44cb320 (patch) | |
tree | 760f2a4c5eeda2f53a86c2e1e35e8d82d5fbb2bf | |
parent | c23dcd72a0a0f5c4e0b63418a8602aa7f131219a (diff) | |
parent | 754c3f6ba4a33cfd4f573c1fee3b5df74ba00ee2 (diff) | |
download | nixlib-7004641566c6c1e11464cec360613341e44cb320.tar nixlib-7004641566c6c1e11464cec360613341e44cb320.tar.gz nixlib-7004641566c6c1e11464cec360613341e44cb320.tar.bz2 nixlib-7004641566c6c1e11464cec360613341e44cb320.tar.lz nixlib-7004641566c6c1e11464cec360613341e44cb320.tar.xz nixlib-7004641566c6c1e11464cec360613341e44cb320.tar.zst nixlib-7004641566c6c1e11464cec360613341e44cb320.zip |
Merge pull request #26974 from obsidiansystems/response-file-parsing-speed
cc-wrapper: improve response file parsing speed
-rw-r--r-- | pkgs/build-support/cc-wrapper/default.nix | 19 | ||||
-rw-r--r-- | pkgs/build-support/cc-wrapper/expand-response-params.c | 84 | ||||
-rw-r--r-- | pkgs/build-support/cc-wrapper/utils.sh | 56 | ||||
-rw-r--r-- | pkgs/os-specific/darwin/ios-cross/default.nix | 6 | ||||
-rw-r--r-- | pkgs/stdenv/darwin/default.nix | 16 | ||||
-rw-r--r-- | pkgs/stdenv/linux/default.nix | 6 |
6 files changed, 134 insertions, 53 deletions
diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 935e6fb6267c..de0fcf953e47 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -10,7 +10,7 @@ , zlib ? null, extraPackages ? [], extraBuildCommands ? "" , dyld ? null # TODO: should this be a setup-hook on dyld? , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null -, hostPlatform, targetPlatform +, buildPackages ? {}, hostPlatform, targetPlatform , runCommand ? null }: @@ -120,6 +120,17 @@ let null) else ""; + expand-response-params = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null" + then buildPackages.stdenv.mkDerivation { + name = "expand-response-params"; + src = ./expand-response-params.c; + buildCommand = '' + # Work around "stdenv-darwin-boot-2 is not allowed to refer to path /nix/store/...-expand-response-params.c" + cp "$src" expand-response-params.c + "$CC" -std=c99 -O3 -o "$out" expand-response-params.c + ''; + } else ""; + in stdenv.mkDerivation { @@ -368,11 +379,13 @@ stdenv.mkDerivation { + '' substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh - cp -p ${preWrap ./utils.sh} $out/nix-support/utils.sh + substituteAll ${preWrap ./utils.sh} $out/nix-support/utils.sh '' + extraBuildCommands; - inherit dynamicLinker; + inherit dynamicLinker expand-response-params; + + expandResponseParams = expand-response-params; # for substitution in utils.sh crossAttrs = { shell = shell.crossDrv + shell.crossDrv.shellPath; diff --git a/pkgs/build-support/cc-wrapper/expand-response-params.c b/pkgs/build-support/cc-wrapper/expand-response-params.c new file mode 100644 index 000000000000..05b9c62b1e8d --- /dev/null +++ b/pkgs/build-support/cc-wrapper/expand-response-params.c @@ -0,0 +1,84 @@ +#include <assert.h> +#include <ctype.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +typedef struct { char *data; size_t len, cap; } String; + +void resize(String *s, size_t len) { + s->len = len; + if (s->cap < s->len) { + s->cap = s->len * 2; + s->data = (char *)realloc(s->data, s->cap); + assert(s->data); + } +} + +void append(String *s, const char *data, size_t len) { + resize(s, s->len + len); + memcpy(s->data + s->len - len, data, len); +} + +typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass; +typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State; + +// current State -> CharClass -> next State +const State transitions[][5] = { + [outside] = {outside, unq, unq_esc, sq, dq}, + [unq] = {outside, unq, unq_esc, sq, dq}, + [unq_esc] = {unq, unq, unq, unq, unq}, + [sq] = {sq, sq, sq_esc, unq, sq}, + [sq_esc] = {sq, sq, sq, sq, sq}, + [dq] = {dq, dq, dq_esc, dq, unq}, + [dq_esc] = {dq, dq, dq, dq, dq}, +}; + +CharClass charClass(int c) { + return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark : + isspace(c) ? space : other; +} + +// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated +// string, to stdout. If arg does not begin with `@' or does not refer to a +// file, it is written as is. Otherwise the contents of the file are +// recursively expanded. On unexpected EOF in malformed response files an +// incomplete final argument is written, even if it is empty, to parse like GCC. +void expandArg(String *arg) { + FILE *f; + if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) { + fwrite(arg->data, 1, arg->len, stdout); + return; + } + + resize(arg, 0); + State cur = outside; + int c; + do { + c = fgetc(f); + State next = transitions[cur][charClass(c)]; + if ((cur == unq && next == outside) || (cur != outside && c == EOF)) { + append(arg, "", 1); + expandArg(arg); + resize(arg, 0); + } else if (cur == unq_esc || cur == sq_esc || cur == dq_esc || + (cur == outside ? next == unq : cur == next)) { + char s = c; + append(arg, &s, 1); + } + cur = next; + } while (c != EOF); + + fclose(f); +} + +int main(int argc, char **argv) { + String arg = { 0 }; + while (*++argv) { + resize(&arg, 0); + append(&arg, *argv, strlen(*argv) + 1); + expandArg(&arg); + } + free(arg.data); + return EXIT_SUCCESS; +} diff --git a/pkgs/build-support/cc-wrapper/utils.sh b/pkgs/build-support/cc-wrapper/utils.sh index 8cefc47816f8..87e48da9c8d5 100644 --- a/pkgs/build-support/cc-wrapper/utils.sh +++ b/pkgs/build-support/cc-wrapper/utils.sh @@ -23,52 +23,18 @@ badPath() { "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP" } -# @args.rsp parser. -# Char classes: space, other, backslash, single quote, double quote. -# States: 0 - outside, 1/2 - unquoted arg/slash, 3/4 - 'arg'/slash, 5/6 - "arg"/slash. -# State transitions: -rspT=(01235 01235 11111 33413 33333 55651 55555) -# Push (a) arg or (c) char on transition: -rspP[10]=a rspP[01]=c rspP[11]=c rspP[21]=c rspP[33]=c rspP[43]=c rspP[55]=c rspP[65]=c - -rspParse() { - rsp=() - local state=0 - local arg='' - local c - - while read -r -N1 c; do - local cls=1 - case "$c" in - ' ' | $'\t' | $'\r' | $'\n') cls=0 ;; - '\') cls=2 ;; - "'") cls=3 ;; - '"') cls=4 ;; - esac - local nextstates="${rspT[$state]}" - local nextstate="${nextstates:$cls:1}" - case "${rspP[$state$nextstate]}" in - 'c') arg+="$c" ;; - 'a') rsp+=("$arg"); arg='' ;; - esac - state="$nextstate" - done - - if [ "$state" -ne 0 ]; then - rsp+=("$arg") - fi -} - expandResponseParams() { - params=() - while [ $# -gt 0 ]; do - local p="$1" - shift - if [ "${p:0:1}" = '@' -a -e "${p:1}" ]; then - rspParse <"${p:1}" - set -- "${rsp[@]}" "$@" - else - params+=("$p") + params=("$@") + local arg + for arg in "$@"; do + if [[ "$arg" == @* ]]; then + if [ -n "@expandResponseParams@" ]; then + readarray -d '' params < <("@expandResponseParams@" "$@") + return 0 + else + echo "Response files aren't supported during bootstrapping" >&2 + return 1 + fi fi done } diff --git a/pkgs/os-specific/darwin/ios-cross/default.nix b/pkgs/os-specific/darwin/ios-cross/default.nix index e0981d48c8c9..f56c8148317d 100644 --- a/pkgs/os-specific/darwin/ios-cross/default.nix +++ b/pkgs/os-specific/darwin/ios-cross/default.nix @@ -6,7 +6,9 @@ , stdenv , coreutils , gnugrep -, hostPlatform, targetPlatform +, buildPackages +, hostPlatform +, targetPlatform }: /* As of this writing, known-good prefix/arch/simulator triples: @@ -29,7 +31,7 @@ let sdk = "/Applications/Xcode.app/Contents/Developer/Platforms/iPhone${sdkType}.platform/Developer/SDKs/iPhone${sdkType}${sdkVer}.sdk"; in (import ../../../build-support/cc-wrapper { - inherit stdenv coreutils gnugrep runCommand; + inherit stdenv coreutils gnugrep runCommand buildPackages; nativeTools = false; nativeLibc = false; inherit binutils; diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index 03a815109c25..1c0b42886ca3 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -54,14 +54,17 @@ in rec { __sandboxProfile = binShClosure + libSystemProfile; }; - stageFun = step: last: {shell ? "${bootstrapTools}/bin/sh", + stageFun = step: last: {shell ? "${bootstrapTools}/bin/bash", overrides ? (self: super: {}), extraPreHook ? "", extraBuildInputs, allowedRequisites ? null}: let thisStdenv = import ../generic { - inherit config shell extraBuildInputs allowedRequisites; + inherit config shell extraBuildInputs; + allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [ + thisStdenv.cc.expand-response-params + ]; name = "stdenv-darwin-boot-${toString step}"; @@ -73,6 +76,9 @@ in rec { nativeTools = true; nativePrefix = bootstrapTools; nativeLibc = false; + buildPackages = lib.optionalAttrs (last ? stdenv) { + inherit (last) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; libc = last.pkgs.darwin.Libsystem; @@ -80,7 +86,7 @@ in rec { cc = { name = "clang-9.9.9"; outPath = bootstrapTools; }; }; - preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/sh") '' + preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/bash") '' # Don't patch #!/interpreter because it leads to retained # dependencies on the bootstrapTools in the final stdenv. dontPatchShebangs=1 @@ -297,6 +303,9 @@ in rec { inherit shell; nativeTools = false; nativeLibc = false; + buildPackages = { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; inherit (pkgs) coreutils binutils gnugrep; @@ -319,6 +328,7 @@ in rec { gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out binutils-raw.dev binutils gettext + cc.expand-response-params ]) ++ (with pkgs.darwin; [ dyld Libsystem CF cctools ICU libiconv locale ]); diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index b116a48a2bd6..c2879d93e17c 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -76,6 +76,9 @@ let else lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; + buildPackages = lib.optionalAttrs (prevStage ? stdenv) { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; cc = prevStage.gcc-unwrapped; @@ -241,6 +244,9 @@ in nativeTools = false; nativeLibc = false; isGNU = true; + buildPackages = { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; cc = prevStage.gcc-unwrapped; |