From d07f30f628f52186aa68b0c918c88aa9ed36f988 Mon Sep 17 00:00:00 2001 From: Ryan Trinkle Date: Thu, 29 Jun 2017 12:06:55 -0400 Subject: cc-wrapper: improve response file parsing speed --- pkgs/stdenv/darwin/default.nix | 12 +++++++++++- pkgs/stdenv/linux/default.nix | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'pkgs/stdenv') diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index 03a815109c25..e62aee9a02da 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -61,7 +61,10 @@ in rec { allowedRequisites ? null}: let thisStdenv = import ../generic { - inherit config shell extraBuildInputs allowedRequisites; + inherit config shell extraBuildInputs; + allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [ + thisStdenv.cc.parseResponseFile + ]; name = "stdenv-darwin-boot-${toString step}"; @@ -73,6 +76,9 @@ in rec { nativeTools = true; nativePrefix = bootstrapTools; nativeLibc = false; + buildPackages = lib.optionalAttrs (last ? stdenv) { + inherit (last) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; libc = last.pkgs.darwin.Libsystem; @@ -297,6 +303,9 @@ in rec { inherit shell; nativeTools = false; nativeLibc = false; + buildPackages = { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; inherit (pkgs) coreutils binutils gnugrep; @@ -319,6 +328,7 @@ in rec { gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out binutils-raw.dev binutils gettext + cc.parseResponseFile ]) ++ (with pkgs.darwin; [ dyld Libsystem CF cctools ICU libiconv locale ]); diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index b116a48a2bd6..c2879d93e17c 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -76,6 +76,9 @@ let else lib.makeOverridable (import ../../build-support/cc-wrapper) { nativeTools = false; nativeLibc = false; + buildPackages = lib.optionalAttrs (prevStage ? stdenv) { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; cc = prevStage.gcc-unwrapped; @@ -241,6 +244,9 @@ in nativeTools = false; nativeLibc = false; isGNU = true; + buildPackages = { + inherit (prevStage) stdenv; + }; hostPlatform = localSystem; targetPlatform = localSystem; cc = prevStage.gcc-unwrapped; -- cgit 1.4.1 From 2bc7b4e134079cf72307538e57b8968cfb27d70c Mon Sep 17 00:00:00 2001 From: Orivej Desh Date: Sat, 1 Jul 2017 00:27:48 +0000 Subject: cc-wrapper: simplify expandResponseParams parser Import from https://github.com/orivej/expand-compiler-args/tree/b2446902fe7816f34c3f21d29a66da3ee2b1601e --- pkgs/build-support/cc-wrapper/default.nix | 16 +- .../cc-wrapper/expand-response-params.c | 84 ++++++++++ pkgs/build-support/cc-wrapper/parseResponseFile.c | 181 --------------------- pkgs/build-support/cc-wrapper/utils.sh | 33 ++-- pkgs/stdenv/darwin/default.nix | 8 +- 5 files changed, 109 insertions(+), 213 deletions(-) create mode 100644 pkgs/build-support/cc-wrapper/expand-response-params.c delete mode 100644 pkgs/build-support/cc-wrapper/parseResponseFile.c (limited to 'pkgs/stdenv') diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 61afa56fc490..de0fcf953e47 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -120,14 +120,14 @@ let null) else ""; - parseResponseFile = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null" + expand-response-params = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null" then buildPackages.stdenv.mkDerivation { - name = "parse-response-file"; - src = ./parseResponseFile.c; + name = "expand-response-params"; + src = ./expand-response-params.c; buildCommand = '' - # Make sure the output file doesn't refer to the input nix path - cp "$src" parseResponseFile.c - "$CC" -O3 -o "$out" parseResponseFile.c + # Work around "stdenv-darwin-boot-2 is not allowed to refer to path /nix/store/...-expand-response-params.c" + cp "$src" expand-response-params.c + "$CC" -std=c99 -O3 -o "$out" expand-response-params.c ''; } else ""; @@ -383,7 +383,9 @@ stdenv.mkDerivation { '' + extraBuildCommands; - inherit dynamicLinker parseResponseFile; + inherit dynamicLinker expand-response-params; + + expandResponseParams = expand-response-params; # for substitution in utils.sh crossAttrs = { shell = shell.crossDrv + shell.crossDrv.shellPath; diff --git a/pkgs/build-support/cc-wrapper/expand-response-params.c b/pkgs/build-support/cc-wrapper/expand-response-params.c new file mode 100644 index 000000000000..ca4ba65ec45b --- /dev/null +++ b/pkgs/build-support/cc-wrapper/expand-response-params.c @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include + +typedef struct { char *data; size_t len, cap; } String; + +void resize(String *s, size_t len) { + s->len = len; + if (s->cap < s->len) { + s->cap = s->len * 2; + s->data = (char *)realloc(s->data, s->cap); + assert(s->data); + } +} + +void append(String *s, const char *data, size_t len) { + resize(s, s->len + len); + memcpy(s->data + s->len - len, data, len); +} + +typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass; +typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State; + +// current State -> CharClass -> next State +const State transitions[][5] = { + [outside] = {outside, unq, unq_esc, sq, dq}, + [unq] = {outside, unq, unq_esc, sq, dq}, + [unq_esc] = {unq, unq, unq, unq, unq}, + [sq] = {sq, sq, sq_esc, unq, sq}, + [sq_esc] = {sq, sq, sq, sq, sq}, + [dq] = {dq, dq, dq_esc, dq, unq}, + [dq_esc] = {dq, dq, dq, dq, dq}, +}; + +CharClass charClass(int c) { + return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark : + isspace(c) ? space : other; +} + +// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated +// string, to stdout. If arg does not begin with `@' or does not refer to a +// file, it is written as is. Otherwise the contents of the file are +// recursively expanded. On unexpected EOF in malformed response files an +// incomplete final argument is written, even if it is empty, to parse like GCC. +void expandArg(String *arg) { + FILE *f; + if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) { + fwrite(arg->data, 1, arg->len, stdout); + return; + } + + resize(arg, 0); + State cur = outside; + int c; + do { + c = fgetc(f); + State next = transitions[cur][charClass(c)]; + if ((cur == unq && next == outside) || (cur != outside && c == EOF)) { + append(arg, "", 1); + expandArg(arg); + resize(arg, 0); + } else if (cur == unq_esc || cur == sq_esc || cur == dq_esc || + cur == outside ? next == unq : cur == next) { + char s = c; + append(arg, &s, 1); + } + cur = next; + } while (c != EOF); + + fclose(f); +} + +int main(int argc, char **argv) { + String arg = { 0 }; + while (*++argv) { + resize(&arg, 0); + append(&arg, *argv, strlen(*argv) + 1); + expandArg(&arg); + } + free(arg.data); + return EXIT_SUCCESS; +} diff --git a/pkgs/build-support/cc-wrapper/parseResponseFile.c b/pkgs/build-support/cc-wrapper/parseResponseFile.c deleted file mode 100644 index 4a2a21e5ba31..000000000000 --- a/pkgs/build-support/cc-wrapper/parseResponseFile.c +++ /dev/null @@ -1,181 +0,0 @@ -#include -#include -#include - -typedef enum { - UNQUOTED, - SINGLE_QUOTED, - DOUBLE_QUOTED -} quote_state; - -typedef enum { - BEFORE_FIRST_WORD, - IN_WORD, - AFTER_WORD -} word_break_state; - -void emitWordChar(word_break_state *w, char c) { - switch(*w) { // Note: These all fall through - case AFTER_WORD: - putchar(' '); - case BEFORE_FIRST_WORD: - putchar('\''); - *w = IN_WORD; - case IN_WORD: - if(c == '\'') { - printf("'\\''"); - } else { - putchar(c); - } - } -} - -void emitWordEnd(word_break_state *w) { - if(*w == IN_WORD) { - putchar('\''); - *w = AFTER_WORD; - } // Otherwise, the state remains the same -} - -typedef struct { - word_break_state *w; - char *subFilename; // Non-null if we're currently accumulating a response file name - size_t subFilenameUsed; // If subFilename == 0, this should be 0; this should always be less than (subFilenameAllocated - 1), to allow room for the null byte - size_t subFilenameAllocated; // If subFilename == 0, this should be 0 -} file_state; // The state of parsing a single file - -static const unsigned int INITIAL_SUB_FILENAME_CHARS = 32; // Arbitrary, but must be > 0 - -void *exitIfNull(void *p) { - if(!p) { - fprintf(stderr, "Out of memory"); - exit(2); - } - return p; -} - -void wordChar(file_state *s, char c) { - if(s->subFilename) { // We're accumulating a file to recursively process - // Allocate more space if we need to - if(s->subFilenameUsed >= s->subFilenameAllocated - 1) { - size_t newSize = s->subFilenameAllocated * 2; - s->subFilename = exitIfNull(realloc(s->subFilename, newSize)); - s->subFilenameAllocated = newSize; - } - s->subFilename[s->subFilenameUsed++] = c; - } else if(*s->w != IN_WORD && c == '@') { // This is the first letter in the word; note that even quoted or escaped @'s are recursively interpreted - s->subFilename = exitIfNull(malloc(INITIAL_SUB_FILENAME_CHARS * sizeof(*(s->subFilename)))); - assert(s->subFilenameUsed == 0); - assert(s->subFilenameAllocated == 0); - s->subFilenameAllocated = INITIAL_SUB_FILENAME_CHARS; - } else { - emitWordChar(s->w, c); - } -} - -void processFile(word_break_state *w, const char *filename); - -void endWord(file_state *s) { - if(s->subFilename) { - s->subFilename[s->subFilenameUsed] = '\0'; - - processFile(s->w, s->subFilename); - - free(s->subFilename); - s->subFilename = 0; - s->subFilenameUsed = 0; - s->subFilenameAllocated = 0; - } else { - emitWordEnd(s->w); - } -} - -void processFile(word_break_state *w, const char *filename) { - FILE *h = fopen(filename, "r"); - if(!h) { //TODO: We assume it's because the file doesn't exist, but perhaps we should check for other failure cases - emitWordChar(w, '@'); - while(*filename) { - emitWordChar(w, *filename); - ++filename; - } - emitWordEnd(w); - return; - } - - char c; - quote_state q = UNQUOTED; - file_state s = { - .w = w, - .subFilename = 0, - .subFilenameUsed = 0, - .subFilenameAllocated = 0 - }; - while((c = fgetc(h)) != EOF) { - //fprintf(stderr, "%d\n", c); - switch(c) { - case '\'': - switch(q) { - case UNQUOTED: - q = SINGLE_QUOTED; - break; - case SINGLE_QUOTED: - q = UNQUOTED; - break; - case DOUBLE_QUOTED: - wordChar(&s, '\''); - break; - } - break; - case '"': - switch(q) { - case UNQUOTED: - q = DOUBLE_QUOTED; - break; - case SINGLE_QUOTED: - wordChar(&s, '"'); - break; - case DOUBLE_QUOTED: - q = UNQUOTED; - break; - } - break; - case '\\': - c = fgetc(h); - if(c != EOF) { - wordChar(&s, c); - } - break; - case ' ': - case '\t': - case '\n': - case '\v': - case '\f': - case '\r': - if(q == UNQUOTED) { - endWord(&s); - } else { - wordChar(&s, c); - } - break; - default: - wordChar(&s, c); - break; - } - } - - endWord(&s); - - fclose(h); -} - -int main(int argc, const char *argv[]) { - if(argc != 2) { - fprintf(stderr, "Usage: %s [responsefile]", argv[0]); - return 1; - } - - word_break_state w = BEFORE_FIRST_WORD; - processFile(&w, argv[1]); - - return 0; -} diff --git a/pkgs/build-support/cc-wrapper/utils.sh b/pkgs/build-support/cc-wrapper/utils.sh index 3d8ea6842f11..87e48da9c8d5 100644 --- a/pkgs/build-support/cc-wrapper/utils.sh +++ b/pkgs/build-support/cc-wrapper/utils.sh @@ -24,26 +24,17 @@ badPath() { } expandResponseParams() { - local inparams=("$@") - local n=0 - local p - params=() - while [ $n -lt ${#inparams[*]} ]; do - p=${inparams[n]} - case $p in - @*) - local parseResponseFile="@parseResponseFile@" - if [ -n "$parseResponseFile" ] ; then - eval "params+=($("$parseResponseFile" "${p:1}"))" - else - echo "Response files aren't supported during bootstrapping" >&2 - exit 1 - fi - ;; - *) - params+=("$p") - ;; - esac - n=$((n + 1)) + params=("$@") + local arg + for arg in "$@"; do + if [[ "$arg" == @* ]]; then + if [ -n "@expandResponseParams@" ]; then + readarray -d '' params < <("@expandResponseParams@" "$@") + return 0 + else + echo "Response files aren't supported during bootstrapping" >&2 + return 1 + fi + fi done } diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index e62aee9a02da..1c0b42886ca3 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -54,7 +54,7 @@ in rec { __sandboxProfile = binShClosure + libSystemProfile; }; - stageFun = step: last: {shell ? "${bootstrapTools}/bin/sh", + stageFun = step: last: {shell ? "${bootstrapTools}/bin/bash", overrides ? (self: super: {}), extraPreHook ? "", extraBuildInputs, @@ -63,7 +63,7 @@ in rec { thisStdenv = import ../generic { inherit config shell extraBuildInputs; allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [ - thisStdenv.cc.parseResponseFile + thisStdenv.cc.expand-response-params ]; name = "stdenv-darwin-boot-${toString step}"; @@ -86,7 +86,7 @@ in rec { cc = { name = "clang-9.9.9"; outPath = bootstrapTools; }; }; - preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/sh") '' + preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/bash") '' # Don't patch #!/interpreter because it leads to retained # dependencies on the bootstrapTools in the final stdenv. dontPatchShebangs=1 @@ -328,7 +328,7 @@ in rec { gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out binutils-raw.dev binutils gettext - cc.parseResponseFile + cc.expand-response-params ]) ++ (with pkgs.darwin; [ dyld Libsystem CF cctools ICU libiconv locale ]); -- cgit 1.4.1