cc-wrapper: improve response file parsing speed

pkgs/build-support/cc-wrapper/default.nix

@@ -10,7 +10,7 @@
 , zlib ? null, extraPackages ? [], extraBuildCommands ? ""
 , dyld ? null # TODO: should this be a setup-hook on dyld?
 , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
-, hostPlatform, targetPlatform
+, buildPackages ? {}, hostPlatform, targetPlatform
 , runCommand ? null
 }:
 
@@ -120,6 +120,17 @@ let
          null)
     else "";
 
+  parseResponseFile = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
+  then buildPackages.stdenv.mkDerivation {
+    name = "parse-response-file";
+    src = ./parseResponseFile.c;
+    buildCommand = ''
+      # Make sure the output file doesn't refer to the input nix path
+      cp "$src" parseResponseFile.c
+      "$CC" -O3 -o "$out" parseResponseFile.c
+    '';
+  } else "";
+
 in
 
 stdenv.mkDerivation {
@@ -368,11 +379,11 @@ stdenv.mkDerivation {
     + ''
       substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh
       substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh
-      cp -p ${preWrap ./utils.sh} $out/nix-support/utils.sh
+      substituteAll ${preWrap ./utils.sh} $out/nix-support/utils.sh
     ''
     + extraBuildCommands;
 
-  inherit dynamicLinker;
+  inherit dynamicLinker parseResponseFile;
 
   crossAttrs = {
     shell = shell.crossDrv + shell.crossDrv.shellPath;

pkgs/build-support/cc-wrapper/parseResponseFile.c

@@ -0,0 +1,181 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+typedef enum {
+  UNQUOTED,
+  SINGLE_QUOTED,
+  DOUBLE_QUOTED
+} quote_state;
+
+typedef enum {
+  BEFORE_FIRST_WORD,
+  IN_WORD,
+  AFTER_WORD
+} word_break_state;
+
+void emitWordChar(word_break_state *w, char c) {
+  switch(*w) { // Note: These all fall through
+  case AFTER_WORD:
+    putchar(' ');
+  case BEFORE_FIRST_WORD:
+    putchar('\'');
+    *w = IN_WORD;
+  case IN_WORD:
+    if(c == '\'') {
+      printf("'\\''");
+    } else {
+      putchar(c);
+    }
+  }
+}
+
+void emitWordEnd(word_break_state *w) {
+  if(*w == IN_WORD) {
+    putchar('\'');
+    *w = AFTER_WORD;
+  } // Otherwise, the state remains the same
+}
+
+typedef struct {
+  word_break_state *w;
+  char *subFilename; // Non-null if we're currently accumulating a response file name
+  size_t subFilenameUsed; // If subFilename == 0, this should be 0; this should always be less than (subFilenameAllocated - 1), to allow room for the null byte
+  size_t subFilenameAllocated; // If subFilename == 0, this should be 0
+} file_state; // The state of parsing a single file
+
+static const unsigned int INITIAL_SUB_FILENAME_CHARS = 32; // Arbitrary, but must be > 0
+
+void *exitIfNull(void *p) {
+  if(!p) {
+    fprintf(stderr, "Out of memory");
+    exit(2);
+  }
+  return p;
+}
+
+void wordChar(file_state *s, char c) {
+  if(s->subFilename) { // We're accumulating a file to recursively process
+    // Allocate more space if we need to
+    if(s->subFilenameUsed >= s->subFilenameAllocated - 1) {
+      size_t newSize = s->subFilenameAllocated * 2;
+      s->subFilename = exitIfNull(realloc(s->subFilename, newSize));
+      s->subFilenameAllocated = newSize;
+    }
+    s->subFilename[s->subFilenameUsed++] = c;
+  } else if(*s->w != IN_WORD && c == '@') { // This is the first letter in the word; note that even quoted or escaped @'s are recursively interpreted
+    s->subFilename = exitIfNull(malloc(INITIAL_SUB_FILENAME_CHARS * sizeof(*(s->subFilename))));
+    assert(s->subFilenameUsed == 0);
+    assert(s->subFilenameAllocated == 0);
+    s->subFilenameAllocated = INITIAL_SUB_FILENAME_CHARS;
+  } else {
+    emitWordChar(s->w, c);
+  }
+}
+
+void processFile(word_break_state *w, const char *filename);
+
+void endWord(file_state *s) {
+  if(s->subFilename) {
+    s->subFilename[s->subFilenameUsed] = '\0';
+
+    processFile(s->w, s->subFilename);
+
+    free(s->subFilename);
+    s->subFilename = 0;
+    s->subFilenameUsed = 0;
+    s->subFilenameAllocated = 0;
+  } else {
+    emitWordEnd(s->w);
+  }
+}
+
+void processFile(word_break_state *w, const char *filename) {
+  FILE *h = fopen(filename, "r");
+  if(!h) { //TODO: We assume it's because the file doesn't exist, but perhaps we should check for other failure cases
+    emitWordChar(w, '@');
+    while(*filename) {
+      emitWordChar(w, *filename);
+      ++filename;
+    }
+    emitWordEnd(w);
+    return;
+  }
+
+  char c;
+  quote_state q = UNQUOTED;
+  file_state s = {
+    .w = w,
+    .subFilename = 0,
+    .subFilenameUsed = 0,
+    .subFilenameAllocated = 0
+  };
+  while((c = fgetc(h)) != EOF) {
+    //fprintf(stderr, "%d\n", c);
+    switch(c) {
+    case '\'':
+      switch(q) {
+      case UNQUOTED:
+        q = SINGLE_QUOTED;
+        break;
+      case SINGLE_QUOTED:
+        q = UNQUOTED;
+        break;
+      case DOUBLE_QUOTED:
+        wordChar(&s, '\'');
+        break;
+      }
+      break;
+    case '"':
+      switch(q) {
+      case UNQUOTED:
+        q = DOUBLE_QUOTED;
+        break;
+      case SINGLE_QUOTED:
+        wordChar(&s, '"');
+        break;
+      case DOUBLE_QUOTED:
+        q = UNQUOTED;
+        break;
+      }
+      break;
+    case '\\':
+      c = fgetc(h);
+      if(c != EOF) {
+        wordChar(&s, c);
+      }
+      break;
+    case ' ':
+    case '\t':
+    case '\n':
+    case '\v':
+    case '\f':
+    case '\r':
+      if(q == UNQUOTED) {
+        endWord(&s);
+      } else {
+        wordChar(&s, c);
+      }
+      break;
+    default:
+      wordChar(&s, c);
+      break;
+    }
+  }
+
+  endWord(&s);
+
+  fclose(h);
+}
+
+int main(int argc, const char *argv[]) {
+  if(argc != 2) {
+    fprintf(stderr, "Usage: %s [responsefile]", argv[0]);
+    return 1;
+  }
+
+  word_break_state w = BEFORE_FIRST_WORD;
+  processFile(&w, argv[1]);
+
+  return 0;
+}

pkgs/build-support/cc-wrapper/utils.sh

@@ -23,52 +23,27 @@ badPath() {
         "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
 }
 
-# @args.rsp parser.
-# Char classes: space, other, backslash, single quote, double quote.
-# States: 0 - outside, 1/2 - unquoted arg/slash, 3/4 - 'arg'/slash, 5/6 - "arg"/slash.
-# State transitions:
-rspT=(01235 01235 11111 33413 33333 55651 55555)
-# Push (a) arg or (c) char on transition:
-rspP[10]=a rspP[01]=c rspP[11]=c rspP[21]=c rspP[33]=c rspP[43]=c rspP[55]=c rspP[65]=c
-
-rspParse() {
-    rsp=()
-    local state=0
-    local arg=''
-    local c
-
-    while read -r -N1 c; do
-        local cls=1
-        case "$c" in
-            ' ' | $'\t' | $'\r' | $'\n') cls=0 ;;
-            '\') cls=2 ;;
-            "'") cls=3 ;;
-            '"') cls=4 ;;
-        esac
-        local nextstates="${rspT[$state]}"
-        local nextstate="${nextstates:$cls:1}"
-        case "${rspP[$state$nextstate]}" in
-            'c') arg+="$c" ;;
-            'a') rsp+=("$arg"); arg='' ;;
-        esac
-        state="$nextstate"
-    done
-
-    if [ "$state" -ne 0 ]; then
-        rsp+=("$arg")
-    fi
-}
-
 expandResponseParams() {
+    local inparams=("$@")
+    local n=0
+    local p
     params=()
-    while [ $# -gt 0 ]; do
-        local p="$1"
-        shift
-        if [ "${p:0:1}" = '@' -a -e "${p:1}" ]; then
-            rspParse <"${p:1}"
-            set -- "${rsp[@]}" "$@"
-        else
-            params+=("$p")
-        fi
+    while [ $n -lt ${#inparams[*]} ]; do
+        p=${inparams[n]}
+        case $p in
+            @*)
+                local parseResponseFile="@parseResponseFile@"
+                if [ -n "$parseResponseFile" ] ; then
+                    eval "params+=($("$parseResponseFile" "${p:1}"))"
+                else
+                    echo "Response files aren't supported during bootstrapping" >&2
+                    exit 1
+                fi
+                ;;
+            *)
+                params+=("$p")
+                ;;
+        esac
+        n=$((n + 1))
     done
 }

pkgs/stdenv/darwin/default.nix

@@ -61,7 +61,10 @@ in rec {
                           allowedRequisites ? null}:
     let
       thisStdenv = import ../generic {
-        inherit config shell extraBuildInputs allowedRequisites;
+        inherit config shell extraBuildInputs;
+        allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [
+          thisStdenv.cc.parseResponseFile
+        ];
 
         name = "stdenv-darwin-boot-${toString step}";
 
@@ -73,6 +76,9 @@ in rec {
           nativeTools  = true;
           nativePrefix = bootstrapTools;
           nativeLibc   = false;
+          buildPackages = lib.optionalAttrs (last ? stdenv) {
+            inherit (last) stdenv;
+          };
           hostPlatform = localSystem;
           targetPlatform = localSystem;
           libc         = last.pkgs.darwin.Libsystem;
@@ -297,6 +303,9 @@ in rec {
       inherit shell;
       nativeTools = false;
       nativeLibc  = false;
+      buildPackages = {
+        inherit (prevStage) stdenv;
+      };
       hostPlatform = localSystem;
       targetPlatform = localSystem;
       inherit (pkgs) coreutils binutils gnugrep;
@@ -319,6 +328,7 @@ in rec {
       gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
       gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
       binutils-raw.dev binutils gettext
+      cc.parseResponseFile
     ]) ++ (with pkgs.darwin; [
       dyld Libsystem CF cctools ICU libiconv locale
     ]);

pkgs/stdenv/linux/default.nix

@@ -76,6 +76,9 @@ let
              else lib.makeOverridable (import ../../build-support/cc-wrapper) {
           nativeTools = false;
           nativeLibc = false;
+          buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
+            inherit (prevStage) stdenv;
+          };
           hostPlatform = localSystem;
           targetPlatform = localSystem;
           cc = prevStage.gcc-unwrapped;
@@ -241,6 +244,9 @@ in
         nativeTools = false;
         nativeLibc = false;
         isGNU = true;
+        buildPackages = {
+          inherit (prevStage) stdenv;
+        };
         hostPlatform = localSystem;
         targetPlatform = localSystem;
         cc = prevStage.gcc-unwrapped;