cc-wrapper: improve response file parsing speed
This commit is contained in:
parent
c53449ce57
commit
d07f30f628
@ -10,7 +10,7 @@
|
|||||||
, zlib ? null, extraPackages ? [], extraBuildCommands ? ""
|
, zlib ? null, extraPackages ? [], extraBuildCommands ? ""
|
||||||
, dyld ? null # TODO: should this be a setup-hook on dyld?
|
, dyld ? null # TODO: should this be a setup-hook on dyld?
|
||||||
, isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
|
, isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
|
||||||
, hostPlatform, targetPlatform
|
, buildPackages ? {}, hostPlatform, targetPlatform
|
||||||
, runCommand ? null
|
, runCommand ? null
|
||||||
}:
|
}:
|
||||||
|
|
||||||
@ -120,6 +120,17 @@ let
|
|||||||
null)
|
null)
|
||||||
else "";
|
else "";
|
||||||
|
|
||||||
|
parseResponseFile = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
|
||||||
|
then buildPackages.stdenv.mkDerivation {
|
||||||
|
name = "parse-response-file";
|
||||||
|
src = ./parseResponseFile.c;
|
||||||
|
buildCommand = ''
|
||||||
|
# Make sure the output file doesn't refer to the input nix path
|
||||||
|
cp "$src" parseResponseFile.c
|
||||||
|
"$CC" -O3 -o "$out" parseResponseFile.c
|
||||||
|
'';
|
||||||
|
} else "";
|
||||||
|
|
||||||
in
|
in
|
||||||
|
|
||||||
stdenv.mkDerivation {
|
stdenv.mkDerivation {
|
||||||
@ -368,11 +379,11 @@ stdenv.mkDerivation {
|
|||||||
+ ''
|
+ ''
|
||||||
substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh
|
substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh
|
||||||
substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh
|
substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh
|
||||||
cp -p ${preWrap ./utils.sh} $out/nix-support/utils.sh
|
substituteAll ${preWrap ./utils.sh} $out/nix-support/utils.sh
|
||||||
''
|
''
|
||||||
+ extraBuildCommands;
|
+ extraBuildCommands;
|
||||||
|
|
||||||
inherit dynamicLinker;
|
inherit dynamicLinker parseResponseFile;
|
||||||
|
|
||||||
crossAttrs = {
|
crossAttrs = {
|
||||||
shell = shell.crossDrv + shell.crossDrv.shellPath;
|
shell = shell.crossDrv + shell.crossDrv.shellPath;
|
||||||
|
181
pkgs/build-support/cc-wrapper/parseResponseFile.c
Normal file
181
pkgs/build-support/cc-wrapper/parseResponseFile.c
Normal file
@ -0,0 +1,181 @@
|
|||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
UNQUOTED,
|
||||||
|
SINGLE_QUOTED,
|
||||||
|
DOUBLE_QUOTED
|
||||||
|
} quote_state;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
BEFORE_FIRST_WORD,
|
||||||
|
IN_WORD,
|
||||||
|
AFTER_WORD
|
||||||
|
} word_break_state;
|
||||||
|
|
||||||
|
void emitWordChar(word_break_state *w, char c) {
|
||||||
|
switch(*w) { // Note: These all fall through
|
||||||
|
case AFTER_WORD:
|
||||||
|
putchar(' ');
|
||||||
|
case BEFORE_FIRST_WORD:
|
||||||
|
putchar('\'');
|
||||||
|
*w = IN_WORD;
|
||||||
|
case IN_WORD:
|
||||||
|
if(c == '\'') {
|
||||||
|
printf("'\\''");
|
||||||
|
} else {
|
||||||
|
putchar(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void emitWordEnd(word_break_state *w) {
|
||||||
|
if(*w == IN_WORD) {
|
||||||
|
putchar('\'');
|
||||||
|
*w = AFTER_WORD;
|
||||||
|
} // Otherwise, the state remains the same
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
word_break_state *w;
|
||||||
|
char *subFilename; // Non-null if we're currently accumulating a response file name
|
||||||
|
size_t subFilenameUsed; // If subFilename == 0, this should be 0; this should always be less than (subFilenameAllocated - 1), to allow room for the null byte
|
||||||
|
size_t subFilenameAllocated; // If subFilename == 0, this should be 0
|
||||||
|
} file_state; // The state of parsing a single file
|
||||||
|
|
||||||
|
static const unsigned int INITIAL_SUB_FILENAME_CHARS = 32; // Arbitrary, but must be > 0
|
||||||
|
|
||||||
|
void *exitIfNull(void *p) {
|
||||||
|
if(!p) {
|
||||||
|
fprintf(stderr, "Out of memory");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
void wordChar(file_state *s, char c) {
|
||||||
|
if(s->subFilename) { // We're accumulating a file to recursively process
|
||||||
|
// Allocate more space if we need to
|
||||||
|
if(s->subFilenameUsed >= s->subFilenameAllocated - 1) {
|
||||||
|
size_t newSize = s->subFilenameAllocated * 2;
|
||||||
|
s->subFilename = exitIfNull(realloc(s->subFilename, newSize));
|
||||||
|
s->subFilenameAllocated = newSize;
|
||||||
|
}
|
||||||
|
s->subFilename[s->subFilenameUsed++] = c;
|
||||||
|
} else if(*s->w != IN_WORD && c == '@') { // This is the first letter in the word; note that even quoted or escaped @'s are recursively interpreted
|
||||||
|
s->subFilename = exitIfNull(malloc(INITIAL_SUB_FILENAME_CHARS * sizeof(*(s->subFilename))));
|
||||||
|
assert(s->subFilenameUsed == 0);
|
||||||
|
assert(s->subFilenameAllocated == 0);
|
||||||
|
s->subFilenameAllocated = INITIAL_SUB_FILENAME_CHARS;
|
||||||
|
} else {
|
||||||
|
emitWordChar(s->w, c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void processFile(word_break_state *w, const char *filename);
|
||||||
|
|
||||||
|
void endWord(file_state *s) {
|
||||||
|
if(s->subFilename) {
|
||||||
|
s->subFilename[s->subFilenameUsed] = '\0';
|
||||||
|
|
||||||
|
processFile(s->w, s->subFilename);
|
||||||
|
|
||||||
|
free(s->subFilename);
|
||||||
|
s->subFilename = 0;
|
||||||
|
s->subFilenameUsed = 0;
|
||||||
|
s->subFilenameAllocated = 0;
|
||||||
|
} else {
|
||||||
|
emitWordEnd(s->w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void processFile(word_break_state *w, const char *filename) {
|
||||||
|
FILE *h = fopen(filename, "r");
|
||||||
|
if(!h) { //TODO: We assume it's because the file doesn't exist, but perhaps we should check for other failure cases
|
||||||
|
emitWordChar(w, '@');
|
||||||
|
while(*filename) {
|
||||||
|
emitWordChar(w, *filename);
|
||||||
|
++filename;
|
||||||
|
}
|
||||||
|
emitWordEnd(w);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char c;
|
||||||
|
quote_state q = UNQUOTED;
|
||||||
|
file_state s = {
|
||||||
|
.w = w,
|
||||||
|
.subFilename = 0,
|
||||||
|
.subFilenameUsed = 0,
|
||||||
|
.subFilenameAllocated = 0
|
||||||
|
};
|
||||||
|
while((c = fgetc(h)) != EOF) {
|
||||||
|
//fprintf(stderr, "%d\n", c);
|
||||||
|
switch(c) {
|
||||||
|
case '\'':
|
||||||
|
switch(q) {
|
||||||
|
case UNQUOTED:
|
||||||
|
q = SINGLE_QUOTED;
|
||||||
|
break;
|
||||||
|
case SINGLE_QUOTED:
|
||||||
|
q = UNQUOTED;
|
||||||
|
break;
|
||||||
|
case DOUBLE_QUOTED:
|
||||||
|
wordChar(&s, '\'');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case '"':
|
||||||
|
switch(q) {
|
||||||
|
case UNQUOTED:
|
||||||
|
q = DOUBLE_QUOTED;
|
||||||
|
break;
|
||||||
|
case SINGLE_QUOTED:
|
||||||
|
wordChar(&s, '"');
|
||||||
|
break;
|
||||||
|
case DOUBLE_QUOTED:
|
||||||
|
q = UNQUOTED;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case '\\':
|
||||||
|
c = fgetc(h);
|
||||||
|
if(c != EOF) {
|
||||||
|
wordChar(&s, c);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case ' ':
|
||||||
|
case '\t':
|
||||||
|
case '\n':
|
||||||
|
case '\v':
|
||||||
|
case '\f':
|
||||||
|
case '\r':
|
||||||
|
if(q == UNQUOTED) {
|
||||||
|
endWord(&s);
|
||||||
|
} else {
|
||||||
|
wordChar(&s, c);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
wordChar(&s, c);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
endWord(&s);
|
||||||
|
|
||||||
|
fclose(h);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, const char *argv[]) {
|
||||||
|
if(argc != 2) {
|
||||||
|
fprintf(stderr, "Usage: %s [responsefile]", argv[0]);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
word_break_state w = BEFORE_FIRST_WORD;
|
||||||
|
processFile(&w, argv[1]);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
@ -23,52 +23,27 @@ badPath() {
|
|||||||
"${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
|
"${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
|
||||||
}
|
}
|
||||||
|
|
||||||
# @args.rsp parser.
|
|
||||||
# Char classes: space, other, backslash, single quote, double quote.
|
|
||||||
# States: 0 - outside, 1/2 - unquoted arg/slash, 3/4 - 'arg'/slash, 5/6 - "arg"/slash.
|
|
||||||
# State transitions:
|
|
||||||
rspT=(01235 01235 11111 33413 33333 55651 55555)
|
|
||||||
# Push (a) arg or (c) char on transition:
|
|
||||||
rspP[10]=a rspP[01]=c rspP[11]=c rspP[21]=c rspP[33]=c rspP[43]=c rspP[55]=c rspP[65]=c
|
|
||||||
|
|
||||||
rspParse() {
|
|
||||||
rsp=()
|
|
||||||
local state=0
|
|
||||||
local arg=''
|
|
||||||
local c
|
|
||||||
|
|
||||||
while read -r -N1 c; do
|
|
||||||
local cls=1
|
|
||||||
case "$c" in
|
|
||||||
' ' | $'\t' | $'\r' | $'\n') cls=0 ;;
|
|
||||||
'\') cls=2 ;;
|
|
||||||
"'") cls=3 ;;
|
|
||||||
'"') cls=4 ;;
|
|
||||||
esac
|
|
||||||
local nextstates="${rspT[$state]}"
|
|
||||||
local nextstate="${nextstates:$cls:1}"
|
|
||||||
case "${rspP[$state$nextstate]}" in
|
|
||||||
'c') arg+="$c" ;;
|
|
||||||
'a') rsp+=("$arg"); arg='' ;;
|
|
||||||
esac
|
|
||||||
state="$nextstate"
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$state" -ne 0 ]; then
|
|
||||||
rsp+=("$arg")
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
expandResponseParams() {
|
expandResponseParams() {
|
||||||
|
local inparams=("$@")
|
||||||
|
local n=0
|
||||||
|
local p
|
||||||
params=()
|
params=()
|
||||||
while [ $# -gt 0 ]; do
|
while [ $n -lt ${#inparams[*]} ]; do
|
||||||
local p="$1"
|
p=${inparams[n]}
|
||||||
shift
|
case $p in
|
||||||
if [ "${p:0:1}" = '@' -a -e "${p:1}" ]; then
|
@*)
|
||||||
rspParse <"${p:1}"
|
local parseResponseFile="@parseResponseFile@"
|
||||||
set -- "${rsp[@]}" "$@"
|
if [ -n "$parseResponseFile" ] ; then
|
||||||
else
|
eval "params+=($("$parseResponseFile" "${p:1}"))"
|
||||||
params+=("$p")
|
else
|
||||||
fi
|
echo "Response files aren't supported during bootstrapping" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
params+=("$p")
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
n=$((n + 1))
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
@ -61,7 +61,10 @@ in rec {
|
|||||||
allowedRequisites ? null}:
|
allowedRequisites ? null}:
|
||||||
let
|
let
|
||||||
thisStdenv = import ../generic {
|
thisStdenv = import ../generic {
|
||||||
inherit config shell extraBuildInputs allowedRequisites;
|
inherit config shell extraBuildInputs;
|
||||||
|
allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [
|
||||||
|
thisStdenv.cc.parseResponseFile
|
||||||
|
];
|
||||||
|
|
||||||
name = "stdenv-darwin-boot-${toString step}";
|
name = "stdenv-darwin-boot-${toString step}";
|
||||||
|
|
||||||
@ -73,6 +76,9 @@ in rec {
|
|||||||
nativeTools = true;
|
nativeTools = true;
|
||||||
nativePrefix = bootstrapTools;
|
nativePrefix = bootstrapTools;
|
||||||
nativeLibc = false;
|
nativeLibc = false;
|
||||||
|
buildPackages = lib.optionalAttrs (last ? stdenv) {
|
||||||
|
inherit (last) stdenv;
|
||||||
|
};
|
||||||
hostPlatform = localSystem;
|
hostPlatform = localSystem;
|
||||||
targetPlatform = localSystem;
|
targetPlatform = localSystem;
|
||||||
libc = last.pkgs.darwin.Libsystem;
|
libc = last.pkgs.darwin.Libsystem;
|
||||||
@ -297,6 +303,9 @@ in rec {
|
|||||||
inherit shell;
|
inherit shell;
|
||||||
nativeTools = false;
|
nativeTools = false;
|
||||||
nativeLibc = false;
|
nativeLibc = false;
|
||||||
|
buildPackages = {
|
||||||
|
inherit (prevStage) stdenv;
|
||||||
|
};
|
||||||
hostPlatform = localSystem;
|
hostPlatform = localSystem;
|
||||||
targetPlatform = localSystem;
|
targetPlatform = localSystem;
|
||||||
inherit (pkgs) coreutils binutils gnugrep;
|
inherit (pkgs) coreutils binutils gnugrep;
|
||||||
@ -319,6 +328,7 @@ in rec {
|
|||||||
gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
|
gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
|
||||||
gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
|
gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
|
||||||
binutils-raw.dev binutils gettext
|
binutils-raw.dev binutils gettext
|
||||||
|
cc.parseResponseFile
|
||||||
]) ++ (with pkgs.darwin; [
|
]) ++ (with pkgs.darwin; [
|
||||||
dyld Libsystem CF cctools ICU libiconv locale
|
dyld Libsystem CF cctools ICU libiconv locale
|
||||||
]);
|
]);
|
||||||
|
@ -76,6 +76,9 @@ let
|
|||||||
else lib.makeOverridable (import ../../build-support/cc-wrapper) {
|
else lib.makeOverridable (import ../../build-support/cc-wrapper) {
|
||||||
nativeTools = false;
|
nativeTools = false;
|
||||||
nativeLibc = false;
|
nativeLibc = false;
|
||||||
|
buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
|
||||||
|
inherit (prevStage) stdenv;
|
||||||
|
};
|
||||||
hostPlatform = localSystem;
|
hostPlatform = localSystem;
|
||||||
targetPlatform = localSystem;
|
targetPlatform = localSystem;
|
||||||
cc = prevStage.gcc-unwrapped;
|
cc = prevStage.gcc-unwrapped;
|
||||||
@ -241,6 +244,9 @@ in
|
|||||||
nativeTools = false;
|
nativeTools = false;
|
||||||
nativeLibc = false;
|
nativeLibc = false;
|
||||||
isGNU = true;
|
isGNU = true;
|
||||||
|
buildPackages = {
|
||||||
|
inherit (prevStage) stdenv;
|
||||||
|
};
|
||||||
hostPlatform = localSystem;
|
hostPlatform = localSystem;
|
||||||
targetPlatform = localSystem;
|
targetPlatform = localSystem;
|
||||||
cc = prevStage.gcc-unwrapped;
|
cc = prevStage.gcc-unwrapped;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user