diff options
author | Ryan Trinkle <ryan.trinkle@gmail.com> | 2017-11-03 10:53:00 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-11-03 10:53:00 -0400 |
commit | ded1281f4519bfdeea6c79eb6857f00619d9859d (patch) | |
tree | 0562858f7368f6e9ae65a730d2e4ea750d40f31e /pkgs/build-support | |
parent | dce2c258ac33ee1cd3b3e852b36e1d8fa11f33a1 (diff) | |
parent | 74260a4922e678348eac91f4aa5767a3f5a039a4 (diff) | |
download | nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar.gz nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar.bz2 nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar.lz nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar.xz nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.tar.zst nixlib-ded1281f4519bfdeea6c79eb6857f00619d9859d.zip |
Merge branch 'master' into docker-dirlinks
Diffstat (limited to 'pkgs/build-support')
85 files changed, 1859 insertions, 1886 deletions
diff --git a/pkgs/build-support/binutils-wrapper/macos-sierra-reexport-hack.bash b/pkgs/build-support/binutils-wrapper/macos-sierra-reexport-hack.bash new file mode 100644 index 000000000000..c3077e869e75 --- /dev/null +++ b/pkgs/build-support/binutils-wrapper/macos-sierra-reexport-hack.bash @@ -0,0 +1,108 @@ +#! @shell@ + +set -eu -o pipefail + +path_backup="$PATH" +if [ -n "@coreutils_bin@" ]; then + PATH="@coreutils_bin@/bin" +fi + +declare -r recurThreshold=300 + +declare overflowCount=0 +for ((n=0; n < $#; ++n)); do + case "${!n}" in + -l*) let overflowCount+=1 ;; + -reexport-l*) let overflowCount+=1 ;; + *) ;; + esac +done + +declare -a allArgs=() + +if (( "$overflowCount" <= "$recurThreshold" )); then + allArgs=("$@") +else + declare -a childrenLookup=() childrenLink=() + + while (( $# )); do + case "$1" in + -L/*) + childrenLookup+=("$1") + allArgs+=("$1") + ;; + -L) + echo "cctools LD does not support '-L foo' or '-l foo'" >&2 + exit 1 + ;; + -l) + echo "cctools LD does not support '-L foo' or '-l foo'" >&2 + exit 1 + ;; + -lazy_library | -lazy_framework | -lto_library) + # We aren't linking any "azy_library", "to_library", etc. + allArgs+=("$1") + ;; + -lazy-l | -weak-l) allArgs+=("$1") ;; + # We can't so easily prevent header issues from these. + -lSystem) allArgs+=("$1") ;; + # Special case as indirection seems like a bad idea for something + # so fundamental. Can be removed for simplicity. + -l?* | -reexport-l?*) childrenLink+=("$1") ;; + *) allArgs+=("$1") ;; + esac + + shift + done + + declare n=0 + while (( $n < "${#childrenLink[@]}" )); do + if [[ "${childrenLink[n]}" = -l* ]]; then + childrenLink[n]="-reexport${childrenLink[n]}" + fi + let ++n + done + unset n + + declare -r outputNameLibless=$(basename $( \ + if [[ -z "${outputName:+isUndefined}" ]]; then + echo unnamed + elif [[ "${outputName:0:3}" = lib ]]; then + echo "${outputName:3}" + else + echo "${outputName}" + fi)) + declare -ra children=("$outputNameLibless-reexport-delegate-0" \ + "$outputNameLibless-reexport-delegate-1") + + mkdir -p "$out/lib" + + PATH="$PATH:@out@/bin" + + symbolBloatObject=$outputNameLibless-symbol-hack.o + if [[ ! -e $symbolBloatObject ]]; then + # `-Q` means use GNU Assembler rather than Clang, avoiding an awkward + # dependency cycle. + printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' \ + | @binPrefix@as -Q -- -o $symbolBloatObject + fi + + # first half of libs + @binPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[0]}.dylib" \ + -install_name "$out/lib/lib${children[0]}.dylib" \ + "${childrenLookup[@]}" "$symbolBloatObject" \ + "${childrenLink[@]:0:$((${#childrenLink[@]} / 2 ))}" + + # second half of libs + @binPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[1]}.dylib" \ + -install_name "$out/lib/lib${children[1]}.dylib" \ + "${childrenLookup[@]}" "$symbolBloatObject" \ + "${childrenLink[@]:$((${#childrenLink[@]} / 2 ))}" + + allArgs+=("-L$out/lib" "-l${children[0]}" "-l${children[1]}") +fi + +PATH="$path_backup" +exec @prog@ "${allArgs[@]}" diff --git a/pkgs/build-support/build-dotnet-package/default.nix b/pkgs/build-support/build-dotnet-package/default.nix index 2b1b34429c57..f36c69e43392 100644 --- a/pkgs/build-support/build-dotnet-package/default.nix +++ b/pkgs/build-support/build-dotnet-package/default.nix @@ -19,8 +19,8 @@ attrsOrig @ attrs = { name = "${baseName}-${version}"; + nativeBuildInputs = [ pkgconfig ]; buildInputs = [ - pkgconfig mono dotnetbuildhelpers makeWrapper diff --git a/pkgs/build-support/build-fhs-userenv/chroot-user.rb b/pkgs/build-support/build-fhs-userenv/chroot-user.rb index 11f672acb9ff..833aab16ceb1 100755 --- a/pkgs/build-support/build-fhs-userenv/chroot-user.rb +++ b/pkgs/build-support/build-fhs-userenv/chroot-user.rb @@ -16,6 +16,7 @@ mounts = { '/' => 'host', # Propagate environment variables envvars = [ 'TERM', 'DISPLAY', + 'XAUTHORITY', 'HOME', 'XDG_RUNTIME_DIR', 'LANG', diff --git a/pkgs/build-support/build-fhs-userenv/env.nix b/pkgs/build-support/build-fhs-userenv/env.nix index 9c228b391147..da8a86b68f5d 100644 --- a/pkgs/build-support/build-fhs-userenv/env.nix +++ b/pkgs/build-support/build-fhs-userenv/env.nix @@ -54,6 +54,7 @@ let export PATH='/run/wrappers/bin:/usr/bin:/usr/sbin' # Force compilers and other tools to look in default search paths + export NIX_CC_WRAPPER_${stdenv.cc.infixSalt}_TARGET_HOST=1 export NIX_CFLAGS_COMPILE='-idirafter /usr/include' export NIX_LDFLAGS_BEFORE='-L/usr/lib -L/usr/lib32' export PKG_CONFIG_PATH=/usr/lib/pkgconfig diff --git a/pkgs/build-support/build-pecl.nix b/pkgs/build-support/build-pecl.nix index 5837d413abc9..ce948739c320 100644 --- a/pkgs/build-support/build-pecl.nix +++ b/pkgs/build-support/build-pecl.nix @@ -2,6 +2,7 @@ { name , buildInputs ? [] +, nativeBuildInputs ? [] , makeFlags ? [] , src ? fetchurl { url = "http://pecl.php.net/get/${name}.tgz"; @@ -15,7 +16,8 @@ stdenv.mkDerivation (args // { inherit src; - buildInputs = [ php autoreconfHook ] ++ buildInputs; + nativeBuildInputs = [ autoreconfHook ] ++ nativeBuildInputs; + buildInputs = [ php ] ++ buildInputs; makeFlags = [ "EXTENSION_DIR=$(out)/lib/php/extensions" ] ++ makeFlags; diff --git a/pkgs/build-support/buildenv/default.nix b/pkgs/build-support/buildenv/default.nix index 47e2c1b904c3..41a1e67ef428 100644 --- a/pkgs/build-support/buildenv/default.nix +++ b/pkgs/build-support/buildenv/default.nix @@ -4,7 +4,8 @@ { buildPackages, runCommand, lib }: -{ name +lib.makeOverridable +({ name , # The manifest file (if any). A symlink $out/manifest will be # created to it. @@ -68,4 +69,4 @@ runCommand name '' ${buildPackages.perl}/bin/perl -w ${./builder.pl} eval "$postBuild" - '' + '') diff --git a/pkgs/build-support/cc-wrapper/add-flags.sh b/pkgs/build-support/cc-wrapper/add-flags.sh index 5634c82aa285..39633fce69a8 100644 --- a/pkgs/build-support/cc-wrapper/add-flags.sh +++ b/pkgs/build-support/cc-wrapper/add-flags.sh @@ -1,28 +1,82 @@ +# N.B. It may be a surprise that the derivation-specific variables are exported, +# since this is just sourced by the wrapped binaries---the end consumers. This +# is because one wrapper binary may invoke another (e.g. cc invoking ld). In +# that case, it is cheaper/better to not repeat this step and let the forked +# wrapped binary just inherit the work of the forker's wrapper script. + +var_templates=( + NIX+CFLAGS_COMPILE + NIX+CFLAGS_LINK + NIX+CXXSTDLIB_COMPILE + NIX+CXXSTDLIB_LINK + NIX+GNATFLAGS_COMPILE + NIX+IGNORE_LD_THROUGH_GCC + NIX+LDFLAGS + NIX+LDFLAGS_BEFORE + NIX+LDFLAGS_AFTER + + NIX+SET_BUILD_ID + NIX+DONT_SET_RPATH + NIX+ENFORCE_NO_NATIVE +) + +# Accumulate infixes for taking in the right input parameters. See setup-hook +# for details. +declare -a role_infixes=() +if [ "${NIX_CC_WRAPPER_@infixSalt@_TARGET_BUILD:-}" ]; then + role_infixes+=(_BUILD_) +fi +if [ "${NIX_CC_WRAPPER_@infixSalt@_TARGET_HOST:-}" ]; then + role_infixes+=(_) +fi +if [ "${NIX_CC_WRAPPER_@infixSalt@_TARGET_TARGET:-}" ]; then + role_infixes+=(_TARGET_) +fi + +# We need to mangle names for hygiene, but also take parameters/overrides +# from the environment. +for var in "${var_templates[@]}"; do + outputVar="${var/+/_@infixSalt@_}" + export ${outputVar}+='' + # For each role we serve, we accumulate the input parameters into our own + # cc-wrapper-derivation-specific environment variables. + for infix in "${role_infixes[@]}"; do + inputVar="${var/+/${infix}}" + if [ -v "$inputVar" ]; then + export ${outputVar}+="${!outputVar:+ }${!inputVar}" + fi + done +done + # `-B@out@/bin' forces cc to use ld-wrapper.sh when calling ld. -export NIX_CFLAGS_COMPILE="-B@out@/bin/ $NIX_CFLAGS_COMPILE" +NIX_@infixSalt@_CFLAGS_COMPILE="-B@out@/bin/ $NIX_@infixSalt@_CFLAGS_COMPILE" + +# Export and assign separately in order that a failing $(..) will fail +# the script. if [ -e @out@/nix-support/libc-cflags ]; then - export NIX_CFLAGS_COMPILE="$(cat @out@/nix-support/libc-cflags) $NIX_CFLAGS_COMPILE" + NIX_@infixSalt@_CFLAGS_COMPILE="$(< @out@/nix-support/libc-cflags) $NIX_@infixSalt@_CFLAGS_COMPILE" fi if [ -e @out@/nix-support/cc-cflags ]; then - export NIX_CFLAGS_COMPILE="$(cat @out@/nix-support/cc-cflags) $NIX_CFLAGS_COMPILE" + NIX_@infixSalt@_CFLAGS_COMPILE="$(< @out@/nix-support/cc-cflags) $NIX_@infixSalt@_CFLAGS_COMPILE" fi if [ -e @out@/nix-support/gnat-cflags ]; then - export NIX_GNATFLAGS_COMPILE="$(cat @out@/nix-support/gnat-cflags) $NIX_GNATFLAGS_COMPILE" + NIX_@infixSalt@_GNATFLAGS_COMPILE="$(< @out@/nix-support/gnat-cflags) $NIX_@infixSalt@_GNATFLAGS_COMPILE" fi if [ -e @out@/nix-support/libc-ldflags ]; then - export NIX_LDFLAGS+=" $(cat @out@/nix-support/libc-ldflags)" + NIX_@infixSalt@_LDFLAGS+=" $(< @out@/nix-support/libc-ldflags)" fi if [ -e @out@/nix-support/cc-ldflags ]; then - export NIX_LDFLAGS+=" $(cat @out@/nix-support/cc-ldflags)" + NIX_@infixSalt@_LDFLAGS+=" $(< @out@/nix-support/cc-ldflags)" fi if [ -e @out@/nix-support/libc-ldflags-before ]; then - export NIX_LDFLAGS_BEFORE="$(cat @out@/nix-support/libc-ldflags-before) $NIX_LDFLAGS_BEFORE" + NIX_@infixSalt@_LDFLAGS_BEFORE="$(< @out@/nix-support/libc-ldflags-before) $NIX_@infixSalt@_LDFLAGS_BEFORE" fi -export NIX_CC_WRAPPER_FLAGS_SET=1 +# That way forked processes will not extend these environment variables again. +export NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET=1 diff --git a/pkgs/build-support/cc-wrapper/add-hardening.sh b/pkgs/build-support/cc-wrapper/add-hardening.sh index b98833b3513b..b0e39e455ffc 100644 --- a/pkgs/build-support/cc-wrapper/add-hardening.sh +++ b/pkgs/build-support/cc-wrapper/add-hardening.sh @@ -1,57 +1,75 @@ hardeningFlags=(fortify stackprotector pic strictoverflow format relro bindnow) -hardeningFlags+=("${hardeningEnable[@]}") +# Intentionally word-split in case 'hardeningEnable' is defined in +# Nix. Also, our bootstrap tools version of bash is old enough that +# undefined arrays trip `set -u`. +if [[ -v hardeningEnable[@] ]]; then + hardeningFlags+=(${hardeningEnable[@]}) +fi hardeningCFlags=() hardeningLDFlags=() -hardeningDisable=${hardeningDisable:-""} -hardeningDisable+=" @hardening_unsupported_flags@" +declare -A hardeningDisableMap + +# Intentionally word-split in case 'hardeningDisable' is defined in Nix. +for flag in ${hardeningDisable[@]:-IGNORED_KEY} @hardening_unsupported_flags@ +do + hardeningDisableMap[$flag]=1 +done -if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: Value of '$hardeningDisable': $hardeningDisable >&2; fi +if (( "${NIX_DEBUG:-0}" >= 1 )); then + printf 'HARDENING: disabled flags:' >&2 + (( "${#hardeningDisableMap[@]}" )) && printf ' %q' "${!hardeningDisableMap[@]}" >&2 + echo >&2 +fi -if [[ ! $hardeningDisable =~ "all" ]]; then - if [[ -n "$NIX_DEBUG" ]]; then echo 'HARDENING: Is active (not completely disabled with "all" flag)' >&2; fi +if [[ -z "${hardeningDisableMap[all]:-}" ]]; then + if (( "${NIX_DEBUG:-0}" >= 1 )); then + echo 'HARDENING: Is active (not completely disabled with "all" flag)' >&2; + fi for flag in "${hardeningFlags[@]}" do - if [[ ! "${hardeningDisable}" =~ "$flag" ]]; then + if [[ -z "${hardeningDisableMap[$flag]:-}" ]]; then case $flag in fortify) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling fortify >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling fortify >&2; fi hardeningCFlags+=('-O2' '-D_FORTIFY_SOURCE=2') ;; stackprotector) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling stackprotector >&2; fi - hardeningCFlags+=('-fstack-protector-strong' '--param ssp-buffer-size=4') + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling stackprotector >&2; fi + hardeningCFlags+=('-fstack-protector-strong' '--param' 'ssp-buffer-size=4') ;; pie) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling CFlags -fPIE >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling CFlags -fPIE >&2; fi hardeningCFlags+=('-fPIE') if [[ ! ("$*" =~ " -shared " || "$*" =~ " -static ") ]]; then - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling LDFlags -pie >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling LDFlags -pie >&2; fi + hardeningCFlags+=('-pie') hardeningLDFlags+=('-pie') fi ;; pic) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling pic >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling pic >&2; fi hardeningCFlags+=('-fPIC') ;; strictoverflow) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling strictoverflow >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling strictoverflow >&2; fi hardeningCFlags+=('-fno-strict-overflow') ;; format) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling format >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling format >&2; fi hardeningCFlags+=('-Wformat' '-Wformat-security' '-Werror=format-security') ;; relro) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling relro >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling relro >&2; fi hardeningLDFlags+=('-z' 'relro') ;; bindnow) - if [[ -n "$NIX_DEBUG" ]]; then echo HARDENING: enabling bindnow >&2; fi + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo HARDENING: enabling bindnow >&2; fi hardeningLDFlags+=('-z' 'now') ;; *) - echo "Hardening flag unknown: $flag" >&2 + # Ignore unsupported. Checked in Nix that at least *some* + # tool supports each flag. ;; esac fi diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh index 3ccdc34db5b2..f1f56cf5cec4 100644 --- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh @@ -1,14 +1,21 @@ -#! @shell@ -e -path_backup="$PATH" -if [ -n "@coreutils_bin@" ]; then - PATH="@coreutils_bin@/bin:@gnugrep_bin@/bin" +#! @shell@ +set -eu -o pipefail +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x fi -if [ -n "$NIX_CC_WRAPPER_START_HOOK" ]; then - source "$NIX_CC_WRAPPER_START_HOOK" +path_backup="$PATH" + +# That @-vars are substituted separately from bash evaluation makes +# shellcheck think this, and others like it, are useless conditionals. +# shellcheck disable=SC2157 +if [[ -n "@coreutils_bin@" && -n "@gnugrep_bin@" ]]; then + PATH="@coreutils_bin@/bin:@gnugrep_bin@/bin" fi -if [ -z "$NIX_CC_WRAPPER_FLAGS_SET" ]; then +if [ -z "${NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -19,16 +26,17 @@ source @out@/nix-support/utils.sh # For instance, figure out if linker flags should be passed. # GCC prints annoying warnings when they are not needed. dontLink=0 -getVersion=0 nonFlagArgs=0 +# shellcheck disable=SC2193 [[ "@prog@" = *++ ]] && isCpp=1 || isCpp=0 cppInclude=1 expandResponseParams "$@" -n=0 -while [ $n -lt ${#params[*]} ]; do +declare -i n=0 +nParams=${#params[@]} +while (( "$n" < "$nParams" )); do p=${params[n]} - p2=${params[$((n+1))]} + p2=${params[n+1]:-} # handle `p` being last one if [ "$p" = -c ]; then dontLink=1 elif [ "$p" = -S ]; then @@ -53,12 +61,8 @@ while [ $n -lt ${#params[*]} ]; do cppInclude=0 elif [ "${p:0:1}" != - ]; then nonFlagArgs=1 - elif [ "$p" = -m32 ]; then - if [ -e @out@/nix-support/dynamic-linker-m32 ]; then - NIX_LDFLAGS="$NIX_LDFLAGS -dynamic-linker $(cat @out@/nix-support/dynamic-linker-m32)" - fi fi - n=$((n + 1)) + n+=1 done # If we pass a flag like -Wl, then gcc will call the linker unless it @@ -71,77 +75,80 @@ if [ "$nonFlagArgs" = 0 ]; then fi # Optionally filter out paths not refering to the store. -if [ "$NIX_ENFORCE_PURITY" = 1 -a -n "$NIX_STORE" ]; then +if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "$NIX_STORE" ]]; then rest=() - n=0 - while [ $n -lt ${#params[*]} ]; do + nParams=${#params[@]} + declare -i n=0 + while (( "$n" < "$nParams" )); do p=${params[n]} - p2=${params[$((n+1))]} + p2=${params[n+1]:-} # handle `p` being last one if [ "${p:0:3}" = -L/ ] && badPath "${p:2}"; then - skip $p + skip "${p:2}" elif [ "$p" = -L ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" elif [ "${p:0:3}" = -I/ ] && badPath "${p:2}"; then - skip $p + skip "${p:2}" elif [ "$p" = -I ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" elif [ "$p" = -isystem ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" else rest+=("$p") fi - n=$((n + 1)) + n+=1 done - params=("${rest[@]}") + # Old bash empty array hack + params=(${rest+"${rest[@]}"}) fi # Clear march/mtune=native -- they bring impurity. -if [ "$NIX_ENFORCE_NO_NATIVE" = 1 ]; then +if [ "$NIX_@infixSalt@_ENFORCE_NO_NATIVE" = 1 ]; then rest=() - for i in "${params[@]}"; do - if [[ "$i" = -m*=native ]]; then - skip $i + # Old bash empty array hack + for p in ${params+"${params[@]}"}; do + if [[ "$p" = -m*=native ]]; then + skip "$p" else - rest+=("$i") + rest+=("$p") fi done - params=("${rest[@]}") + # Old bash empty array hack + params=(${rest+"${rest[@]}"}) fi if [[ "$isCpp" = 1 ]]; then if [[ "$cppInclude" = 1 ]]; then - NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE ${NIX_CXXSTDLIB_COMPILE-@default_cxx_stdlib_compile@}" + NIX_@infixSalt@_CFLAGS_COMPILE+=" ${NIX_@infixSalt@_CXXSTDLIB_COMPILE-@default_cxx_stdlib_compile@}" fi - NIX_CFLAGS_LINK="$NIX_CFLAGS_LINK $NIX_CXXSTDLIB_LINK" + NIX_@infixSalt@_CFLAGS_LINK+=" $NIX_@infixSalt@_CXXSTDLIB_LINK" fi -LD=@ldPath@/ld source @out@/nix-support/add-hardening.sh # Add the flags for the C compiler proper. -extraAfter=($NIX_CFLAGS_COMPILE ${hardeningCFlags[@]}) +extraAfter=($NIX_@infixSalt@_CFLAGS_COMPILE "${hardeningCFlags[@]}") extraBefore=() if [ "$dontLink" != 1 ]; then # Add the flags that should only be passed to the compiler when # linking. - extraAfter+=($NIX_CFLAGS_LINK ${hardeningLDFlags[@]}) + extraAfter+=($NIX_@infixSalt@_CFLAGS_LINK) # Add the flags that should be passed to the linker (and prevent - # `ld-wrapper' from adding NIX_LDFLAGS again). - for i in $NIX_LDFLAGS_BEFORE; do - extraBefore=(${extraBefore[@]} "-Wl,$i") + # `ld-wrapper' from adding NIX_@infixSalt@_LDFLAGS again). + for i in $NIX_@infixSalt@_LDFLAGS_BEFORE; do + extraBefore+=("-Wl,$i") done - for i in $NIX_LDFLAGS; do + for i in $NIX_@infixSalt@_LDFLAGS; do if [ "${i:0:3}" = -L/ ]; then extraAfter+=("$i") else extraAfter+=("-Wl,$i") fi done - export NIX_LDFLAGS_SET=1 + export NIX_@infixSalt@_LDFLAGS_SET=1 fi # As a very special hack, if the arguments are just `-v', then don't @@ -154,24 +161,19 @@ if [ "$*" = -v ]; then fi # Optionally print debug info. -if [ -n "$NIX_DEBUG" ]; then - echo "original flags to @prog@:" >&2 - for i in "${params[@]}"; do - echo " $i" >&2 - done - echo "extraBefore flags to @prog@:" >&2 - for i in ${extraBefore[@]}; do - echo " $i" >&2 - done - echo "extraAfter flags to @prog@:" >&2 - for i in ${extraAfter[@]}; do - echo " $i" >&2 - done -fi - -if [ -n "$NIX_CC_WRAPPER_EXEC_HOOK" ]; then - source "$NIX_CC_WRAPPER_EXEC_HOOK" +if (( "${NIX_DEBUG:-0}" >= 1 )); then + # Old bash workaround, see ld-wrapper for explanation. + echo "extra flags before to @prog@:" >&2 + printf " %q\n" ${extraBefore+"${extraBefore[@]}"} >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" ${params+"${params[@]}"} >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" ${extraAfter+"${extraAfter[@]}"} >&2 fi PATH="$path_backup" -exec @prog@ ${extraBefore[@]} "${params[@]}" "${extraAfter[@]}" +# Old bash workaround, see above. +exec @prog@ \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 95c6bee3cc77..489fb02dcb5d 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -5,65 +5,125 @@ # script that sets up the right environment variables so that the # compiler and the linker just "work". -{ name ? "", stdenv, nativeTools, nativeLibc, nativePrefix ? "" -, cc ? null, libc ? null, binutils ? null, coreutils ? null, shell ? stdenv.shell +{ name ? "", stdenvNoCC, nativeTools, noLibc ? false, nativeLibc, nativePrefix ? "" +, cc ? null, libc ? null, binutils ? null, coreutils ? null, shell ? stdenvNoCC.shell , zlib ? null, extraPackages ? [], extraBuildCommands ? "" -, dyld ? null # TODO: should this be a setup-hook on dyld? , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null +, buildPackages ? {} +, useMacosReexportHack ? false }: -with stdenv.lib; +with stdenvNoCC.lib; assert nativeTools -> nativePrefix != ""; assert !nativeTools -> cc != null && binutils != null && coreutils != null && gnugrep != null; -assert !nativeLibc -> libc != null; +assert !(nativeLibc && noLibc); +assert (noLibc || nativeLibc) == (libc == null); # For ghdl (the vhdl language provider to gcc) we need zlib in the wrapper. assert cc.langVhdl or false -> zlib != null; let + stdenv = stdenvNoCC; + inherit (stdenv) hostPlatform targetPlatform; + + # Prefix for binaries. Customarily ends with a dash separator. + # + # TODO(@Ericson2314) Make unconditional, or optional but always true by + # default. + prefix = stdenv.lib.optionalString (targetPlatform != hostPlatform) + (targetPlatform.config + "-"); ccVersion = (builtins.parseDrvName cc.name).version; ccName = (builtins.parseDrvName cc.name).name; - libc_bin = if nativeLibc then null else getBin libc; - libc_dev = if nativeLibc then null else getDev libc; - libc_lib = if nativeLibc then null else getLib libc; + libc_bin = if libc == null then null else getBin libc; + libc_dev = if libc == null then null else getDev libc; + libc_lib = if libc == null then null else getLib libc; cc_solib = getLib cc; binutils_bin = if nativeTools then "" else getBin binutils; # The wrapper scripts use 'cat' and 'grep', so we may need coreutils. coreutils_bin = if nativeTools then "" else getBin coreutils; + + default_cxx_stdlib_compile=optionalString (targetPlatform.isLinux && !(cc.isGNU or false)) + "-isystem $(echo -n ${cc.gcc}/include/c++/*) -isystem $(echo -n ${cc.gcc}/include/c++/*)/$(${cc.gcc}/bin/gcc -dumpmachine)"; + + dashlessTarget = stdenv.lib.replaceStrings ["-"] ["_"] targetPlatform.config; + + # The "infix salt" is a arbitrary string added in the middle of env vars + # defined by cc-wrapper's hooks so that multiple cc-wrappers can be used + # without interfering. For the moment, it is defined as the target triple, + # adjusted to be a valid bash identifier. This should be considered an + # unstable implementation detail, however. + infixSalt = dashlessTarget; + + # The dynamic linker has different names on different platforms. This is a + # shell glob that ought to match it. + dynamicLinker = + /**/ if libc == null then null + else if targetPlatform.system == "i686-linux" then "${libc_lib}/lib/ld-linux.so.2" + else if targetPlatform.system == "x86_64-linux" then "${libc_lib}/lib/ld-linux-x86-64.so.2" + # ARM with a wildcard, which can be "" or "-armhf". + else if (with targetPlatform; isArm && isLinux) then "${libc_lib}/lib/ld-linux*.so.3" + else if targetPlatform.system == "aarch64-linux" then "${libc_lib}/lib/ld-linux-aarch64.so.1" + else if targetPlatform.system == "powerpc-linux" then "${libc_lib}/lib/ld.so.1" + else if targetPlatform.system == "mips64el-linux" then "${libc_lib}/lib/ld.so.1" + else if targetPlatform.isDarwin then "/usr/lib/dyld" + else if stdenv.lib.hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1" + else null; + + expand-response-params = + if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null" + then import ../expand-response-params { inherit (buildPackages) stdenv; } + else ""; + in stdenv.mkDerivation { - name = - (if name != "" then name else ccName + "-wrapper") + - (if cc != null && ccVersion != "" then "-" + ccVersion else ""); + name = prefix + + (if name != "" then name else "${ccName}-wrapper") + + (stdenv.lib.optionalString (cc != null && ccVersion != "") "-${ccVersion}"); preferLocalBuild = true; - inherit cc shell libc_bin libc_dev libc_lib binutils_bin coreutils_bin; + inherit cc libc_bin libc_dev libc_lib binutils_bin coreutils_bin; + shell = getBin shell + shell.shellPath or ""; gnugrep_bin = if nativeTools then "" else gnugrep; + binPrefix = prefix; + inherit infixSalt; + + outputs = [ "out" "man" ]; + passthru = { - inherit libc nativeTools nativeLibc nativePrefix isGNU isClang; + inherit libc nativeTools nativeLibc nativePrefix isGNU isClang default_cxx_stdlib_compile + prefix; emacsBufferSetup = pkgs: '' ; We should handle propagation here too (mapc (lambda (arg) (when (file-directory-p (concat arg "/include")) - (setenv "NIX_CFLAGS_COMPILE" (concat (getenv "NIX_CFLAGS_COMPILE") " -isystem " arg "/include"))) + (setenv "NIX_${infixSalt}_CFLAGS_COMPILE" (concat (getenv "NIX_${infixSalt}_CFLAGS_COMPILE") " -isystem " arg "/include"))) (when (file-directory-p (concat arg "/lib")) - (setenv "NIX_LDFLAGS" (concat (getenv "NIX_LDFLAGS") " -L" arg "/lib"))) + (setenv "NIX_${infixSalt}_LDFLAGS" (concat (getenv "NIX_${infixSalt}_LDFLAGS") " -L" arg "/lib"))) (when (file-directory-p (concat arg "/lib64")) - (setenv "NIX_LDFLAGS" (concat (getenv "NIX_LDFLAGS") " -L" arg "/lib64")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) + (setenv "NIX_${infixSalt}_LDFLAGS" (concat (getenv "NIX_${infixSalt}_LDFLAGS") " -L" arg "/lib64")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) ''; }; - buildCommand = + dontBuild = true; + dontConfigure = true; + + unpackPhase = '' + src=$PWD + ''; + + installPhase = '' - mkdir -p $out/bin $out/nix-support + set -u + + mkdir -p $out/bin $out/nix-support $man/nix-support wrap() { local dst="$1" @@ -74,25 +134,122 @@ stdenv.mkDerivation { } '' - + optionalString (!nativeLibc) (if (!stdenv.isDarwin) then '' - dynamicLinker="${libc_lib}/lib/$dynamicLinker" - echo $dynamicLinker > $out/nix-support/dynamic-linker + + (if nativeTools then '' + echo ${if targetPlatform.isDarwin then cc else nativePrefix} > $out/nix-support/orig-cc + + ccPath="${if targetPlatform.isDarwin then cc else nativePrefix}/bin" + ldPath="${nativePrefix}/bin" + '' else '' + echo $cc > $out/nix-support/orig-cc + + ccPath="${cc}/bin" + ldPath="${binutils_bin}/bin" + '' + + + optionalString (targetPlatform.isSunOS && nativePrefix != "") '' + # Solaris needs an additional ld wrapper. + ldPath="${nativePrefix}/bin" + exec="$ldPath/${prefix}ld" + wrap ld-solaris ${./ld-solaris-wrapper.sh} + '') - if [ -e ${libc_lib}/lib/32/ld-linux.so.2 ]; then - echo ${libc_lib}/lib/32/ld-linux.so.2 > $out/nix-support/dynamic-linker-m32 + + '' + # Create a symlink to as (the assembler). This is useful when a + # cc-wrapper is installed in a user environment, as it ensures that + # the right assembler is called. + if [ -e $ldPath/${prefix}as ]; then + ln -s $ldPath/${prefix}as $out/bin/${prefix}as fi - # The dynamic linker is passed in `ldflagsBefore' to allow - # explicit overrides of the dynamic linker by callers to gcc/ld - # (the *last* value counts, so ours should come first). - echo "-dynamic-linker" $dynamicLinker > $out/nix-support/libc-ldflags-before + '' + (if !useMacosReexportHack then '' + wrap ${prefix}ld ${./ld-wrapper.sh} ''${ld:-$ldPath/${prefix}ld} '' else '' - echo $dynamicLinker > $out/nix-support/dynamic-linker + ldInner="${prefix}ld-reexport-delegate" + wrap "$ldInner" ${./macos-sierra-reexport-hack.bash} ''${ld:-$ldPath/${prefix}ld} + wrap "${prefix}ld" ${./ld-wrapper.sh} "$out/bin/$ldInner" + unset ldInner + '') + '' + + if [ -e ${binutils_bin}/bin/${prefix}ld.gold ]; then + wrap ${prefix}ld.gold ${./ld-wrapper.sh} ${binutils_bin}/bin/${prefix}ld.gold + fi - echo "export LD_DYLD_PATH=\"$dynamicLinker\"" >> $out/nix-support/setup-hook - '') + if [ -e ${binutils_bin}/bin/ld.bfd ]; then + wrap ${prefix}ld.bfd ${./ld-wrapper.sh} ${binutils_bin}/bin/${prefix}ld.bfd + fi + + # We export environment variables pointing to the wrapped nonstandard + # cmds, lest some lousy configure script use those to guess compiler + # version. + export named_cc=${prefix}cc + export named_cxx=${prefix}c++ + + export default_cxx_stdlib_compile="${default_cxx_stdlib_compile}" + + if [ -e $ccPath/${prefix}gcc ]; then + wrap ${prefix}gcc ${./cc-wrapper.sh} $ccPath/${prefix}gcc + ln -s ${prefix}gcc $out/bin/${prefix}cc + export named_cc=${prefix}gcc + export named_cxx=${prefix}g++ + elif [ -e $ccPath/clang ]; then + wrap ${prefix}clang ${./cc-wrapper.sh} $ccPath/clang + ln -s ${prefix}clang $out/bin/${prefix}cc + export named_cc=${prefix}clang + export named_cxx=${prefix}clang++ + fi + + if [ -e $ccPath/${prefix}g++ ]; then + wrap ${prefix}g++ ${./cc-wrapper.sh} $ccPath/${prefix}g++ + ln -s ${prefix}g++ $out/bin/${prefix}c++ + elif [ -e $ccPath/clang++ ]; then + wrap ${prefix}clang++ ${./cc-wrapper.sh} $ccPath/clang++ + ln -s ${prefix}clang++ $out/bin/${prefix}c++ + fi + + if [ -e $ccPath/cpp ]; then + wrap ${prefix}cpp ${./cc-wrapper.sh} $ccPath/cpp + fi + '' + + + optionalString cc.langFortran or false '' + wrap ${prefix}gfortran ${./cc-wrapper.sh} $ccPath/${prefix}gfortran + ln -sv ${prefix}gfortran $out/bin/${prefix}g77 + ln -sv ${prefix}gfortran $out/bin/${prefix}f77 + '' + + + optionalString cc.langJava or false '' + wrap ${prefix}gcj ${./cc-wrapper.sh} $ccPath/${prefix}gcj + '' + + + optionalString cc.langGo or false '' + wrap ${prefix}gccgo ${./cc-wrapper.sh} $ccPath/${prefix}gccgo + '' + + + optionalString cc.langAda or false '' + wrap ${prefix}gnatgcc ${./cc-wrapper.sh} $ccPath/${prefix}gnatgcc + wrap ${prefix}gnatmake ${./gnat-wrapper.sh} $ccPath/${prefix}gnatmake + wrap ${prefix}gnatbind ${./gnat-wrapper.sh} $ccPath/${prefix}gnatbind + wrap ${prefix}gnatlink ${./gnatlink-wrapper.sh} $ccPath/${prefix}gnatlink + '' + + + optionalString cc.langVhdl or false '' + ln -s $ccPath/${prefix}ghdl $out/bin/${prefix}ghdl + ''; + + propagatedBuildInputs = extraPackages; + + setupHook = ./setup-hook.sh; + + postFixup = + '' + set -u + '' + + + optionalString (libc != null) ('' + ## + ## General libc support + ## - + optionalString (!nativeLibc) '' # The "-B${libc_lib}/lib/" flag is a quick hack to force gcc to link # against the crt1.o from our own glibc, rather than the one in # /usr/lib. (This is only an issue when using an `impure' @@ -110,13 +267,50 @@ stdenv.mkDerivation { echo "${libc_lib}" > $out/nix-support/orig-libc echo "${libc_dev}" > $out/nix-support/orig-libc-dev - '' - + (if nativeTools then '' - ccPath="${if stdenv.isDarwin then cc else nativePrefix}/bin" - ldPath="${nativePrefix}/bin" + ## + ## Dynamic linker support + ## + + if [[ -z ''${dynamicLinker+x} ]]; then + echo "Don't know the name of the dynamic linker for platform '${targetPlatform.config}', so guessing instead." >&2 + local dynamicLinker="${libc_lib}/lib/ld*.so.?" + fi + + # Expand globs to fill array of options + dynamicLinker=($dynamicLinker) + + case ''${#dynamicLinker[@]} in + 0) echo "No dynamic linker found for platform '${targetPlatform.config}'." >&2;; + 1) echo "Using dynamic linker: '$dynamicLinker'" >&2;; + *) echo "Multiple dynamic linkers found for platform '${targetPlatform.config}'." >&2;; + esac + + if [ -n "$dynamicLinker" ]; then + echo $dynamicLinker > $out/nix-support/dynamic-linker + + '' + (if targetPlatform.isDarwin then '' + printf "export LD_DYLD_PATH=%q\n" "$dynamicLinker" >> $out/nix-support/setup-hook '' else '' - echo $cc > $out/nix-support/orig-cc + if [ -e ${libc_lib}/lib/32/ld-linux.so.2 ]; then + echo ${libc_lib}/lib/32/ld-linux.so.2 > $out/nix-support/dynamic-linker-m32 + fi + + local ldflagsBefore=(-dynamic-linker "$dynamicLinker") + '') + '' + fi + + # The dynamic linker is passed in `ldflagsBefore' to allow + # explicit overrides of the dynamic linker by callers to gcc/ld + # (the *last* value counts, so ours should come first). + printWords "''${ldflagsBefore[@]}" > $out/nix-support/libc-ldflags-before + '') + + + optionalString (!nativeTools) '' + + ## + ## Initial CFLAGS + ## # GCC shows ${cc_solib}/lib in `gcc -print-search-dirs', but not # ${cc_solib}/lib64 (even though it does actually search there...).. @@ -141,157 +335,56 @@ stdenv.mkDerivation { echo "$gnatCFlags" > $out/nix-support/gnat-cflags ''} - if [ -e $ccPath/clang ]; then - # Need files like crtbegin.o from gcc - # It's unclear if these will ever be provided by an LLVM project - ccCFlags="$ccCFlags -B$basePath" - ccCFlags="$ccCFlags -isystem$cc/lib/clang/$ccVersion/include" - fi - echo "$ccLDFlags" > $out/nix-support/cc-ldflags echo "$ccCFlags" > $out/nix-support/cc-cflags - ccPath="${cc}/bin" - ldPath="${binutils_bin}/bin" + ## + ## User env support + ## # Propagate the wrapped cc so that if you install the wrapper, # you get tools like gcov, the manpages, etc. as well (including # for binutils and Glibc). - echo ${cc} ${cc.man or ""} ${binutils_bin} ${libc_bin} > $out/nix-support/propagated-user-env-packages - - echo ${toString extraPackages} > $out/nix-support/propagated-native-build-inputs + printWords ${cc} ${binutils_bin} ${if libc == null then "" else libc_bin} > $out/nix-support/propagated-user-env-packages + printWords ${cc.man or ""} > $man/nix-support/propagated-user-env-packages '' - + optionalString (stdenv.isSunOS && nativePrefix != "") '' - # Solaris needs an additional ld wrapper. - ldPath="${nativePrefix}/bin" - exec="$ldPath/ld" - wrap ld-solaris ${./ld-solaris-wrapper.sh} - '') - + '' - # Create a symlink to as (the assembler). This is useful when a - # cc-wrapper is installed in a user environment, as it ensures that - # the right assembler is called. - if [ -e $ldPath/as ]; then - ln -s $ldPath/as $out/bin/as - fi - - wrap ld ${./ld-wrapper.sh} ''${ld:-$ldPath/ld} - - if [ -e ${binutils_bin}/bin/ld.gold ]; then - wrap ld.gold ${./ld-wrapper.sh} ${binutils_bin}/bin/ld.gold - fi - - if [ -e ${binutils_bin}/bin/ld.bfd ]; then - wrap ld.bfd ${./ld-wrapper.sh} ${binutils_bin}/bin/ld.bfd - fi - export real_cc=cc - export real_cxx=c++ - export default_cxx_stdlib_compile="${ - if stdenv.isLinux && !(cc.isGNU or false) - then "-isystem $(echo -n ${cc.gcc}/include/c++/*) -isystem $(echo -n ${cc.gcc}/include/c++/*)/$(${cc.gcc}/bin/gcc -dumpmachine)" - else "" - }" - - if [ -e $ccPath/gcc ]; then - wrap gcc ${./cc-wrapper.sh} $ccPath/gcc - ln -s gcc $out/bin/cc - export real_cc=gcc - export real_cxx=g++ - elif [ -e $ccPath/clang ]; then - wrap clang ${./cc-wrapper.sh} $ccPath/clang - ln -s clang $out/bin/cc - export real_cc=clang - export real_cxx=clang++ - fi + ## + ## Hardening support + ## - if [ -e $ccPath/g++ ]; then - wrap g++ ${./cc-wrapper.sh} $ccPath/g++ - ln -s g++ $out/bin/c++ - elif [ -e $ccPath/clang++ ]; then - wrap clang++ ${./cc-wrapper.sh} $ccPath/clang++ - ln -s clang++ $out/bin/c++ + # some linkers on some platforms don't support specific -z flags + export hardening_unsupported_flags="" + if [[ "$($ldPath/${prefix}ld -z now 2>&1 || true)" =~ un(recognized|known)\ option ]]; then + hardening_unsupported_flags+=" bindnow" fi - - if [ -e $ccPath/cpp ]; then - wrap cpp ${./cc-wrapper.sh} $ccPath/cpp + if [[ "$($ldPath/${prefix}ld -z relro 2>&1 || true)" =~ un(recognized|known)\ option ]]; then + hardening_unsupported_flags+=" relro" fi '' - + optionalString cc.langFortran or false '' - wrap gfortran ${./cc-wrapper.sh} $ccPath/gfortran - ln -sv gfortran $out/bin/g77 - ln -sv gfortran $out/bin/f77 - '' - - + optionalString cc.langJava or false '' - wrap gcj ${./cc-wrapper.sh} $ccPath/gcj - '' - - + optionalString cc.langGo or false '' - wrap gccgo ${./cc-wrapper.sh} $ccPath/gccgo - '' - - + optionalString cc.langAda or false '' - wrap gnatgcc ${./cc-wrapper.sh} $ccPath/gnatgcc - wrap gnatmake ${./gnat-wrapper.sh} $ccPath/gnatmake - wrap gnatbind ${./gnat-wrapper.sh} $ccPath/gnatbind - wrap gnatlink ${./gnatlink-wrapper.sh} $ccPath/gnatlink - '' - - + optionalString cc.langVhdl or false '' - ln -s $ccPath/ghdl $out/bin/ghdl + + optionalString hostPlatform.isCygwin '' + hardening_unsupported_flags+=" pic" '' + '' - substituteAll ${./setup-hook.sh} $out/nix-support/setup-hook.tmp - cat $out/nix-support/setup-hook.tmp >> $out/nix-support/setup-hook - rm $out/nix-support/setup-hook.tmp - - # some linkers on some platforms don't support specific -z flags - hardening_unsupported_flags="" - if [[ "$($ldPath/ld -z now 2>&1 || true)" =~ un(recognized|known)\ option ]]; then - hardening_unsupported_flags+=" bindnow" - fi - if [[ "$($ldPath/ld -z relro 2>&1 || true)" =~ un(recognized|known)\ option ]]; then - hardening_unsupported_flags+=" relro" - fi - substituteAll ${./add-flags.sh} $out/nix-support/add-flags.sh substituteAll ${./add-hardening.sh} $out/nix-support/add-hardening.sh - cp -p ${./utils.sh} $out/nix-support/utils.sh + substituteAll ${./utils.sh} $out/nix-support/utils.sh + + ## + ## Extra custom steps + ## + '' + extraBuildCommands; - # The dynamic linker has different names on different Linux platforms. - dynamicLinker = - if !nativeLibc then - (if stdenv.system == "i686-linux" then "ld-linux.so.2" else - if stdenv.system == "x86_64-linux" then "ld-linux-x86-64.so.2" else - # ARM with a wildcard, which can be "" or "-armhf". - if stdenv.isArm then "ld-linux*.so.3" else - if stdenv.system == "aarch64-linux" then "ld-linux-aarch64.so.1" else - if stdenv.system == "powerpc-linux" then "ld.so.1" else - if stdenv.system == "mips64el-linux" then "ld.so.1" else - if stdenv.system == "x86_64-darwin" then "/usr/lib/dyld" else - abort "Don't know the name of the dynamic linker for this platform.") - else ""; + inherit dynamicLinker expand-response-params; - crossAttrs = { - shell = shell.crossDrv + shell.crossDrv.shellPath; - libc = stdenv.ccCross.libc; - # - # This is not the best way to do this. I think the reference should be - # the style in the gcc-cross-wrapper, but to keep a stable stdenv now I - # do this sufficient if/else. - dynamicLinker = - (if stdenv.cross.arch == "arm" then "ld-linux.so.3" else - if stdenv.cross.arch == "mips" then "ld.so.1" else - if stdenv.lib.hasSuffix "pc-gnu" stdenv.cross.config then "ld.so.1" else - abort "don't know the name of the dynamic linker for this platform"); - }; + # for substitution in utils.sh + expandResponseParams = "${expand-response-params}/bin/expand-response-params"; meta = let cc_ = if cc != null then cc else {}; in @@ -299,5 +392,7 @@ stdenv.mkDerivation { { description = stdenv.lib.attrByPath ["meta" "description"] "System C compiler" cc_ + " (wrapper script)"; - }; + } // optionalAttrs useMacosReexportHack { + platforms = stdenv.lib.platforms.darwin; + }; } diff --git a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh index 0d74527dd8ad..b15f1e461e5f 100644 --- a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh @@ -1,14 +1,23 @@ -#! @shell@ -e -path_backup="$PATH" -if [ -n "@coreutils_bin@" ]; then - PATH="@coreutils_bin@/bin" +#! @shell@ +set -eu -o pipefail +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x fi -if [ -n "$NIX_GNAT_WRAPPER_START_HOOK" ]; then - source "$NIX_GNAT_WRAPPER_START_HOOK" +# N.B. Gnat is not used during bootstrapping, so we don't need to +# worry about the old bash empty array `set -u` workarounds. + +path_backup="$PATH" + +# phase separation makes this look useless +# shellcheck disable=SC2157 +if [ -n "@coreutils_bin@" ]; then + PATH="@coreutils_bin@/bin" fi -if [ -z "$NIX_GNAT_WRAPPER_FLAGS_SET" ]; then +if [ -z "${NIX_@infixSalt@_GNAT_WRAPPER_FLAGS_SET:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -18,7 +27,6 @@ source @out@/nix-support/utils.sh # Figure out if linker flags should be passed. GCC prints annoying # warnings when they are not needed. dontLink=0 -getVersion=0 nonFlagArgs=0 for i in "$@"; do @@ -30,7 +38,7 @@ for i in "$@"; do nonFlagArgs=1 elif [ "$i" = -m32 ]; then if [ -e @out@/nix-support/dynamic-linker-m32 ]; then - NIX_LDFLAGS="$NIX_LDFLAGS -dynamic-linker $(cat @out@/nix-support/dynamic-linker-m32)" + NIX_@infixSalt@_LDFLAGS+=" -dynamic-linker $(< @out@/nix-support/dynamic-linker-m32)" fi fi done @@ -47,37 +55,33 @@ fi # Optionally filter out paths not refering to the store. params=("$@") -if [ "$NIX_ENFORCE_PURITY" = 1 -a -n "$NIX_STORE" ]; then +if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "$NIX_STORE" ]]; then rest=() - n=0 - while [ $n -lt ${#params[*]} ]; do - p=${params[n]} - p2=${params[$((n+1))]} + for p in "${params[@]}"; do if [ "${p:0:3}" = -L/ ] && badPath "${p:2}"; then - skip $p + skip "${p:2}" elif [ "${p:0:3}" = -I/ ] && badPath "${p:2}"; then - skip $p + skip "${p:2}" elif [ "${p:0:4}" = -aI/ ] && badPath "${p:3}"; then - skip $p + skip "${p:2}" elif [ "${p:0:4}" = -aO/ ] && badPath "${p:3}"; then - skip $p + skip "${p:2}" else rest+=("$p") fi - n=$((n + 1)) done params=("${rest[@]}") fi # Clear march/mtune=native -- they bring impurity. -if [ "$NIX_ENFORCE_NO_NATIVE" = 1 ]; then +if [ "$NIX_@infixSalt@_ENFORCE_NO_NATIVE" = 1 ]; then rest=() - for i in "${params[@]}"; do - if [[ "$i" = -m*=native ]]; then - skip $i + for p in "${params[@]}"; do + if [[ "$p" = -m*=native ]]; then + skip "$p" else - rest+=("$i") + rest+=("$p") fi done params=("${rest[@]}") @@ -85,38 +89,38 @@ fi # Add the flags for the GNAT compiler proper. -extraAfter=($NIX_GNATFLAGS_COMPILE) +extraAfter=($NIX_@infixSalt@_GNATFLAGS_COMPILE) extraBefore=() -if [ "`basename $0`x" = "gnatmakex" ]; then - extraBefore=("--GNATBIND=@out@/bin/gnatbind --GNATLINK=@out@/bin/gnatlink ") +if [ "$(basename "$0")x" = "gnatmakex" ]; then + extraBefore=("--GNATBIND=@out@/bin/gnatbind" "--GNATLINK=@out@/bin/gnatlink ") fi -# Add the flags that should be passed to the linker (and prevent -# `ld-wrapper' from adding NIX_LDFLAGS again). -#for i in $NIX_LDFLAGS_BEFORE; do -# extraBefore=(${extraBefore[@]} "-largs $i") -#done +#if [ "$dontLink" != 1 ]; then +# # Add the flags that should be passed to the linker (and prevent +# # `ld-wrapper' from adding NIX_@infixSalt@_LDFLAGS again). +# for i in $NIX_@infixSalt@_LDFLAGS_BEFORE; do +# extraBefore+=("-largs" "$i") +# done +# for i in $NIX_@infixSalt@_LDFLAGS; do +# if [ "${i:0:3}" = -L/ ]; then +# extraAfter+=("$i") +# else +# extraAfter+=("-largs" "$i") +# fi +# done +# export NIX_@infixSalt@_LDFLAGS_SET=1 +#fi # Optionally print debug info. -if [ -n "$NIX_DEBUG" ]; then - echo "original flags to @prog@:" >&2 - for i in "${params[@]}"; do - echo " $i" >&2 - done - echo "extraBefore flags to @prog@:" >&2 - for i in ${extraBefore[@]}; do - echo " $i" >&2 - done - echo "extraAfter flags to @prog@:" >&2 - for i in ${extraAfter[@]}; do - echo " $i" >&2 - done -fi - -if [ -n "$NIX_GNAT_WRAPPER_EXEC_HOOK" ]; then - source "$NIX_GNAT_WRAPPER_EXEC_HOOK" +if (( "${NIX_DEBUG:-0}" >= 1 )); then + echo "extra flags before to @prog@:" >&2 + printf " %q\n" "${extraBefore[@]}" >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" "${params[@]}" >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" "${extraAfter[@]}" >&2 fi PATH="$path_backup" -exec @prog@ ${extraBefore[@]} "${params[@]}" ${extraAfter[@]} +exec @prog@ "${extraBefore[@]}" "${params[@]}" "${extraAfter[@]}" diff --git a/pkgs/build-support/cc-wrapper/gnatlink-wrapper.sh b/pkgs/build-support/cc-wrapper/gnatlink-wrapper.sh index c9958dbbb413..88e644dc54dc 100644 --- a/pkgs/build-support/cc-wrapper/gnatlink-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/gnatlink-wrapper.sh @@ -1,33 +1,40 @@ -#! @shell@ -e +#! @shell@ +set -eu -o pipefail +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x +fi + +# N.B. Gnat is not used during bootstrapping, so we don't need to +# worry about the old bash empty array `set -u` workarounds. # Add the flags for the GNAT compiler proper. -extraAfter="--GCC=@out@/bin/gcc" +extraAfter=("--GCC=@out@/bin/gcc") extraBefore=() -# Add the flags that should be passed to the linker (and prevent -# `ld-wrapper' from adding NIX_LDFLAGS again). -#for i in $NIX_LDFLAGS_BEFORE; do -# extraBefore=(${extraBefore[@]} "-largs $i") +## Add the flags that should be passed to the linker (and prevent +## `ld-wrapper' from adding NIX_@infixSalt@_LDFLAGS again). +#for i in $NIX_@infixSalt@_LDFLAGS_BEFORE; do +# extraBefore+=("-largs" "$i") #done +#for i in $NIX_@infixSalt@_LDFLAGS; do +# if [ "${i:0:3}" = -L/ ]; then +# extraAfter+=("$i") +# else +# extraAfter+=("-largs" "$i") +# fi +#done +#export NIX_@infixSalt@_LDFLAGS_SET=1 # Optionally print debug info. -if [ -n "$NIX_DEBUG" ]; then - echo "original flags to @prog@:" >&2 - for i in "$@"; do - echo " $i" >&2 - done - echo "extraBefore flags to @prog@:" >&2 - for i in ${extraBefore[@]}; do - echo " $i" >&2 - done - echo "extraAfter flags to @prog@:" >&2 - for i in ${extraAfter[@]}; do - echo " $i" >&2 - done -fi - -if [ -n "$NIX_GNAT_WRAPPER_EXEC_HOOK" ]; then - source "$NIX_GNAT_WRAPPER_EXEC_HOOK" +if (( "${NIX_DEBUG:-0}" >= 1 )); then + echo "extra flags before to @prog@:" >&2 + printf " %q\n" "${extraBefore[@]}" >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" "$@" >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" "${extraAfter[@]}" >&2 fi -exec @prog@ ${extraBefore[@]} "$@" ${extraAfter[@]} +exec @prog@ "${extraBefore[@]}" "$@" "${extraAfter[@]}" diff --git a/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh b/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh index 263ea5408e9a..5d81e34a047f 100755 --- a/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh @@ -1,40 +1,29 @@ #!@shell@ +set -eu -o pipefail +shopt -s nullglob -set -e -set -u +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x +fi +declare -a args=("$@") # I've also tried adding -z direct and -z lazyload, but it gave too many problems with C++ exceptions :'( # Also made sure libgcc would not be lazy-loaded, as suggested here: https://www.illumos.org/issues/2534#note-3 # but still no success. -cmd="@ld@ -z ignore" - -args=("$@"); +declare -a argsBefore=(-z ignore) argsAfter=() # This loop makes sure all -L arguments are before -l arguments, or ld may complain it cannot find a library. # GNU binutils does not have this problem: # http://stackoverflow.com/questions/5817269/does-the-order-of-l-and-l-options-in-the-gnu-linker-matter -i=0; -while [[ $i -lt $# ]]; do - case "${args[$i]}" in - -L) cmd="$cmd ${args[$i]} ${args[($i+1)]}"; i=($i+1); ;; - -L*) cmd="$cmd ${args[$i]}" ;; - *) ;; - esac - i=($i+1); -done - -i=0; -while [[ $i -lt $# ]]; do +while (( $# )); do case "${args[$i]}" in - -L) i=($i+1); ;; - -L*) ;; - *) cmd="$cmd ${args[$i]}" ;; + -L) argsBefore+=("$1" "$2"); shift ;; + -L?*) argsBefore+=("$1") ;; + *) argsAfter+=("$1") ;; esac - i=($i+1); + shift done # Trace: set -x -exec $cmd - -exit 0 +exec "@ld@" "${argsBefore[@]}" "${argsAfter[@]}" diff --git a/pkgs/build-support/cc-wrapper/ld-wrapper.sh b/pkgs/build-support/cc-wrapper/ld-wrapper.sh index 056cfa920535..232e1245af07 100644 --- a/pkgs/build-support/cc-wrapper/ld-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/ld-wrapper.sh @@ -1,14 +1,20 @@ -#! @shell@ -e -path_backup="$PATH" -if [ -n "@coreutils_bin@" ]; then - PATH="@coreutils_bin@/bin" +#! @shell@ +set -eu -o pipefail +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x fi -if [ -n "$NIX_LD_WRAPPER_START_HOOK" ]; then - source "$NIX_LD_WRAPPER_START_HOOK" +path_backup="$PATH" + +# phase separation makes this look useless +# shellcheck disable=SC2157 +if [ -n "@coreutils_bin@" ]; then + PATH="@coreutils_bin@/bin" fi -if [ -z "$NIX_CC_WRAPPER_FLAGS_SET" ]; then +if [ -z "${NIX_CC_WRAPPER_@infixSalt@_FLAGS_SET:-}" ]; then source @out@/nix-support/add-flags.sh fi @@ -17,21 +23,22 @@ source @out@/nix-support/utils.sh # Optionally filter out paths not refering to the store. expandResponseParams "$@" -if [ "$NIX_ENFORCE_PURITY" = 1 -a -n "$NIX_STORE" \ - -a \( -z "$NIX_IGNORE_LD_THROUGH_GCC" -o -z "$NIX_LDFLAGS_SET" \) ]; then +if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "${NIX_STORE:-}" + && ( -z "$NIX_@infixSalt@_IGNORE_LD_THROUGH_GCC" || -z "${NIX_@infixSalt@_LDFLAGS_SET:-}" ) ]]; then rest=() - n=0 - while [ $n -lt ${#params[*]} ]; do + nParams=${#params[@]} + declare -i n=0 + while (( "$n" < "$nParams" )); do p=${params[n]} - p2=${params[$((n+1))]} + p2=${params[n+1]:-} # handle `p` being last one if [ "${p:0:3}" = -L/ ] && badPath "${p:2}"; then - skip $p + skip "${p:2}" elif [ "$p" = -L ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" elif [ "$p" = -rpath ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" elif [ "$p" = -dynamic-linker ] && badPath "$p2"; then - n=$((n + 1)); skip $p2 + n+=1; skip "$p2" elif [ "${p:0:1}" = / ] && badPath "$p"; then # We cannot skip this; barf. echo "impure path \`$p' used in link" >&2 @@ -40,149 +47,159 @@ if [ "$NIX_ENFORCE_PURITY" = 1 -a -n "$NIX_STORE" \ # Our ld is not built with sysroot support (Can we fix that?) : else - rest=("${rest[@]}" "$p") + rest+=("$p") fi - n=$((n + 1)) + n+=1 done - params=("${rest[@]}") + # Old bash empty array hack + params=(${rest+"${rest[@]}"}) fi -LD=@prog@ source @out@/nix-support/add-hardening.sh -extra=(${hardeningLDFlags[@]}) +extraAfter=("${hardeningLDFlags[@]}") extraBefore=() -if [ -z "$NIX_LDFLAGS_SET" ]; then - extra+=($NIX_LDFLAGS) - extraBefore+=($NIX_LDFLAGS_BEFORE) +if [ -z "${NIX_@infixSalt@_LDFLAGS_SET:-}" ]; then + extraAfter+=($NIX_@infixSalt@_LDFLAGS) + extraBefore+=($NIX_@infixSalt@_LDFLAGS_BEFORE) fi -extra+=($NIX_LDFLAGS_AFTER $NIX_LDFLAGS_HARDEN) - - -# Add all used dynamic libraries to the rpath. -if [ "$NIX_DONT_SET_RPATH" != 1 ]; then - - libPath="" - addToLibPath() { - local path="$1" - if [ "${path:0:1}" != / ]; then return 0; fi - case "$path" in - *..*|*./*|*/.*|*//*) - local path2 - if path2=$(readlink -f "$path"); then - path="$path2" - fi +extraAfter+=($NIX_@infixSalt@_LDFLAGS_AFTER) + +# Three tasks: +# +# 1. Find all -L... switches for rpath +# +# 2. Find relocatable flag for build id. +# +# 3. Choose 32-bit dynamic linker if needed +declare -a libDirs +declare -A libs +declare -i relocatable=0 link32=0 + +if + [ "$NIX_@infixSalt@_DONT_SET_RPATH" != 1 ] \ + || [ "$NIX_@infixSalt@_SET_BUILD_ID" = 1 ] \ + || [ -e @out@/nix-support/dynamic-linker-m32 ] +then + prev= + # Old bash thinks empty arrays are undefined, ugh. + for p in \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} + do + case "$prev" in + -L) + libDirs+=("$p") + ;; + -l) + libs["lib${p}.so"]=1 + ;; + -m) + # Presumably only the last `-m` flag has any effect. + case "$p" in + elf_i386) link32=1;; + *) link32=0;; + esac + ;; + -dynamic-linker | -plugin) + # Ignore this argument, or it will match *.so and be added to rpath. + ;; + *) + case "$p" in + -L/*) + libDirs+=("${p:2}") + ;; + -l?*) + libs["lib${p:2}.so"]=1 + ;; + "${NIX_STORE:-}"/*.so | "${NIX_STORE:-}"/*.so.*) + # This is a direct reference to a shared library. + libDirs+=("${p%/*}") + libs["${p##*/}"]=1 + ;; + -r | --relocatable | -i) + relocatable=1 + esac ;; esac - case $libPath in - *\ $path\ *) return 0 ;; - esac - libPath="$libPath $path " - } - - addToRPath() { - # If the path is not in the store, don't add it to the rpath. - # This typically happens for libraries in /tmp that are later - # copied to $out/lib. If not, we're screwed. - if [ "${1:0:${#NIX_STORE}}" != "$NIX_STORE" ]; then return 0; fi - case $rpath in - *\ $1\ *) return 0 ;; - esac - rpath="$rpath $1 " - } - - libs="" - addToLibs() { - libs="$libs $1" - } - - rpath="" - - # First, find all -L... switches. - allParams=("${params[@]}" ${extra[@]}) - n=0 - while [ $n -lt ${#allParams[*]} ]; do - p=${allParams[n]} - p2=${allParams[$((n+1))]} - if [ "${p:0:3}" = -L/ ]; then - addToLibPath ${p:2} - elif [ "$p" = -L ]; then - addToLibPath ${p2} - n=$((n + 1)) - elif [ "$p" = -l ]; then - addToLibs ${p2} - n=$((n + 1)) - elif [ "${p:0:2}" = -l ]; then - addToLibs ${p:2} - elif [ "$p" = -dynamic-linker ]; then - # Ignore the dynamic linker argument, or it - # will get into the next 'elif'. We don't want - # the dynamic linker path rpath to go always first. - n=$((n + 1)) - elif [[ "$p" =~ ^[^-].*\.so($|\.) ]]; then - # This is a direct reference to a shared library, so add - # its directory to the rpath. - path="$(dirname "$p")"; - addToRPath "${path}" - fi - n=$((n + 1)) + prev="$p" done +fi + +if [ -e "@out@/nix-support/dynamic-linker-m32" ] && (( "$link32" )); then + # We have an alternate 32-bit linker and we're producing a 32-bit ELF, let's + # use it. + extraAfter+=( + '-dynamic-linker' + "$(< @out@/nix-support/dynamic-linker-m32)" + ) +fi - # Second, for each directory in the library search path (-L...), +# Add all used dynamic libraries to the rpath. +if [ "$NIX_@infixSalt@_DONT_SET_RPATH" != 1 ]; then + # For each directory in the library search path (-L...), # see if it contains a dynamic library used by a -l... flag. If # so, add the directory to the rpath. # It's important to add the rpath in the order of -L..., so # the link time chosen objects will be those of runtime linking. - - for i in $libPath; do - for j in $libs; do - if [ -f "$i/lib$j.so" ]; then - addToRPath $i + declare -A rpaths + for dir in ${libDirs+"${libDirs[@]}"}; do + if [[ "$dir" =~ [/.][/.] ]] && dir2=$(readlink -f "$dir"); then + dir="$dir2" + fi + if [ -n "${rpaths[$dir]:-}" ] || [[ "$dir" != "${NIX_STORE:-}"/* ]]; then + # If the path is not in the store, don't add it to the rpath. + # This typically happens for libraries in /tmp that are later + # copied to $out/lib. If not, we're screwed. + continue + fi + for path in "$dir"/*; do + file="${path##*/}" + if [ "${libs[$file]:-}" ]; then + # This library may have been provided by a previous directory, + # but if that library file is inside an output of the current + # derivation, it can be deleted after this compilation and + # should be found in a later directory, so we add all + # directories that contain any of the libraries to rpath. + rpaths["$dir"]=1 + extraAfter+=(-rpath "$dir") break fi done done - - # Finally, add `-rpath' switches. - for i in $rpath; do - extra+=(-rpath $i) - done fi +# This is outside the DONT_SET_RPATH branch because it's more targeted and we +# usually want it (on Darwin) even if DONT_SET_RPATH is set. +if [ -n "${NIX_COREFOUNDATION_RPATH:-}" ]; then + extraAfter+=(-rpath $NIX_COREFOUNDATION_RPATH) +fi # Only add --build-id if this is a final link. FIXME: should build gcc # with --enable-linker-build-id instead? -if [ "$NIX_SET_BUILD_ID" = 1 ]; then - for p in "${params[@]}"; do - if [ "$p" = "-r" -o "$p" = "--relocatable" -o "$p" = "-i" ]; then - relocatable=1 - break - fi - done - if [ -z "$relocatable" ]; then - extra+=(--build-id) - fi +if [ "$NIX_@infixSalt@_SET_BUILD_ID" = 1 ] && ! (( "$relocatable" )); then + extraAfter+=(--build-id) fi # Optionally print debug info. -if [ -n "$NIX_DEBUG" ]; then - echo "original flags to @prog@:" >&2 - for i in "${params[@]}"; do - echo " $i" >&2 - done - echo "extra flags to @prog@:" >&2 - for i in ${extra[@]}; do - echo " $i" >&2 - done -fi - -if [ -n "$NIX_LD_WRAPPER_EXEC_HOOK" ]; then - source "$NIX_LD_WRAPPER_EXEC_HOOK" +if (( "${NIX_DEBUG:-0}" >= 1 )); then + # Old bash workaround, see above. + echo "extra flags before to @prog@:" >&2 + printf " %q\n" ${extraBefore+"${extraBefore[@]}"} >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" ${params+"${params[@]}"} >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" ${extraAfter+"${extraAfter[@]}"} >&2 fi PATH="$path_backup" -exec @prog@ ${extraBefore[@]} "${params[@]}" ${extra[@]} +# Old bash workaround, see above. +exec @prog@ \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} diff --git a/pkgs/build-support/cc-wrapper/macos-sierra-reexport-hack.bash b/pkgs/build-support/cc-wrapper/macos-sierra-reexport-hack.bash new file mode 100644 index 000000000000..b7aa7ea5c092 --- /dev/null +++ b/pkgs/build-support/cc-wrapper/macos-sierra-reexport-hack.bash @@ -0,0 +1,106 @@ +#! @shell@ + +set -eu -o pipefail + +path_backup="$PATH" +if [ -n "@coreutils_bin@" ]; then + PATH="@coreutils_bin@/bin" +fi + +declare -r recurThreshold=300 + +declare overflowCount=0 +for ((n=0; n < $#; ++n)); do + case "${!n}" in + -l*) let overflowCount+=1 ;; + -reexport-l*) let overflowCount+=1 ;; + *) ;; + esac +done + +declare -a allArgs=() + +if (( "$overflowCount" <= "$recurThreshold" )); then + allArgs=("$@") +else + declare -a childrenLookup=() childrenLink=() + + while (( $# )); do + case "$1" in + -L/*) + childrenLookup+=("$1") + allArgs+=("$1") + ;; + -L) + echo "cctools LD does not support '-L foo' or '-l foo'" >&2 + exit 1 + ;; + -l) + echo "cctools LD does not support '-L foo' or '-l foo'" >&2 + exit 1 + ;; + -lazy_library | -lazy_framework | -lto_library) + # We aren't linking any "azy_library", "to_library", etc. + allArgs+=("$1") + ;; + -lazy-l | -weak-l) allArgs+=("$1") ;; + # We can't so easily prevent header issues from these. + -lSystem) allArgs+=("$1") ;; + # Special case as indirection seems like a bad idea for something + # so fundamental. Can be removed for simplicity. + -l?* | -reexport-l?*) childrenLink+=("$1") ;; + *) allArgs+=("$1") ;; + esac + + shift + done + + declare n=0 + while (( $n < "${#childrenLink[@]}" )); do + if [[ "${childrenLink[n]}" = -l* ]]; then + childrenLink[n]="-reexport${childrenLink[n]}" + fi + let ++n + done + unset n + + declare -r outputNameLibless=$(basename $( \ + if [[ -z "${outputName:+isUndefined}" ]]; then + echo unnamed + elif [[ "${outputName:0:3}" = lib ]]; then + echo "${outputName:3}" + else + echo "${outputName}" + fi)) + declare -ra children=("$outputNameLibless-reexport-delegate-0" \ + "$outputNameLibless-reexport-delegate-1") + + mkdir -p "$out/lib" + + PATH="$PATH:@out@/bin" + + symbolBloatObject=$outputNameLibless-symbol-hack.o + if [[ ! -e $symbolBloatObject ]]; then + printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' \ + | @binPrefix@as -- -o $symbolBloatObject + fi + + # first half of libs + @binPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[0]}.dylib" \ + -install_name "$out/lib/lib${children[0]}.dylib" \ + "${childrenLookup[@]}" "$symbolBloatObject" \ + "${childrenLink[@]:0:$((${#childrenLink[@]} / 2 ))}" + + # second half of libs + @binPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[1]}.dylib" \ + -install_name "$out/lib/lib${children[1]}.dylib" \ + "${childrenLookup[@]}" "$symbolBloatObject" \ + "${childrenLink[@]:$((${#childrenLink[@]} / 2 ))}" + + allArgs+=("-L$out/lib" "-l${children[0]}" "-l${children[1]}") +fi + +PATH="$path_backup" +exec @prog@ "${allArgs[@]}" diff --git a/pkgs/build-support/cc-wrapper/setup-hook.sh b/pkgs/build-support/cc-wrapper/setup-hook.sh index f4f7ab181d3e..e43c1609edb1 100644 --- a/pkgs/build-support/cc-wrapper/setup-hook.sh +++ b/pkgs/build-support/cc-wrapper/setup-hook.sh @@ -1,47 +1,152 @@ -export NIX_CC=@out@ +# CC Wrapper hygiene +# +# For at least cross compilation, we need to depend on multiple cc-wrappers at +# once---specifically up to one per sort of dependency. This follows from having +# different tools targeting different platforms, and different flags for those +# tools. For example: +# +# # Flags for compiling (whether or not linking) C code for the... +# NIX_BUILD_CFLAGS_COMPILE # ...build platform +# NIX_CFLAGS_COMPILE # ...host platform +# NIX_TARGET_CFLAGS_COMPILE # ...target platform +# +# Notice that these platforms are the 3 *relative* to the package using +# cc-wrapper, not absolute like `x86_64-pc-linux-gnu`. +# +# The simplest solution would be to have separate cc-wrappers per (3 intended +# use-cases * n absolute concrete platforms). For the use-case axis, we would +# @-splice in 'BUILD_' '' 'TARGET_' to use the write environment variables when +# building the cc-wrapper, and likewise prefix the binaries' names so they didn't +# clobber each other on the PATH. But the need for 3x cc-wrappers, along with +# non-standard name prefixes, is annoying and liable to break packages' build +# systems. +# +# Instead, we opt to have just one cc-wrapper per absolute platform. Matching +# convention, the binaries' names can just be prefixed with their target +# platform. On the other hand, that means packages will depend on not just +# multiple cc-wrappers, but the exact same cc-wrapper derivation multiple ways. +# That means the exact same cc-wrapper derivation must be able to avoid +# conflicting with itself, despite the fact that `setup-hook.sh`, the `addCvars` +# function, and `add-flags.sh` are all communicating with each other with +# environment variables. Yuck. +# +# The basic strategy is: +# +# - Everyone exclusively *adds information* to relative-platform-specific +# environment variables, like `NIX_TARGET_CFLAGS_COMPILE`, to communicate +# with the wrapped binaries. +# +# - The wrapped binaries will exclusively *read* cc-wrapper-derivation-specific +# environment variables distinguished with with `infixSalt`, like +# `NIX_@infixSalt@_CFLAGS_COMPILE`. +# +# - `add-flags`, beyond its old task of reading extra flags stuck inside the +# cc-wrapper derivation, will convert the relative-platform-specific +# variables to cc-wrapper-derivation-specific variables. This conversion is +# the only time all but one of the cc-wrapper-derivation-specific variables +# are set. +# +# This ensures the flow of information is exclusive from +# relative-platform-specific variables to cc-wrapper-derivation-specific +# variables. This allows us to support the general case of a many--many relation +# between relative platforms and cc-wrapper derivations. +# +# For more details, read the individual files where the mechanisms used to +# accomplish this will be individually documented. -addCVars () { - if [ -d $1/include ]; then - export NIX_CFLAGS_COMPILE+=" ${ccIncludeFlag:--isystem} $1/include" + +# It's fine that any other cc-wrapper will redefine this. Bash functions close +# over no state, and there's no @-substitutions within, so any redefined +# function is guaranteed to be exactly the same. +ccWrapper_addCVars () { + # The `depOffset` describes how the platforms of the dependencies are slid + # relative to the depending package. It is brought into scope of the + # environment hook defined as the role of the dependency being applied. + case $depOffset in + -1) local role='BUILD_' ;; + 0) local role='' ;; + 1) local role='TARGET_' ;; + *) echo "cc-wrapper: Error: Cannot be used with $depOffset-offset deps, " >2; + return 1 ;; + esac + + if [[ -d "$1/include" ]]; then + export NIX_${role}CFLAGS_COMPILE+=" ${ccIncludeFlag:--isystem} $1/include" fi - if [ -d $1/lib64 -a ! -L $1/lib64 ]; then - export NIX_LDFLAGS+=" -L$1/lib64" + if [[ -d "$1/lib64" && ! -L "$1/lib64" ]]; then + export NIX_${role}LDFLAGS+=" -L$1/lib64" fi - if [ -d $1/lib ]; then - export NIX_LDFLAGS+=" -L$1/lib" + if [[ -d "$1/lib" ]]; then + export NIX_${role}LDFLAGS+=" -L$1/lib" fi - if test -d $1/Library/Frameworks; then - export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -F$1/Library/Frameworks" + if [[ -d "$1/Library/Frameworks" ]]; then + export NIX_${role}CFLAGS_COMPILE+=" -F$1/Library/Frameworks" fi } -envHooks+=(addCVars) +# Since the same cc-wrapper derivation can be depend on in multiple ways, we +# need to accumulate *each* role (i.e. target platform relative the depending +# derivation) in which the cc-wrapper derivation is used. +# `NIX_CC_WRAPPER_@infixSalt@_TARGET_*` tracks this (needs to be an exported env +# var so can't use fancier data structures). +# +# We also need to worry about what role is being added on *this* invocation of +# setup-hook, which `role` tracks. +if [ -n "${crossConfig:-}" ]; then + export NIX_CC_WRAPPER_@infixSalt@_TARGET_BUILD=1 + role="BUILD_" +else + export NIX_CC_WRAPPER_@infixSalt@_TARGET_HOST=1 + role="" +fi + +# Eventually the exact sort of env-hook we create will depend on the role. This +# is because based on what relative platform we are targeting, we use different +# dependencies. +envHooks+=(ccWrapper_addCVars) -# Note: these come *after* $out in the PATH (see setup.sh). +# Note 1: these come *after* $out in the PATH (see setup.sh). +# Note 2: phase separation makes this look useless to shellcheck. +# shellcheck disable=SC2157 if [ -n "@cc@" ]; then addToSearchPath _PATH @cc@/bin fi +# shellcheck disable=SC2157 if [ -n "@binutils_bin@" ]; then addToSearchPath _PATH @binutils_bin@/bin fi +# shellcheck disable=SC2157 if [ -n "@libc_bin@" ]; then addToSearchPath _PATH @libc_bin@/bin fi +# shellcheck disable=SC2157 if [ -n "@coreutils_bin@" ]; then addToSearchPath _PATH @coreutils_bin@/bin fi -if [ -z "$crossConfig" ]; then - export CC=@real_cc@ - export CXX=@real_cxx@ -else - export BUILD_CC=@real_cc@ - export BUILD_CXX=@real_cxx@ -fi +# Export tool environment variables so various build systems use the right ones. + +export NIX_${role}CC=@out@ + +export ${role}CC=@named_cc@ +export ${role}CXX=@named_cxx@ + +for CMD in \ + ar as nm objcopy ranlib strip strings size ld windres +do + if + PATH=$_PATH type -p "@binPrefix@$CMD" > /dev/null + then + export "${role}$(echo "$CMD" | tr "[:lower:]" "[:upper:]")=@binPrefix@${CMD}"; + fi +done + +# No local scope in sourced file +unset role diff --git a/pkgs/build-support/cc-wrapper/utils.sh b/pkgs/build-support/cc-wrapper/utils.sh index aba5f3295a98..c43c2e12d74a 100644 --- a/pkgs/build-support/cc-wrapper/utils.sh +++ b/pkgs/build-support/cc-wrapper/utils.sh @@ -1,5 +1,5 @@ skip () { - if [ -n "$NIX_DEBUG" ]; then + if (( "${NIX_DEBUG:-0}" >= 1 )); then echo "skipping impure path $1" >&2 fi } @@ -24,25 +24,21 @@ badPath() { } expandResponseParams() { - local inparams=("$@") - local n=0 - local p - params=() - while [ $n -lt ${#inparams[*]} ]; do - p=${inparams[n]} - case $p in - @*) - if [ -e "${p:1}" ]; then - args=$(<"${p:1}") - eval 'for arg in '${args//$/\\$}'; do params+=("$arg"); done' - else - params+=("$p") - fi - ;; - *) - params+=("$p") - ;; - esac - n=$((n + 1)) + declare -ga params=("$@") + local arg + for arg in "$@"; do + if [[ "$arg" == @* ]]; then + # phase separation makes this look useless + # shellcheck disable=SC2157 + if [ -x "@expandResponseParams@" ]; then + # params is used by caller + #shellcheck disable=SC2034 + readarray -d '' params < <("@expandResponseParams@" "$@") + return 0 + else + echo "Response files aren't supported during bootstrapping" >&2 + return 1 + fi + fi done } diff --git a/pkgs/build-support/closure-info.nix b/pkgs/build-support/closure-info.nix new file mode 100644 index 000000000000..4d178ac96c5d --- /dev/null +++ b/pkgs/build-support/closure-info.nix @@ -0,0 +1,58 @@ +# This derivation builds two files containing information about the +# closure of 'rootPaths': $out/store-paths contains the paths in the +# closure, and $out/registration contains a file suitable for use with +# "nix-store --load-db" and "nix-store --register-validity +# --hash-given". + +{ stdenv, coreutils, jq, perl, pathsFromGraph }: + +{ rootPaths }: + +#if builtins.langVersion >= 5 then +# FIXME: it doesn't work on Hydra, failing to find mkdir; +# perhaps .attrs.sh clobbers PATH with new nix? +if false then + + # Nix >= 1.12: Include NAR hash / size info. + + stdenv.mkDerivation { + name = "closure-info"; + + __structuredAttrs = true; + + exportReferencesGraph.closure = rootPaths; + + PATH = "${coreutils}/bin:${jq}/bin"; + + builder = builtins.toFile "builder" + '' + if [ -e .attrs.sh ]; then . .attrs.sh; fi + + out=''${outputs[out]} + + mkdir $out + + jq -r '.closure | map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < .attrs.json | head -n -1 > $out/registration + jq -r .closure[].path < .attrs.json > $out/store-paths + ''; + } + +else + + # Nix < 1.12 + + stdenv.mkDerivation { + name = "closure-info"; + + exportReferencesGraph = + map (x: [("closure-" + baseNameOf x) x]) rootPaths; + + buildInputs = [ perl ]; + + buildCommand = + '' + mkdir $out + printRegistration=1 perl ${pathsFromGraph} closure-* > $out/registration + perl ${pathsFromGraph} closure-* > $out/store-paths + ''; + } diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 8230eb69c29c..b9d9fd2a54c8 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -6,9 +6,12 @@ findutils, go, jshon, + jq, lib, pkgs, pigz, + nixUnstable, + perl, runCommand, rsync, shadow, @@ -26,7 +29,7 @@ rec { examples = import ./examples.nix { - inherit pkgs buildImage pullImage shadowSetup; + inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb; }; pullImage = callPackage ./pull.nix {}; @@ -42,7 +45,7 @@ rec { cp ${./tarsum.go} tarsum.go export GOPATH=$(pwd) mkdir src - ln -sT ${docker.src}/pkg/tarsum src/tarsum + ln -sT ${docker.src}/components/engine/pkg/tarsum src/tarsum go build cp tarsum $out @@ -82,7 +85,7 @@ rec { export PATH=${shadow}/bin:$PATH mkdir -p /etc/pam.d if [[ ! -f /etc/passwd ]]; then - echo "root:x:0:0::/root:/bin/sh" > /etc/passwd + echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd echo "root:!x:::::::" > /etc/shadow fi if [[ ! -f /etc/group ]]; then @@ -225,6 +228,19 @@ rec { ${text} ''; + nixRegistration = contents: runCommand "nix-registration" { + buildInputs = [ nixUnstable perl ]; + # For obtaining the closure of `contents'. + exportReferencesGraph = + let contentsList = if builtins.isList contents then contents else [ contents ]; + in map (x: [("closure-" + baseNameOf x) x]) contentsList; + } + '' + mkdir $out + printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump + perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths + ''; + # Create a "layer" (set of files). mkPureLayer = { # Name of the layer @@ -238,11 +254,10 @@ rec { # into directories. keepContentsDirlinks ? false, # Additional commands to run on the layer before it is tar'd up. - extraCommands ? "" + extraCommands ? "", uid ? 0, gid ? 0 }: runCommand "docker-layer-${name}" { inherit baseJson contents extraCommands; - buildInputs = [ jshon rsync ]; } '' @@ -257,6 +272,8 @@ rec { echo "No contents to add to layer." fi + chmod ug+w layer + if [[ -n $extraCommands ]]; then (cd layer; eval "$extraCommands") fi @@ -264,7 +281,7 @@ rec { # Tar up the layer and throw it into 'layer.tar'. echo "Packing layer..." mkdir $out - tar -C layer --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar . + tar -C layer --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . # Compute a checksum of the tarball. echo "Computing layer checksum..." @@ -320,6 +337,8 @@ rec { echo "Adding $item..." rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/ done + + chmod ug+w layer ''; postMount = '' @@ -387,11 +406,13 @@ rec { # Docker config; e.g. what command to run on the container. config ? null, # Optional bash script to run on the files prior to fixturizing the layer. - extraCommands ? "", + extraCommands ? "", uid ? 0, gid ? 0, # Optional bash script to run as root on the image when provisioning. runAsRoot ? null, # Size of the virtual machine disk to provision when building the image. diskSize ? 1024, + # Time of creation of the image. + created ? "1970-01-01T00:00:01Z", }: let @@ -399,17 +420,16 @@ rec { # Create a JSON blob of the configuration. Set the date to unix zero. baseJson = writeText "${baseName}-config.json" (builtins.toJSON { - created = "1970-01-01T00:00:01Z"; + inherit created config; architecture = "amd64"; os = "linux"; - config = config; }); layer = if runAsRoot == null then mkPureLayer { name = baseName; - inherit baseJson contents keepContentsDirlinks extraCommands; + inherit baseJson contents keepContentsDirlinks extraCommands uid gid; } else mkRootLayer { name = baseName; inherit baseJson fromImage fromImageName fromImageTag @@ -417,9 +437,10 @@ rec { extraCommands; }; result = runCommand "docker-image-${baseName}.tar.gz" { - buildInputs = [ jshon pigz coreutils findutils ]; - imageName = name; - imageTag = tag; + buildInputs = [ jshon pigz coreutils findutils jq ]; + # Image name and tag must be lowercase + imageName = lib.toLower name; + imageTag = lib.toLower tag; inherit fromImage baseJson; layerClosure = writeReferencesToFile layer; passthru.buildArgs = args; @@ -443,6 +464,9 @@ rec { if [[ -n "$fromImage" ]]; then echo "Unpacking base image..." tar -C image -xpf "$fromImage" + # Do not import the base image configuration and manifest + chmod a+w image image/*.json + rm -f image/*.json if [[ -z "$fromImageName" ]]; then fromImageName=$(jshon -k < image/repositories|head -n1) @@ -501,6 +525,24 @@ rec { # Use the temp folder we've been working on to create a new image. mv temp image/$layerID + # Create image json and image manifest + imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") + manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") + currentID=$layerID + while [[ -n "$currentID" ]]; do + layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1) + imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .") + imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .") + manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .") + + currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true)) + done + + imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) + echo "$imageJson" > "image/$imageJsonChecksum.json" + manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"") + echo "$manifestJson" > image/manifest.json + # Store the json under the name image/repositories. jshon -n object \ -n object -s "$layerID" -i "$imageTag" \ @@ -510,11 +552,44 @@ rec { chmod -R a-w image echo "Cooking the image..." - tar -C image --mtime="@$SOURCE_DATE_EPOCH" -c . | pigz -nT > $out + tar -C image --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out echo "Finished." ''; in result; + + # Build an image and populate its nix database with the provided + # contents. The main purpose is to be able to use nix commands in + # the container. + # Be careful since this doesn't work well with multilayer. + buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: + buildImage (args // { + extraCommands = '' + echo "Generating the nix database..." + echo "Warning: only the database of the deepest Nix layer is loaded." + echo " If you want to use nix commands in the container, it would" + echo " be better to only have one layer that contains a nix store." + # This requires Nix 1.12 or higher + export NIX_REMOTE=local?root=$PWD + ${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump + + # We fill the store in order to run the 'verify' command that + # generates hash and size of output paths. + # Note when Nix 1.12 is be the stable one, the database dump + # generated by the exportReferencesGraph function will + # contains sha and size. See + # https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a + storePaths=$(cat ${nixRegistration contents}/storePaths) + echo "Copying everything to /nix/store (will take a while)..." + cp -prd $storePaths nix/store/ + ${nixUnstable}/bin/nix-store --verify --check-contents + + mkdir -p nix/var/nix/gcroots/docker/ + for i in ${lib.concatStringsSep " " contents}; do + ln -s $i nix/var/nix/gcroots/docker/$(basename $i) + done; + '' + extraCommands; + }); } diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 05b4a9b4f2d2..b9a334971744 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -7,7 +7,7 @@ # $ nix-build '<nixpkgs>' -A dockerTools.examples.redis # $ docker load < result -{ pkgs, buildImage, pullImage, shadowSetup }: +{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }: rec { # 1. basic example @@ -83,16 +83,12 @@ rec { }; # 4. example of pulling an image. could be used as a base for other images - # - # ***** Currently broken, getting 404s. Perhaps the docker API has changed? - # - # - # debian = pullImage { - # imageName = "debian"; - # imageTag = "jessie"; - # # this hash will need change if the tag is updated at docker hub - # sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q"; - # }; + nixFromDockerHub = pullImage { + imageName = "nixos/nix"; + imageTag = "1.11"; + # this hash will need change if the tag is updated at docker hub + sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83"; + }; # 5. example of multiple contents, emacs and vi happily coexisting editors = buildImage { @@ -105,4 +101,19 @@ rec { pkgs.nano ]; }; + + # 5. nix example to play with the container nix store + # docker run -it --rm nix nix-store -qR $(nix-build '<nixpkgs>' -A nix) + nix = buildImageWithNixDb { + name = "nix"; + contents = [ + # nix-store uses cat program to display results as specified by + # the image env variable NIX_PAGER. + pkgs.coreutils + pkgs.nix + ]; + config = { + Env = [ "NIX_PAGER=cat" ]; + }; + }; } diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix index 0e1b147f6e18..5ccd0a41c5e4 100644 --- a/pkgs/build-support/docker/pull.nix +++ b/pkgs/build-support/docker/pull.nix @@ -1,41 +1,32 @@ -{ stdenv, lib, curl, jshon, python, runCommand }: - -# Inspired and simplified version of fetchurl. +{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }: +let + nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name; +in # For simplicity we only support sha256. +{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}" +, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }: +let + pullImage = vmTools.runInLinuxVM ( + stdenv.mkDerivation { + inherit name imageId; -# Currently only registry v1 is supported, compatible with Docker Hub. - -{ imageName, imageTag ? "latest", imageId ? null -, sha256, name ? "${imageName}-${imageTag}" -, indexUrl ? "https://index.docker.io" -, registryVersion ? "v1" -, curlOpts ? "" }: - -assert registryVersion == "v1"; - -let layer = stdenv.mkDerivation { - inherit name imageName imageTag imageId - indexUrl registryVersion curlOpts; + certs = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - builder = ./pull.sh; - detjson = ./detjson.py; + builder = ./pull.sh; - buildInputs = [ curl jshon python ]; + buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ]; - outputHashAlgo = "sha256"; - outputHash = sha256; - outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = sha256; - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - # This variable allows the user to pass additional options to curl - "NIX_CURL_FLAGS" - ]; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; - # Doing the download on a remote machine just duplicates network - # traffic, so don't do that. - preferLocalBuild = true; -}; + preVM = vmTools.createEmptyImage { + size = 2048; + fullName = "${name}-disk"; + }; -in runCommand "${name}.tar.gz" {} '' - tar -C ${layer} -czf $out . -'' + QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0"; + }); +in + pullImage diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh index 7ba146e9de09..0b1e9f310ee9 100644 --- a/pkgs/build-support/docker/pull.sh +++ b/pkgs/build-support/docker/pull.sh @@ -1,86 +1,36 @@ -# Reference: docker src contrib/download-frozen-image.sh - source $stdenv/setup -# Curl flags to handle redirects, not use EPSV, handle cookies for -# servers to need them during redirects, and work on SSL without a -# certificate (this isn't a security problem because we check the -# cryptographic hash of the output anyway). -curl=$(command -v curl) -curl() { - [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@" - $curl \ - --location --max-redirs 20 \ - --retry 3 \ - --fail \ - --disable-epsv \ - --cookie-jar cookies \ - --insecure \ - $curlOpts \ - $NIX_CURL_FLAGS \ - "$@" -} - -fetchLayer() { - local url="$1" - local dest="$2" - local curlexit=18; - - # if we get error code 18, resume partial download - while [ $curlexit -eq 18 ]; do - # keep this inside an if statement, since on failure it doesn't abort the script - if curl -C - "$url" --output "$dest"; then - return 0 - else - curlexit=$?; - fi - done - - return $curlexit -} - -headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \ - "$indexUrl/$registryVersion/repositories/$imageName/images") +mkdir -p /var/lib/docker +mkfs.ext4 /dev/vda +mount -t ext4 /dev/vda /var/lib/docker -header() { - grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2 -} +modprobe virtio_net +dhclient eth0 -# this only takes the first endpoint, more may be provided -# https://docs.docker.com/v1.6/reference/api/docker-io_api/ -if ! registryUrl=$(header X-Docker-Endpoints); then - echo "error: index returned no endpoint" - exit 1 -fi -baseUrl="https://$registryUrl/$registryVersion" +mkdir -p /etc/ssl/certs/ +cp "$certs" "/etc/ssl/certs/" -token="$(header X-Docker-Token || true)"; - -if [ -z "$imageId" ]; then - imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")" - imageId="${imageId//\"/}" - if [ -z "$imageId" ]; then - echo "error: no image ID found for ${imageName}:${imageTag}" - exit 1 +# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount +mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup +cd /sys/fs/cgroup +for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do + mkdir -p $sys + if ! mountpoint -q $sys; then + if ! mount -n -t cgroup -o $sys cgroup $sys; then + rmdir $sys || true fi + fi +done - echo "found image ${imageName}:${imageTag}@$imageId" -fi - -mkdir -p $out +# run docker daemon +dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock & -jshon -n object \ - -n object -s "$imageId" -i "$imageTag" \ - -i "$imageName" > $out/repositories +until docker ps 2>/dev/null; do + printf '.' + sleep 1 +done -curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json +rm -r $out -layerIds=$(jshon -a -u < ancestry.json) -for layerId in $layerIds; do - echo "fetching layer $layerId" - - mkdir "$out/$layerId" - echo '1.0' > "$out/$layerId/VERSION" - curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json" - fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar" -done +docker pull ${imageId} +docker save ${imageId} > $out diff --git a/pkgs/build-support/emacs/buffer.nix b/pkgs/build-support/emacs/buffer.nix index 6c5e0570fd0d..75e660d02143 100644 --- a/pkgs/build-support/emacs/buffer.nix +++ b/pkgs/build-support/emacs/buffer.nix @@ -3,7 +3,7 @@ { lib, writeText, inherit-local }: -{ +rec { withPackages = pkgs: let extras = map (x: x.emacsBufferSetup pkgs) (builtins.filter (builtins.hasAttr "emacsBufferSetup") pkgs); in writeText "dir-locals.el" '' @@ -49,4 +49,28 @@ ${lib.concatStringsSep "\n" extras} ''; + # nix-buffer function for a project with a bunch of haskell packages + # in one directory + haskellMonoRepo = { project-root # The monorepo root + , haskellPackages # The composed haskell packages set that contains all of the packages + }: { root }: + let # The haskell paths. + haskell-paths = lib.filesystem.haskellPathsInDir project-root; + # Find the haskell package that the 'root' is in, if any. + haskell-path-parent = + let filtered = builtins.filter (name: + lib.hasPrefix (toString (project-root + "/${name}")) (toString root) + ) (builtins.attrNames haskell-paths); + in + if filtered == [] then null else builtins.head filtered; + # We're in the directory of a haskell package + is-haskell-package = haskell-path-parent != null; + haskell-package = haskellPackages.${haskell-path-parent}; + # GHC environment with all needed deps for the haskell package + haskell-package-env = + builtins.head haskell-package.env.nativeBuildInputs; + in + if is-haskell-package + then withPackages [ haskell-package-env ] + else {}; } diff --git a/pkgs/build-support/emacs/elpa2nix.el b/pkgs/build-support/emacs/elpa2nix.el index 7eef81b9e7ab..64587c0fad1a 100644 --- a/pkgs/build-support/emacs/elpa2nix.el +++ b/pkgs/build-support/emacs/elpa2nix.el @@ -28,3 +28,6 @@ The file can either be a tar file or an Emacs Lisp file." (insert-file-contents file)) (when is-tar (tar-mode)) (elpa2nix-install-from-buffer)))) + +;; Allow installing package tarfiles larger than 10MB +(setq large-file-warning-threshold nil) diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix index b13def07bb88..bd733f1b9baf 100644 --- a/pkgs/build-support/emacs/wrapper.nix +++ b/pkgs/build-support/emacs/wrapper.nix @@ -21,7 +21,7 @@ set which contains `emacsWithPackages`. For example, to override `emacsPackagesNg.emacsWithPackages`, ``` let customEmacsPackages = - emacsPackagesNg.override (super: self: { + emacsPackagesNg.overrideScope (super: self: { # use a custom version of emacs emacs = ...; # use the unstable MELPA version of magit @@ -80,7 +80,8 @@ stdenv.mkDerivation { linkPath "$1" "share/emacs/site-lisp" "share/emacs/site-lisp" } - for pkg in $requires; do + # Iterate over the array of inputs (avoiding nix's own interpolation) + for pkg in "''${requires[@]}"; do linkEmacsPackage $pkg done @@ -115,6 +116,19 @@ EOF --suffix EMACSLOADPATH ":" "$deps/share/emacs/site-lisp:" done + # Wrap MacOS app + # this has to pick up resources and metadata + # to recognize it as an "app" + if [ -d "$emacs/Applications/Emacs.app" ]; then + mkdir -p $out/Applications/Emacs.app/Contents/MacOS + cp -r $emacs/Applications/Emacs.app/Contents/Info.plist \ + $emacs/Applications/Emacs.app/Contents/PkgInfo \ + $emacs/Applications/Emacs.app/Contents/Resources \ + $out/Applications/Emacs.app/Contents + makeWrapper $emacs/Applications/Emacs.app/Contents/MacOS/Emacs $out/Applications/Emacs.app/Contents/MacOS/Emacs \ + --suffix EMACSLOADPATH ":" "$deps/share/emacs/site-lisp:" + fi + mkdir -p $out/share # Link icons and desktop files into place for dir in applications icons info man; do diff --git a/pkgs/build-support/expand-response-params/default.nix b/pkgs/build-support/expand-response-params/default.nix new file mode 100644 index 000000000000..2a4bee74197b --- /dev/null +++ b/pkgs/build-support/expand-response-params/default.nix @@ -0,0 +1,19 @@ +{ stdenv }: + +stdenv.mkDerivation { + name = "expand-response-params"; + src = ./expand-response-params.c; + # Work around "stdenv-darwin-boot-2 is not allowed to refer to path + # /nix/store/...-expand-response-params.c" + unpackPhase = '' + cp "$src" expand-response-params.c + src=$PWD + ''; + buildPhase = '' + "$CC" -std=c99 -O3 -o "expand-response-params" expand-response-params.c + ''; + installPhase = '' + mkdir -p $prefix/bin + mv expand-response-params $prefix/bin/ + ''; +} diff --git a/pkgs/build-support/expand-response-params/expand-response-params.c b/pkgs/build-support/expand-response-params/expand-response-params.c new file mode 100644 index 000000000000..05b9c62b1e8d --- /dev/null +++ b/pkgs/build-support/expand-response-params/expand-response-params.c @@ -0,0 +1,84 @@ +#include <assert.h> +#include <ctype.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +typedef struct { char *data; size_t len, cap; } String; + +void resize(String *s, size_t len) { + s->len = len; + if (s->cap < s->len) { + s->cap = s->len * 2; + s->data = (char *)realloc(s->data, s->cap); + assert(s->data); + } +} + +void append(String *s, const char *data, size_t len) { + resize(s, s->len + len); + memcpy(s->data + s->len - len, data, len); +} + +typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass; +typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State; + +// current State -> CharClass -> next State +const State transitions[][5] = { + [outside] = {outside, unq, unq_esc, sq, dq}, + [unq] = {outside, unq, unq_esc, sq, dq}, + [unq_esc] = {unq, unq, unq, unq, unq}, + [sq] = {sq, sq, sq_esc, unq, sq}, + [sq_esc] = {sq, sq, sq, sq, sq}, + [dq] = {dq, dq, dq_esc, dq, unq}, + [dq_esc] = {dq, dq, dq, dq, dq}, +}; + +CharClass charClass(int c) { + return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark : + isspace(c) ? space : other; +} + +// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated +// string, to stdout. If arg does not begin with `@' or does not refer to a +// file, it is written as is. Otherwise the contents of the file are +// recursively expanded. On unexpected EOF in malformed response files an +// incomplete final argument is written, even if it is empty, to parse like GCC. +void expandArg(String *arg) { + FILE *f; + if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) { + fwrite(arg->data, 1, arg->len, stdout); + return; + } + + resize(arg, 0); + State cur = outside; + int c; + do { + c = fgetc(f); + State next = transitions[cur][charClass(c)]; + if ((cur == unq && next == outside) || (cur != outside && c == EOF)) { + append(arg, "", 1); + expandArg(arg); + resize(arg, 0); + } else if (cur == unq_esc || cur == sq_esc || cur == dq_esc || + (cur == outside ? next == unq : cur == next)) { + char s = c; + append(arg, &s, 1); + } + cur = next; + } while (c != EOF); + + fclose(f); +} + +int main(int argc, char **argv) { + String arg = { 0 }; + while (*++argv) { + resize(&arg, 0); + append(&arg, *argv, strlen(*argv) + 1); + expandArg(&arg); + } + free(arg.data); + return EXIT_SUCCESS; +} diff --git a/pkgs/build-support/fetchadc/builder.sh b/pkgs/build-support/fetchadc/builder.sh deleted file mode 100644 index ceeaa9213d65..000000000000 --- a/pkgs/build-support/fetchadc/builder.sh +++ /dev/null @@ -1,7 +0,0 @@ -source $stdenv/setup - -loginpage=`curl --insecure -s -L -b cookies.txt "$url"` - -[[ $loginpage =~ form[^\>]+action=\"([^\"]+)\" ]] && loginurl=${BASH_REMATCH[1]} - -curl --insecure -s --output "$out" -L -b cookies.txt --data "appleId=${adc_user}&accountPassword=${adc_pass}" "https://idmsa.apple.com/IDMSWebAuth/${loginurl}" diff --git a/pkgs/build-support/fetchadc/default.nix b/pkgs/build-support/fetchadc/default.nix deleted file mode 100644 index 4d759e6f7f18..000000000000 --- a/pkgs/build-support/fetchadc/default.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ stdenv, curl, adc_user, adc_pass }: - -{ # Path to fetch. - path - - # Hash of the downloaded file -, sha256 - -, # Additional curl options needed for the download to succeed. - curlOpts ? "" - -, # Name of the file. If empty, use the basename of `path'. - name ? "" -}: - -stdenv.mkDerivation { - url = "https://developer.apple.com/downloads/download.action?path=${path}"; - - name = if name != "" then name else baseNameOf path; - builder = ./builder.sh; - - buildInputs = [ curl ]; - - meta = { - # Password-guarded files from ADC are certainly unfree, as far as we're concerned! - license = stdenv.lib.licenses.unfree; - }; - - outputHashAlgo = "sha256"; - outputHash = sha256; - outputHashMode = "flat"; - - inherit curlOpts adc_user adc_pass; - - preferLocalBuild = true; -} diff --git a/pkgs/build-support/fetchbower/default.nix b/pkgs/build-support/fetchbower/default.nix index 835fbec6bf0e..3e1f0eff84af 100644 --- a/pkgs/build-support/fetchbower/default.nix +++ b/pkgs/build-support/fetchbower/default.nix @@ -4,11 +4,13 @@ let let components = lib.splitString "#" version; hash = lib.last components; - ver = if builtins.length components == 1 then version else hash; + ver = if builtins.length components == 1 then (cleanName version) else hash; in ver; + cleanName = name: lib.replaceStrings ["/" ":"] ["-" "-"] name; + fetchbower = name: version: target: outputHash: stdenv.mkDerivation { - name = "${name}-${bowerVersion version}"; + name = "${cleanName name}-${bowerVersion version}"; SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; buildCommand = '' fetch-bower --quiet --out=$PWD/out "${name}" "${target}" "${version}" diff --git a/pkgs/build-support/fetchbzr/builder.sh b/pkgs/build-support/fetchbzr/builder.sh index af1257d3688c..e424fd92d51e 100644 --- a/pkgs/build-support/fetchbzr/builder.sh +++ b/pkgs/build-support/fetchbzr/builder.sh @@ -4,6 +4,6 @@ header "exporting \`$url' (revision $rev) into \`$out'" # Perform a lightweight checkout so that we don't end up importing # all the repository's history. -bzr -Ossl.cert_reqs=none export -r "$rev" --format=dir "$out" "$url" +BZR_LOG=/dev/null bzr -Ossl.cert_reqs=none export -r "$rev" --format=dir "$out" "$url" stopNest diff --git a/pkgs/build-support/fetchdarcs/default.nix b/pkgs/build-support/fetchdarcs/default.nix index 2644a20d0a53..2df1b136c559 100644 --- a/pkgs/build-support/fetchdarcs/default.nix +++ b/pkgs/build-support/fetchdarcs/default.nix @@ -1,10 +1,13 @@ -{stdenv, darcs, nix}: {url, rev ? null, context ? null, md5 ? "", sha256 ? ""}: +{stdenv, darcs, nix, cacert}: + +{url, rev ? null, context ? null, md5 ? "", sha256 ? ""}: if md5 != "" then throw "fetchdarcs does not support md5 anymore, please use sha256" else stdenv.mkDerivation { name = "fetchdarcs"; + NIX_SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; builder = ./builder.sh; buildInputs = [darcs]; diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 4bbef1d6e62a..6ae46469738a 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -12,4 +12,5 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" \ ${fetchSubmodules:+--fetch-submodules} \ ${branchName:+--branch-name "$branchName"} +runHook postFetch stopNest diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index d85d2c893c52..8e060b87ebd3 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -1,6 +1,7 @@ {stdenv, git, cacert}: let urlToName = url: rev: let - base = baseNameOf (stdenv.lib.removeSuffix "/" url); + inherit (stdenv.lib) removeSuffix splitString last; + base = last (splitString ":" (baseNameOf (removeSuffix "/" url))); matched = builtins.match "(.*).git" base; @@ -15,6 +16,9 @@ in , fetchSubmodules ? true, deepClone ? false , branchName ? null , name ? urlToName url rev +, # Shell code executed after the file has been fetched + # successfully. This can do things like check or transform the file. + postFetch ? "" }: /* NOTE: @@ -54,7 +58,7 @@ stdenv.mkDerivation { outputHashMode = "recursive"; outputHash = sha256; - inherit url rev leaveDotGit fetchSubmodules deepClone branchName; + inherit url rev leaveDotGit fetchSubmodules deepClone branchName postFetch; GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index 3d656eba5fff..17962d08acc3 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -39,7 +39,7 @@ usage(){ Options: --out path Path where the output would be stored. - --url url Any url understand by 'git clone'. + --url url Any url understood by 'git clone'. --rev ref Any sha1 or references (such as refs/heads/master) --hash h Expected hash. --deepClone Clone the entire repository. @@ -47,6 +47,7 @@ Options: --leave-dotGit Keep the .git directories. --fetch-submodules Fetch submodules. --builder Clone as fetchgit does, but url, rev, and out option are mandatory. + --quiet Only print the final json summary. " exit 1 } @@ -120,9 +121,8 @@ hash_from_ref(){ url_to_name(){ local url=$1 local ref=$2 - # basename removes the / and .git suffixes local base - base=$(basename "$url" .git) + base=$(basename "$url" .git | cut -d: -f2) if [[ $ref =~ ^[a-z0-9]+$ ]]; then echo "$base-${ref:0:7}" @@ -283,8 +283,8 @@ _clone_user_rev() { if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then clone "$dir" "$url" "$rev" "" 1>&2 else - echo 1>&2 "Bad commit hash or bad reference." - exit 1 + # if revision is not hexadecimal it might be a tag + clone "$dir" "$url" "" "refs/tags/$rev" 1>&2 fi;; esac diff --git a/pkgs/build-support/fetchgitrevision/default.nix b/pkgs/build-support/fetchgitrevision/default.nix deleted file mode 100644 index e877648978d9..000000000000 --- a/pkgs/build-support/fetchgitrevision/default.nix +++ /dev/null @@ -1,10 +0,0 @@ -runCommand: git: repository: branch: - import (runCommand "head-revision" - { buildInputs = [ git ]; - dummy = builtins.currentTime; - } - '' - rev=$(git ls-remote ${repository} | grep "refs/${branch}$" | awk '{ print $1 }') - echo "[ \"$rev\" ]" > $out - echo Latest revision in ${branch} is $rev - '') diff --git a/pkgs/build-support/fetchgx/default.nix b/pkgs/build-support/fetchgx/default.nix index 6d209cec2549..ea91a0854d16 100644 --- a/pkgs/build-support/fetchgx/default.nix +++ b/pkgs/build-support/fetchgx/default.nix @@ -14,7 +14,7 @@ stdenv.mkDerivation { phases = [ "unpackPhase" "buildPhase" "installPhase" ]; - SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + NIX_SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; buildPhase = '' export GOPATH=$(pwd)/vendor diff --git a/pkgs/build-support/fetchpatch/default.nix b/pkgs/build-support/fetchpatch/default.nix index a6ddf132cd5b..e3159d205301 100644 --- a/pkgs/build-support/fetchpatch/default.nix +++ b/pkgs/build-support/fetchpatch/default.nix @@ -5,7 +5,7 @@ # stripLen acts as the -p parameter when applying a patch. { lib, fetchurl, patchutils }: -{ stripLen ? 0, addPrefixes ? false, ... }@args: +{ stripLen ? 0, addPrefixes ? false, excludes ? [], ... }@args: fetchurl ({ postFetch = '' @@ -21,7 +21,10 @@ fetchurl ({ --addnewprefix=b/ \ ''} \ --clean "$out" > "$tmpfile" - mv "$tmpfile" "$out" + ${patchutils}/bin/filterdiff \ + -p1 \ + ${builtins.toString (builtins.map (x: "-x ${x}") excludes)} \ + "$tmpfile" > "$out" ${args.postFetch or ""} ''; -} // builtins.removeAttrs args ["stripLen" "addPrefixes"]) +} // builtins.removeAttrs args ["stripLen" "addPrefixes" "excludes" "postFetch"]) diff --git a/pkgs/build-support/fetchrepoproject/default.nix b/pkgs/build-support/fetchrepoproject/default.nix index 8c55db5372d9..199c029d3b64 100644 --- a/pkgs/build-support/fetchrepoproject/default.nix +++ b/pkgs/build-support/fetchrepoproject/default.nix @@ -1,7 +1,9 @@ -{ stdenv, git, gitRepo, gnupg ? null, cacert, copyPathsToStore }: +{ stdenv, gitRepo, cacert, copyPathsToStore }: -{ name, manifest, rev ? "HEAD", sha256, repoRepoURL ? "", repoRepoRev ? "", referenceDir ? "" -, localManifests ? [], createMirror ? false, useArchive ? !createMirror +{ name, manifest, rev ? "HEAD", sha256 +# Optional parameters: +, repoRepoURL ? "", repoRepoRev ? "", referenceDir ? "" +, localManifests ? [], createMirror ? false, useArchive ? false }: assert repoRepoRev != "" -> repoRepoURL != ""; @@ -10,64 +12,66 @@ assert createMirror -> !useArchive; with stdenv.lib; let + extraRepoInitFlags = [ + (optionalString (repoRepoURL != "") "--repo-url=${repoRepoURL}") + (optionalString (repoRepoRev != "") "--repo-branch=${repoRepoRev}") + (optionalString (referenceDir != "") "--reference=${referenceDir}") + ]; + repoInitFlags = [ "--manifest-url=${manifest}" "--manifest-branch=${rev}" "--depth=1" - #TODO: fetching clone.bundle seems to fail spectacularly inside a sandbox. - "--no-clone-bundle" (optionalString createMirror "--mirror") (optionalString useArchive "--archive") - (optionalString (repoRepoURL != "") "--repo-url=${repoRepoURL}") - (optionalString (repoRepoRev != "") "--repo-branch=${repoRepoRev}") - (optionalString (referenceDir != "") "--reference=${referenceDir}") - ]; + ] ++ extraRepoInitFlags; local_manifests = copyPathsToStore localManifests; -in +in stdenv.mkDerivation { + inherit name; -with stdenv.lib; + inherit cacert manifest rev repoRepoURL repoRepoRev referenceDir; # TODO -let - extraRepoInitFlags = [ - (optionalString (repoRepoURL != "") "--repo-url=${repoRepoURL}") - (optionalString (repoRepoRev != "") "--repo-branch=${repoRepoRev}") - (optionalString (referenceDir != "") "--reference=${referenceDir}") + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = sha256; + + preferLocalBuild = true; + enableParallelBuilding = true; + + impureEnvVars = fetchers.proxyImpureEnvVars ++ [ + "GIT_PROXY_COMMAND" "SOCKS_SERVER" ]; -in -stdenv.mkDerivation { + buildInputs = [ gitRepo cacert ]; + + GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + buildCommand = '' + # Path must be absolute (e.g. for GnuPG: ~/.repoconfig/gnupg/pubring.kbx) + export HOME="$(pwd)" + + mkdir $out + cd $out + mkdir .repo ${optionalString (local_manifests != []) '' - mkdir ./.repo/local_manifests - for local_manifest in ${concatMapStringsSep " " toString local_manifests} - - do - cp $local_manifest ./.repo/local_manifests/$(stripHash $local_manifest; echo $strippedName) - done + mkdir .repo/local_manifests + for local_manifest in ${concatMapStringsSep " " toString local_manifests}; do + cp $local_manifest .repo/local_manifests/$(stripHash $local_manifest; echo $strippedName) + done ''} - export HOME=.repo repo init ${concatStringsSep " " repoInitFlags} - repo sync --jobs=$NIX_BUILD_CORES --current-branch - ${optionalString (!createMirror) "rm -rf $out/.repo"} - ''; - - GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - impureEnvVars = stdenv.lib.fetchers.proxyImpureEnvVars ++ [ - "GIT_PROXY_COMMAND" "SOCKS_SERVER" - ]; - - buildInputs = [git gitRepo cacert] ++ optional (gnupg != null) [gnupg] ; - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = sha256; - - preferLocalBuild = true; - enableParallelBuilding = true; - inherit name cacert manifest rev repoRepoURL repoRepoRev referenceDir; + # TODO: The git-index files (and probably the files in .repo as well) have + # different contents each time and will therefore change the final hash + # (i.e. creating a mirror probably won't work). + ${optionalString (!createMirror) '' + rm -rf .repo + find -type d -name '.git' -prune -exec rm -rf {} + + ''} + ''; } diff --git a/pkgs/build-support/fetchs3/default.nix b/pkgs/build-support/fetchs3/default.nix new file mode 100644 index 000000000000..a5cdbd150b8b --- /dev/null +++ b/pkgs/build-support/fetchs3/default.nix @@ -0,0 +1,29 @@ +{ stdenv, runCommand, awscli }: + +{ s3url +, sha256 +, region ? "us-east-1" +, credentials ? null # Default to looking at local EC2 metadata service +, executable ? false +, recursiveHash ? false +, postFetch ? null +}: + +let + credentialAttrs = stdenv.lib.optionalAttrs (credentials != null) { + AWS_ACCESS_KEY_ID = credentials.access_key_id; + AWS_SECRET_ACCESS_KEY = credentials.secret_access_key; + AWS_SESSION_TOKEN = credentials.session_token ? null; + }; +in runCommand "foo" ({ + buildInputs = [ awscli ]; + outputHashAlgo = "sha256"; + outputHash = sha256; + outputHashMode = if recursiveHash then "recursive" else "flat"; +} // credentialAttrs) (if postFetch != null then '' + downloadedFile="$(mktemp)" + aws s3 cp ${s3url} $downloadedFile + ${postFetch} +'' else '' + aws s3 cp ${s3url} $out +'') diff --git a/pkgs/build-support/fetchsvn/builder.sh b/pkgs/build-support/fetchsvn/builder.sh index 7a8a161712d6..8ed30b37fc7f 100644 --- a/pkgs/build-support/fetchsvn/builder.sh +++ b/pkgs/build-support/fetchsvn/builder.sh @@ -18,6 +18,10 @@ if test -n "$http_proxy"; then export HOME="$PWD" fi; +if test -z "$LC_ALL"; then + export LC_ALL="en_US.UTF-8" +fi; + # Pipe the "p" character into Subversion to force it to accept the # server's certificate. This is perfectly safe: we don't care # whether the server is being spoofed --- only the cryptographic diff --git a/pkgs/build-support/fetchsvn/default.nix b/pkgs/build-support/fetchsvn/default.nix index 6ed34ec02763..8a1085affd3c 100644 --- a/pkgs/build-support/fetchsvn/default.nix +++ b/pkgs/build-support/fetchsvn/default.nix @@ -1,4 +1,4 @@ -{stdenv, subversion, sshSupport ? false, openssh ? null}: +{stdenv, subversion, glibcLocales, sshSupport ? false, openssh ? null}: {url, rev ? "HEAD", md5 ? "", sha256 ? "", ignoreExternals ? false, ignoreKeywords ? false, name ? null}: @@ -31,7 +31,7 @@ else stdenv.mkDerivation { name = name_; builder = ./builder.sh; - buildInputs = [subversion]; + buildInputs = [ subversion glibcLocales ]; outputHashAlgo = "sha256"; outputHashMode = "recursive"; diff --git a/pkgs/build-support/fetchurl/boot.nix b/pkgs/build-support/fetchurl/boot.nix index 722fd2566ef3..bd71f93c5291 100644 --- a/pkgs/build-support/fetchurl/boot.nix +++ b/pkgs/build-support/fetchurl/boot.nix @@ -5,10 +5,11 @@ let mirrors = import ./mirrors.nix; in { url ? builtins.head urls , urls ? [] , sha256 +, name ? baseNameOf (toString url) }: import <nix/fetchurl.nix> { - inherit system sha256; + inherit system sha256 name; url = # Handle mirror:// URIs. Since <nix/fetchurl.nix> currently diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index c4fd18e46caf..7c2bdf260b4e 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -39,7 +39,6 @@ tryDownload() { curlexit=$?; fi done - stopNest } @@ -51,7 +50,6 @@ finish() { fi runHook postFetch - stopNest exit 0 } diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index fea06e22ab46..5811950bf053 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -59,6 +59,13 @@ in , recursiveHash ? false +, # Shell code to build a netrc file for BASIC auth + netrcPhase ? null + +, # Impure env vars (http://nixos.org/nix/manual/#sec-advanced-attributes) + # needed for netrcPhase + netrcImpureEnvVars ? [] + , # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. postFetch ? "" @@ -118,11 +125,18 @@ else stdenv.mkDerivation { outputHashMode = if (recursiveHash || executable) then "recursive" else "flat"; - inherit curlOpts showURLs mirrorsFile impureEnvVars postFetch downloadToTemp executable; + inherit curlOpts showURLs mirrorsFile postFetch downloadToTemp executable; + + impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; # Doing the download on a remote machine just duplicates network # traffic, so don't do that. preferLocalBuild = true; + postHook = if netrcPhase == null then null else '' + ${netrcPhase} + curlOpts="$curlOpts --netrc-file $PWD/netrc" + ''; + inherit meta; } diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 1dfe968f129e..d612db64c122 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -132,7 +132,7 @@ rec { http://ftp.riken.jp/net/samba ]; - # BitlBee mirrors, see http://www.bitlbee.org/main.php/mirrors.html . + # BitlBee mirrors, see https://www.bitlbee.org/main.php/mirrors.html . bitlbee = [ http://get.bitlbee.org/ http://get.bitlbee.be/ @@ -141,9 +141,11 @@ rec { http://bitlbee.intergenia.de/ ]; - # ImageMagick mirrors, see http://www.imagemagick.org/script/download.php. + # ImageMagick mirrors, see https://www.imagemagick.org/script/mirror.php imagemagick = [ - http://www.imagemagick.org/download/ + https://www.imagemagick.org/download/ + https://mirror.checkdomain.de/imagemagick/ + https://ftp.nluug.nl/ImageMagick/ ftp://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/ # also contains older versions removed from most mirrors http://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/ ftp://ftp.imagemagick.org/pub/ImageMagick/ @@ -157,8 +159,9 @@ rec { cpan = [ http://ftp.gwdg.de/pub/languages/perl/CPAN/ ftp://download.xs4all.nl/pub/mirror/CPAN/ - ftp://ftp.nl.uu.net/pub/CPAN/ + http://ftp.tuwien.ac.at/pub/CPAN/ http://ftp.funet.fi/pub/CPAN/ + https://cpan.metacpan.org/ http://cpan.perl.org/ http://backpan.perl.org/ # for old releases ]; @@ -264,15 +267,14 @@ rec { # Apache mirrors (see http://www.apache.org/mirrors/). apache = [ http://www.eu.apache.org/dist/ - ftp://ftp.inria.fr/pub/Apache/ - http://apache.cict.fr/ + http://wwwftp.ciril.fr/pub/apache/ ftp://ftp.fu-berlin.de/unix/www/apache/ - ftp://crysys.hit.bme.hu/pub/apache/dist/ + http://ftp.tudelft.nl/apache/ http://mirror.cc.columbia.edu/pub/software/apache/ http://www.apache.org/dist/ http://archive.apache.org/dist/ # fallback for old releases ftp://ftp.funet.fi/pub/mirrors/apache.org/ - http://apache.cs.uu.nl/dist/ + http://apache.cs.uu.nl/ http://apache.cs.utah.edu/ ]; diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index 1145d32ba022..4a5381d71f20 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -11,10 +11,11 @@ stripRoot ? true , url , extraPostFetch ? "" +, name ? "source" , ... } @ args: lib.overrideDerivation (fetchurl ({ - name = args.name or (baseNameOf url); + inherit name; recursiveHash = true; @@ -23,7 +24,6 @@ lib.overrideDerivation (fetchurl ({ postFetch = '' export PATH=${unzip}/bin:$PATH - mkdir $out unpackDir="$TMPDIR/unpack" mkdir "$unpackDir" @@ -32,8 +32,6 @@ lib.overrideDerivation (fetchurl ({ renamed="$TMPDIR/${baseNameOf url}" mv "$downloadedFile" "$renamed" unpackFile "$renamed" - - shopt -s dotglob '' + (if stripRoot then '' if [ $(ls "$unpackDir" | wc -l) != 1 ]; then @@ -43,12 +41,11 @@ lib.overrideDerivation (fetchurl ({ fi fn=$(cd "$unpackDir" && echo *) if [ -f "$unpackDir/$fn" ]; then - mv "$unpackDir/$fn" "$out" - else - mv "$unpackDir/$fn"/* "$out/" + mkdir $out fi + mv "$unpackDir/$fn" "$out" '' else '' - mv "$unpackDir"/* "$out/" + mv "$unpackDir" "$out" '') #*/ + extraPostFetch; } // removeAttrs args [ "stripRoot" "extraPostFetch" ])) diff --git a/pkgs/build-support/gcc-cross-wrapper/add-flags b/pkgs/build-support/gcc-cross-wrapper/add-flags deleted file mode 100644 index 9ff4522e800b..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/add-flags +++ /dev/null @@ -1,5 +0,0 @@ -export NIX_CROSS_CFLAGS_COMPILE="@cflagsCompile@ $NIX_CROSS_CFLAGS_COMPILE" -export NIX_CROSS_CFLAGS_LINK="@cflagsLink@ $NIX_CROSS_CFLAGS_LINK" -export NIX_CROSS_LDFLAGS="@ldflags@ $NIX_CROSS_LDFLAGS" -export NIX_CROSS_LDFLAGS_BEFORE="@ldflagsBefore@ $NIX_CROSS_LDFLAGS_BEFORE" -export NIX_CROSS_GLIBC_FLAGS_SET=1 diff --git a/pkgs/build-support/gcc-cross-wrapper/builder.sh b/pkgs/build-support/gcc-cross-wrapper/builder.sh deleted file mode 100644 index b729144b8601..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/builder.sh +++ /dev/null @@ -1,120 +0,0 @@ -source $stdenv/setup - -mkdir $out -mkdir $out/bin -mkdir $out/nix-support - -# Force gcc to use ld-wrapper.sh when calling ld. -cflagsCompile="-B$out/bin/" - -if test -z "$nativeLibc" -a -n "$libc"; then - cflagsCompile="$cflagsCompile -B$gccLibs/lib -B$libc/lib/ -isystem $libc_dev/include" - ldflags="$ldflags -L$libc/lib" - # Get the proper dynamic linker for glibc and uclibc. - dlinker=`eval 'echo $libc/lib/ld*.so.?'` - if [ -n "$dlinker" ]; then - ldflagsBefore="-dynamic-linker $dlinker" - - # The same as above, but put into files, useful for the gcc builder. - echo $dlinker > $out/nix-support/dynamic-linker - # This trick is to avoid dependencies on the cross-toolchain gcc - # for libgcc, libstdc++, ... - # -L is for libtool's .la files, and -rpath for the usual fixupPhase - # shrinking rpaths. - if [ -n "$gccLibs" ]; then - ldflagsBefore="$ldflagsBefore -rpath $gccLibs/lib" - fi - fi - - if [ -n "$osxMinVersion" ]; then - cflagsCompile="$cflagsCompile -mmacosx-version-min=$osxMinVersion" - fi - - echo "$cflagsCompile -B$libc/lib/ -idirafter $libc/include -idirafter $gcc/lib/gcc/*/*/include-fixed" > $out/nix-support/libc-cflags - - echo "-L$libc/lib -rpath $libc/lib -rpath-link $libc/lib" > $out/nix-support/libc-ldflags - - # The dynamic linker is passed in `ldflagsBefore' to allow - # explicit overrides of the dynamic linker by callers to gcc/ld - # (the *last* value counts, so ours should come first). - echo "$ldflagsBefore" > $out/nix-support/libc-ldflags-before -fi - -if test -n "$nativeTools"; then - gccPath="$nativePrefix/bin" - ldPath="$nativePrefix/bin" -else - ldflags="$ldflags -L$gcc/lib -L$gcc/lib64" - gccPath="$gcc/bin" - ldPath="$binutils/$crossConfig/bin" -fi - - -doSubstitute() { - local src=$1 - local dst=$2 - substitute "$src" "$dst" \ - --subst-var "out" \ - --subst-var "shell" \ - --subst-var "gcc" \ - --subst-var "gccProg" \ - --subst-var "binutils" \ - --subst-var "libc" \ - --subst-var "cflagsCompile" \ - --subst-var "cflagsLink" \ - --subst-var "ldflags" \ - --subst-var "ldflagsBefore" \ - --subst-var "ldPath" \ - --subst-var-by "ld" "$ldPath/ld" -} - - -# Make wrapper scripts around gcc, g++, and g77. Also make symlinks -# cc, c++, and f77. -mkGccWrapper() { - local dst=$1 - local src=$2 - - if ! test -f "$src"; then - echo "$src does not exist (skipping)" - return - fi - - gccProg="$src" - doSubstitute "$gccWrapper" "$dst" - chmod +x "$dst" -} - -mkGccWrapper $out/bin/$crossConfig-gcc $gccPath/$crossConfig-gcc -#ln -s gcc $out/bin/cc - -mkGccWrapper $out/bin/$crossConfig-g++ $gccPath/$crossConfig-g++ -ln -s $crossConfig-g++ $out/bin/$crossConfig-c++ - -mkGccWrapper $out/bin/$crossConfig-cpp $gccPath/$crossConfig-cpp - -mkGccWrapper $out/bin/$crossConfig-g77 $gccPath/$crossConfig-g77 -ln -s $crossConfig-g77 $out/bin/$crossConfig-f77 - -ln -s $binutils/bin/$crossConfig-ar $out/bin/$crossConfig-ar -ln -s $binutils/bin/$crossConfig-as $out/bin/$crossConfig-as -ln -s $binutils/bin/$crossConfig-nm $out/bin/$crossConfig-nm -ln -s $binutils/bin/$crossConfig-strip $out/bin/$crossConfig-strip - - -# Make a wrapper around the linker. -doSubstitute "$ldWrapper" "$out/bin/$crossConfig-ld" -chmod +x "$out/bin/$crossConfig-ld" - - -# Emit a setup hook. Also store the path to the original GCC and -# Glibc. -test -n "$gcc" && echo $gcc > $out/nix-support/orig-cc -test -n "$libc" && echo $libc > $out/nix-support/orig-libc -test -n "$libc_dev" && echo $libc_dev > $out/nix-support/orig-libc-dev - -doSubstitute "$addFlags" "$out/nix-support/add-flags" - -doSubstitute "$setupHook" "$out/nix-support/setup-hook" - -cp -p $utils $out/nix-support/utils diff --git a/pkgs/build-support/gcc-cross-wrapper/default.nix b/pkgs/build-support/gcc-cross-wrapper/default.nix deleted file mode 100644 index 505d80a6b2ac..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/default.nix +++ /dev/null @@ -1,65 +0,0 @@ -# The Nix `gcc' stdenv.mkDerivation is not directly usable, since it doesn't -# know where the C library and standard header files are. Therefore -# the compiler produced by that package cannot be installed directly -# in a user environment and used from the command line. This -# stdenv.mkDerivation provides a wrapper that sets up the right environment -# variables so that the compiler and the linker just "work". - -{ name ? "", stdenv, nativeTools, nativeLibc, noLibc ? false, nativePrefix ? "" -, gcc ? null, libc ? null, binutils ? null, shell ? "", cross -}: - -assert nativeTools -> nativePrefix != ""; -assert !nativeTools -> gcc != null && binutils != null; -assert !noLibc -> (!nativeLibc -> libc != null); - -let - chosenName = if name == "" then gcc.name else name; - gccLibs = stdenv.mkDerivation { - name = chosenName + "-libs"; - phases = [ "installPhase" ]; - installPhase = '' - echo $out - mkdir -p "$out" - - if [ -d "${gcc}/${cross.config}/lib" ] - then - cp -Rd "${gcc}/${cross.config}/lib" "$out/lib" - chmod -R +w "$out/lib" - for a in "$out/lib/"*.la; do - sed -i -e "s,${gcc}/${cross.config}/lib,$out/lib,g" $a - done - rm -f "$out/lib/"*.py - else - # The MinGW cross-compiler falls into this category. - mkdir "$out/lib" - fi - ''; - }; -in -stdenv.mkDerivation { - builder = ./builder.sh; - setupHook = ./setup-hook.sh; - gccWrapper = ./gcc-wrapper.sh; - ldWrapper = ./ld-wrapper.sh; - utils = ./utils.sh; - addFlags = ./add-flags; - inherit nativeTools nativeLibc nativePrefix gcc binutils; - libc = if libc ? out then libc.out else libc; - libc_dev = if libc ? dev then libc.dev else libc; - crossConfig = if cross != null then cross.config else null; - osxMinVersion = cross.osxMinVersion or null; - gccLibs = if gcc != null then gccLibs else null; - name = chosenName; - langC = if nativeTools then true else gcc.langC; - langCC = if nativeTools then true else gcc.langCC; - langF77 = if nativeTools then false else gcc ? langFortran; - shell = if shell == "" then stdenv.shell else shell; - meta = if gcc != null then gcc.meta else - { description = "System C compiler wrapper"; - }; - - passthru = { - target = cross; - }; -} diff --git a/pkgs/build-support/gcc-cross-wrapper/gcc-wrapper.sh b/pkgs/build-support/gcc-cross-wrapper/gcc-wrapper.sh deleted file mode 100644 index c15777144e11..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/gcc-wrapper.sh +++ /dev/null @@ -1,117 +0,0 @@ -#! @shell@ -e - -if test -n "$NIX_CC_WRAPPER_START_HOOK"; then - source "$NIX_CC_WRAPPER_START_HOOK" -fi - -if test -z "$NIX_CROSS_GLIBC_FLAGS_SET"; then - source @out@/nix-support/add-flags -fi - -source @out@/nix-support/utils - - -# Figure out if linker flags should be passed. GCC prints annoying -# warnings when they are not needed. -dontLink=0 -if test "$*" = "-v" -o -z "$*"; then - dontLink=1 -else - for i in "$@"; do - if test "$i" = "-c"; then - dontLink=1 - elif test "$i" = "-S"; then - dontLink=1 - elif test "$i" = "-E"; then - dontLink=1 - elif test "$i" = "-E"; then - dontLink=1 - elif test "$i" = "-M"; then - dontLink=1 - elif test "$i" = "-MM"; then - dontLink=1 - fi - done -fi - - -# Optionally filter out paths not refering to the store. -params=("$@") -if test "$NIX_ENFORCE_PURITY" = "1" -a -n "$NIX_STORE"; then - rest=() - n=0 - while test $n -lt ${#params[*]}; do - p=${params[n]} - p2=${params[$((n+1))]} - if test "${p:0:3}" = "-L/" && badPath "${p:2}"; then - skip $p - elif test "$p" = "-L" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - elif test "${p:0:3}" = "-I/" && badPath "${p:2}"; then - skip $p - elif test "$p" = "-I" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - elif test "$p" = "-isystem" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - else - rest=("${rest[@]}" "$p") - fi - n=$((n + 1)) - done - params=("${rest[@]}") -fi - - -# Add the flags for the C compiler proper. -extraAfter=($NIX_CROSS_CFLAGS_COMPILE) -extraBefore=() - -if test "$dontLink" != "1"; then - - # Add the flags that should only be passed to the compiler when - # linking. - extraAfter=(${extraAfter[@]} $NIX_CROSS_CFLAGS_LINK) - - # Add the flags that should be passed to the linker (and prevent - # `ld-wrapper' from adding NIX_CROSS_LDFLAGS again). - for i in $NIX_CROSS_LDFLAGS_BEFORE; do - if test "${i:0:3}" = "-L/"; then - extraBefore=(${extraBefore[@]} "$i") - else - extraBefore=(${extraBefore[@]} "-Wl,$i") - fi - done - for i in $NIX_CROSS_LDFLAGS; do - if test "${i:0:3}" = "-L/"; then - extraAfter=(${extraAfter[@]} "$i") - else - extraAfter=(${extraAfter[@]} "-Wl,$i") - fi - done - export NIX_CROSS_LDFLAGS_SET=1 -fi - -# Optionally print debug info. -if test "$NIX_DEBUG" = "1"; then - echo "original flags to @gccProg@:" >&2 - for i in "${params[@]}"; do - echo " $i" >&2 - done - echo "extraBefore flags to @gccProg@:" >&2 - for i in ${extraBefore[@]}; do - echo " $i" >&2 - done - echo "extraAfter flags to @gccProg@:" >&2 - for i in ${extraAfter[@]}; do - echo " $i" >&2 - done -fi - -if test -n "$NIX_CC_WRAPPER_EXEC_HOOK"; then - source "$NIX_CC_WRAPPER_EXEC_HOOK" -fi - -# We want gcc to call the wrapper linker, not that of binutils. -export PATH="@ldPath@:$PATH" - -exec @gccProg@ ${extraBefore[@]} "${params[@]}" ${extraAfter[@]} diff --git a/pkgs/build-support/gcc-cross-wrapper/ld-wrapper.sh b/pkgs/build-support/gcc-cross-wrapper/ld-wrapper.sh deleted file mode 100644 index 226fad833599..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/ld-wrapper.sh +++ /dev/null @@ -1,145 +0,0 @@ -#! @shell@ -e - -if test -n "$NIX_LD_WRAPPER_START_HOOK"; then - source "$NIX_LD_WRAPPER_START_HOOK" -fi - -if test -z "$NIX_CROSS_GLIBC_FLAGS_SET"; then - source @out@/nix-support/add-flags -fi - -source @out@/nix-support/utils - - -# Optionally filter out paths not refering to the store. -params=("$@") -if test "$NIX_ENFORCE_PURITY" = "1" -a -n "$NIX_STORE" \ - -a \( -z "$NIX_IGNORE_LD_THROUGH_GCC" -o -z "$NIX_CROSS_LDFLAGS_SET" \); then - rest=() - n=0 - while test $n -lt ${#params[*]}; do - p=${params[n]} - p2=${params[$((n+1))]} - if test "${p:0:3}" = "-L/" && badPath "${p:2}"; then - skip $p - elif test "$p" = "-L" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - elif test "$p" = "-rpath" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - elif test "$p" = "-dynamic-linker" && badPath "$p2"; then - n=$((n + 1)); skip $p2 - elif test "${p:0:1}" = "/" && badPath "$p"; then - # We cannot skip this; barf. - echo "impure path \`$p' used in link" >&2 - exit 1 - else - rest=("${rest[@]}" "$p") - fi - n=$((n + 1)) - done - params=("${rest[@]}") -fi - - -extra=() -extraBefore=() - -if test -z "$NIX_CROSS_LDFLAGS_SET"; then - extra=(${extra[@]} $NIX_CROSS_LDFLAGS) - extraBefore=(${extraBefore[@]} $NIX_CROSS_LDFLAGS_BEFORE) -fi - - -# Add all used dynamic libraries to the rpath. -if test "$NIX_DONT_SET_RPATH" != "1"; then - - # First, find all -L... switches. - allParams=("${params[@]}" ${extra[@]}) - libPath="" - addToLibPath() { - local path="$1" - if test "${path:0:1}" != "/"; then return 0; fi - case "$path" in - *..*|*./*|*/.*|*//*) - local path2 - if path2=$(readlink -f "$path"); then - path="$path2" - fi - ;; - esac - case $libPath in - *\ $path\ *) return 0 ;; - esac - libPath="$libPath $path " - } - n=0 - while test $n -lt ${#allParams[*]}; do - p=${allParams[n]} - p2=${allParams[$((n+1))]} - if test "${p:0:3}" = "-L/"; then - addToLibPath ${p:2} - elif test "$p" = "-L"; then - addToLibPath ${p2} - n=$((n + 1)) - fi - n=$((n + 1)) - done - - # Second, for each -l... switch, find the directory containing the - # library and add it to the rpath. - rpath="" - addToRPath() { - # If the path is not in the store, don't add it to the rpath. - # This typically happens for libraries in /tmp that are later - # copied to $out/lib. If not, we're screwed. - if test "${1:0:${#NIX_STORE}}" != "$NIX_STORE"; then return 0; fi - case $rpath in - *\ $1\ *) return 0 ;; - esac - rpath="$rpath $1 " - } - findLib() { - for i in $libPath; do - if test -f $i/lib$1.so; then - addToRPath $i - fi - done - } - n=0 - while test $n -lt ${#allParams[*]}; do - p=${allParams[n]} - p2=${allParams[$((n+1))]} - if test "${p:0:2}" = "-l"; then - findLib ${p:2} - elif test "$p" = "-l"; then - # I haven't seen `-l foo', but you never know... - findLib ${p2} - n=$((n + 1)) - fi - n=$((n + 1)) - done - - # Finally, add `-rpath' switches. - for i in $rpath; do - extra=(${extra[@]} -rpath $i -rpath-link $i) - done -fi - - -# Optionally print debug info. -if test "$NIX_DEBUG" = "1"; then - echo "original flags to @ld@:" >&2 - for i in "${params[@]}"; do - echo " $i" >&2 - done - echo "extra flags to @ld@:" >&2 - for i in ${extra[@]}; do - echo " $i" >&2 - done -fi - -if test -n "$NIX_LD_WRAPPER_EXEC_HOOK"; then - source "$NIX_LD_WRAPPER_EXEC_HOOK" -fi - -exec @ld@ ${extraBefore[@]} "${params[@]}" ${extra[@]} diff --git a/pkgs/build-support/gcc-cross-wrapper/setup-hook.sh b/pkgs/build-support/gcc-cross-wrapper/setup-hook.sh deleted file mode 100644 index 599954bd127d..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/setup-hook.sh +++ /dev/null @@ -1,90 +0,0 @@ -NIX_CROSS_CFLAGS_COMPILE="" -NIX_CROSS_LDFLAGS="" - -crossAddCVars () { - if test -d $1/include; then - export NIX_CROSS_CFLAGS_COMPILE="$NIX_CROSS_CFLAGS_COMPILE -I$1/include" - fi - - if test -d $1/lib; then - export NIX_CROSS_LDFLAGS="$NIX_CROSS_LDFLAGS -L$1/lib -rpath-link $1/lib" - fi -} - -crossEnvHooks+=(crossAddCVars) - -crossStripDirs() { - local dirs="$1" - local stripFlags="$2" - local dirsNew= - - for d in ${dirs}; do - if test -d "$prefix/$d"; then - dirsNew="${dirsNew} $prefix/$d " - fi - done - dirs=${dirsNew} - - if test -n "${dirs}"; then - header "cross stripping (with flags $stripFlags) in $dirs" - # libc_nonshared.a should never be stripped, or builds will break. - find $dirs -type f -print0 | xargs -0 ${xargsFlags:--r} $crossConfig-strip $stripFlags || true - stopNest - fi -} - -crossStrip () { - # In cross_renaming we may rename dontCrossStrip to dontStrip, and - # dontStrip to dontNativeStrip. - # TODO: strip _only_ ELF executables, and return || fail here... - if test -z "$dontCrossStrip"; then - stripDebugList=${stripDebugList:-lib lib64 libexec bin sbin} - if test -n "$stripDebugList"; then - crossStripDirs "$stripDebugList" "${stripDebugFlags:--S}" - fi - - stripAllList=${stripAllList:-} - if test -n "$stripAllList"; then - crossStripDirs "$stripAllList" "${stripAllFlags:--s}" - fi - fi -} - -preDistPhases=(${preDistPhases[@]} crossStrip) - - -# Note: these come *after* $out in the PATH (see setup.sh). - -if test -n "@gcc@"; then - PATH=$PATH:@gcc@/bin -fi - -if test -n "@binutils@"; then - PATH=$PATH:@binutils@/bin -fi - -if test -n "@libc@"; then - PATH=$PATH:@libc@/bin - crossAddCVars @libc@ -fi - -if test "$dontSetConfigureCross" != "1"; then - configureFlags="$configureFlags --build=$system --host=$crossConfig" -fi -# Disabling the tests when cross compiling, as usually the tests are meant for -# native compilations. -doCheck="" - -# Don't strip foreign binaries with native "strip" tool. -dontStrip=1 - -# Add the output as an rpath. -if test "$NIX_NO_SELF_RPATH" != "1"; then - export NIX_CROSS_LDFLAGS="-rpath $out/lib -rpath-link $out/lib $NIX_CROSS_LDFLAGS" - if test -n "$NIX_LIB64_IN_SELF_RPATH"; then - export NIX_CROSS_LDFLAGS="-rpath $out/lib64 -rpath-link $out/lib $NIX_CROSS_LDFLAGS" - fi -fi - -export CC=${crossConfig}-gcc -export CXX=${crossConfig}-g++ diff --git a/pkgs/build-support/gcc-cross-wrapper/utils.sh b/pkgs/build-support/gcc-cross-wrapper/utils.sh deleted file mode 100644 index 753b3772e956..000000000000 --- a/pkgs/build-support/gcc-cross-wrapper/utils.sh +++ /dev/null @@ -1,24 +0,0 @@ -skip () { - if test "$NIX_DEBUG" = "1"; then - echo "skipping impure path $1" >&2 - fi -} - - -# Checks whether a path is impure. E.g., `/lib/foo.so' is impure, but -# `/nix/store/.../lib/foo.so' isn't. -badPath() { - local p=$1 - - # Relative paths are okay (since they're presumably relative to - # the temporary build directory). - if test "${p:0:1}" != "/"; then return 1; fi - - # Otherwise, the path should refer to the store or some temporary - # directory (including the build directory). - test \ - "$p" != "/dev/null" -a \ - "${p:0:${#NIX_STORE}}" != "$NIX_STORE" -a \ - "${p:0:4}" != "/tmp" -a \ - "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP" -} diff --git a/pkgs/build-support/gcc-wrapper-old/builder.sh b/pkgs/build-support/gcc-wrapper-old/builder.sh index a8e8a370ec0d..22e32814927e 100644 --- a/pkgs/build-support/gcc-wrapper-old/builder.sh +++ b/pkgs/build-support/gcc-wrapper-old/builder.sh @@ -211,5 +211,5 @@ cp -p $utils $out/nix-support/utils.sh # tools like gcov, the manpages, etc. as well (including for binutils # and Glibc). if test -z "$nativeTools"; then - echo $gcc $binutils $libc $libc_bin > $out/nix-support/propagated-user-env-packages + printWords $gcc $binutils $libc $libc_bin > $out/nix-support/propagated-user-env-packages fi diff --git a/pkgs/build-support/gcc-wrapper-old/default.nix b/pkgs/build-support/gcc-wrapper-old/default.nix index a87c726e0a8b..ae17989d932b 100644 --- a/pkgs/build-support/gcc-wrapper-old/default.nix +++ b/pkgs/build-support/gcc-wrapper-old/default.nix @@ -6,8 +6,9 @@ # variables so that the compiler and the linker just "work". { name ? "", stdenv, lib, nativeTools, nativeLibc, nativePrefix ? "" -, gcc ? null, libc ? null, binutils ? null, coreutils ? null, shell ? "" +, gcc ? null, libc ? null, binutils ? null, coreutils ? null, shell ? stdenv.shell , zlib ? null +, hostPlatform, targetPlatform }: assert nativeTools -> nativePrefix != ""; @@ -55,24 +56,17 @@ stdenv.mkDerivation { langAda = if nativeTools then false else gcc ? langAda && gcc.langAda; langVhdl = if nativeTools then false else gcc ? langVhdl && gcc.langVhdl; zlib = if gcc != null && gcc ? langVhdl then zlib else null; - shell = if shell == "" then stdenv.shell else - if builtins.isAttrs shell then (shell + shell.shellPath) - else shell; + shell = shell + shell.shellPath or ""; crossAttrs = { - shell = shell.crossDrv + shell.crossDrv.shellPath; - libc = stdenv.ccCross.libc; - coreutils = coreutils.crossDrv; - binutils = binutils.crossDrv; - gcc = gcc.crossDrv; # # This is not the best way to do this. I think the reference should be # the style in the gcc-cross-wrapper, but to keep a stable stdenv now I # do this sufficient if/else. dynamicLinker = - (if stdenv.cross.arch == "arm" then "ld-linux.so.3" else - if stdenv.cross.arch == "mips" then "ld.so.1" else - if stdenv.lib.hasSuffix "pc-gnu" stdenv.cross.config then "ld.so.1" else + (if hostPlatform.arch == "arm" then "ld-linux.so.3" else + if hostPlatform.arch == "mips" then "ld.so.1" else + if stdenv.lib.hasSuffix "pc-gnu" hostPlatform.config then "ld.so.1" else abort "don't know the name of the dynamic linker for this platform"); }; @@ -86,15 +80,20 @@ stdenv.mkDerivation { + " (wrapper script)"; }; - # The dynamic linker has different names on different Linux platforms. + # The dynamic linker has different names on different platforms. dynamicLinker = if !nativeLibc then - (if stdenv.system == "i686-linux" then "ld-linux.so.2" else - if stdenv.system == "x86_64-linux" then "ld-linux-x86-64.so.2" else + (if targetPlatform.system == "i686-linux" then "ld-linux.so.2" else + if targetPlatform.system == "x86_64-linux" then "ld-linux-x86-64.so.2" else # ARM with a wildcard, which can be "" or "-armhf". - if stdenv.isArm then "ld-linux*.so.3" else - if stdenv.system == "powerpc-linux" then "ld.so.1" else - if stdenv.system == "mips64el-linux" then "ld.so.1" else - abort "don't know the name of the dynamic linker for this platform") + if targetPlatform.isArm then "ld-linux*.so.3" else + if targetPlatform.system == "aarch64-linux" then "ld-linux-aarch64.so.1" else + if targetPlatform.system == "powerpc-linux" then "ld.so.1" else + if targetPlatform.system == "mips64el-linux" then "ld.so.1" else + if targetPlatform.system == "x86_64-darwin" then "/usr/lib/dyld" else + if stdenv.lib.hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1" else + builtins.trace + "Don't know the name of the dynamic linker for platform ${targetPlatform.config}, so guessing instead." + null) else ""; } diff --git a/pkgs/build-support/grsecurity/default.nix b/pkgs/build-support/grsecurity/default.nix deleted file mode 100644 index ccd46e20654f..000000000000 --- a/pkgs/build-support/grsecurity/default.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ stdenv -, lib -, overrideDerivation - -# required for gcc plugins -, gmp, libmpc, mpfr - -# the base kernel -, kernel - -, grsecPatch -, kernelPatches ? [] - -, localver ? "-grsec" -, modDirVersion ? "${kernel.version}${localver}" -, extraConfig ? "" -, ... -} @ args: - -assert (kernel.version == grsecPatch.kver); - -overrideDerivation (kernel.override { - inherit modDirVersion; - kernelPatches = lib.unique ([ grsecPatch ] ++ kernelPatches ++ (kernel.kernelPatches or [])); - extraConfig = '' - GRKERNSEC y - PAX y - ${extraConfig} - ''; - ignoreConfigErrors = true; -}) (attrs: { - nativeBuildInputs = (lib.chooseDevOutputs [ gmp libmpc mpfr ]) ++ (attrs.nativeBuildInputs or []); - preConfigure = '' - echo ${localver} >localversion-grsec - ${attrs.preConfigure or ""} - ''; -}) diff --git a/pkgs/build-support/kde/derivation.nix b/pkgs/build-support/kde/derivation.nix deleted file mode 100644 index 2e29a0a661e1..000000000000 --- a/pkgs/build-support/kde/derivation.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ stdenv, lib, debug ? false }: - -args: - -stdenv.mkDerivation (args // { - - outputs = args.outputs or [ "out" "dev" ]; - - propagatedUserEnvPkgs = - builtins.map lib.getBin (args.propagatedBuildInputs or []); - - cmakeFlags = - (args.cmakeFlags or []) - ++ [ "-DBUILD_TESTING=OFF" ] - ++ lib.optional debug "-DCMAKE_BUILD_TYPE=Debug"; - -}) diff --git a/pkgs/build-support/kde/wrapper.nix b/pkgs/build-support/kde/wrapper.nix deleted file mode 100644 index 228eb696bd9a..000000000000 --- a/pkgs/build-support/kde/wrapper.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ stdenv, lib, makeWrapper, buildEnv, gtk3, dconf }: - -packages: - -let - packages_ = if builtins.isList packages then packages else [packages]; - - unwrapped = lib.concatMap (p: if builtins.isList p.unwrapped then p.unwrapped else [p.unwrapped]) packages_; - targets = lib.concatMap (p: p.targets) packages_; - paths = lib.concatMap (p: p.paths or []) packages_; - - name = - if builtins.length unwrapped == 1 - then (lib.head unwrapped).name - else "kde-application"; - meta = - if builtins.length unwrapped == 1 - then (lib.head unwrapped).meta - else {}; - - env = buildEnv { - inherit name meta; - paths = builtins.map lib.getBin (unwrapped ++ paths); - pathsToLink = [ "/bin" "/share" "/lib/qt5" "/etc/xdg" ]; - }; -in - -stdenv.mkDerivation { - inherit name meta; - preferLocalBuild = true; - - inherit unwrapped env targets; - - passthru = { - inherit targets paths; - unwrapped = if builtins.length unwrapped == 1 then lib.head unwrapped else unwrapped; - }; - - nativeBuildInputs = [ makeWrapper ]; - - buildCommand = '' - for t in $targets; do - good="" - for drv in $unwrapped; do - if [ -a "$drv/$t" ]; then - makeWrapper "$drv/$t" "$out/$t" \ - --argv0 '"$0"' \ - --suffix PATH : "$env/bin" \ - --prefix XDG_CONFIG_DIRS : "$env/etc/xdg" \ - --prefix XDG_DATA_DIRS : "$env/share:${gtk3}/share/gsettings-schemas/${gtk3.name}" \ - --set QML_IMPORT_PATH "$env/lib/qt5/imports" \ - --set QML2_IMPORT_PATH "$env/lib/qt5/qml" \ - --set QT_PLUGIN_PATH "$env/lib/qt5/plugins" \ - --prefix GIO_EXTRA_MODULES : "${dconf.lib}/lib/gio/modules" - good="1" - break - fi - done - if [ -z "$good" ]; then - echo "file or directory not found in derivations: $t" - exit 1 - fi - done - - mkdir -p "$out/nix-support" - echo "$unwrapped" > "$out/nix-support/propagated-user-env-packages" - ''; -} diff --git a/pkgs/build-support/kernel/make-initrd.nix b/pkgs/build-support/kernel/make-initrd.nix index 092ab4586b38..70727f9b49dc 100644 --- a/pkgs/build-support/kernel/make-initrd.nix +++ b/pkgs/build-support/kernel/make-initrd.nix @@ -12,7 +12,9 @@ # `contents = {object = ...; symlink = /init;}' is a typical # argument. -{ stdenv, perl, cpio, contents, ubootChooser, compressor, prepend }: +{ stdenv, perl, cpio, contents, ubootChooser, compressor, prepend +, hostPlatform +}: let inputsFun = ubootName : [ perl cpio ] @@ -22,9 +24,9 @@ in stdenv.mkDerivation { name = "initrd"; builder = ./make-initrd.sh; - nativeBuildInputs = inputsFun stdenv.platform.uboot; + nativeBuildInputs = inputsFun hostPlatform.platform.uboot; - makeUInitrd = makeUInitrdFun stdenv.platform.uboot; + makeUInitrd = makeUInitrdFun hostPlatform.platform.uboot; # !!! should use XML. objects = map (x: x.object) contents; @@ -36,9 +38,5 @@ stdenv.mkDerivation { map (x: [("closure-" + baseNameOf x.symlink) x.object]) contents; pathsFromGraph = ./paths-from-graph.pl; - crossAttrs = { - nativeBuildInputs = inputsFun stdenv.cross.platform.uboot; - makeUInitrd = makeUInitrdFun stdenv.cross.platform.uboot; - }; inherit compressor prepend; } diff --git a/pkgs/build-support/kernel/paths-from-graph.pl b/pkgs/build-support/kernel/paths-from-graph.pl index 9a199a2b3044..747e1edec811 100644 --- a/pkgs/build-support/kernel/paths-from-graph.pl +++ b/pkgs/build-support/kernel/paths-from-graph.pl @@ -1,8 +1,6 @@ # Parses a /nix/store/*-closure file and prints # various information. # By default, the nodes in the graph are printed to stdout. -# If the environment variable printManifest is set, -# then the graph is written as a manifest. # If printRegistration is set, then the graph is written # as a registration file for a manifest is written # in the `nix-store --load-db' format. @@ -46,26 +44,7 @@ foreach my $graph (@ARGV) { } -if ($ENV{"printManifest"} eq "1") { - print "version {\n"; - print " ManifestVersion: 3\n"; - print "}\n"; - - foreach my $storePath (sort (keys %storePaths)) { - my $base = basename $storePath; - print "localPath {\n"; - print " StorePath: $storePath\n"; - print " CopyFrom: /tmp/inst-store/$base\n"; - print " References: "; - foreach my $ref (@{$refs{$storePath}}) { - print "$ref "; - } - print "\n"; - print "}\n"; - } -} - -elsif ($ENV{"printRegistration"} eq "1") { +if ($ENV{"printRegistration"} eq "1") { # This is the format used by `nix-store --register-validity # --hash-given' / `nix-store --load-db'. foreach my $storePath (sort (keys %storePaths)) { diff --git a/pkgs/build-support/libredirect/libredirect.c b/pkgs/build-support/libredirect/libredirect.c index ed0d5b0043d5..d1e8f77fb1f6 100644 --- a/pkgs/build-support/libredirect/libredirect.c +++ b/pkgs/build-support/libredirect/libredirect.c @@ -47,6 +47,7 @@ static void init() static const char * rewrite(const char * path, char * buf) { + if (path == NULL) return path; for (int n = 0; n < nrRedirects; ++n) { int len = strlen(from[n]); if (strncmp(path, from[n], len) != 0) continue; diff --git a/pkgs/build-support/make-startupitem/default.nix b/pkgs/build-support/make-startupitem/default.nix index fad6f00d8211..da1d4105c89f 100644 --- a/pkgs/build-support/make-startupitem/default.nix +++ b/pkgs/build-support/make-startupitem/default.nix @@ -19,14 +19,14 @@ stdenv.mkDerivation { priority = 5; buildCommand = '' - mkdir -p $out/share/autostart + mkdir -p $out/etc/xdg/autostart target=${name}.desktop cp ${package}/share/applications/${srcPrefix}${name}.desktop $target chmod +rw $target echo "X-KDE-autostart-phase=${phase}" >> $target ${lib.optionalString (after != null) ''echo "${after}" >> $target''} ${lib.optionalString (condition != null) ''echo "${condition}" >> $target''} - cp $target $out/share/autostart + cp $target $out/etc/xdg/autostart ''; # this will automatically put 'package' in the environment when you diff --git a/pkgs/build-support/plugins.nix b/pkgs/build-support/plugins.nix new file mode 100644 index 000000000000..bf8a982a88f9 --- /dev/null +++ b/pkgs/build-support/plugins.nix @@ -0,0 +1,29 @@ +{ stdenv }: +# helper functions for packaging programs with plugin systems +{ + + /* Takes a list of expected plugin names + * and compares it to the found plugins given in the file, + * one plugin per line. + * If the lists differ, the build fails with a nice message. + * + * This is helpful to ensure maintainers don’t miss + * the addition or removal of a plugin. + */ + diffPlugins = expectedPlugins: foundPluginsFilePath: '' + # sort both lists first + plugins_expected=$(mktemp) + (${stdenv.lib.concatMapStrings (s: "echo \"${s}\";") expectedPlugins}) \ + | sort -u > "$plugins_expected" + plugins_found=$(mktemp) + sort -u "${foundPluginsFilePath}" > "$plugins_found" + + if ! mismatches="$(diff -y "$plugins_expected" "$plugins_found")"; then + echo "The the list of expected plugins (left side) doesn't match" \ + "the list of plugins we found (right side):" >&2 + echo "$mismatches" >&2 + exit 1 + fi + ''; + +} diff --git a/pkgs/build-support/release/ant-build.nix b/pkgs/build-support/release/ant-build.nix index c77db30a81ce..5ab24132290a 100644 --- a/pkgs/build-support/release/ant-build.nix +++ b/pkgs/build-support/release/ant-build.nix @@ -69,7 +69,7 @@ stdenv.mkDerivation ( mkdir -p $out/bin cat >> $out/bin/${w.name} <<EOF - #! /bin/sh + #!${stdenv.shell} export JAVA_HOME=$jre $jre/bin/java ${cp w} ${if w ? mainClass then w.mainClass else "-jar ${w.jar}"} \$@ EOF diff --git a/pkgs/build-support/release/default.nix b/pkgs/build-support/release/default.nix index 82919cf819fb..5e3eb751b81e 100644 --- a/pkgs/build-support/release/default.nix +++ b/pkgs/build-support/release/default.nix @@ -73,4 +73,56 @@ rec { done ''; + /* Create a channel job which success depends on the success of all of + its contituents. Channel jobs are a special type of jobs that are + listed in the channel tab of Hydra and that can be suscribed. + A tarball of the src attribute is distributed via the channel. + + - constituents: a list of derivations on which the channel success depends. + - name: the channel name that will be used in the hydra interface. + - src: should point to the root folder of the nix-expressions used by the + channel, typically a folder containing a `default.nix`. + + channel { + constituents = [ foo bar baz ]; + name = "my-channel"; + src = ./.; + }; + + */ + channel = + { name, src, constituents ? [], meta ? {}, isNixOS ? true, ... }@args: + stdenv.mkDerivation ({ + preferLocalBuild = true; + _hydraAggregate = true; + + phases = [ "unpackPhase" "patchPhase" "installPhase" ]; + + patchPhase = stdenv.lib.optionalString isNixOS '' + touch .update-on-nixos-rebuild + ''; + + installPhase = '' + mkdir -p $out/{tarballs,nix-support} + + tar cJf "$out/tarballs/nixexprs.tar.xz" \ + --owner=0 --group=0 --mtime="1970-01-01 00:00:00 UTC" \ + --transform='s!^\.!${name}!' . + + echo "channel - $out/tarballs/nixexprs.tar.xz" > "$out/nix-support/hydra-build-products" + echo $constituents > "$out/nix-support/hydra-aggregate-constituents" + + # Propagate build failures. + for i in $constituents; do + if [ -e "$i/nix-support/failed" ]; then + touch "$out/nix-support/failed" + fi + done + ''; + + meta = meta // { + isHydraChannel = true; + }; + } // removeAttrs args [ "meta" ]); + } diff --git a/pkgs/build-support/rust/cargo-vendor.nix b/pkgs/build-support/rust/cargo-vendor.nix new file mode 100644 index 000000000000..6b50f8b83e73 --- /dev/null +++ b/pkgs/build-support/rust/cargo-vendor.nix @@ -0,0 +1,32 @@ +{ fetchurl, stdenv }: +let + inherit (stdenv) system; + + version = "0.1.12"; + + hashes = { + x86_64-linux = "1hxlavcxy374yypfamlkygjg662lhll8j434qcvdawkvlidg5ii5"; + x86_64-darwin = "1jkvhh710gwjnnjx59kaplx2ncfvkx9agfa76rr94sbjqq4igddm"; + }; + hash = hashes. ${system} or (throw "missing bootstrap hash for platform ${system}"); + + platforms = { + x86_64-linux = "x86_64-unknown-linux-musl"; + x86_64-darwin = "x86_64-apple-darwin"; + }; + platform = platforms . ${system}; + +in stdenv.mkDerivation { + name = "cargo-vendor-${version}"; + + src = fetchurl { + url = "https://github.com/alexcrichton/cargo-vendor/releases/download/${version}/cargo-vendor-${version}-${platform}.tar.gz"; + sha256 = hash; + }; + + phases = "unpackPhase installPhase"; + + installPhase = '' + install -Dm755 cargo-vendor $out/bin/cargo-vendor + ''; +} diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 2275a065594f..37ea264a53fb 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -1,5 +1,14 @@ -{ stdenv, cacert, git, rust, rustRegistry }: -{ name, depsSha256 +{ fetchurl, stdenv, path, cacert, git, rust }: +let + cargoVendor = import ./cargo-vendor.nix { + inherit fetchurl stdenv; + }; + + fetchcargo = import ./fetchcargo.nix { + inherit stdenv cacert git rust cargoVendor; + }; +in +{ name, cargoSha256 , src ? null , srcs ? null , sourceRoot ? null @@ -11,17 +20,15 @@ , ... } @ args: let - fetchDeps = import ./fetchcargo.nix { - inherit stdenv cacert git rust rustRegistry; - }; + lib = stdenv.lib; - cargoDeps = fetchDeps { + cargoDeps = fetchcargo { inherit name src srcs sourceRoot cargoUpdateHook; - sha256 = depsSha256; + sha256 = cargoSha256; }; in stdenv.mkDerivation (args // { - inherit cargoDeps rustRegistry; + inherit cargoDeps; patchRegistryDeps = ./patch-registry-deps; @@ -36,71 +43,24 @@ in stdenv.mkDerivation (args // { postUnpack = '' eval "$cargoDepsHook" - echo "Using cargo deps from $cargoDeps" + mkdir .cargo + cat >.cargo/config <<-EOF + [source.crates-io] + registry = 'https://github.com/rust-lang/crates.io-index' + replace-with = 'vendored-sources' - cp -a "$cargoDeps" deps - chmod +w deps -R - - # It's OK to use /dev/null as the URL because by the time we do this, cargo - # won't attempt to update the registry anymore, so the URL is more or less - # irrelevant - - cat <<EOF > deps/config - [registry] - index = "file:///dev/null" + [source.vendored-sources] + directory = '$cargoDeps' EOF - export CARGO_HOME="$(realpath deps)" export RUST_LOG=${logLevel} export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt - - # Let's find out which $indexHash cargo uses for file:///dev/null - (cd $sourceRoot && cargo fetch &>/dev/null) || true - cd deps - indexHash="$(basename $(echo registry/index/*))" - - echo "Using indexHash '$indexHash'" - - rm -rf -- "registry/cache/$indexHash" \ - "registry/index/$indexHash" - - mv registry/cache/HASH "registry/cache/$indexHash" - - echo "Using rust registry from $rustRegistry" - ln -s "$rustRegistry" "registry/index/$indexHash" - - # Retrieved the Cargo.lock file which we saved during the fetch - cd .. - mv deps/Cargo.lock $sourceRoot/ - - ( - cd $sourceRoot - - cargo fetch - cargo clean - ) '' + (args.postUnpack or ""); - prePatch = '' - # Patch registry dependencies, using the scripts in $patchRegistryDeps - ( - set -euo pipefail - - cd $NIX_BUILD_TOP/deps/registry/src/* - - for script in $patchRegistryDeps/*; do - # Run in a subshell so that directory changes and shell options don't - # affect any following commands - - ( . $script) - done - ) - '' + (args.prePatch or ""); - buildPhase = with builtins; args.buildPhase or '' runHook preBuild echo "Running cargo build --release ${concatStringsSep " " cargoBuildFlags}" - cargo build --release ${concatStringsSep " " cargoBuildFlags} + cargo build --release --frozen ${concatStringsSep " " cargoBuildFlags} runHook postBuild ''; @@ -119,4 +79,6 @@ in stdenv.mkDerivation (args // { find target/release -maxdepth 1 -executable -exec cp "{}" $out/bin \; runHook postInstall ''; + + passthru = { inherit cargoDeps; } // (args.passthru or {}); }) diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps deleted file mode 100755 index 76661a4f9ecc..000000000000 --- a/pkgs/build-support/rust/fetch-cargo-deps +++ /dev/null @@ -1,209 +0,0 @@ -# copied from libgit2 source code 'repo-template.h' -makeGitTemplate() { - local target="$1" - mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs" - mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack" - cat <<'EOF' > "$target/description" -Unnamed repository; edit this file 'description' to name the repository. -EOF - chmod 666 "$target/description" - cat <<'EOF' > "$target/info/exclude" -# File patterns to ignore; see `git help ignore` for more information. -# Lines that start with '#' are comments. -EOF -} - -fetchCargoDeps() { - src=$(realpath $1) - out=$(realpath $2) - - echo "Fetching $src to $out" - - mkdir $out - - # Configure git template dir to make libgit2 more deterministic - # - # Without a template dir, libgit2 defaults to /usr/share/git-core/templates, - # which can vary between systems if sandboxed builds aren't used. - # - # Note: we explictly set --tmpdir for mktemp here to make it more friendly - # for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP - echo "Setting up git templatedir" - export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)" - makeGitTemplate "$GIT_TEMPLATE_DIR" - export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)" - mkdir -p $XDG_CONFIG_HOME/git - cat <<EOF > $XDG_CONFIG_HOME/git/config -[init] - templatedir = $GIT_TEMPLATE_DIR -EOF - - # Configure cargo to fetch from a local copy of the crates.io registry - - echo "Using rust registry from $rustRegistry" - - cat <<EOF > $out/config -[registry] -index = "file://$rustRegistry" -EOF - - export CARGO_HOME=$out - cd $src - - if [[ ! -f Cargo.lock ]]; then - echo - echo "ERROR: The Cargo.lock file doesn't exist" - echo - echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" - echo "when the registry is updated." - echo - - exit 1 - fi - - # We need to do the following string replacement so that 'cargo fetch' - # doesn't ignore the versions specified in Cargo.lock - substituteInPlace Cargo.lock \ - --replace "registry+https://github.com/rust-lang/crates.io-index" \ - "registry+file://$rustRegistry" - - # Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates - eval "$cargoUpdateHook" - - # Do the fetch - cargo fetch --verbose - - # Now that we have fetched everything, let's make the output deterministic - - # Cargo uses the following directory structure for fetched data, where - # $indexHash is a hash of the registry index URL: - # - # - # /config: - # - # Cargo config file. We'll delete this because it's not deterministic, - # and instead recreate it just before running 'cargo build'. - # - # /registry/cache/$indexHash/: - # - # This is where tarballs of registry package dependencies are kept - # We'll need to keep this, but make sure $indexHash is a fixed name. - # - # /registry/index/$indexHash/: - # - # A copy of the registry index is kept here. We can delete this, and - # instead, just before running 'cargo build', we'll symlink this - # directory to our static copy of the registry in the Nix store. - # - # /registry/src/$indexHash/{pkgName-pkgVersion}/: - # - # Here cargo keeps extracted sources of the cached tarballs. - # We'll just delete this because cargo will re-populate them from the - # tarballs. - # - # /git/db/{domain-hash}/: - # - # Here cargo keeps the `.git` directories of git dependencies. - # We'll need to keep these, but make them deterministic. - # - # /git/checkouts/{domain-hash}/{branchName}/: - # - # Here cargo keeps checked-out sources of the git dependencies. - # We can delete this, because cargo will re-populate them from the above - # `.git` directories. - # - # Let's start - - # Remove cargo config file, which points to the ever-changing registry - rm $out/config - - # Save the Cargo.lock file into the output, so that we don't have to do another - # 'cargo update' during the build (which would try to access the network) for - # any ad-hoc package updates (through $cargoUpdateHook). - # - # We need to replace the rustRegistry URL with something deterministic. - # Since the URL won't actually be accessed anymore, it's fine to use /dev/null. - - substituteInPlace Cargo.lock \ - --replace "registry+file://$rustRegistry" \ - "registry+file:///dev/null" - mv Cargo.lock $out/ - - - # Let's replace $indexHash with something more deterministic - mv $out/registry/cache/* $out/registry/cache/HASH - - # The registry index changes all the time, so it's not deterministic - # We'll symlink it before running 'cargo build' - rm -rf $out/registry/index/* - - # Make git DBs deterministic - # TODO: test with git submodules - [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do - revs="" - cd "$out/git/checkouts/$name" - while read dir; do - # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] - branch="${dir:2:$((${#dir}-7))}" - - cd "$out/git/checkouts/$name/$branch" - rev="$(git rev-parse HEAD)" - revs="$revs $rev" - done < <(find . -type d -name .git -print) - - echo "List of revs to keep for git db $name: $revs" - - ( - # The following code was adapted from nix-prefetch-git - - cd "$out/git/db/$name" - - export GIT_DIR=. - - # Remove all remote branches - git branch -r | while read branch; do - git branch -rD "$branch" >&2 - done - - # Remove all tags - git tag | while read tag; do - git tag -d "$tag" >&2 - done - - # Remove all local branches - branchrefs=() - eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" - - for branchref in "${branchrefs[@]}"; do - git update-ref -d "$branchref" >&2 - done - - # Create ad-hoc branches for the revs we need - echo "$revs" | tr " " "\n" | while read -d " " rev; do - echo "Creating git branch b_$rev $rev" - git branch b_$rev $rev - done - - # Remove files that have timestamps or otherwise have non-deterministic - # properties. - rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config - - # Do a full repack. Must run single-threaded, or else we lose determinism. - git config pack.threads 1 - git repack -A -d -f - rm -f config - - # Garbage collect unreferenced objects. - git gc --prune=all - ) - done) - - # Remove unneeded outputs - [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src - [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts - - # XXX: provide some debugging output to see find out why we are seeing - # sporadic hash mismatches - find $out ! -type f - find $out -type f -exec sha256sum {} + -} diff --git a/pkgs/build-support/rust/fetchcargo.nix b/pkgs/build-support/rust/fetchcargo.nix index 0c9625e51405..9b3ba5303398 100644 --- a/pkgs/build-support/rust/fetchcargo.nix +++ b/pkgs/build-support/rust/fetchcargo.nix @@ -1,19 +1,30 @@ -{ stdenv, cacert, git, rust, rustRegistry }: +{ stdenv, cacert, git, rust, cargoVendor }: { name ? "cargo-deps", src, srcs, sourceRoot, sha256, cargoUpdateHook ? "" }: - stdenv.mkDerivation { - name = "${name}-fetch"; - buildInputs = [ rust.cargo rust.rustc git ]; - inherit src srcs sourceRoot rustRegistry cargoUpdateHook; + name = "${name}-vendor"; + buildInputs = [ cacert cargoVendor git rust.cargo ]; + inherit src srcs sourceRoot; phases = "unpackPhase installPhase"; installPhase = '' - source ${./fetch-cargo-deps} + if [[ ! -f Cargo.lock ]]; then + echo + echo "ERROR: The Cargo.lock file doesn't exist" + echo + echo "Cargo.lock is needed to make sure that cargoSha256 doesn't change" + echo "when the registry is updated." + echo + + exit 1 + fi export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt + export CARGO_HOME=$(mktemp -d cargo-home.XXX) + + cargo vendor - fetchCargoDeps . "$out" + cp -ar vendor $out ''; outputHashAlgo = "sha256"; diff --git a/pkgs/build-support/setup-hooks/audit-tmpdir.sh b/pkgs/build-support/setup-hooks/audit-tmpdir.sh new file mode 100644 index 000000000000..ffaa61f2d809 --- /dev/null +++ b/pkgs/build-support/setup-hooks/audit-tmpdir.sh @@ -0,0 +1,41 @@ +# Check whether RPATHs or wrapper scripts contain references to +# $TMPDIR. This is a serious security bug because it allows any user +# to inject files into search paths of other users' processes. +# +# It might be better to have Nix scan build output for any occurrence +# of $TMPDIR (which would also be good for reproducibility), but at +# the moment that would produce too many spurious errors (e.g. debug +# info or assertion messages that refer to $TMPDIR). + +fixupOutputHooks+=('if [ -z "$noAuditTmpdir" -a -e "$prefix" ]; then auditTmpdir "$prefix"; fi') + +auditTmpdir() { + local dir="$1" + [ -e "$dir" ] || return 0 + + header "checking for references to $TMPDIR in $dir..." + + local i + while IFS= read -r -d $'\0' i; do + if [[ "$i" =~ .build-id ]]; then continue; fi + + if isELF "$i"; then + if patchelf --print-rpath "$i" | grep -q -F "$TMPDIR"; then + echo "RPATH of binary $i contains a forbidden reference to $TMPDIR" + exit 1 + fi + fi + + if isScript "$i"; then + if [ -e "$(dirname $i)/.$(basename $i)-wrapped" ]; then + if grep -q -F "$TMPDIR" "$i"; then + echo "wrapper script $i contains a forbidden reference to $TMPDIR" + exit 1 + fi + fi + fi + + done < <(find "$dir" -type f -print0) + + stopNest +} diff --git a/pkgs/build-support/setup-hooks/autoreconf.sh b/pkgs/build-support/setup-hooks/autoreconf.sh index 441d6b43baa2..c08cab158688 100644 --- a/pkgs/build-support/setup-hooks/autoreconf.sh +++ b/pkgs/build-support/setup-hooks/autoreconf.sh @@ -1,9 +1,5 @@ preConfigurePhases+=" autoreconfPhase" -for i in @autoconf@ @automake@ @libtool@ @gettext@; do - findInputs $i nativePkgs propagated-native-build-inputs -done - autoreconfPhase() { runHook preAutoreconf autoreconf ${autoreconfFlags:---install --force --verbose} diff --git a/pkgs/build-support/setup-hooks/die.sh b/pkgs/build-support/setup-hooks/die.sh new file mode 100644 index 000000000000..0db41e030f4c --- /dev/null +++ b/pkgs/build-support/setup-hooks/die.sh @@ -0,0 +1,21 @@ +# Exit with backtrace and error message +# +# Usage: die "Error message" +die() { + # Let us be a little sloppy with errors, because otherwise the final + # invocation of `caller` below will cause the script to exit. + set +e + + # Print our error message + printf "\nBuilder called die: %b\n" "$*" + printf "Backtrace:\n" + + # Print a backtrace. + local frame=0 + while caller $frame; do + ((frame++)); + done + printf "\n" + + exit 1 +} diff --git a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh index 8fe661026774..1b36f5f555da 100644 --- a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh +++ b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh @@ -1,4 +1,4 @@ -# On Mac OS X, binaries refer to dynamic library dependencies using +# On macOS, binaries refer to dynamic library dependencies using # either relative paths (e.g. "libicudata.dylib", searched relative to # $DYLD_LIBRARY_PATH) or absolute paths # (e.g. "/nix/store/.../lib/libicudata.dylib"). In Nix, the latter is diff --git a/pkgs/build-support/setup-hooks/fix-darwin-frameworks.sh b/pkgs/build-support/setup-hooks/fix-darwin-frameworks.sh deleted file mode 100644 index e3a08b2598d9..000000000000 --- a/pkgs/build-support/setup-hooks/fix-darwin-frameworks.sh +++ /dev/null @@ -1,31 +0,0 @@ -# On Mac OS X, frameworks are linked to the system CoreFoundation but -# dynamic libraries built with nix use a pure version of CF this -# causes segfaults for binaries that depend on it at runtime. This -# can be solved in two ways. -# 1. Rewrite references to the pure CF using this setup hook, this -# works for the simple case but this can still cause problems if other -# dependencies (eg. python) use the pure CF. -# 2. Create a wrapper for the binary that sets DYLD_FRAMEWORK_PATH to -# /System/Library/Frameworks. This will make everything load the -# system's CoreFoundation framework while still keeping the -# dependencies pure for other packages. - -fixupOutputHooks+=('fixDarwinFrameworksIn $prefix') - -fixDarwinFrameworks() { - local systemPrefix='/System/Library/Frameworks' - - for fn in "$@"; do - if [ -L "$fn" ]; then continue; fi - echo "$fn: fixing dylib" - - for framework in $(otool -L "$fn" | awk '/CoreFoundation\.framework/ {print $1}'); do - install_name_tool -change "$framework" "$systemPrefix/CoreFoundation.framework/Versions/A/CoreFoundation" "$fn" >&2 - done - done -} - -fixDarwinFrameworksIn() { - local dir="$1" - fixDarwinFrameworks $(find "$dir" -name "*.dylib") -} diff --git a/pkgs/build-support/setup-hooks/make-wrapper.sh b/pkgs/build-support/setup-hooks/make-wrapper.sh index 96e50773138b..f9d435df655b 100644 --- a/pkgs/build-support/setup-hooks/make-wrapper.sh +++ b/pkgs/build-support/setup-hooks/make-wrapper.sh @@ -1,3 +1,12 @@ +# Assert that FILE exists and is executable +# +# assertExecutable FILE +assertExecutable() { + local file="$1" + [[ -f "$file" && -x "$file" ]] || \ + die "Cannot wrap '$file' because it is not an executable file" +} + # construct an executable file that wraps the actual executable # makeWrapper EXECUTABLE ARGS @@ -24,6 +33,8 @@ makeWrapper() { local params varName value command separator n fileNames local argv0 flagsBefore flags + assertExecutable "$original" + mkdir -p "$(dirname "$wrapper")" echo "#! $SHELL -e" > "$wrapper" @@ -32,26 +43,20 @@ makeWrapper() { for ((n = 2; n < ${#params[*]}; n += 1)); do p="${params[$n]}" - if test "$p" = "--set"; then + if [[ "$p" == "--set" ]]; then varName="${params[$((n + 1))]}" value="${params[$((n + 2))]}" n=$((n + 2)) echo "export $varName=\"$value\"" >> "$wrapper" - fi - - if test "$p" = "--unset"; then + elif [[ "$p" == "--unset" ]]; then varName="${params[$((n + 1))]}" n=$((n + 1)) echo "unset $varName" >> "$wrapper" - fi - - if test "$p" = "--run"; then + elif [[ "$p" == "--run" ]]; then command="${params[$((n + 1))]}" n=$((n + 1)) echo "$command" >> "$wrapper" - fi - - if test "$p" = "--suffix" -o "$p" = "--prefix"; then + elif [[ ("$p" == "--suffix") || ("$p" == "--prefix") ]]; then varName="${params[$((n + 1))]}" separator="${params[$((n + 2))]}" value="${params[$((n + 3))]}" @@ -63,9 +68,7 @@ makeWrapper() { echo "export $varName=$value\${$varName:+$separator}\$$varName" >> "$wrapper" fi fi - fi - - if test "$p" = "--suffix-each"; then + elif [[ "$p" == "--suffix-each" ]]; then varName="${params[$((n + 1))]}" separator="${params[$((n + 2))]}" values="${params[$((n + 3))]}" @@ -73,9 +76,7 @@ makeWrapper() { for value in $values; do echo "export $varName=\$$varName\${$varName:+$separator}$value" >> "$wrapper" done - fi - - if test "$p" = "--suffix-contents" -o "$p" = "--prefix-contents"; then + elif [[ ("$p" == "--suffix-contents") || ("$p" == "--prefix-contents") ]]; then varName="${params[$((n + 1))]}" separator="${params[$((n + 2))]}" fileNames="${params[$((n + 3))]}" @@ -87,17 +88,15 @@ makeWrapper() { echo "export $varName=$(cat "$fileName")\${$varName:+$separator}\$$varName" >> "$wrapper" fi done - fi - - if test "$p" = "--add-flags"; then + elif [[ "$p" == "--add-flags" ]]; then flags="${params[$((n + 1))]}" n=$((n + 1)) flagsBefore="$flagsBefore $flags" - fi - - if test "$p" = "--argv0"; then + elif [[ "$p" == "--argv0" ]]; then argv0="${params[$((n + 1))]}" n=$((n + 1)) + else + die "makeWrapper doesn't understand the arg $p" fi done @@ -131,9 +130,15 @@ filterExisting() { wrapProgram() { local prog="$1" local hidden + + assertExecutable "$prog" + hidden="$(dirname "$prog")/.$(basename "$prog")"-wrapped + while [ -e "$hidden" ]; do + hidden="${hidden}_" + done mv "$prog" "$hidden" # Silence warning about unexpanded $0: # shellcheck disable=SC2016 - makeWrapper "$hidden" "$prog" --argv0 '$0' "$@" + makeWrapper "$hidden" "$prog" --argv0 '$0' "${@:2}" } diff --git a/pkgs/build-support/setup-hooks/multiple-outputs.sh b/pkgs/build-support/setup-hooks/multiple-outputs.sh index eafc770a8e17..60d4ccf99ed1 100644 --- a/pkgs/build-support/setup-hooks/multiple-outputs.sh +++ b/pkgs/build-support/setup-hooks/multiple-outputs.sh @@ -40,9 +40,9 @@ _overrideFirst outputLib "lib" "out" _overrideFirst outputDoc "doc" "out" _overrideFirst outputDevdoc "devdoc" REMOVE # documentation for developers # man and info pages are small and often useful to distribute with binaries -_overrideFirst outputMan "man" "doc" "$outputBin" +_overrideFirst outputMan "man" "$outputBin" _overrideFirst outputDevman "devman" "devdoc" "$outputMan" -_overrideFirst outputInfo "info" "doc" "$outputMan" +_overrideFirst outputInfo "info" "$outputBin" # Add standard flags to put files into the desired outputs. @@ -61,7 +61,7 @@ _multioutConfig() { local shareDocName="$(sed -n "s/^PACKAGE_TARNAME='\(.*\)'$/\1/p" < "$confScript")" fi # PACKAGE_TARNAME sometimes contains garbage. - if [ -n "$shareDocName" ] || echo "$shareDocName" | grep -q '[^a-zA-Z-_0-9]'; then + if [ -n "$shareDocName" ] || echo "$shareDocName" | grep -q '[^a-zA-Z0-9_-]'; then shareDocName="$(echo "$name" | sed 's/-[^a-zA-Z].*//')" fi fi diff --git a/pkgs/build-support/setup-hooks/set-source-date-epoch-to-latest.sh b/pkgs/build-support/setup-hooks/set-source-date-epoch-to-latest.sh index fe3458cd21e8..84e40cd0514f 100644 --- a/pkgs/build-support/setup-hooks/set-source-date-epoch-to-latest.sh +++ b/pkgs/build-support/setup-hooks/set-source-date-epoch-to-latest.sh @@ -4,8 +4,9 @@ updateSourceDateEpoch() { # Get the last modification time of all regular files, sort them, # and get the most recent. Maybe we should use # https://github.com/0-wiz-0/findnewest here. - local -a res=($(find "$path" -type f -print0 | xargs -0 -r stat -c '%Y %n' | sort -n | tail -n1)) - local time="${res[0]}" + local -a res=($(find "$path" -type f -not -newer "$NIX_BUILD_TOP/.." -printf '%T@ %p\0' \ + | sort -n --zero-terminated | tail -n1 --zero-terminated | head -c -1)) + local time="${res[0]//\.[0-9]*/}" # remove the fraction part local newestFile="${res[1]}" # Update $SOURCE_DATE_EPOCH if the most recent file we found is newer. diff --git a/pkgs/build-support/setup-hooks/setup-debug-info-dirs.sh b/pkgs/build-support/setup-hooks/setup-debug-info-dirs.sh new file mode 100644 index 000000000000..2fd2a2d6da6f --- /dev/null +++ b/pkgs/build-support/setup-hooks/setup-debug-info-dirs.sh @@ -0,0 +1,5 @@ +setupDebugInfoDirs () { + addToSearchPath NIX_DEBUG_INFO_DIRS $1/lib/debug +} + +envHooks+=(setupDebugInfoDirs) diff --git a/pkgs/build-support/setup-hooks/win-dll-link.sh b/pkgs/build-support/setup-hooks/win-dll-link.sh index 634a9d18f00d..9658b9f82595 100644 --- a/pkgs/build-support/setup-hooks/win-dll-link.sh +++ b/pkgs/build-support/setup-hooks/win-dll-link.sh @@ -35,7 +35,7 @@ _linkDLLs() { local dllPath2 for dllPath2 in "$dllPath" "$(dirname $(readlink "$dllPath" || echo "$dllPath"))"/*.dll; do if [ -e ./"$(basename "$dllPath2")" ]; then continue; fi - ln -sr "$dllPath2" . + CYGWIN+=\ winsymlinks:nativestrict ln -sr "$dllPath2" . linkCount=$(($linkCount+1)) done done diff --git a/pkgs/build-support/setup-hooks/wrap-gapps-hook.sh b/pkgs/build-support/setup-hooks/wrap-gapps-hook.sh index 3cad1838d260..79b8d5b73fa1 100644 --- a/pkgs/build-support/setup-hooks/wrap-gapps-hook.sh +++ b/pkgs/build-support/setup-hooks/wrap-gapps-hook.sh @@ -35,10 +35,16 @@ wrapGAppsHook() { gappsWrapperArgs+=(--prefix $v : "$dummy") done - if [ -z "$dontWrapGApps" ]; then - for i in $prefix/bin/* $prefix/libexec/*; do - echo "Wrapping app $i" - wrapProgram "$i" "${gappsWrapperArgs[@]}" + if [[ -z "$dontWrapGApps" ]]; then + targetDirs=( "${prefix}/bin" "${prefix}/libexec" ) + for targetDir in "${targetDirs[@]}"; do + if [[ -d "${targetDir}" ]]; then + find -L "${targetDir}" -type f -executable -print0 \ + | while IFS= read -r -d '' file; do + echo "Wrapping program ${file}" + wrapProgram "${file}" "${gappsWrapperArgs[@]}" + done + fi done fi } diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 1529869aa331..32214db65842 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -26,6 +26,7 @@ rec { , text , executable ? false # run chmod +x ? , destination ? "" # relative path appended to $out eg "/bin/foo" + , checkPhase ? "" # syntax checks, e.g. for scripts }: runCommand name { inherit text executable; @@ -44,6 +45,8 @@ rec { echo -n "$text" > "$n" fi + ${checkPhase} + (test -n "$executable" && chmod +x "$n") || true ''; @@ -54,6 +57,20 @@ rec { writeScript = name: text: writeTextFile {inherit name text; executable = true;}; writeScriptBin = name: text: writeTextFile {inherit name text; executable = true; destination = "/bin/${name}";}; + # Create a Shell script, check its syntax + writeShellScriptBin = name : text : + writeTextFile { + inherit name; + executable = true; + destination = "/bin/${name}"; + text = '' + #!${stdenv.shell} + ${text} + ''; + checkPhase = '' + ${stdenv.shell} -n $out/bin/${name} + ''; + }; # Create a forest of symlinks to the files in `paths'. symlinkJoin = @@ -71,7 +88,7 @@ rec { '' mkdir -p $out for i in $paths; do - ${lndir}/bin/lndir $i $out + ${lndir}/bin/lndir -silent $i $out done ${postBuild} ''; @@ -84,7 +101,7 @@ rec { mkdir -p $out/nix-support cp ${script} $out/nix-support/setup-hook '' + lib.optionalString (deps != []) '' - echo ${toString deps} > $out/nix-support/propagated-native-build-inputs + printWords ${toString deps} > $out/nix-support/propagated-native-build-inputs '' + lib.optionalString (substitutions != {}) '' substituteAll ${script} $out/nix-support/setup-hook ''); diff --git a/pkgs/build-support/vm/default.nix b/pkgs/build-support/vm/default.nix index 19a3f24a470d..d886e9a56fa4 100644 --- a/pkgs/build-support/vm/default.nix +++ b/pkgs/build-support/vm/default.nix @@ -750,6 +750,7 @@ rec { { name, fullName, size ? 4096, urlPrefix , packagesList ? "", packagesLists ? [packagesList] , packages, extraPackages ? [], postInstall ? "" + , extraDebs ? [] , QEMU_OPTS ? "", memSize ? 512 }: let @@ -760,7 +761,7 @@ rec { in (fillDiskWithDebs { inherit name fullName size postInstall QEMU_OPTS memSize; - debs = import expr {inherit fetchurl;}; + debs = import expr {inherit fetchurl;} ++ extraDebs; }) // {inherit expr;}; @@ -1954,22 +1955,22 @@ rec { }; debian8i386 = { - name = "debian-8.7-jessie-i386"; - fullName = "Debian 8.7 Jessie (i386)"; + name = "debian-8.9-jessie-i386"; + fullName = "Debian 8.9 Jessie (i386)"; packagesList = fetchurl { url = mirror://debian/dists/jessie/main/binary-i386/Packages.xz; - sha256 = "71cacb934dc4ab2e67a5ed215ccbc9836cf8d95687edec7e7fe8d3916e3b3fe8"; + sha256 = "3c78bdf3b693f2f37737c52d6a7718b3a545956f2a853da79f04a2d15541e811"; }; urlPrefix = mirror://debian; packages = commonDebianPackages; }; debian8x86_64 = { - name = "debian-8.7-jessie-amd64"; - fullName = "Debian 8.7 Jessie (amd64)"; + name = "debian-8.9-jessie-amd64"; + fullName = "Debian 8.9 Jessie (amd64)"; packagesList = fetchurl { url = mirror://debian/dists/jessie/main/binary-amd64/Packages.xz; - sha256 = "b4cfbaaef31f05ce1726d00f0a173f5b6f33a9192513302319a49848884a17f3"; + sha256 = "0605589ae7a63c690f37bd2567dc12e02a2eb279d9dc200a7310072ad3593e53"; }; urlPrefix = mirror://debian; packages = commonDebianPackages; diff --git a/pkgs/build-support/vm/windows/controller/default.nix b/pkgs/build-support/vm/windows/controller/default.nix index 9009702113ea..17803a28330f 100644 --- a/pkgs/build-support/vm/windows/controller/default.nix +++ b/pkgs/build-support/vm/windows/controller/default.nix @@ -185,7 +185,7 @@ let MONITOR_SOCKET="$(pwd)/monitor" WINVM_PIDFILE="$(pwd)/winvm.pid" CTRLVM_PIDFILE="$(pwd)/ctrlvm.pid" - ${vde2}/bin/vde_switch -s "$QEMU_VDE_SOCKET" & + ${vde2}/bin/vde_switch -s "$QEMU_VDE_SOCKET" --dirmode 0700 & echo 'alive?' | ${socat}/bin/socat - \ UNIX-CONNECT:$QEMU_VDE_SOCKET/ctl,retry=20 ''; diff --git a/pkgs/build-support/vm/windows/default.nix b/pkgs/build-support/vm/windows/default.nix index c668e7569a44..f15752199822 100644 --- a/pkgs/build-support/vm/windows/default.nix +++ b/pkgs/build-support/vm/windows/default.nix @@ -1,3 +1,4 @@ +#note: the hardcoded /bin/sh is required for the VM's cygwin shell pkgs: let |