From 30bff42231ae088d7f567bda5b9af9a3905999bf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 5 Apr 2018 16:58:50 +0200 Subject: linux module handling: support kernels without modules --- pkgs/build-support/kernel/modules-closure.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh index 8287c1672d06..9a49d42cdc1a 100644 --- a/pkgs/build-support/kernel/modules-closure.sh +++ b/pkgs/build-support/kernel/modules-closure.sh @@ -1,5 +1,19 @@ source $stdenv/setup +# When no modules are built, the $out/lib/modules directory will not +# exist. Because the rest of the script assumes it does exist, we +# handle this special case first. +if ! test -d "$out/lib/modules"; then + if test -z "$rootModules" || test -n "$allowMissing"; then + mkdir -p "$out" + exit 0 + else + echo "Required modules: $rootModules" + echo "Can not derive a closure of kernel modules because no modules were provided." + exit 1 + fi +fi + version=$(cd $kernel/lib/modules && ls -d *) echo "kernel version is $version" -- cgit 1.4.1 From de581b99cabc8af056aa718fd1585fdced58b757 Mon Sep 17 00:00:00 2001 From: aszlig Date: Thu, 12 Apr 2018 15:38:16 +0200 Subject: kernel: Fix running kernels *with* modules Pull request #38470 added support for running/building kernels without modules. This got merged in 38e04bbf29fe3b6af26b3505a42ce5871aeac17d but unfortunately while this works perfectly on kernels without modules it also makes sure that *every* kernel gets no modules. So all of our VM tests fail since that merge with something like this: machine# loading module loop... machine# modprobe: FATAL: Module loop not found in directory /lib/modules/4.14.33 machine# loading module vfat... machine# modprobe: FATAL: Module vfat not found in directory /lib/modules/4.14.33 machine# loading module nls_cp437... machine# modprobe: FATAL: Module nls_cp437 not found in directory /lib/modules/4.14.33 machine# loading module nls_iso8859-1... machine# modprobe: FATAL: Module nls_iso8859-1 not found in directory /lib/modules/4.14.33 machine# loading module fuse... machine# modprobe: FATAL: Module fuse not found in directory /lib/modules/4.14.33 machine# loading module dm_mod... machine# modprobe: FATAL: Module dm_mod not found in directory /lib/modules/4.14.33 I shortly tested this against the "misc" VM test and the test is working again. In the long term (and I currently don't have time for this) it would be better to also have a VM test which tests a kernel without modules. Signed-off-by: aszlig Cc: @roberth, @7c6f434c --- pkgs/build-support/kernel/modules-closure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh index 9a49d42cdc1a..5658094872c8 100644 --- a/pkgs/build-support/kernel/modules-closure.sh +++ b/pkgs/build-support/kernel/modules-closure.sh @@ -3,7 +3,7 @@ source $stdenv/setup # When no modules are built, the $out/lib/modules directory will not # exist. Because the rest of the script assumes it does exist, we # handle this special case first. -if ! test -d "$out/lib/modules"; then +if ! test -d "$kernel/lib/modules"; then if test -z "$rootModules" || test -n "$allowMissing"; then mkdir -p "$out" exit 0 -- cgit 1.4.1 From da8fc391a0897a3897630ce322ad19a825a4ac93 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 13 Apr 2018 12:25:10 -0400 Subject: pythonPackages.buildSetupcfg: Allow disabling tests. --- pkgs/build-support/build-setupcfg/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/build-setupcfg/default.nix b/pkgs/build-support/build-setupcfg/default.nix index 62dda59f1d52..bc6482f0a88e 100644 --- a/pkgs/build-support/build-setupcfg/default.nix +++ b/pkgs/build-support/build-setupcfg/default.nix @@ -5,18 +5,20 @@ # * meta: Standard nixpkgs metadata. # * application: Whether this package is a python library or an # application which happens to be written in python. -pythonPackages: { src, info, meta ? {}, application ? false }: let +# * doCheck: Whether to run the test suites. +pythonPackages: +{ src, info, meta ? {}, application ? false, doCheck ? true }: let build = if application then pythonPackages.buildPythonApplication else pythonPackages.buildPythonPackage; in build { inherit (info) pname version; - inherit src meta; + inherit src meta doCheck; nativeBuildInputs = map (p: pythonPackages.${p}) ( (info.setup_requires or []) ++ - (info.tests_require or [])); + (if doCheck then (info.tests_require or []) else [])); propagatedBuildInputs = map (p: pythonPackages.${p}) (info.install_requires or []); -- cgit 1.4.1 From 1a72330ab067cb3c48c6e6ef3d5e0d2d24a95417 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2017 15:05:30 -0400 Subject: cc-wrapper: Utilize patched cctools ld for more robust macOS Sierra hack Also fix numberous bugs, such as: - Not getting confused on more flags taking file arguments. - Ensuring children reexport their children, but the original binary/library doesn't. - Not spawning children when it turns out we just dynamically link under the threshold but our total number of inputs exceeeds it. - Children were always named `libunnamed-*`, when that name was supposed to be the last resort only. ld-wrapper's own RPATH check hardcodes `.so`, but darwin uses `.dylib` *and* (in practice due to lousy build systems) `.so`. We don't care however because we never inject `--rpath` like that in practice on Darwin. Hopefully someday we won't on linux either. --- .../macos-sierra-reexport-hack.bash | 312 +++++++++++++++------ 1 file changed, 225 insertions(+), 87 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/bintools-wrapper/macos-sierra-reexport-hack.bash b/pkgs/build-support/bintools-wrapper/macos-sierra-reexport-hack.bash index a0c4e9edfcdb..71b9471cbc83 100644 --- a/pkgs/build-support/bintools-wrapper/macos-sierra-reexport-hack.bash +++ b/pkgs/build-support/bintools-wrapper/macos-sierra-reexport-hack.bash @@ -2,107 +2,245 @@ set -eu -o pipefail +# For cmd | while read; do ...; done +shopt -s lastpipe + path_backup="$PATH" if [ -n "@coreutils_bin@" ]; then PATH="@coreutils_bin@/bin" fi -declare -r recurThreshold=300 - -declare overflowCount=0 -for ((n=0; n < $#; ++n)); do - case "${!n}" in - -l*) let overflowCount+=1 ;; - -reexport-l*) let overflowCount+=1 ;; - *) ;; +declare -ri recurThreshold=200 +declare -i overflowCount=0 + +declare -ar origArgs=("$@") + +# Throw away what we won't need +declare -a parentArgs=() + +while (( $# )); do + case "$1" in + -l) + echo "cctools LD does not support '-l foo'" >&2 + exit 1 + ;; + -lazy_library | -reexport_library | -upward_library | -weak_library) + overflowCount+=1 + shift 2 + ;; + -l* | *.so.* | *.dylib | -lazy-l* | -reexport-l* | -upward-l* | -weak-l*) + overflowCount+=1 + shift 1 + ;; + *.a | *.o) + shift 1 + ;; + -L | -F) + # Evidentally ld doesn't like using the child's RPATH, so it still + # needs these. + parentArgs+=("$1" "$2") + shift 2 + ;; + -L?* | -F?*) + parentArgs+=("$1") + shift 1 + ;; + -o) + outputName="$2" + parentArgs+=("$1" "$2") + shift 2 + ;; + -install_name | -dylib_install_name | -dynamic-linker | -plugin) + parentArgs+=("$1" "$2") + shift 2 + ;; + -rpath) + # Only an rpath to the child is needed, which we will add + shift 2 + ;; + *) + if [[ -f "$1" ]]; then + # Propabably a non-standard object file like Haskell's + # `.dyn_o`. Skip it like other inputs + : + else + parentArgs+=("$1") + fi + shift 1 + ;; esac done -declare -a allArgs=() + if (( "$overflowCount" <= "$recurThreshold" )); then - allArgs=("$@") -else - declare -a childrenLookup=() childrenLink=() - - while (( $# )); do - case "$1" in - -L/*) - childrenLookup+=("$1") - allArgs+=("$1") - ;; - -L) - echo "cctools LD does not support '-L foo' or '-l foo'" >&2 - exit 1 - ;; - -l) - echo "cctools LD does not support '-L foo' or '-l foo'" >&2 - exit 1 - ;; - -lazy_library | -lazy_framework | -lto_library) - # We aren't linking any "azy_library", "to_library", etc. - allArgs+=("$1") - ;; - -lazy-l | -weak-l) allArgs+=("$1") ;; - # We can't so easily prevent header issues from these. - -lSystem) allArgs+=("$1") ;; - # Special case as indirection seems like a bad idea for something - # so fundamental. Can be removed for simplicity. - -l?* | -reexport-l?*) childrenLink+=("$1") ;; - *) allArgs+=("$1") ;; - esac - - shift - done + if [ -n "${NIX_DEBUG:-}" ]; then + echo "ld-wrapper: Only ${overflowCount} inputs counted while ${recurThreshold} is the ceiling, linking normally. " >&2 + fi + PATH="$path_backup" + exec @prog@ "${origArgs[@]}" +fi + + + +if [ -n "${NIX_DEBUG:-}" ]; then + echo "ld-wrapper: ${overflowCount} inputs counted when ${recurThreshold} is the ceiling, inspecting further. " >&2 +fi + +# Collect the normalized linker input +declare -a norm=() - declare n=0 - while (( $n < "${#childrenLink[@]}" )); do - if [[ "${childrenLink[n]}" = -l* ]]; then - childrenLink[n]="-reexport${childrenLink[n]}" - fi - let ++n +# Arguments are null-separated +@prog@ --dump-normalized-lib-args "${origArgs[@]}" | + while IFS= read -r -d '' input; do + norm+=("$input") done - unset n - - declare -r outputNameLibless=$(basename $( \ - if [[ -z "${outputName:+isUndefined}" ]]; then - echo unnamed - elif [[ "${outputName:0:3}" = lib ]]; then - echo "${outputName:3}" - else - echo "${outputName}" - fi)) - declare -ra children=("$outputNameLibless-reexport-delegate-0" \ - "$outputNameLibless-reexport-delegate-1") - - mkdir -p "$out/lib" - - PATH="$PATH:@out@/bin" - - symbolBloatObject=$outputNameLibless-symbol-hack.o - if [[ ! -e $symbolBloatObject ]]; then - # `-Q` means use GNU Assembler rather than Clang, avoiding an awkward - # dependency cycle. - printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' \ - | @targetPrefix@as -Q -- -o $symbolBloatObject + +declare -i leafCount=0 +declare lastLeaf='' +declare -a childrenInputs=() trailingInputs=() +while (( "${#norm[@]}" )); do + case "${norm[0]}" in + -lazy_library | -upward_library) + # TODO(@Ericson2314): Don't do that, but intersperse children + # between such args. + echo "ld-wrapper: Warning: Potentially changing link order" >&2 + trailingInputs+=("${norm[0]}" "${norm[1]}") + norm=("${norm[@]:2}") + ;; + -reexport_library | -weak_library) + childrenInputs+=("${norm[0]}" "${norm[1]}") + if [[ "${norm[1]}" != "$lastLeaf" ]]; then + leafCount+=1 + lastLeaf="${norm[1]}" + fi + norm=("${norm[@]:2}") + ;; + *.so | *.dylib) + childrenInputs+=(-reexport_library "${norm[0]}") + if [[ "${norm[0]}" != "$lastLeaf" ]]; then + leafCount+=1 + lastLeaf="${norm[0]}" + fi + norm=("${norm[@]:1}") + ;; + *.o | *.a) + # Don't delegate object files or static libs + parentArgs+=("${norm[0]}") + norm=("${norm[@]:1}") + ;; + *) + if [[ -f "${norm[0]}" ]]; then + # Propabably a non-standard object file. We'll let it by. + parentArgs+=("${norm[0]}") + norm=("${norm[@]:1}") + else + echo "ld-wrapper: Internal Error: Invalid normalized argument" >&2 + exit -1 + fi + ;; + esac +done + + + +if (( "$leafCount" <= "$recurThreshold" )); then + if [ -n "${NIX_DEBUG:-}" ]; then + echo "ld-wrapper: Only ${leafCount} *dynamic* inputs counted while ${recurThreshold} is the ceiling, linking normally. " >&2 + fi + PATH="$path_backup" + exec @prog@ "${origArgs[@]}" +fi + + + +if [ -n "${NIX_DEBUG:-}" ]; then + echo "ld-wrapper: ${leafCount} *dynamic* inputs counted when ${recurThreshold} is the ceiling, delegating to children. " >&2 +fi + +declare -r outputNameLibless=$( \ + if [[ -z "${outputName:+isUndefined}" ]]; then + echo unnamed + return 0; fi + baseName=$(basename ${outputName}) + if [[ "$baseName" = lib* ]]; then + baseName="${baseName:3}" + fi + echo "$baseName") + +declare -ra children=( + "$outputNameLibless-reexport-delegate-0" + "$outputNameLibless-reexport-delegate-1" +) + +mkdir -p "$out/lib" + +symbolBloatObject=$outputNameLibless-symbol-hack.o +if [[ ! -f $symbolBloatObject ]]; then + # `-Q` means use GNU Assembler rather than Clang, avoiding an awkward + # dependency cycle. + printf '.private_extern _______child_hack_foo\nchild_hack_foo:\n' | + PATH="$PATH:@out@/bin" @targetPrefix@as -Q -- -o $symbolBloatObject +fi + +# Split inputs between children +declare -a child0Inputs=() child1Inputs=("${childrenInputs[@]}") +let "countFirstChild = $leafCount / 2" || true +lastLeaf='' +while (( "$countFirstChild" )); do + case "${child1Inputs[0]}" in + -reexport_library | -weak_library) + child0Inputs+=("${child1Inputs[0]}" "${child1Inputs[1]}") + if [[ "${child1Inputs[1]}" != "$lastLeaf" ]]; then + let countFirstChild-=1 || true + lastLeaf="${child1Inputs[1]}" + fi + child1Inputs=("${child1Inputs[@]:2}") + ;; + *.so | *.dylib) + child0Inputs+=(-reexport_library "${child1Inputs[0]}") + if [[ "${child1Inputs[0]}" != "$lastLeaf" ]]; then + let countFirstChild-=1 || true + lastLeaf="${child1Inputs[1]}" + fi + child1Inputs=("${child1Inputs[@]:2}") + ;; + *) + echo "ld-wrapper: Internal Error: Invalid delegated input" >&2 + exit -1 + ;; + esac +done + + +# First half of libs +@out@/bin/@targetPrefix@ld \ + -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[0]}.dylib" \ + -install_name "$out/lib/lib${children[0]}.dylib" \ + "$symbolBloatObject" "${child0Inputs[@]}" "${trailingInputs[@]}" + +# Second half of libs +@out@/bin/@targetPrefix@ld \ + -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ + -o "$out/lib/lib${children[1]}.dylib" \ + -install_name "$out/lib/lib${children[1]}.dylib" \ + "$symbolBloatObject" "${child1Inputs[@]}" "${trailingInputs[@]}" + +parentArgs+=("-L$out/lib" -rpath "$out/lib") +if [[ $outputName != *reexport-delegate* ]]; then + parentArgs+=("-l${children[0]}" "-l${children[1]}") +else + parentArgs+=("-reexport-l${children[0]}" "-reexport-l${children[1]}") +fi + +parentArgs+=("${trailingInputs[@]}") - # first half of libs - @targetPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ - -o "$out/lib/lib${children[0]}.dylib" \ - -install_name "$out/lib/lib${children[0]}.dylib" \ - "${childrenLookup[@]}" "$symbolBloatObject" \ - "${childrenLink[@]:0:$((${#childrenLink[@]} / 2 ))}" - - # second half of libs - @targetPrefix@ld -macosx_version_min $MACOSX_DEPLOYMENT_TARGET -arch x86_64 -dylib \ - -o "$out/lib/lib${children[1]}.dylib" \ - -install_name "$out/lib/lib${children[1]}.dylib" \ - "${childrenLookup[@]}" "$symbolBloatObject" \ - "${childrenLink[@]:$((${#childrenLink[@]} / 2 ))}" - - allArgs+=("-L$out/lib" "-l${children[0]}" "-l${children[1]}") +if [ -n "${NIX_DEBUG:-}" ]; then + echo "flags using delegated children to @prog@:" >&2 + printf " %q\n" "${parentArgs[@]}" >&2 fi PATH="$path_backup" -exec @prog@ "${allArgs[@]}" +exec @prog@ "${parentArgs[@]}" -- cgit 1.4.1 From 0d3eb701333006ce73b6abadea4d95f52d1105f3 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 14 Apr 2018 20:09:51 -0400 Subject: buildSetupcfg: Include unzip for zip sources. --- pkgs/build-support/build-setupcfg/default.nix | 6 ++++-- pkgs/top-level/python-packages.nix | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/build-setupcfg/default.nix b/pkgs/build-support/build-setupcfg/default.nix index bc6482f0a88e..c2d58c3c7857 100644 --- a/pkgs/build-support/build-setupcfg/default.nix +++ b/pkgs/build-support/build-setupcfg/default.nix @@ -6,7 +6,7 @@ # * application: Whether this package is a python library or an # application which happens to be written in python. # * doCheck: Whether to run the test suites. -pythonPackages: +{ pkgs, pythonPackages }: { src, info, meta ? {}, application ? false, doCheck ? true }: let build = if application then pythonPackages.buildPythonApplication @@ -16,7 +16,9 @@ in build { inherit src meta doCheck; - nativeBuildInputs = map (p: pythonPackages.${p}) ( + nativeBuildInputs = [ + pkgs.unzip + ] ++ map (p: pythonPackages.${p}) ( (info.setup_requires or []) ++ (if doCheck then (info.tests_require or []) else [])); diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d5e3a11236c4..db508f054b31 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -67,7 +67,8 @@ let })); # See build-setupcfg/default.nix for documentation. - buildSetupcfg = import ../build-support/build-setupcfg self; + buildSetupcfg = import ../build-support/build-setupcfg + { inherit pkgs; pythonPackages = self; }; graphiteVersion = "1.0.2"; -- cgit 1.4.1 From 603a369b8944c2b2f15c67218f345211fafda6fa Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 14 Apr 2018 20:12:22 -0400 Subject: Revert "buildSetupcfg: Include unzip for zip sources." Misunderstood the error I was seeing. This reverts commit 0d3eb701333006ce73b6abadea4d95f52d1105f3. --- pkgs/build-support/build-setupcfg/default.nix | 6 ++---- pkgs/top-level/python-packages.nix | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/build-setupcfg/default.nix b/pkgs/build-support/build-setupcfg/default.nix index c2d58c3c7857..bc6482f0a88e 100644 --- a/pkgs/build-support/build-setupcfg/default.nix +++ b/pkgs/build-support/build-setupcfg/default.nix @@ -6,7 +6,7 @@ # * application: Whether this package is a python library or an # application which happens to be written in python. # * doCheck: Whether to run the test suites. -{ pkgs, pythonPackages }: +pythonPackages: { src, info, meta ? {}, application ? false, doCheck ? true }: let build = if application then pythonPackages.buildPythonApplication @@ -16,9 +16,7 @@ in build { inherit src meta doCheck; - nativeBuildInputs = [ - pkgs.unzip - ] ++ map (p: pythonPackages.${p}) ( + nativeBuildInputs = map (p: pythonPackages.${p}) ( (info.setup_requires or []) ++ (if doCheck then (info.tests_require or []) else [])); diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index db508f054b31..d5e3a11236c4 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -67,8 +67,7 @@ let })); # See build-setupcfg/default.nix for documentation. - buildSetupcfg = import ../build-support/build-setupcfg - { inherit pkgs; pythonPackages = self; }; + buildSetupcfg = import ../build-support/build-setupcfg self; graphiteVersion = "1.0.2"; -- cgit 1.4.1 From 4a30f2efecc8bea1ad605388baa5db7668a9f580 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 15 Apr 2018 13:58:05 +0100 Subject: requireFile: exit with non-zero error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the script running is a failure condition, we should fail the build properly, not leaving it up to the missing output to determine that the build went wrong. This should partly address #38952 — nix build will print out the build log on non-zero exits. --- pkgs/build-support/trivial-builders.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'pkgs/build-support') diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 9664abeb4651..d092efb556e5 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -170,6 +170,7 @@ rec { *** _EOF_ + exit 1 ''; }; -- cgit 1.4.1