diff options
Diffstat (limited to 'pkgs/build-support')
32 files changed, 260 insertions, 224 deletions
diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index 66027485ee71..1a32f365bab2 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -325,6 +325,12 @@ stdenv.mkDerivation { hardening_unsupported_flags+=" relro bindnow" '' + + optionalString (libc != null && targetPlatform.isAvr) '' + for isa in avr5 avr3 avr4 avr6 avr25 avr31 avr35 avr51 avrxmega2 avrxmega4 avrxmega5 avrxmega6 avrxmega7 tiny-stack; do + echo "-L${getLib libc}/avr/lib/$isa" >> $out/nix-support/libc-cflags + done + '' + + optionalString stdenv.targetPlatform.isDarwin '' echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/libc-ldflags '' diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index d788076dda51..569f6875e1fb 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -53,33 +53,54 @@ , gccForLibs ? if useCcForLibs then cc else null , fortify-headers ? null , includeFortifyHeaders ? null -}: -with lib; +# https://github.com/NixOS/nixpkgs/issues/295322 +# should -march flag be used +, disableMarch ? false +}: assert nativeTools -> !propagateDoc && nativePrefix != ""; -assert !nativeTools -> - cc != null && coreutils != null && gnugrep != null; +assert !nativeTools -> cc != null && coreutils != null && gnugrep != null; assert !(nativeLibc && noLibc); assert (noLibc || nativeLibc) == (libc == null); let - stdenv = stdenvNoCC; + inherit (lib) + attrByPath + concatMapStrings + concatStringsSep + escapeShellArg + getBin + getDev + getLib + getName + getVersion + mapAttrsToList + optional + optionalAttrs + optionals + optionalString + removePrefix + replaceStrings + toList + versionAtLeast + ; + inherit (stdenv) hostPlatform targetPlatform; + stdenv = stdenvNoCC; + includeFortifyHeaders' = if includeFortifyHeaders != null then includeFortifyHeaders else (targetPlatform.libc == "musl" && isGNU); # Prefix for binaries. Customarily ends with a dash separator. # - # TODO(@Ericson2314) Make unconditional, or optional but always true by - # default. - targetPrefix = lib.optionalString (targetPlatform != hostPlatform) - (targetPlatform.config + "-"); + # TODO(@Ericson2314) Make unconditional, or optional but always true by default. + targetPrefix = optionalString (targetPlatform != hostPlatform) (targetPlatform.config + "-"); - ccVersion = lib.getVersion cc; - ccName = lib.removePrefix targetPrefix (lib.getName cc); + ccVersion = getVersion cc; + ccName = removePrefix targetPrefix (getName cc); libc_bin = optionalString (libc != null) (getBin libc); libc_dev = optionalString (libc != null) (getDev libc); @@ -98,7 +119,7 @@ let suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; expand-response-params = - lib.optionalString ((buildPackages.stdenv.hasCC or false) && buildPackages.stdenv.cc != "/dev/null") (import ../expand-response-params { inherit (buildPackages) stdenv; }); + optionalString ((buildPackages.stdenv.hasCC or false) && buildPackages.stdenv.cc != "/dev/null") (import ../expand-response-params { inherit (buildPackages) stdenv; }); useGccForLibs = useCcForLibs && libcxx == null @@ -111,7 +132,7 @@ let + optionalString (targetPlatform != hostPlatform) "/${targetPlatform.config}"; # Analogously to cc_solib and gccForLibs_solib - libcxx_solib = "${lib.getLib libcxx}/lib"; + libcxx_solib = "${getLib libcxx}/lib"; # The following two functions, `isGccArchSupported` and # `isGccTuneSupported`, only handle those situations where a flag @@ -407,9 +428,9 @@ stdenv.mkDerivation { setupHooks = [ ../setup-hooks/role.bash - ] ++ lib.optional (cc.langC or true) ./setup-hook.sh - ++ lib.optional (cc.langFortran or false) ./fortran-hook.sh - ++ lib.optional (targetPlatform.isWindows) (stdenv.mkDerivation { + ] ++ optional (cc.langC or true) ./setup-hook.sh + ++ optional (cc.langFortran or false) ./fortran-hook.sh + ++ optional (targetPlatform.isWindows) (stdenv.mkDerivation { name = "win-dll-hook.sh"; dontUnpack = true; installPhase = '' @@ -476,7 +497,7 @@ stdenv.mkDerivation { # when building e.g. firefox), lld is able to find libgcc_s.so + concatMapStrings (libgcc: '' echo "-L${libgcc}/lib" >> $out/nix-support/cc-ldflags - '') (lib.toList (gccForLibs.libgcc or []))) + '') (toList (gccForLibs.libgcc or []))) ## ## General libc support @@ -542,9 +563,12 @@ stdenv.mkDerivation { done '' + optionalString (libcxx.isLLVM or false) '' - echo "-isystem ${lib.getDev libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags - echo "-isystem ${lib.getDev libcxx.cxxabi}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags + echo "-isystem ${getDev libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags echo "-stdlib=libc++" >> $out/nix-support/libcxx-ldflags + '' + # can remove once LLVM9 and LLVM11 are dropped from nixpkgs + + optionalString (libcxx.isLLVM or false && lib.versionOlder (lib.getVersion libcxx) "12" && libcxx ? cxxabi.libName) '' + echo "-isystem ${lib.getDev libcxx.cxxabi}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags echo "-l${libcxx.cxxabi.libName}" >> $out/nix-support/libcxx-ldflags '' @@ -593,7 +617,7 @@ stdenv.mkDerivation { ## Hardening support ## + '' - export hardening_unsupported_flags="${builtins.concatStringsSep " " ccHardeningUnsupportedFlags}" + export hardening_unsupported_flags="${concatStringsSep " " ccHardeningUnsupportedFlags}" '' # Machine flags. These are necessary to support @@ -609,7 +633,7 @@ stdenv.mkDerivation { # TODO: aarch64-darwin has mcpu incompatible with gcc + optionalString ((targetPlatform ? gcc.arch) && !isClang && !(stdenv.isDarwin && stdenv.isAarch64) && - isGccArchSupported targetPlatform.gcc.arch) '' + isGccArchSupported targetPlatform.gcc.arch && !disableMarch) '' echo "-march=${targetPlatform.gcc.arch}" >> $out/nix-support/cc-cflags-before '' @@ -668,6 +692,12 @@ stdenv.mkDerivation { hardening_unsupported_flags+=" stackprotector" '' + + optionalString (libc != null && targetPlatform.isAvr) '' + for isa in avr5 avr3 avr4 avr6 avr25 avr31 avr35 avr51 avrxmega2 avrxmega4 avrxmega5 avrxmega6 avrxmega7 tiny-stack; do + echo "-B${getLib libc}/avr/lib/$isa" >> $out/nix-support/libc-crt1-cflags + done + '' + + optionalString stdenv.targetPlatform.isDarwin '' echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/cc-cflags '' @@ -698,9 +728,9 @@ stdenv.mkDerivation { ## + optionalString isClang '' # Escape twice: once for this script, once for the one it gets substituted into. - export march=${lib.escapeShellArg - (lib.optionalString (targetPlatform ? gcc.arch) - (lib.escapeShellArg "-march=${targetPlatform.gcc.arch}"))} + export march=${escapeShellArg + (optionalString (targetPlatform ? gcc.arch && !disableMarch) + (escapeShellArg "-march=${targetPlatform.gcc.arch}"))} export defaultTarget=${targetPlatform.config} substituteAll ${./add-clang-cc-cflags-before.sh} $out/nix-support/add-local-cc-cflags-before.sh '' @@ -709,8 +739,8 @@ stdenv.mkDerivation { ## Extra custom steps ## + extraBuildCommands - + lib.strings.concatStringsSep "; " - (lib.attrsets.mapAttrsToList + + concatStringsSep "; " + (mapAttrsToList (name: value: "echo ${toString value} >> $out/nix-support/${name}") nixSupport); @@ -733,11 +763,9 @@ stdenv.mkDerivation { }; meta = - let cc_ = lib.optionalAttrs (cc != null) cc; in - (lib.optionalAttrs (cc_ ? meta) (removeAttrs cc.meta ["priority"])) // - { description = - lib.attrByPath ["meta" "description"] "System C compiler" cc_ - + " (wrapper script)"; + let cc_ = optionalAttrs (cc != null) cc; in + (optionalAttrs (cc_ ? meta) (removeAttrs cc.meta ["priority"])) // + { description = attrByPath ["meta" "description"] "System C compiler" cc_ + " (wrapper script)"; priority = 10; mainProgram = if name != "" then name else ccName; }; diff --git a/pkgs/build-support/deterministic-uname/default.nix b/pkgs/build-support/deterministic-uname/default.nix index 164136c937b9..6d150557aa9d 100644 --- a/pkgs/build-support/deterministic-uname/default.nix +++ b/pkgs/build-support/deterministic-uname/default.nix @@ -39,6 +39,7 @@ substituteAll { meta = with lib; { description = "Print certain system information (hardcoded with lib/system values)"; + mainProgram = "uname"; longDescription = '' This package provides a replacement for `uname` whose output depends only on `stdenv.buildPlatform`. It is meant to be used from within derivations. diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index ea464ebea18c..6b8a03932f5b 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -29,7 +29,7 @@ , tarsum , util-linux , vmTools -, writeReferencesToFile +, writeClosure , writeScript , writeShellScriptBin , writeText @@ -523,7 +523,7 @@ rec { runCommand "${baseNameOf name}.tar${compress.ext}" { inherit (stream) imageName; - passthru = { inherit (stream) imageTag; }; + passthru = { inherit (stream) imageTag; inherit stream; }; nativeBuildInputs = compress.nativeInputs; } "${stream} | ${compress.compress} > $out" ); @@ -630,7 +630,7 @@ rec { imageName = lib.toLower name; imageTag = lib.optionalString (tag != null) tag; inherit fromImage baseJson; - layerClosure = writeReferencesToFile layer; + layerClosure = writeClosure [ layer ]; passthru.buildArgs = args; passthru.layer = layer; passthru.imageTag = diff --git a/pkgs/build-support/docker/nix-prefetch-docker.nix b/pkgs/build-support/docker/nix-prefetch-docker.nix index 61e917461ed9..18accd135c04 100644 --- a/pkgs/build-support/docker/nix-prefetch-docker.nix +++ b/pkgs/build-support/docker/nix-prefetch-docker.nix @@ -18,6 +18,7 @@ stdenv.mkDerivation { meta = with lib; { description = "Script used to obtain source hashes for dockerTools.pullImage"; + mainProgram = "nix-prefetch-docker"; maintainers = with maintainers; [ offline ]; platforms = platforms.unix; }; diff --git a/pkgs/build-support/docker/tarsum.nix b/pkgs/build-support/docker/tarsum.nix index 734c6b3d5aeb..f62a8d49389a 100644 --- a/pkgs/build-support/docker/tarsum.nix +++ b/pkgs/build-support/docker/tarsum.nix @@ -39,4 +39,5 @@ stdenv.mkDerivation { }; meta.platforms = go.meta.platforms; + meta.mainProgram = "tarsum"; } diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index a3ea0297673d..7ac3dec91f7f 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -120,7 +120,7 @@ init_remote(){ git config remote.origin.partialclonefilter "blob:none" echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone} fi - ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true + ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true } # Return the reference of an hash if it exists on the remote repository. diff --git a/pkgs/build-support/fetchnextcloudapp/default.nix b/pkgs/build-support/fetchnextcloudapp/default.nix index 0ec64bbe1df6..059003df903d 100644 --- a/pkgs/build-support/fetchnextcloudapp/default.nix +++ b/pkgs/build-support/fetchnextcloudapp/default.nix @@ -1,6 +1,7 @@ { stdenv, fetchzip, applyPatches, lib, ... }: { url -, sha256 +, hash ? "" +, sha256 ? "" , appName ? null , appVersion ? null , license @@ -11,7 +12,7 @@ applyPatches ({ inherit patches; src = fetchzip { - inherit url sha256; + inherit url hash sha256; postFetch = '' pushd $out &>/dev/null if [ ! -f ./appinfo/info.xml ]; then diff --git a/pkgs/build-support/java/canonicalize-jar.nix b/pkgs/build-support/java/canonicalize-jar.nix deleted file mode 100644 index 1edd9a6e0d20..000000000000 --- a/pkgs/build-support/java/canonicalize-jar.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ substituteAll, unzip, zip }: - -substituteAll { - name = "canonicalize-jar"; - src = ./canonicalize-jar.sh; - - unzip = "${unzip}/bin/unzip"; - zip = "${zip}/bin/zip"; -} diff --git a/pkgs/build-support/java/canonicalize-jar.sh b/pkgs/build-support/java/canonicalize-jar.sh deleted file mode 100644 index af010bcd2b26..000000000000 --- a/pkgs/build-support/java/canonicalize-jar.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Canonicalize the manifest & repack with deterministic timestamps. -canonicalizeJar() { - local input='' outer='' - input="$(realpath -sm -- "$1")" - outer="$(pwd)" - # -qq: even quieter - @unzip@ -qq "$input" -d "$input-tmp" - canonicalizeJarManifest "$input-tmp/META-INF/MANIFEST.MF" - # Sets all timestamps to Jan 1 1980, the earliest mtime zips support. - find -- "$input-tmp" -exec touch -t 198001010000.00 {} + - rm "$input" - pushd "$input-tmp" 2>/dev/null - # -q|--quiet, -r|--recurse-paths - # -o|--latest-time: canonicalizes overall archive mtime - # -X|--no-extra: don't store platform-specific extra file attribute fields - @zip@ -qroX "$outer/tmp-out.jar" . 2> /dev/null - popd 2>/dev/null - rm -rf "$input-tmp" - mv "$outer/tmp-out.jar" "$input" -} - -# See also the Java specification's JAR requirements: -# https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Notes_on_Manifest_and_Signature_Files -canonicalizeJarManifest() { - local input='' - input="$(realpath -sm -- "$1")" - (head -n 1 "$input" && tail -n +2 "$input" | sort | grep -v '^\s*$') > "$input-tmp" - mv "$input-tmp" "$input" -} diff --git a/pkgs/build-support/kernel/make-initrd-ng-tool.nix b/pkgs/build-support/kernel/make-initrd-ng-tool.nix index b1fbee92b32e..5e08c091c054 100644 --- a/pkgs/build-support/kernel/make-initrd-ng-tool.nix +++ b/pkgs/build-support/kernel/make-initrd-ng-tool.nix @@ -11,6 +11,7 @@ rustPlatform.buildRustPackage { meta = { description = "Tool for copying binaries and their dependencies"; + mainProgram = "make-initrd-ng"; maintainers = with lib.maintainers; [ das_j elvishjerricco k900 lheckemann ]; license = lib.licenses.mit; }; diff --git a/pkgs/build-support/lib/cmake.nix b/pkgs/build-support/lib/cmake.nix new file mode 100644 index 000000000000..eff7bbca61a2 --- /dev/null +++ b/pkgs/build-support/lib/cmake.nix @@ -0,0 +1,30 @@ +{ stdenv, lib }: + +let + inherit (lib) findFirst isString optional optionals; + + makeCMakeFlags = { cmakeFlags ? [], ... }: + cmakeFlags + ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) ([ + "-DCMAKE_SYSTEM_NAME=${findFirst isString "Generic" (optional (!stdenv.hostPlatform.isRedox) stdenv.hostPlatform.uname.system)}" + ] ++ optionals (stdenv.hostPlatform.uname.processor != null) [ + "-DCMAKE_SYSTEM_PROCESSOR=${stdenv.hostPlatform.uname.processor}" + ] ++ optionals (stdenv.hostPlatform.uname.release != null) [ + "-DCMAKE_SYSTEM_VERSION=${stdenv.hostPlatform.uname.release}" + ] ++ optionals (stdenv.hostPlatform.isDarwin) [ + "-DCMAKE_OSX_ARCHITECTURES=${stdenv.hostPlatform.darwinArch}" + ] ++ optionals (stdenv.buildPlatform.uname.system != null) [ + "-DCMAKE_HOST_SYSTEM_NAME=${stdenv.buildPlatform.uname.system}" + ] ++ optionals (stdenv.buildPlatform.uname.processor != null) [ + "-DCMAKE_HOST_SYSTEM_PROCESSOR=${stdenv.buildPlatform.uname.processor}" + ] ++ optionals (stdenv.buildPlatform.uname.release != null) [ + "-DCMAKE_HOST_SYSTEM_VERSION=${stdenv.buildPlatform.uname.release}" + ] ++ optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + "-DCMAKE_CROSSCOMPILING_EMULATOR=env" + ] ++ optionals stdenv.hostPlatform.isStatic [ + "-DCMAKE_LINK_SEARCH_START_STATIC=ON" + ]); +in +{ + inherit makeCMakeFlags; +} diff --git a/pkgs/build-support/lib/meson.nix b/pkgs/build-support/lib/meson.nix new file mode 100644 index 000000000000..395b573f8587 --- /dev/null +++ b/pkgs/build-support/lib/meson.nix @@ -0,0 +1,35 @@ +{ stdenv, lib }: + +let + inherit (lib) boolToString optionals; + + # See https://mesonbuild.com/Reference-tables.html#cpu-families + cpuFamily = platform: with platform; + /**/ if isAarch32 then "arm" + else if isx86_32 then "x86" + else platform.uname.processor; + + makeMesonFlags = { mesonFlags ? [], ... }: + let + crossFile = builtins.toFile "cross-file.conf" '' + [properties] + bindgen_clang_arguments = ['-target', '${stdenv.targetPlatform.config}'] + needs_exe_wrapper = ${boolToString (!stdenv.buildPlatform.canExecute stdenv.hostPlatform)} + + [host_machine] + system = '${stdenv.targetPlatform.parsed.kernel.name}' + cpu_family = '${cpuFamily stdenv.targetPlatform}' + cpu = '${stdenv.targetPlatform.parsed.cpu.name}' + endian = ${if stdenv.targetPlatform.isLittleEndian then "'little'" else "'big'"} + + [binaries] + llvm-config = 'llvm-config-native' + rust = ['rustc', '--target', '${stdenv.targetPlatform.rust.rustcTargetSpec}'] + ''; + crossFlags = optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ "--cross-file=${crossFile}" ]; + in crossFlags ++ mesonFlags; + +in +{ + inherit makeMesonFlags; +} diff --git a/pkgs/build-support/libredirect/test.c b/pkgs/build-support/libredirect/test.c index 20b27759f019..e5685fcb31b7 100644 --- a/pkgs/build-support/libredirect/test.c +++ b/pkgs/build-support/libredirect/test.c @@ -74,7 +74,7 @@ int main(int argc, char *argv[]) FILE *testfp; int testfd; struct stat testsb; -#ifndef __APPLE__ +#ifdef __GLIBC__ struct stat64 testsb64; #endif #if defined(__linux__) && defined(STATX_TYPE) @@ -93,11 +93,11 @@ int main(int argc, char *argv[]) assert(access(TESTPATH, X_OK) == 0); assert(stat(TESTPATH, &testsb) != -1); -#ifndef __APPLE__ +#ifdef __GLIBC__ assert(stat64(TESTPATH, &testsb64) != -1); #endif assert(fstatat(123, TESTPATH, &testsb, 0) != -1); -#ifndef __APPLE__ +#ifdef __GLIBC__ assert(fstatat64(123, TESTPATH, &testsb64, 0) != -1); #endif #if defined(__linux__) && defined(STATX_TYPE) diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh index 56e3a883b99a..750ed421789f 100644 --- a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh @@ -12,7 +12,7 @@ npmInstallHook() { local dest="$packageOut/$(dirname "$file")" mkdir -p "$dest" cp "${npmWorkspace-.}/$file" "$dest" - done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm_config_cache="$HOME/.npm" npm pack --json --dry-run --loglevel=warn --no-foreground-scripts ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + done < <(@jq@ --raw-output '.[0].files | map(.path | select(. | startswith("node_modules/") | not)) | join("\n")' <<< "$(npm_config_cache="$HOME/.npm" npm pack --json --dry-run --loglevel=warn --no-foreground-scripts ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") # Based on code from Python's buildPythonPackage wrap.sh script, for # supporting both the case when makeWrapperArgs is an array and a diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix index 725f9ba3bb01..373d63cc59b8 100644 --- a/pkgs/build-support/node/fetch-npm-deps/default.nix +++ b/pkgs/build-support/node/fetch-npm-deps/default.nix @@ -141,6 +141,7 @@ meta = with lib; { description = "Prefetch dependencies from npm (for use with `fetchNpmDeps`)"; + mainProgram = "prefetch-npm-deps"; maintainers = with maintainers; [ lilyinstarlight winter ]; license = licenses.mit; }; diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix index 67e081522d64..1f5507f2eb75 100644 --- a/pkgs/build-support/oci-tools/default.nix +++ b/pkgs/build-support/oci-tools/default.nix @@ -1,4 +1,4 @@ -{ lib, writeText, runCommand, writeReferencesToFile }: +{ lib, writeText, runCommand, writeClosure }: { buildContainer = @@ -72,7 +72,7 @@ set -o pipefail mkdir -p $out/rootfs/{dev,proc,sys} cp ${config} $out/config.json - xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/ + xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/ ''; } diff --git a/pkgs/build-support/php/hooks/php-script-utils.bash b/pkgs/build-support/php/hooks/php-script-utils.bash index 9b5978865634..bba0242e65d1 100644 --- a/pkgs/build-support/php/hooks/php-script-utils.bash +++ b/pkgs/build-support/php/hooks/php-script-utils.bash @@ -21,6 +21,10 @@ checkComposerValidate() { echo -e '\e[31mThe validation of the composer.json failed.\e[0m' echo -e '\e[31mMake sure that the file composer.json is valid.\e[0m' echo + echo -e '\e[31mTo address the issue efficiently, follow one of these steps:\e[0m' + echo -e '\e[31m 1. File an issue in the project'\''s issue tracker with detailed information, and apply any available remote patches as a temporary solution '\('with fetchpatch'\)'.\e[0m' + echo -e '\e[31m 2. If an immediate fix is needed or if reporting upstream isn'\''t suitable, develop a temporary local patch.\e[0m' + echo exit 1 else echo @@ -29,6 +33,10 @@ checkComposerValidate() { echo -e '\e[33mThe validation of the composer.json failed.\e[0m' echo -e '\e[33mMake sure that the file composer.json is valid.\e[0m' echo + echo -e '\e[33mTo address the issue efficiently, follow one of these steps:\e[0m' + echo -e '\e[33m 1. File an issue in the project'\''s issue tracker with detailed information, and apply any available remote patches as a temporary solution with '\('with fetchpatch'\)'.\e[0m' + echo -e '\e[33m 2. If an immediate fix is needed or if reporting upstream isn'\''t suitable, develop a temporary local patch.\e[0m' + echo echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' echo fi @@ -42,6 +50,12 @@ checkComposerValidate() { echo -e '\e[31mThe validation of the composer.json and composer.lock failed.\e[0m' echo -e '\e[31mMake sure that the file composer.lock is consistent with composer.json.\e[0m' echo + echo -e '\e[31mThis often indicates an issue with the upstream project, which can typically be resolved by reporting the issue to the relevant project maintainers.\e[0m' + echo + echo -e '\e[31mTo address the issue efficiently, follow one of these steps:\e[0m' + echo -e '\e[31m 1. File an issue in the project'\''s issue tracker with detailed information '\('run '\''composer update --lock --no-install'\'' to fix the issue'\)', and apply any available remote patches as a temporary solution with '\('with fetchpatch'\)'.\e[0m' + echo -e '\e[31m 2. If an immediate fix is needed or if reporting upstream isn'\''t suitable, develop a temporary local patch.\e[0m' + echo exit 1 else echo @@ -50,6 +64,12 @@ checkComposerValidate() { echo -e '\e[33mThe validation of the composer.json and composer.lock failed.\e[0m' echo -e '\e[33mMake sure that the file composer.lock is consistent with composer.json.\e[0m' echo + echo -e '\e[33mThis often indicates an issue with the upstream project, which can typically be resolved by reporting the issue to the relevant project maintainers.\e[0m' + echo + echo -e '\e[33mTo address the issue efficiently, follow one of these steps:\e[0m' + echo -e '\e[33m 1. File an issue in the project'\''s issue tracker with detailed information '\('run '\''composer update --lock --no-install'\'' to fix the issue'\)', and apply any available remote patches as a temporary solution with '\('with fetchpatch'\)'.\e[0m' + echo -e '\e[33m 2. If an immediate fix is needed or if reporting upstream isn'\''t suitable, develop a temporary local patch.\e[0m' + echo echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' echo fi diff --git a/pkgs/build-support/references-by-popularity/closure-graph.py b/pkgs/build-support/references-by-popularity/closure-graph.py index 579f3b041fa8..4f8efd42ed81 100644 --- a/pkgs/build-support/references-by-popularity/closure-graph.py +++ b/pkgs/build-support/references-by-popularity/closure-graph.py @@ -8,8 +8,8 @@ # and how deep in the tree they live. Equally-"popular" paths are then # sorted by name. # -# The existing writeReferencesToFile prints the paths in a simple -# ascii-based sorting of the paths. +# The existing writeClosure prints the paths in a simple ascii-based +# sorting of the paths. # # Sorting the paths by graph improves the chances that the difference # between two builds appear near the end of the list, instead of near diff --git a/pkgs/build-support/remove-references-to/default.nix b/pkgs/build-support/remove-references-to/default.nix index f022611ef913..1277cdb04fd7 100644 --- a/pkgs/build-support/remove-references-to/default.nix +++ b/pkgs/build-support/remove-references-to/default.nix @@ -32,4 +32,5 @@ stdenv.mkDerivation { inherit (builtins) storeDir; shell = lib.getBin shell + (shell.shellPath or ""); signingUtils = if darwinCodeSign then signingUtils else null; + meta.mainProgram = "remove-references-to"; } diff --git a/pkgs/build-support/setup-hooks/canonicalize-jars.sh b/pkgs/build-support/setup-hooks/canonicalize-jars.sh deleted file mode 100644 index 5137bfc94b01..000000000000 --- a/pkgs/build-support/setup-hooks/canonicalize-jars.sh +++ /dev/null @@ -1,16 +0,0 @@ -# This setup hook causes the fixup phase to repack all JAR files in a -# canonical & deterministic fashion, e.g. resetting mtimes (like with normal -# store files) and avoiding impure metadata. - -fixupOutputHooks+=('if [ -z "$dontCanonicalizeJars" -a -e "$prefix" ]; then canonicalizeJarsIn "$prefix"; fi') - -canonicalizeJarsIn() { - local dir="$1" - echo "canonicalizing jars in $dir" - dir="$(realpath -sm -- "$dir")" - while IFS= read -rd '' f; do - canonicalizeJar "$f" - done < <(find -- "$dir" -type f -name '*.jar' -print0) -} - -source @canonicalize_jar@ diff --git a/pkgs/build-support/setup-hooks/strip-java-archives.sh b/pkgs/build-support/setup-hooks/strip-java-archives.sh new file mode 100644 index 000000000000..22322468f76d --- /dev/null +++ b/pkgs/build-support/setup-hooks/strip-java-archives.sh @@ -0,0 +1,16 @@ +# This setup hook makes the fixup phase to repack all java archives in a +# deterministic fashion. The most important change being done is the resetting +# of the modification times of the archive entries + +fixupOutputHooks+=('stripJavaArchivesIn $prefix') + +stripJavaArchivesIn() { + local dir="$1" + echo "stripping java archives in $dir" + find $dir -type f -regextype posix-egrep -regex ".*\.(jar|war|hpi|apk)$" -print0 | + while IFS= read -rd '' f; do + echo "stripping java archive $f" + strip-nondeterminism --type jar "$f" + done +} + diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix index 8d7ad9e742a1..c9e53a4cb706 100644 --- a/pkgs/build-support/singularity-tools/default.nix +++ b/pkgs/build-support/singularity-tools/default.nix @@ -4,7 +4,7 @@ , storeDir ? builtins.storeDir , writeScript , singularity -, writeReferencesToFile +, writeClosure , bash , vmTools , gawk @@ -50,18 +50,13 @@ rec { }: let projectName = singularity.projectName or "singularity"; - layer = mkLayer { - inherit name; - contents = contents ++ [ bash runScriptFile ]; - inherit projectName; - }; runAsRootFile = shellScript "run-as-root.sh" runAsRoot; runScriptFile = shellScript "run-script.sh" runScript; result = vmTools.runInLinuxVM ( runCommand "${projectName}-image-${name}.img" { buildInputs = [ singularity e2fsprogs util-linux gawk ]; - layerClosure = writeReferencesToFile layer; + layerClosure = writeClosure contents; preVM = vmTools.createEmptyImage { size = diskSize; fullName = "${projectName}-run-disk"; diff --git a/pkgs/build-support/snap/default.nix b/pkgs/build-support/snap/default.nix deleted file mode 100644 index ba5271868911..000000000000 --- a/pkgs/build-support/snap/default.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ callPackage, hello }: -{ - makeSnap = callPackage ./make-snap.nix { }; -} diff --git a/pkgs/build-support/snap/make-snap.nix b/pkgs/build-support/snap/make-snap.nix deleted file mode 100644 index cef7500bcbaf..000000000000 --- a/pkgs/build-support/snap/make-snap.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - runCommand, squashfsTools, closureInfo, lib, jq, writeText -}: - -{ - # The meta parameter is the contents of the `snap.yaml`, NOT the - # `snapcraft.yaml`. - # - # - `snap.yaml` is what is inside of the final Snap, - # - `snapcraft.yaml` is used by `snapcraft` to build snaps - # - # Since we skip the `snapcraft` tool, we skip the `snapcraft.yaml` - # file. For more information: - # - # https://docs.snapcraft.io/snap-format - # - # Note: unsquashfs'ing an existing snap from the store can be helpful - # for determining what you you're missing. - # - meta -}: let - snap_yaml = let - # Validate the snap's meta contains a name. - # Also: automatically set the `base` parameter and the layout for - # the `/nix` bind. - validate = { name, ... } @ args: - args // { - # Combine the provided arguments with the required options. - - # base: built from https://github.com/NixOS/snapd-nix-base - # and published as The NixOS Foundation on the Snapcraft store. - base = "nix-base"; - layout = (args.layout or {}) // { - # Bind mount the Snap's root nix directory to `/nix` in the - # execution environment's filesystem namespace. - "/nix".bind = "$SNAP/nix"; - }; - }; - in writeText "snap.yaml" - (builtins.toJSON (validate meta)); - - # These are specifically required by snapd, so don't change them - # unless you've verified snapcraft / snapd can handle them. Best bet - # is to just mirror this list against how snapcraft creates images. - # from: https://github.com/snapcore/snapcraft/blob/b88e378148134383ffecf3658e3a940b67c9bcc9/snapcraft/internal/lifecycle/_packer.py#L96-L98 - mksquashfs_args = [ - "-noappend" "-comp" "xz" "-no-xattrs" "-no-fragments" - - # Note: We want -all-root every time, since all the files are - # owned by root anyway. This is true for Nix, but not true for - # other builds. - # from: https://github.com/snapcore/snapcraft/blob/b88e378148134383ffecf3658e3a940b67c9bcc9/snapcraft/internal/lifecycle/_packer.py#L100 - "-all-root" - ]; - -in runCommand "squashfs.img" { - nativeBuildInputs = [ squashfsTools jq ]; - - closureInfo = closureInfo { - rootPaths = [ snap_yaml ]; - }; -} '' - root=$PWD/root - mkdir $root - - ( - # Put the snap.yaml in to `/meta/snap.yaml`, setting the version - # to the hash part of the store path - mkdir $root/meta - version=$(echo $out | cut -d/ -f4 | cut -d- -f1) - cat ${snap_yaml} | jq ". + { version: \"$version\" }" \ - > $root/meta/snap.yaml - ) - - ( - # Copy the store closure in to the root - mkdir -p $root/nix/store - cat $closureInfo/store-paths | xargs -I{} cp -r {} $root/nix/store/ - ) - - # Generate the squashfs image. - mksquashfs $root $out \ - ${lib.concatStringsSep " " mksquashfs_args} -'' diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index df81d67d868d..d7438923a54b 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }: +{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }: let inherit (lib) @@ -625,18 +625,22 @@ rec { # Docs in doc/build-helpers/trivial-build-helpers.chapter.md # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile - writeReferencesToFile = path: runCommand "runtime-deps" + # TODO: Convert to throw after Nixpkgs 24.05 branch-off. + writeReferencesToFile = (if config.allowAliases then lib.warn else throw) + "writeReferencesToFile is deprecated in favour of writeClosure" + (path: writeClosure [ path ]); + + # Docs in doc/build-helpers/trivial-build-helpers.chapter.md + # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure + writeClosure = paths: runCommand "runtime-deps" { - exportReferencesGraph = [ "graph" path ]; + # Get the cleaner exportReferencesGraph interface + __structuredAttrs = true; + exportReferencesGraph.graph = paths; + nativeBuildInputs = [ jq ]; } '' - touch $out - while read path; do - echo $path >> $out - read dummy - read nrRefs - for ((i = 0; i < nrRefs; i++)); do read ref; done - done < graph + jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out" ''; # Docs in doc/build-helpers/trivial-build-helpers.chapter.md diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix index 59dbba3f1841..f41372d922bb 100644 --- a/pkgs/build-support/trivial-builders/test/default.nix +++ b/pkgs/build-support/trivial-builders/test/default.nix @@ -26,6 +26,9 @@ recurseIntoAttrs { then references else {}; writeCBin = callPackage ./writeCBin.nix {}; + writeClosure-union = callPackage ./writeClosure-union.nix { + inherit (references) samples; + }; writeShellApplication = callPackage ./writeShellApplication.nix {}; writeScriptBin = callPackage ./writeScriptBin.nix {}; writeShellScript = callPackage ./write-shell-script.nix {}; diff --git a/pkgs/build-support/trivial-builders/test/references/default.nix b/pkgs/build-support/trivial-builders/test/references/default.nix index 3e21c905321e..928cc1d9461f 100644 --- a/pkgs/build-support/trivial-builders/test/references/default.nix +++ b/pkgs/build-support/trivial-builders/test/references/default.nix @@ -12,7 +12,7 @@ , cleanSamples ? lib.filterAttrs (n: lib.isStringLike) # Test targets , writeDirectReferencesToFile -, writeReferencesToFile +, writeClosure }: # -------------------------------------------------------------------------- # @@ -46,8 +46,9 @@ let samplesToString = attrs: lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs); - references = lib.mapAttrs (n: v: writeReferencesToFile v) samples; + closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples; directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples; + collectiveClosure = writeClosure (lib.attrValues samples); testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: { name = "references-test"; @@ -61,8 +62,9 @@ let mkdir -p "$out/bin" substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \ --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \ - --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \ - --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} + --replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \ + --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \ + --replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure} runHook postInstall chmod +x "$out/bin/${finalAttrs.meta.mainProgram}" ''; @@ -79,8 +81,9 @@ let passthru = { inherit + collectiveClosure directReferences - references + closures samples ; }; @@ -109,8 +112,9 @@ testers.nixosTest { ''; passthru = { inherit + collectiveClosure directReferences - references + closures samples testScriptBin ; diff --git a/pkgs/build-support/trivial-builders/test/references/references-test.sh b/pkgs/build-support/trivial-builders/test/references/references-test.sh index 1b8f8e1504ec..92e4467287ed 100755 --- a/pkgs/build-support/trivial-builders/test/references/references-test.sh +++ b/pkgs/build-support/trivial-builders/test/references/references-test.sh @@ -33,16 +33,17 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" # nixpkgs root - # Injected by Nix (to avoid evaluating in a derivation) - # turn them into arrays - # shellcheck disable=SC2206 # deliberately unquoted + # Inject the path to compare from the Nix expression + + # Associative Arrays declare -A samples=( @SAMPLES@ ) - # shellcheck disable=SC2206 # deliberately unquoted declare -A directRefs=( @DIRECT_REFS@ ) - # shellcheck disable=SC2206 # deliberately unquoted - declare -A references=( @REFERENCES@ ) + declare -A closures=( @CLOSURES@ ) + + # Path string + collectiveClosure=@COLLECTIVE_CLOSURE@ -echo >&2 Testing direct references... +echo >&2 Testing direct closures... for i in "${!samples[@]}"; do echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}" diff -U3 \ @@ -52,10 +53,16 @@ done echo >&2 Testing closure... for i in "${!samples[@]}"; do - echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}" + echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}" diff -U3 \ - <(sort <"${references[$i]}") \ + <(sort <"${closures[$i]}") \ <(nix-store -q --requisites "${samples[$i]}" | sort) done +echo >&2 Testing mixed closures... +echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure" +diff -U3 \ + <(sort <"$collectiveClosure") \ + <(nix-store -q --requisites "${samples[@]}" | sort) + echo 'OK!' diff --git a/pkgs/build-support/trivial-builders/test/writeClosure-union.nix b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix new file mode 100644 index 000000000000..92a2bf9f0988 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix @@ -0,0 +1,23 @@ +{ lib +, runCommandLocal + # Test targets +, writeClosure +, samples +}: +runCommandLocal "test-trivial-builders-writeClosure-union" { + __structuredAttrs = true; + closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples; + collectiveClosure = writeClosure (lib.attrValues samples); + inherit samples; + meta.maintainers = with lib.maintainers; [ + ShamrockLee + ]; +} '' + set -eu -o pipefail + echo >&2 Testing mixed closures... + echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure" + diff -U3 \ + <(sort <"$collectiveClosure") \ + <(cat "''${closures[@]}" | sort | uniq) + touch "$out" +'' diff --git a/pkgs/build-support/writers/data.nix b/pkgs/build-support/writers/data.nix index 02f08b9ca0b6..315150378e69 100644 --- a/pkgs/build-support/writers/data.nix +++ b/pkgs/build-support/writers/data.nix @@ -30,7 +30,7 @@ in ``` */ makeDataWriter = lib.warn "pkgs.writers.makeDataWriter is deprecated. Use pkgs.writeTextFile." ({ input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data: - assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); + assert (types.path.check nameOrPath) || (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); let name = last (builtins.split "/" nameOrPath); in diff --git a/pkgs/build-support/writers/scripts.nix b/pkgs/build-support/writers/scripts.nix index 8a23e5dd4a66..1dd25c500719 100644 --- a/pkgs/build-support/writers/scripts.nix +++ b/pkgs/build-support/writers/scripts.nix @@ -19,8 +19,8 @@ rec { # writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; } # makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world" makeScriptWriter = { interpreter, check ? "" }: nameOrPath: content: - assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); - assert lib.or (types.path.check content) (types.str.check content); + assert (types.path.check nameOrPath) || (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); + assert (types.path.check content) || (types.str.check content); let name = last (builtins.split "/" nameOrPath); in @@ -82,8 +82,8 @@ rec { # Examples: # writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; } makeBinWriter = { compileScript, strip ? true }: nameOrPath: content: - assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); - assert lib.or (types.path.check content) (types.str.check content); + assert (types.path.check nameOrPath) || (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); + assert (types.path.check content) || (types.str.check content); let name = last (builtins.split "/" nameOrPath); in |