diff options
Diffstat (limited to 'nixpkgs/pkgs/stdenv/darwin/make-bootstrap-tools.nix')
-rw-r--r-- | nixpkgs/pkgs/stdenv/darwin/make-bootstrap-tools.nix | 429 |
1 files changed, 429 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/nixpkgs/pkgs/stdenv/darwin/make-bootstrap-tools.nix new file mode 100644 index 000000000000..836a938b84ae --- /dev/null +++ b/nixpkgs/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -0,0 +1,429 @@ +{ pkgspath ? ../../.., test-pkgspath ? pkgspath +, localSystem ? { system = builtins.currentSystem; } +, crossSystem ? null +, bootstrapFiles ? null +}: + +let cross = if crossSystem != null + then { inherit crossSystem; } + else {}; + custom-bootstrap = if bootstrapFiles != null + then { stdenvStages = args: + let args' = args // { bootstrapFiles = bootstrapFiles; }; + in (import "${pkgspath}/pkgs/stdenv/darwin" args'); + } + else {}; +in with import pkgspath ({ inherit localSystem; } // cross // custom-bootstrap); + +rec { + build = stdenv.mkDerivation { + name = "stdenv-bootstrap-tools"; + + nativeBuildInputs = [ dumpnar nukeReferences ]; + + buildCommand = let + inherit (lib) + getBin + getDev + getLib + ; + + coreutils_ = (coreutils.override (args: { + # We want coreutils without ACL support. + aclSupport = false; + # Cannot use a single binary build, or it gets dynamically linked against gmp. + singleBinary = false; + })).overrideAttrs (oa: { + # Increase header size to be able to inject extra RPATHs. Otherwise + # x86_64-darwin build fails as: + # https://cache.nixos.org/log/g5wyq9xqshan6m3kl21bjn1z88hx48rh-stdenv-bootstrap-tools.drv + NIX_LDFLAGS = (oa.NIX_LDFLAGS or "") + " -headerpad_max_install_names"; + }); + + cctools_ = darwin.cctools; + + # Avoid messing with libkrb5 and libnghttp2. + curl_ = curlMinimal.override (args: { + gssSupport = false; + http2Support = false; + scpSupport = false; + }); + + gnutar_ = (gnutar.override { libintl = null; }).overrideAttrs (old: { + configureFlags = [ + "--disable-nls" + ] ++ old.configureFlags or []; + }); + + xz_ = xz.override { enableStatic = true; }; + + unpackScript = writeText "bootstrap-tools-unpack.sh" '' + set -euo pipefail + + echo Unpacking the bootstrap tools... >&2 + mkdir $out + tar xf "$1" -C $out + + updateInstallName() { + local path="$1" + + cp "$path" "$path.new" + install_name_tool -id "$path" "$path.new" + codesign -f -i "$(basename "$path")" -s - "$path.new" + mv -f "$path.new" "$path" + } + + find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do + updateInstallName "$lib" + done + + # Provide a gunzip script. + cat > $out/bin/gunzip <<EOF + #!$out/bin/sh + exec $out/bin/gzip -d "\$@" + EOF + chmod +x $out/bin/gunzip + + # Provide fgrep/egrep. + echo "#! $out/bin/sh" > $out/bin/egrep + echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep + echo "#! $out/bin/sh" > $out/bin/fgrep + echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + + cat >$out/bin/dsymutil << EOF + #!$out/bin/sh + EOF + + chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil + ''; + + in + '' + mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin + + ${lib.optionalString stdenv.targetPlatform.isx86_64 '' + # Copy libSystem's .o files for various low-level boot stuff. + cp -d ${getLib darwin.Libsystem}/lib/*.o $out/lib + + # Resolv is actually a link to another package, so let's copy it properly + cp -L ${getLib darwin.Libsystem}/lib/libresolv.9.dylib $out/lib + ''} + + cp -rL ${getDev darwin.Libsystem}/include $out + chmod -R u+w $out/include + cp -rL ${getDev libiconv}/include/* $out/include + cp -rL ${getDev gnugrep.pcre2}/include/* $out/include + mv $out/include $out/include-Libsystem + + # Copy coreutils, bash, etc. + cp ${getBin coreutils_}/bin/* $out/bin + (cd $out/bin && rm vdir dir sha*sum pinky factor pathchk runcon shuf who whoami shred users) + + cp ${getBin bash}/bin/bash $out/bin + ln -s bash $out/bin/sh + cp ${getBin findutils}/bin/find $out/bin + cp ${getBin findutils}/bin/xargs $out/bin + cp -d ${getBin diffutils}/bin/* $out/bin + cp -d ${getBin gnused}/bin/* $out/bin + cp -d ${getBin gnugrep}/bin/grep $out/bin + cp ${getBin gawk}/bin/gawk $out/bin + cp -d ${getBin gawk}/bin/awk $out/bin + cp ${getBin gnutar}/bin/tar $out/bin + cp ${getBin gzip}/bin/.gzip-wrapped $out/bin/gzip + cp ${getBin bzip2}/bin/bzip2 $out/bin + ln -s bzip2 $out/bin/bunzip2 + cp -d ${getBin gnumake}/bin/* $out/bin + cp -d ${getBin patch}/bin/* $out/bin + cp -d ${getBin xz}/bin/xz $out/bin + cp ${getBin cpio}/bin/cpio $out/bin + + # This used to be in-nixpkgs, but now is in the bundle + # because I can't be bothered to make it partially static + cp ${getBin curl_}/bin/curl $out/bin + cp -d ${getLib curl_}/lib/libcurl*.dylib $out/lib + cp -d ${getLib openssl}/lib/*.dylib $out/lib + + cp -d ${getLib gnugrep.pcre2}/lib/libpcre2*.dylib $out/lib + cp -d ${getLib libiconv}/lib/lib*.dylib $out/lib + cp -d ${getLib gettext}/lib/libintl*.dylib $out/lib + chmod +x $out/lib/libintl*.dylib + cp -d ${getLib ncurses}/lib/libncurses*.dylib $out/lib + cp -d ${getLib libxml2}/lib/libxml2*.dylib $out/lib + + # Copy what we need of clang + cp -d ${getBin llvmPackages.clang-unwrapped}/bin/clang{,++,-cl,-cpp,-[0-9]*} $out/bin + cp -d ${getLib llvmPackages.clang-unwrapped}/lib/libclang-cpp*.dylib $out/lib + cp -rd ${getLib llvmPackages.clang-unwrapped}/lib/clang $out/lib + + cp -d ${getLib llvmPackages.libcxx}/lib/libc++*.dylib $out/lib + mkdir -p $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/darwin/libclang_rt.{,profile_}osx.a $out/lib/darwin + cp -d ${getLib llvmPackages.compiler-rt}/lib/libclang_rt.{,profile_}osx.a $out/lib + cp -d ${getLib llvmPackages.llvm}/lib/libLLVM.dylib $out/lib + cp -d ${getLib libffi}/lib/libffi*.dylib $out/lib + + mkdir $out/include + cp -rd ${getDev llvmPackages.libcxx}/include/c++ $out/include + + # copy .tbd assembly utils + cp -d ${getBin pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin + cp -d ${getLib pkgs.libyaml}/lib/libyaml*.dylib $out/lib + + # copy package extraction tools + cp -d ${getBin pkgs.pbzx}/bin/pbzx $out/bin + cp -d ${getLib pkgs.xar}/lib/libxar*.dylib $out/lib + cp -d ${getLib pkgs.bzip2}/lib/libbz2*.dylib $out/lib + + # copy sigtool + cp -d ${getBin pkgs.darwin.sigtool}/bin/sigtool $out/bin + cp -d ${getBin pkgs.darwin.sigtool}/bin/codesign $out/bin + + cp -d ${getLib zlib}/lib/libz.* $out/lib + cp -d ${getLib gmpxx}/lib/libgmp*.* $out/lib + cp -d ${getLib xz}/lib/liblzma*.* $out/lib + + # Copy binutils. + for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do + cp ${getBin cctools_}/bin/$i $out/bin + done + + cp -d ${getLib darwin.libtapi}/lib/libtapi* $out/lib + + # tools needed to unpack bootstrap archive. they should not contain any + # external references. we will process them like the other tools but + # perform some additional checks and will not pack them into the archive. + mkdir -p unpack/bin + cp ${getBin bash}/bin/bash unpack/bin + ln -s bash unpack/bin/sh + cp ${getBin coreutils_}/bin/mkdir unpack/bin + cp ${getBin gnutar_}/bin/tar unpack/bin + cp ${getBin xz_}/bin/xz unpack/bin + cp ${unpackScript} unpack/bootstrap-tools-unpack.sh + + # + # All files copied. Perform processing to update references to point into + # the archive + # + + chmod -R u+w $out unpack + + # - change nix store library paths to use @rpath/library + # - if needed add an rpath containing lib/ + # - strip executable + rpathify() { + local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*" || true) + local lib rpath + for lib in $libs; do + ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" + done + + case "$(dirname "$1")" in + */bin) + # Strip executables even further + ${stdenv.cc.targetPrefix}strip "$i" + rpath='@executable_path/../lib' + ;; + */lib) + # the '/.' suffix is required + rpath='@loader_path/.' + ;; + */lib/darwin) + rpath='@loader_path/..' + ;; + *) + echo unkown executable $1 >&2 + exit 1 + ;; + esac + + # if shared object contains references add an rpath to lib/ + if ${stdenv.cc.targetPrefix}otool -l "$1"| grep -q '@rpath/'; then + ${stdenv.cc.targetPrefix}install_name_tool -add_rpath "$rpath" "$1" + fi + } + + # check that linked library paths exist in $out/lib + # must be run after rpathify is performed + checkDeps() { + local deps=$(${stdenv.cc.targetPrefix}otool -l "$1"| grep -o '@rpath/[^ ]*' || true) + local lib + for lib in $deps; do + if [[ ! -e $out/''${lib/@rpath/lib} ]]; then + echo "error: $1 missing lib for $lib" >&2 + exit 1 + fi + done + } + + for i in $out/bin/* unpack/bin/* $out/lib{,/darwin}/*.dylib; do + if [[ ! -L $i ]] && isMachO "$i"; then + rpathify "$i" + checkDeps "$i" + fi + done + + nuke-refs $out/bin/* + nuke-refs $out/lib/* + nuke-refs $out/lib/darwin/* + nuke-refs $out/lib/system/* + nuke-refs unpack/bin/* + + mkdir $out/.pack + mv $out/* $out/.pack + mv $out/.pack $out/pack + + # validate that tools contain no references into the archive + for tool in unpack/bin/*; do + deps=$(${stdenv.cc.targetPrefix}otool -l "$tool"| grep '@rpath/' || true) + if [[ -n "$deps" ]]; then + printf "error: $tool is not self contained\n$deps\n" >&2 + exit 1 + fi + done + + mkdir $out/on-server + cp -r unpack $out + + XZ_OPT="-9 -T $NIX_BUILD_CORES" tar cvJf $out/on-server/bootstrap-tools.tar.xz \ + --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . + dumpnar $out/unpack | xz -9 -T $NIX_BUILD_CORES > $out/on-server/unpack.nar.xz + ''; + + allowedReferences = []; + + meta = { + maintainers = [ lib.maintainers.copumpkin ]; + }; + }; + + dist = runCommand "stdenv-bootstrap-tools" {} '' + mkdir -p $out/nix-support + echo "file tarball ${build}/on-server/*.tar.xz" >> $out/nix-support/hydra-build-products + echo "file unpack ${build}/on-server/unpack.* " >> $out/nix-support/hydra-build-products + ''; + + bootstrapFiles = { + bootstrapTools = "${build}/on-server/bootstrap-tools.tar.xz"; + unpack = runCommand "unpack" { allowedReferences = []; } '' + cp -r ${build}/unpack $out + ''; + }; + + bootstrapTools = derivation { + inherit (stdenv.hostPlatform) system; + + name = "bootstrap-tools"; + builder = "${bootstrapFiles.unpack}/bin/bash"; + + args = [ + "${bootstrapFiles.unpack}/bootstrap-tools-unpack.sh" + bootstrapFiles.bootstrapTools + ]; + + PATH = lib.makeBinPath [ + (placeholder "out") + bootstrapFiles.unpack + ]; + + allowedReferences = [ "out" ]; + }; + + test = derivation { + name = "test-bootstrap-tools"; + inherit (stdenv.hostPlatform) system; + builder = "${bootstrapTools}/bin/bash"; + args = [ "-euo" "pipefail" "-c" "eval \"$buildCommand\"" ]; + PATH = lib.makeBinPath [ bootstrapTools ]; + tools = bootstrapTools; + "${stdenv.cc.darwinMinVersionVariable}" = stdenv.cc.darwinMinVersion; + + # Create a pure environment where we use just what's in the bootstrap tools. + buildCommand = '' + + ls -l + mkdir $out + mkdir $out/bin + sed --version + find --version + diff --version + patch --version + make --version + awk --version + grep --version + clang --version + xz --version + + # The grep will return a nonzero exit code if there is no match, and we want to assert that we have + # an SSL-capable curl + curl --version | grep SSL + + # This approximates a bootstrap version of libSystem can that be + # assembled via fetchurl. Adapted from main libSystem expression. + mkdir libSystem-boot + cp -vr \ + ${stdenv.cc.libc_dev}/lib/libSystem.B.tbd \ + ${stdenv.cc.libc_dev}/lib/system \ + libSystem-boot + + sed -i "s|/usr/lib/system/|$PWD/libSystem-boot/system/|g" libSystem-boot/libSystem.B.tbd + ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd + # End of bootstrap libSystem + + export flags="-idirafter $tools/include-Libsystem --sysroot=$tools -L$tools/lib -L$PWD/libSystem-boot" + + export CPP="clang -E $flags" + export CC="clang $flags" + export CXX="clang++ $flags --stdlib=libc++ -isystem$tools/include/c++/v1" + + # NOTE: These tests do a separate 'install' step (using cp), because + # having clang write directly to the final location apparently will make + # running the executable fail signature verification. (SIGKILL'd) + # + # Suspect this is creating a corrupt entry in the kernel cache, but it is + # unique to cctools ld. (The problem goes away with `-fuse-ld=lld`.) + + echo '#include <stdio.h>' >> hello1.c + echo '#include <float.h>' >> hello1.c + echo '#include <limits.h>' >> hello1.c + echo 'int main() { printf("Hello World\n"); return 0; }' >> hello1.c + $CC -o hello1 hello1.c + cp hello1 $out/bin/ + $out/bin/hello1 + + echo '#include <iostream>' >> hello3.cc + echo 'int main() { std::cout << "Hello World\n"; }' >> hello3.cc + $CXX -v -o hello3 hello3.cc + cp hello3 $out/bin/ + $out/bin/hello3 + + # test that libc++.dylib rpaths are correct so it can reference libc++abi.dylib when linked. + # using -Wl,-flat_namespace is required to generate an error + mkdir libtest/ + ln -s $tools/lib/libc++.dylib libtest/ + clang++ -Wl,-flat_namespace -idirafter $tools/include-Libsystem -isystem$tools/include/c++/v1 \ + --sysroot=$tools -L./libtest -L$PWD/libSystem-boot hello3.cc + + tar xvf ${hello.src} + cd hello-* + # hello configure detects -liconv is needed but doesn't add to the link step + LDFLAGS=-liconv ./configure --prefix=$out + make + make install + $out/bin/hello + ''; + }; + + # The ultimate test: bootstrap a whole stdenv from the tools specified above and get a package set out of it + # TODO: uncomment once https://github.com/NixOS/nixpkgs/issues/222717 is resolved + /* + test-pkgs = import test-pkgspath { + # if the bootstrap tools are for another platform, we should be testing + # that platform. + localSystem = if crossSystem != null then crossSystem else localSystem; + + stdenvStages = args: let + args' = args // { inherit bootstrapFiles; }; + in (import (test-pkgspath + "/pkgs/stdenv/darwin") args'); + }; + */ +} |