rust: fix cross-compilation

3 files changed, 143 insertions, 85 deletions

diff --git a/pkgs/development/compilers/rust/cargo.nix b/pkgs/development/compilers/rust/cargo.nix
index 65aa4ea9292f..9f576042787a 100644
--- a/pkgs/development/compilers/rust/cargo.nix
+++ b/pkgs/development/compilers/rust/cargo.nix
@@ -1,13 +1,11 @@
 { stdenv, file, curl, pkgconfig, python, openssl, cmake, zlib
 , makeWrapper, libiconv, cacert, rustPlatform, rustc, libgit2
 , CoreFoundation, Security
-, version
-, patches ? []
-, src }:
+}:
 
 rustPlatform.buildRustPackage rec {
-  name = "cargo-${version}";
-  inherit version src patches;
+  name = "cargo-${rustc.version}";
+  inherit (rustc) version src;
 
   # the rust source tarball already has all the dependencies vendored, no need to fetch them again
   cargoVendorDir = "vendor";
@@ -19,8 +17,8 @@ rustPlatform.buildRustPackage rec {
   # changes hash of vendor directory otherwise
   dontUpdateAutotoolsGnuConfigScripts = true;
 
-  nativeBuildInputs = [ pkgconfig ];
-  buildInputs = [ cacert file curl python openssl cmake zlib makeWrapper libgit2 ]
+  nativeBuildInputs = [ pkgconfig cmake makeWrapper ];
+  buildInputs = [ cacert file curl python openssl zlib libgit2 ]
     ++ stdenv.lib.optionals stdenv.isDarwin [ CoreFoundation Security libiconv ];
 
   LIBGIT2_SYS_USE_PKG_CONFIG = 1;
diff --git a/pkgs/development/compilers/rust/default.nix b/pkgs/development/compilers/rust/default.nix
index 9640cd9b577e..3a6ce5821d91 100644
--- a/pkgs/development/compilers/rust/default.nix
+++ b/pkgs/development/compilers/rust/default.nix
@@ -1,47 +1,67 @@
-{ stdenv, callPackage, recurseIntoAttrs, makeRustPlatform, llvm, fetchurl
+{ stdenv, lib, overrideCC
+, buildPackages
+, newScope, callPackage
 , CoreFoundation, Security
-, targets ? []
-, targetToolchains ? []
-, targetPatches ? []
-}:
+, gcc6
+}: rec {
+  makeRustPlatform = { rustc, cargo, ... }: {
+    rust = {
+      inherit rustc cargo;
+    };
 
-let
-  rustPlatform = recurseIntoAttrs (makeRustPlatform (callPackage ./bootstrap.nix {}));
-  version = "1.32.0";
-  cargoVersion = "1.32.0";
-  src = fetchurl {
-    url = "https://static.rust-lang.org/dist/rustc-${version}-src.tar.gz";
-    sha256 = "0ji2l9xv53y27xy72qagggvq47gayr5lcv2jwvmfirx029vlqnac";
-  };
-in rec {
-  rustc = callPackage ./rustc.nix {
-    inherit stdenv llvm targets targetPatches targetToolchains rustPlatform version src;
-
-    patches = [
-      ./patches/net-tcp-disable-tests.patch
-
-      # Re-evaluate if this we need to disable this one
-      #./patches/stdsimd-disable-doctest.patch
-    ];
-
-    withBundledLLVM = false;
-
-    configureFlags = [ "--release-channel=stable" ];
+    buildRustPackage = callPackage ../../../build-support/rust {
+      inherit rustc cargo;
 
-    # 1. Upstream is not running tests on aarch64:
-    # see https://github.com/rust-lang/rust/issues/49807#issuecomment-380860567
-    # So we do the same.
-    # 2. Tests run out of memory for i686
-    #doCheck = !stdenv.isAarch64 && !stdenv.isi686;
+      fetchcargo = buildPackages.callPackage ../../../build-support/rust/fetchcargo.nix {
+        inherit cargo;
+      };
+    };
 
-    # Disabled for now; see https://github.com/NixOS/nixpkgs/pull/42348#issuecomment-402115598.
-    doCheck = false;
+    rustcSrc = callPackage ./rust-src.nix {
+      inherit rustc;
+    };
   };
 
-  cargo = callPackage ./cargo.nix rec {
-    version = cargoVersion;
-    inherit src stdenv CoreFoundation Security;
-    inherit rustc; # the rustc that will be wrapped by cargo
-    inherit rustPlatform; # used to build cargo
+  # This just contains tools for now. But it would conceivably contain
+  # libraries too, say if we picked some default/recommended versions from
+  # `cratesIO` to build by Hydra and/or try to prefer/bias in Cargo.lock for
+  # all vendored Carnix-generated nix.
+  #
+  # In the end game, rustc, the rust standard library (`core`, `std`, etc.),
+  # and cargo would themselves be built with `buildRustCreate` like
+  # everything else. Tools and `build.rs` and procedural macro dependencies
+  # would be taken from `buildRustPackages` (and `bootstrapRustPackages` for
+  # anything provided prebuilt or their build-time dependencies to break
+  # cycles / purify builds). In this way, nixpkgs would be in control of all
+  # bootstrapping.
+  packages = {
+    prebuilt = callPackage ./bootstrap.nix {};
+    stable = lib.makeScope newScope (self: let
+      # Like `buildRustPackages`, but may also contain prebuilt binaries to
+      # break cycle. Just like `bootstrapTools` for nixpkgs as a whole,
+      # nothing in the final package set should refer to this.
+      bootstrapRustPackages = self.buildRustPackages.overrideScope' (_: _:
+        lib.optionalAttrs (stdenv.buildPlatform == stdenv.hostPlatform)
+          buildPackages.rust.packages.prebuilt);
+      bootRustPlatform = makeRustPlatform bootstrapRustPackages;
+    in {
+      # Packages suitable for build-time, e.g. `build.rs`-type stuff.
+      buildRustPackages = buildPackages.rust.packages.stable;
+      # Analogous to stdenv
+      rustPlatform = makeRustPlatform self.buildRustPackages;
+      rustc = self.callPackage ./rustc.nix ({
+        # Use boot package set to break cycle
+        rustPlatform = bootRustPlatform;
+      } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
+        stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
+      });
+      cargo = self.callPackage ./cargo.nix ({
+        # Use boot package set to break cycle
+        rustPlatform = bootRustPlatform;
+        inherit CoreFoundation Security;
+      } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
+        stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
+      });
+    });
   };
 }
diff --git a/pkgs/development/compilers/rust/rustc.nix b/pkgs/development/compilers/rust/rustc.nix
index 8cddb9ea2288..631c20d16c6c 100644
--- a/pkgs/development/compilers/rust/rustc.nix
+++ b/pkgs/development/compilers/rust/rustc.nix
@@ -1,33 +1,28 @@
-{ stdenv, targetPackages, removeReferencesTo
+{ stdenv, removeReferencesTo, pkgsBuildBuild, pkgsBuildHost, pkgsBuildTarget
 , fetchurl, fetchgit, fetchzip, file, python2, tzdata, ps
-, llvm, ncurses, darwin, rustPlatform, git, cmake, curl
+, llvm_7, ncurses, darwin, git, cmake, curl, rustPlatform
 , which, libffi, gdb
-, version
 , withBundledLLVM ? false
-, src
-, configureFlags ? []
-, patches
-, targets
-, targetPatches
-, targetToolchains
-, doCheck ? true
-, broken ? false
 }:
 
 let
   inherit (stdenv.lib) optional optionalString;
   inherit (darwin.apple_sdk.frameworks) Security;
 
-  llvmShared = llvm.override { enableSharedLibraries = true; };
+  llvmSharedForBuild = pkgsBuildBuild.llvm_7.override { enableSharedLibraries = true; };
+  llvmSharedForHost = pkgsBuildHost.llvm_7.override { enableSharedLibraries = true; };
+  llvmSharedForTarget = pkgsBuildTarget.llvm_7.override { enableSharedLibraries = true; };
 
-  target = builtins.replaceStrings [" "] [","] (builtins.toString targets);
-in
+  # For use at runtime
+  llvmShared = llvm_7.override { enableSharedLibraries = true; };
+in stdenv.mkDerivation rec {
+  pname = "rustc";
+  version = "1.32.0";
 
-stdenv.mkDerivation {
-  name = "rustc-${version}";
-  inherit version;
-
-  inherit src;
+  src = fetchurl {
+    url = "https://static.rust-lang.org/dist/rustc-${version}-src.tar.gz";
+    sha256 = "0ji2l9xv53y27xy72qagggvq47gayr5lcv2jwvmfirx029vlqnac";
+  };
 
   __darwinAllowLocalNetworking = true;
 
@@ -40,11 +35,12 @@ stdenv.mkDerivation {
   # See https://github.com/NixOS/nixpkgs/pull/34227
   stripDebugList = if stdenv.isDarwin then [ "bin" ] else null;
 
+
   NIX_LDFLAGS =
        # when linking stage1 libstd: cc: undefined reference to `__cxa_begin_catch'
        optional (stdenv.isLinux && !withBundledLLVM) "--push-state --as-needed -lstdc++ --pop-state"
     ++ optional (stdenv.isDarwin && !withBundledLLVM) "-lc++"
-    ++ optional stdenv.isDarwin "-rpath ${llvmShared}/lib";
+    ++ optional stdenv.isDarwin "-rpath ${llvmSharedForHost}/lib";
 
   # Enable nightly features in stable compiles (used for
   # bootstrapping, see https://github.com/rust-lang/rust/pull/37265).
@@ -57,27 +53,67 @@ stdenv.mkDerivation {
 
   # We need rust to build rust. If we don't provide it, configure will try to download it.
   # Reference: https://github.com/rust-lang/rust/blob/master/src/bootstrap/configure.py
-  configureFlags = configureFlags
-                ++ [ "--enable-local-rust" "--local-rust-root=${rustPlatform.rust.rustc}" "--enable-rpath"
-                     "--enable-vendor"
-                     "--default-linker=${targetPackages.stdenv.cc}/bin/cc" ]
-                ++ optional (!withBundledLLVM) [ "--enable-llvm-link-shared" "--llvm-root=${llvmShared}" ]
-                ++ optional (targets != []) "--target=${target}";
+  configureFlags = let
+    setBuild  = "--set=target.${stdenv.buildPlatform.config}";
+    setHost   = "--set=target.${stdenv.hostPlatform.config}";
+    setTarget = "--set=target.${stdenv.targetPlatform.config}";
+    ccForBuild  = "${pkgsBuildBuild.targetPackages.stdenv.cc}/bin/${pkgsBuildBuild.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForBuild = "${pkgsBuildBuild.targetPackages.stdenv.cc}/bin/${pkgsBuildBuild.targetPackages.stdenv.cc.targetPrefix}c++";
+    ccForHost  = "${pkgsBuildHost.targetPackages.stdenv.cc}/bin/${pkgsBuildHost.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForHost = "${pkgsBuildHost.targetPackages.stdenv.cc}/bin/${pkgsBuildHost.targetPackages.stdenv.cc.targetPrefix}c++";
+    ccForTarget  = "${pkgsBuildTarget.targetPackages.stdenv.cc}/bin/${pkgsBuildTarget.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForTarget = "${pkgsBuildTarget.targetPackages.stdenv.cc}/bin/${pkgsBuildTarget.targetPackages.stdenv.cc.targetPrefix}c++";
+  in [
+    "--release-channel=stable"
+    "--set=build.rustc=${rustPlatform.rust.rustc}/bin/rustc"
+    "--set=build.cargo=${rustPlatform.rust.cargo}/bin/cargo"
+    "--enable-rpath"
+    "--enable-vendor"
+    "--build=${stdenv.buildPlatform.config}"
+    "--host=${stdenv.hostPlatform.config}"
+    "--target=${stdenv.targetPlatform.config}"
+
+    "${setBuild}.cc=${ccForBuild}"
+    "${setHost}.cc=${ccForHost}"
+    "${setTarget}.cc=${ccForTarget}"
+
+    "${setBuild}.linker=${ccForBuild}"
+    "${setHost}.linker=${ccForHost}"
+    "${setTarget}.linker=${ccForTarget}"
+
+    "${setBuild}.cxx=${cxxForBuild}"
+    "${setHost}.cxx=${cxxForHost}"
+    "${setTarget}.cxx=${cxxForTarget}"
+  ] ++ optional (!withBundledLLVM) [
+    "--enable-llvm-link-shared"
+    "${setBuild}.llvm-config=${llvmSharedForBuild}/bin/llvm-config"
+    "${setHost}.llvm-config=${llvmSharedForHost}/bin/llvm-config"
+    "${setTarget}.llvm-config=${llvmSharedForTarget}/bin/llvm-config"
+  ];
 
   # The bootstrap.py will generated a Makefile that then executes the build.
   # The BOOTSTRAP_ARGS used by this Makefile must include all flags to pass
   # to the bootstrap builder.
   postConfigure = ''
-    substituteInPlace Makefile --replace 'BOOTSTRAP_ARGS :=' 'BOOTSTRAP_ARGS := --jobs $(NIX_BUILD_CORES)'
+    substituteInPlace Makefile \
+      --replace 'BOOTSTRAP_ARGS :=' 'BOOTSTRAP_ARGS := --jobs $(NIX_BUILD_CORES)'
   '';
 
-  patches = patches ++ targetPatches;
+  patches = [
+    ./patches/net-tcp-disable-tests.patch
+
+    # Re-evaluate if this we need to disable this one
+    #./patches/stdsimd-disable-doctest.patch
+
+    # Fails on hydra - not locally; the exact reason is unknown.
+    # Comments in the test suggest that some non-reproducible environment
+    # variables such $RANDOM can make it fail.
+    # ./patches/disable-test-inherit-env.patch
+  ];
 
   # the rust build system complains that nix alters the checksums
   dontFixLibtool = true;
 
-  passthru.target = target;
-
   postPatch = ''
     patchShebangs src/etc
 
@@ -120,15 +156,13 @@ stdenv.mkDerivation {
   dontUseCmakeConfigure = true;
 
   # ps is needed for one of the test cases
-  nativeBuildInputs =
-    [ file python2 ps rustPlatform.rust.rustc git cmake
-      which libffi removeReferencesTo
-    ]
-    # Only needed for the debuginfo tests
+  nativeBuildInputs = [
+    file python2 ps rustPlatform.rust.rustc git cmake
+    which libffi removeReferencesTo
+  ] # Only needed for the debuginfo tests
     ++ optional (!stdenv.isDarwin) gdb;
 
-  buildInputs = targetToolchains
-    ++ optional stdenv.isDarwin Security
+  buildInputs = optional stdenv.isDarwin Security
     ++ optional (!withBundledLLVM) llvmShared;
 
   outputs = [ "out" "man" "doc" ];
@@ -148,7 +182,14 @@ stdenv.mkDerivation {
     sed -i '28s/home_dir().is_some()/true/' ./src/test/run-pass/env-home-dir.rs
   '';
 
-  inherit doCheck;
+  # 1. Upstream is not running tests on aarch64:
+  # see https://github.com/rust-lang/rust/issues/49807#issuecomment-380860567
+  # So we do the same.
+  # 2. Tests run out of memory for i686
+  #doCheck = !stdenv.isAarch64 && !stdenv.isi686;
+
+  # Disabled for now; see https://github.com/NixOS/nixpkgs/pull/42348#issuecomment-402115598.
+  doCheck = false;
 
   # remove references to llvm-config in lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/librustc_codegen_llvm-llvm.so
   # and thus a transitive dependency on ncurses
@@ -170,6 +211,5 @@ stdenv.mkDerivation {
     maintainers = with maintainers; [ madjar cstrahan wizeman globin havvy ];
     license = [ licenses.mit licenses.asl20 ];
     platforms = platforms.linux ++ platforms.darwin;
-    broken = broken;
   };
 }