stdenv/darwin: integrate a new CoreFoundation

This also updates the bootstrap tool builder to LLVM 5, but not the ones
we actually use for bootstrap. I'll make that change in a subsequent commit
so as to provide traceable provenance of the bootstrap tools.

10 files changed, 72 insertions, 168 deletions

diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch
deleted file mode 100644
index a1018d389c14..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch
+++ /dev/null
@@ -1,22 +0,0 @@
---- CF-855.17/CoreFoundation.h  2015-01-03 00:17:41.000000000 -0500
-+++ CF-855.17/CoreFoundation.h.new  2015-01-03 00:18:35.000000000 -0500
-@@ -72,6 +72,7 @@
- #include <CoreFoundation/CFDictionary.h>
- #include <CoreFoundation/CFError.h>
- #include <CoreFoundation/CFLocale.h>
-+#include <CoreFoundation/CFMachPort.h>
- #include <CoreFoundation/CFNumber.h>
- #include <CoreFoundation/CFNumberFormatter.h>
- #include <CoreFoundation/CFPreferences.h>
-
---- CF-855.17/Makefile  2015-01-03 00:32:52.000000000 -0500
-+++ CF-855.17/Makefile.new  2015-01-03 00:33:07.000000000 -0500
-@@ -9,7 +9,7 @@
- HFILES = $(wildcard *.h)
- INTERMEDIATE_HFILES = $(addprefix $(OBJBASE)/CoreFoundation/,$(HFILES))
- 
--PUBLIC_HEADERS=CFArray.h CFBag.h CFBase.h CFBinaryHeap.h CFBitVector.h CFBundle.h CFByteOrder.h CFCalendar.h CFCharacterSet.h CFData.h CFDate.h CFDateFormatter.h CFDictionary.h CFError.h CFLocale.h CFMessagePort.h CFNumber.h CFNumberFormatter.h CFPlugIn.h CFPlugInCOM.h CFPreferences.h CFPropertyList.h CFRunLoop.h CFSet.h CFSocket.h CFStream.h CFString.h CFStringEncodingExt.h CFTimeZone.h CFTree.h CFURL.h CFURLAccess.h CFUUID.h CFUserNotification.h CFXMLNode.h CFXMLParser.h CFAvailability.h CFUtilities.h CoreFoundation.h
-+PUBLIC_HEADERS=CFArray.h CFBag.h CFBase.h CFBinaryHeap.h CFBitVector.h CFBundle.h CFByteOrder.h CFCalendar.h CFCharacterSet.h CFData.h CFDate.h CFDateFormatter.h CFDictionary.h CFError.h CFLocale.h CFMachPort.h CFMessagePort.h CFNumber.h CFNumberFormatter.h CFPlugIn.h CFPlugInCOM.h CFPreferences.h CFPropertyList.h CFRunLoop.h CFSet.h CFSocket.h CFStream.h CFString.h CFStringEncodingExt.h CFTimeZone.h CFTree.h CFURL.h CFURLAccess.h CFUUID.h CFUserNotification.h CFXMLNode.h CFXMLParser.h CFAvailability.h CFUtilities.h CoreFoundation.h
- 
- PRIVATE_HEADERS=CFBundlePriv.h CFCharacterSetPriv.h CFError_Private.h CFLogUtilities.h CFPriv.h CFRuntime.h CFStorage.h CFStreamAbstract.h CFStreamPriv.h CFStreamInternal.h CFStringDefaultEncoding.h CFStringEncodingConverter.h CFStringEncodingConverterExt.h CFUniChar.h CFUnicodeDecomposition.h CFUnicodePrecomposition.h ForFoundationOnly.h CFBurstTrie.h CFICULogging.h
- 
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
deleted file mode 100644
index 068a6311a9cb..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
+++ /dev/null
@@ -1,39 +0,0 @@
-diff --git a/CFBase.h b/CFBase.h
-index ffddd2b..e5a926b 100644
---- a/CFBase.h
-+++ b/CFBase.h
-@@ -249,6 +249,33 @@ CF_EXTERN_C_BEGIN
- #endif
- #endif
- 
-+#if __has_attribute(objc_bridge) && __has_feature(objc_bridge_id) && __has_feature(objc_bridge_id_on_typedefs)
-+
-+#ifdef __OBJC__
-+@class NSArray;
-+@class NSAttributedString;
-+@class NSString;
-+@class NSNull;
-+@class NSCharacterSet;
-+@class NSData;
-+@class NSDate;
-+@class NSTimeZone;
-+@class NSDictionary;
-+@class NSError;
-+@class NSLocale;
-+@class NSNumber;
-+@class NSSet;
-+@class NSURL;
-+#endif
-+
-+#define CF_BRIDGED_TYPE(T)		__attribute__((objc_bridge(T)))
-+#define CF_BRIDGED_MUTABLE_TYPE(T)	__attribute__((objc_bridge_mutable(T)))
-+#define CF_RELATED_TYPE(T,C,I)		__attribute__((objc_bridge_related(T,C,I)))
-+#else
-+#define CF_BRIDGED_TYPE(T)
-+#define CF_BRIDGED_MUTABLE_TYPE(T)
-+#define CF_RELATED_TYPE(T,C,I)
-+#endif
- 
- CF_EXPORT double kCFCoreFoundationVersionNumber;
- 
-
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix b/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix
deleted file mode 100644
index 5589d1592f46..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix
+++ /dev/null
@@ -1,51 +0,0 @@
-{ stdenv, appleDerivation, ICU, dyld, libdispatch, libplatform, launchd, libclosure }:
-
-# this project uses blocks, a clang-only extension
-assert stdenv.cc.isClang;
-
-appleDerivation {
-  buildInputs = [ dyld ICU libdispatch libplatform launchd libclosure ];
-
-  patches = [ ./add-cfmachport.patch ./cf-bridging.patch ./remove-xpc.patch ];
-
-  __propagatedImpureHostDeps = [ "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" ];
-
-  preBuild = ''
-    substituteInPlace Makefile \
-      --replace "/usr/bin/clang" "clang" \
-      --replace "-arch i386 " "" \
-      --replace "/usr/bin/" "" \
-      --replace "/usr/sbin/" "" \
-      --replace "/bin/" "" \
-      --replace "INSTALLNAME=/System" "INSTALLNAME=$out" \
-      --replace "install_name_tool -id /System/Library/Frameworks" "install_name_tool -id @rpath" \
-      --replace 'chown -RH -f root:wheel $(DSTBASE)/CoreFoundation.framework' "" \
-      --replace 'chmod -RH' 'chmod -R'
-
-    # with this file present, CoreFoundation gets a _main symbol defined, which can
-    # interfere with linking other programs
-    rm plconvert.c
-
-    replacement=''$'#define __PTK_FRAMEWORK_COREFOUNDATION_KEY5 55\n#define _pthread_getspecific_direct(key) pthread_getspecific((key))\n#define _pthread_setspecific_direct(key, val) pthread_setspecific((key), (val))'
-
-    substituteInPlace CFPlatform.c --replace "#include <pthread/tsd_private.h>" "$replacement"
-
-    substituteInPlace CFRunLoop.c --replace "#include <pthread/private.h>" ""
-
-    substituteInPlace CFURLPriv.h \
-      --replace "#include <CoreFoundation/CFFileSecurity.h>" "" \
-      --replace "#include <CoreFoundation/CFURLEnumerator.h>" "" \
-      --replace "CFFileSecurityRef" "void *" \
-      --replace "CFURLEnumeratorResult" "void *" \
-      --replace "CFURLEnumeratorRef" "void *"
-
-    export DSTROOT=$out
-  '';
-
-  postInstall = ''
-    mv $out/System/* $out
-    rmdir $out/System
-    mv $out/Library/Frameworks/CoreFoundation.framework/Versions/A/PrivateHeaders/* \
-       $out/Library/Frameworks/CoreFoundation.framework/Versions/A/Headers
-  '';
-}
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch
deleted file mode 100644
index a7b9fe486434..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch
+++ /dev/null
@@ -1,17 +0,0 @@
-diff --git a/CFBundlePriv.h b/CFBundlePriv.h
-index d4feb5f..e7b52e8 100644
---- a/CFBundlePriv.h
-+++ b/CFBundlePriv.h
-@@ -254,12 +254,6 @@ Boolean _CFBundleGetStringsFilesShared(CFBundleRef bundle);
- CF_EXPORT
- CFURLRef _CFBundleCopyFrameworkURLForExecutablePath(CFStringRef executablePath);
- 
--#if (TARGET_OS_MAC && !(TARGET_OS_EMBEDDED || TARGET_OS_IPHONE)) || (TARGET_OS_EMBEDDED || TARGET_OS_IPHONE)
--#include <xpc/xpc.h>
--CF_EXPORT
--void _CFBundleSetupXPCBootstrap(xpc_object_t bootstrap) CF_AVAILABLE(10_10, 8_0);
--#endif
--
- /* Functions deprecated as SPI */
- 
- CF_EXPORT
diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix
index e2d62cef1ba4..d490048c4df3 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix
@@ -204,7 +204,6 @@ let
     bootstrap_cmds  = applePackage "bootstrap_cmds"    "dev-tools-7.0"   "1v5dv2q3af1xwj5kz0a5g54fd5dm6j4c9dd2g66n4kc44ixyrhp3" {};
     bsdmake         = applePackage "bsdmake"           "dev-tools-3.2.6" "11a9kkhz5bfgi1i8kpdkis78lhc6b5vxmhd598fcdgra1jw4iac2" {};
     CarbonHeaders   = applePackage "CarbonHeaders"     "osx-10.6.2"      "1zam29847cxr6y9rnl76zqmkbac53nx0szmqm9w5p469a6wzjqar" {};
-    CF              = applePackage "CF"                "osx-10.10.5"     "07f5psjxi7wyd13ci4x83ya5hy6p69sjfqcpp2mmxdlhd8yzkf74" {};
     CommonCrypto    = applePackage "CommonCrypto"      "osx-10.11.6"     "0vllfpb8f4f97wj2vpdd7w5k9ibnsbr6ff1zslpp6q323h01n25y" {};
     configd         = applePackage "configd"           "osx-10.8.5"      "1gxakahk8gallf16xmhxhprdxkh3prrmzxnmxfvj0slr0939mmr2" {};
     copyfile        = applePackage "copyfile"          "osx-10.11.6"     "1rkf3iaxmjz5ycgrmf0g971kh90jb2z1zqxg5vlqz001s4y457gs" {};
diff --git a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
index fe68ee78d1f0..714524e8da58 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
@@ -33,7 +33,6 @@ name: version: sha256: args: let
         pkgs.gnustep.make
         pkgs.darwin.apple_sdk.frameworks.AppKit
         pkgs.darwin.apple_sdk.frameworks.Foundation
-        pkgs.darwin.cf-private
       ];
       makeFlags = [
         "-f${makeFile}"
diff --git a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
index 609d07fda100..724c4788b6cc 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
@@ -39,9 +39,5 @@ appleDerivation {
       --replace 'return mLoginDLDbIdentifier;' 'return mLoginDLDbIdentifier; }' \
       --replace '_xpc_runtime_is_app_sandboxed()' 'false'
       # hope that doesn't hurt anything
-
-    substituteInPlace lib/KCEventNotifier.h --replace \
-      'CoreFoundation/CFNotificationCenter.h' \
-      '${apple_sdk.sdk.out}/Library/Frameworks/CoreFoundation.framework/Versions/A/Headers/CFNotificationCenter.h'
   '';
 }
diff --git a/pkgs/os-specific/darwin/cf-private/default.nix b/pkgs/os-specific/darwin/cf-private/default.nix
index 603c0f652b01..3fac20d23c78 100644
--- a/pkgs/os-specific/darwin/cf-private/default.nix
+++ b/pkgs/os-specific/darwin/cf-private/default.nix
@@ -1,21 +1,59 @@
-{ stdenv, osx_private_sdk, CF }:
+{ CF, apple_sdk }:
 
-stdenv.mkDerivation {
-  name = "${CF.name}-private";
-  phases = [ "installPhase" "fixupPhase" ];
-  installPhase = ''
-    dest=$out/Library/Frameworks/CoreFoundation.framework/Headers
-    mkdir -p $dest
-    pushd $dest
-      for file in ${CF}/Library/Frameworks/CoreFoundation.framework/Headers/*; do
-        ln -sf $file
-      done
-
-      # Copy or overwrite private headers, some of these might already
-      # exist in CF but the private versions have more information.
-      cp -Lfv ${osx_private_sdk}/include/CoreFoundationPrivateHeaders/* $dest
-    popd
-  '';
+# cf-private is a bit weird, but boils down to CF with a weird setup-hook that
+# makes a build link against the system CoreFoundation rather than our pure one.
+# The reason it exists is that although our CF headers and build are pretty legit
+# now, the underlying runtime is quite different. Apple's in a bit of flux around CF
+# right now, and support three different backends for it: swift, "C", and an ObjC
+# one. The former two can be built from public sources, but the ObjC one isn't really
+# public. Unfortunately, it's also one of the core underpinnings of a lot of Mac-
+# specific behavior, and defines a lot of symbols that some Objective C apps depend
+# on, even though one might expect those symbols to derive from Foundation. So if
+# your app relies on NSArray and several other basic ObjC types, it turns out that
+# because of their magic "toll-free bridging" support, the symbols for those types
+# live in CoreFoundation with an ObjC runtime. And because that isn't public, we have
+# this hack in place to let people link properly anyway. Phew!
+# 
+# This can be revisited if Apple ever decide to release the ObjC backend in a publicly
+# buildable form.
+# 
+# This doesn't really need to rebuild CF, but it's cheap, and adding a setup hook to
+# an existing package was annoying. We need a buildEnv that knows how to add those
+CF.overrideAttrs (orig: {
+  # PLEASE if you add things to this derivation, explain in reasonable detail why
+  # you're adding them and when the workaround can go away. This whole derivation is
+  # a workaround and if you don't explain what you're working around, it makes it
+  # very hard for people to clean it up later.
 
+  name = "${orig.name}-private";
   setupHook = ./setup-hook.sh;
-}
+
+  # TODO: consider re-adding https://github.com/NixOS/nixpkgs/blob/master/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
+  # once the missing headers are in and see if that fixes all need for this.
+
+  # This can go away once https://bugs.swift.org/browse/SR-8741 happens, which is
+  # looking more likely these days with the friendly people at Apple! We only need
+  # the header because the setup hook takes care of linking us against a version
+  # of the framework with the functionality built into it. The main user I know of
+  # this is watchman, who can almost certainly switch to the pure CF once the header
+  # and functionality is merged in.
+  installPhase = orig.installPhase + ''
+    basepath="Library/Frameworks/CoreFoundation.framework/Headers"
+    path="$basepath/CFFileDescriptor.h"
+
+    # Append the include at top level or nobody will notice the header we're about to add
+    sed -i '/CFNotificationCenter.h/a #include <CoreFoundation/CFFileDescriptor.h>' \
+      "$out/$basepath/CoreFoundation.h"
+
+    cp ${apple_sdk.frameworks.CoreFoundation}/$path $out/$path
+  '' +
+  # This one is less likely to go away, but I'll mention it anyway. The issue is at
+  # https://bugs.swift.org/browse/SR-8744, and the main user I know of is qtbase
+  ''
+    path="$basepath/CFURLEnumerator.h"    
+    sed -i '/CFNotificationCenter.h/a #include <CoreFoundation/CFURLEnumerator.h>' \
+      "$out/$basepath/CoreFoundation.h"
+
+    cp ${apple_sdk.frameworks.CoreFoundation}/$path $out/$path
+  '';
+})
\ No newline at end of file
diff --git a/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix b/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
index 1dea55cccc9e..f819429f4deb 100644
--- a/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
+++ b/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
@@ -14,11 +14,12 @@ in stdenv.mkDerivation {
   src = fetchFromGitHub {
     owner  = "apple";
     repo   = "swift-corelibs-foundation";
-    rev    = "85c640e7ce50e6ca61a134c72270e214bc63fdba"; # https://github.com/apple/swift-corelibs-foundation/pull/1686
-    sha256 = "0z2v278wy7jh0c92g1dszd8hj8naxari660sqx6yab5dwapd46qc";
+    rev    = "71aaba20e1450a82c516af1342fe23268e15de0a";
+    sha256 = "17kpql0f27xxz4jjw84vpas5f5sn4vdqwv10g151rc3rswbwln1z";
   };
 
-  buildInputs = [ ninja python libxml2 objc4 ICU curl ];
+  nativeBuildInputs = [ ninja python ];
+  buildInputs = [ libxml2 objc4 ICU curl ];
 
   sourceRoot = "source/CoreFoundation";
 
@@ -31,8 +32,7 @@ in stdenv.mkDerivation {
     # 3. Use the legit CoreFoundation.h, not the one telling you not to use it because of Swift
     substituteInPlace build.py \
       --replace "cf.CFLAGS += '-DDEPLOYMENT" '#' \
-      --replace "cf.LDFLAGS += '-ldispatch" '#' \
-      --replace "Base.subproj/SwiftRuntime/CoreFoundation.h" 'Base.subproj/CoreFoundation.h'
+      --replace "cf.LDFLAGS += '-ldispatch" '#'
 
     # Includes xpc for some initialization routine that they don't define anyway, so no harm here
     substituteInPlace PlugIn.subproj/CFBundlePriv.h \
@@ -53,8 +53,11 @@ in stdenv.mkDerivation {
 
   BUILD_DIR = "./Build";
   CFLAGS = "-DINCLUDE_OBJC -I${libxml2.dev}/include/libxml2"; # They seem to assume we include objc in some places and not in others, make a PR; also not sure why but libxml2 include path isn't getting picked up from buildInputs
-  LDFLAGS = "-install_name ${placeholder "out"}/Frameworks/CoreFoundation.framework/CoreFoundation -current_version 1234.56.7 -compatibility_version 150.0.0 -init ___CFInitialize";
-  configurePhase = "../configure --sysroot unused";
+  
+  # I'm guessing at the version here. https://github.com/apple/swift-corelibs-foundation/commit/df3ec55fe6c162d590a7653d89ad669c2b9716b1 imported "high sierra"
+  # and this version is a version from there. No idea how accurate it is.
+  LDFLAGS = "-current_version 1454.90.0 -compatibility_version 150.0.0 -init ___CFInitialize";
+  configurePhase = "../configure release --sysroot UNUSED";
 
   enableParallelBuilding = true;
   buildPhase = "ninja -j $NIX_BUILD_CORES";
@@ -66,6 +69,12 @@ in stdenv.mkDerivation {
     mkdir -p $base/Versions/A/{Headers,PrivateHeaders,Modules}
 
     cp ./Build/CoreFoundation/libCoreFoundation.dylib $base/Versions/A/CoreFoundation
+
+    # Note that this could easily live in the ldflags above as `-install_name @rpath/...` but
+    # https://github.com/NixOS/nixpkgs/issues/46434 thwarts that, so for now I'm hacking it up
+    # after the fact.
+    install_name_tool -id '@rpath/CoreFoundation.framework/Versions/A/CoreFoundation' $base/Versions/A/CoreFoundation
+
     cp ./Build/CoreFoundation/usr/include/CoreFoundation/*.h $base/Versions/A/Headers
     cp ./Build/CoreFoundation/usr/include/CoreFoundation/module.modulemap $base/Versions/A/Modules
 
diff --git a/pkgs/os-specific/darwin/swift-corelibs/default.nix b/pkgs/os-specific/darwin/swift-corelibs/default.nix
deleted file mode 100644
index 0d96b8fd008e..000000000000
--- a/pkgs/os-specific/darwin/swift-corelibs/default.nix
+++ /dev/null
@@ -1,8 +0,0 @@
-{ callPackage, darwin }:
-
-rec {
-  corefoundation = callPackage ./corefoundation.nix { inherit (darwin) objc4 ICU; };
-  libdispatch = callPackage ./libdispatch.nix {
-   inherit (darwin) apple_sdk_sierra xnu;
-  };
-}