macvim: Clean up hybrid compilation environment

MacVim compiles the Vim part using `/usr/bin/clang` and the GUI part
using Xcode. The Xcode portion always uses Xcode's own SDK and we have
no workable alternative. The Vim portion so far has been compiling using
a hybrid compilation environment, where it uses the SDK for most stuff
but picks up a bunch of library linker paths (including libSystem) by
virtue of Ruby's LDFLAGS. This hybrid compilation environment meant that
if the SDK headers referenced a symbol that the library itself didn't
have, this could produce link errors.

Previously we attempted to fix this by synthesizing an include path that
contained just the one header from Nix's Libsystem that referenced the
missing symbol, to get rid of the reference and allow linking to work
again, but this was very hacky and runs the risk of future Xcode SDK
changes producing the same errors with different headers, or of future
SDK versions expecting the intercepted header to contain a definition
that Nix's doesn't.

This new approach is to just clean up the compilation environment such
that the Vim portion is compiling against the Xcode SDK as well, by
sanitizing the LDFLAGS produced by the configure script so it stops
referencing Nix's versions of OS libraries. This means the resulting Vim
binary no longer depends at runtime on Nix for anything except the
scripting language support, but that's how it's been for the MacVim
binary all along anyway, and this approach should keep us insulated
against future Xcode SDK changes.

2 files changed, 27 insertions, 37 deletions

diff --git a/pkgs/applications/editors/vim/macvim.nix b/pkgs/applications/editors/vim/macvim.nix
index 2a13c0ea700f..93ec48e54e0d 100644
--- a/pkgs/applications/editors/vim/macvim.nix
+++ b/pkgs/applications/editors/vim/macvim.nix
@@ -22,14 +22,6 @@ let
     mkdir -p $out/bin
     ln -s /usr/bin/xcrun /usr/bin/xcodebuild /usr/bin/tiffutil /usr/bin/qlmanage $out/bin
   '';
-  # I'm not really sure how configure finds Nix Libsystem, but we get link errors when using
-  # Xcode 11.4 if we're compiling against Xcode's SDK but using Nix Libsystem. We can't just pass
-  # Libsystem's includes though as that causes other issues, particularly with availability macros.
-  # This is a horrible hack but let's just pass the header that's causing problems.
-  fakeLibsystemInclude = runCommand "macvim-libsystem-include-shim" {} ''
-    mkdir -p $out/include/sys/_types
-    ln -s ${darwin.Libsystem}/include/sys/_types/_fd_def.h $out/include/sys/_types
-  '';
 in
 
 stdenv.mkDerivation {
@@ -74,7 +66,6 @@ stdenv.mkDerivation {
       "--with-tlib=ncurses"
       "--with-compiledby=Nix"
       "--disable-sparkle"
-      "LDFLAGS=-headerpad_max_install_names"
   ];
 
   makeFlags = ''PREFIX=$(out) CPPFLAGS="-Wno-error"'';
@@ -89,19 +80,12 @@ stdenv.mkDerivation {
 
   # This is unfortunate, but we need to use the same compiler as Xcode,
   # but Xcode doesn't provide a way to configure the compiler.
-  #
-  # If you're willing to modify the system files, you can do this:
-  #   http://hamelot.co.uk/programming/add-gcc-compiler-to-xcode-6/
-  #
-  # But we don't have that option.
   preConfigure = ''
     CC=/usr/bin/clang
 
     DEV_DIR=$(/usr/bin/xcode-select -print-path)/Platforms/MacOSX.platform/Developer
     configureFlagsArray+=(
       "--with-developer-dir=$DEV_DIR"
-      # Also pass `-g -O` because configure would add those if we weren't setting CFLAGS.
-      "CFLAGS=-g -O -isystem ${fakeLibsystemInclude}/include"
     )
   ''
   # For some reason having LD defined causes PSMTabBarControl to fail at link-time as it
@@ -111,8 +95,34 @@ stdenv.mkDerivation {
   ''
   ;
 
+  # Because we're building with system clang, this means we're building against Xcode's SDK and
+  # linking against system libraries. The configure script is picking up Nix Libsystem (via ruby)
+  # so we need to patch that out or we'll get linker issues. The MacVim binary built by Xcode links
+  # against the system anyway so it doesn't really matter that the Vim binary will too. If we
+  # decide that matters, we can always patch it back to the Nix libsystem post-build.
+  # It also picks up libiconv, libunwind, and objc4 from Nix. These seem relatively harmless but
+  # let's strip them out too.
+  #
+  # Note: If we do add a post-build install_name_tool patch, we need to add the
+  # "LDFLAGS=-headerpad_max_install_names" flag to configureFlags and either patch it into the
+  # Xcode project or pass it as a flag to xcodebuild as well.
   postConfigure = ''
-    substituteInPlace src/auto/config.mk --replace "PERL_CFLAGS	=" "PERL_CFLAGS	= -I${darwin.libutil}/include"
+    substituteInPlace src/auto/config.mk \
+      --replace "PERL_CFLAGS	=" "PERL_CFLAGS	= -I${darwin.libutil}/include" \
+      --replace " -L${stdenv.cc.libc}/lib" "" \
+      --replace " -L${darwin.libobjc}/lib" "" \
+      --replace " -L${darwin.libunwind}/lib" "" \
+      --replace " -L${darwin.libiconv}/lib" ""
+
+    # All the libraries we stripped have -osx- in their name as of this time.
+    # Assert now that this pattern no longer appears in config.mk.
+    ( # scope variable
+      while IFS="" read -r line; do
+        if [[ "$line" == LDFLAGS*-osx-* ]]; then
+          echo "WARNING: src/auto/config.mk contains reference to Nix osx library" >&2
+        fi
+      done <src/auto/config.mk
+    )
 
     substituteInPlace src/MacVim/vimrc --subst-var-by CSCOPE ${cscope}/bin/cscope
   '';
diff --git a/pkgs/applications/editors/vim/macvim.patch b/pkgs/applications/editors/vim/macvim.patch
index e44edd74047f..6b90b623cb68 100644
--- a/pkgs/applications/editors/vim/macvim.patch
+++ b/pkgs/applications/editors/vim/macvim.patch
@@ -1,23 +1,3 @@
-diff --git a/src/MacVim/MacVim.xcodeproj/project.pbxproj b/src/MacVim/MacVim.xcodeproj/project.pbxproj
-index 2bf5687..103b6e4 100644
---- a/src/MacVim/MacVim.xcodeproj/project.pbxproj
-+++ b/src/MacVim/MacVim.xcodeproj/project.pbxproj
-@@ -1087,6 +1081,7 @@
- 				LD_RUNPATH_SEARCH_PATHS = "@loader_path/../Frameworks";
- 				MACOSX_DEPLOYMENT_TARGET = "";
- 				ONLY_ACTIVE_ARCH = YES;
-+				OTHER_LDFLAGS = "-headerpad_max_install_names";
- 				PRODUCT_BUNDLE_IDENTIFIER = org.vim.MacVim;
- 				PRODUCT_NAME = MacVim;
- 				VERSIONING_SYSTEM = "apple-generic";
-@@ -1119,6 +1114,7 @@
- 				LD_RUNPATH_SEARCH_PATHS = "@loader_path/../Frameworks";
- 				MACOSX_DEPLOYMENT_TARGET = "";
- 				ONLY_ACTIVE_ARCH = YES;
-+				OTHER_LDFLAGS = "-headerpad_max_install_names";
- 				PRODUCT_BUNDLE_IDENTIFIER = org.vim.MacVim;
- 				PRODUCT_NAME = MacVim;
- 				VERSIONING_SYSTEM = "apple-generic";
 diff --git a/src/MacVim/vimrc b/src/MacVim/vimrc
 index 23a06bf..dfb10fe 100644
 --- a/src/MacVim/vimrc