darwin.stdenv: use CoreFoundation instead of CF

pkgs/os-specific/darwin/apple-sdk/default.nix

@@ -1,9 +1,6 @@
-{ stdenv, fetchurl, libxml2, xar, cpio, pkgs, python3Minimal, pbzx, lib, darwin-stubs, print-reexports }:
+{ stdenv, fetchurl, cpio, pbzx, pkgs, lib, darwin-stubs, print-reexports }:
 
 let
-  xarMinimal = xar.override {
-    libxml2 = libxml2.override { pythonSupport = false; };
-  };
   # sadly needs to be exported because security_tool needs it
   sdk = stdenv.mkDerivation rec {
     pname = "MacOS_SDK";
@@ -19,27 +16,23 @@ let
       sha256 = "13xq34sb7383b37hwy076gnhf96prpk1b4087p87xnwswxbrisih";
     };
 
-    nativeBuildInputs = [ xarMinimal cpio python3Minimal pbzx ];
+    nativeBuildInputs = [ cpio pbzx ];
 
     outputs = [ "out" "dev" "man" ];
 
     unpackPhase = ''
-      xar -x -f $src
+      pbzx $src | cpio -idm
     '';
 
+    sourceRoot = ".";
+
     installPhase = ''
-      start="$(pwd)"
       mkdir -p $out
-      cd $out
-      pbzx -n $start/Payload | cpio -idm
-
-      mv usr/* .
-      rmdir usr
 
-      mv System/* .
-      rmdir System
+      cp -R System/Library $out
+      cp -R usr/* $out
 
-      pushd lib
+      pushd $out/lib
       cp ${darwin-stubs}/usr/lib/libcups*.tbd .
       ln -s libcups.2.tbd      libcups.tbd
       ln -s libcupscgi.1.tbd   libcupscgi.tbd

pkgs/os-specific/darwin/apple-source-releases/configd/default.nix

@@ -1,11 +1,19 @@
-{ lib, stdenv, appleDerivation', launchd, bootstrap_cmds, xnu, xpc, ppp, IOKit, eap8021x, Security
+{ lib, stdenv, runCommand, appleDerivation', launchd, bootstrap_cmds, swift-corelibs-foundation, xnu, xpc, ppp, IOKit, eap8021x, Security
 , headersOnly ? false }:
 
+let
+  privateHeaders = runCommand "swift-corelibs-foundation-private" { } ''
+    mkdir -p $out/include/CoreFoundation
+
+    cp ${swift-corelibs-foundation}/Library/Frameworks/CoreFoundation.framework/PrivateHeaders/* \
+      $out/include/CoreFoundation
+  '';
+in
 appleDerivation' stdenv {
   meta.broken = stdenv.cc.nativeLibc;
 
   nativeBuildInputs = lib.optionals (!headersOnly) [ bootstrap_cmds ];
-  buildInputs = lib.optionals (!headersOnly) [ launchd ppp xpc IOKit eap8021x ];
+  buildInputs = lib.optionals (!headersOnly) [ privateHeaders launchd ppp xpc IOKit eap8021x ];
 
   propagatedBuildInputs = lib.optionals (!headersOnly) [ Security ];
 
@@ -23,11 +31,6 @@ appleDerivation' stdenv {
 
     substituteInPlace SystemConfiguration.fproj/SCNetworkReachability.c \
       --replace ''$'#define\tHAVE_VPN_STATUS' ""
-
-    # Our neutered CoreFoundation doesn't have this function, but I think we'll live...
-    substituteInPlace SystemConfiguration.fproj/SCNetworkConnectionPrivate.c \
-      --replace 'CFPreferencesAppValueIsForced(serviceID, USER_PREFERENCES_APPLICATION_ID)' 'FALSE' \
-      --replace 'CFPreferencesAppValueIsForced(userPrivate->serviceID, USER_PREFERENCES_APPLICATION_ID)' 'FALSE'
   '';
 
   dontBuild = headersOnly;

pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, fetchFromGitHub, fetchurl, makeSetupHook, cmake, ninja, pkg-config, launchd, libdispatch, python3, libxml2, objc4, icu }:
+{ lib, stdenv, fetchFromGitHub, fetchurl, makeSetupHook, cmake, pkg-config, launchd, libdispatch, python3Minimal, libxml2, objc4, icu }:
 
 let
   # 10.12 adds a new sysdir.h that our version of CF in the main derivation depends on, but
@@ -21,7 +21,7 @@ stdenv.mkDerivation {
     sha256 = "17kpql0f27xxz4jjw84vpas5f5sn4vdqwv10g151rc3rswbwln1z";
   };
 
-  nativeBuildInputs = [ cmake ninja pkg-config python3 ];
+  nativeBuildInputs = [ cmake pkg-config python3Minimal ];
   buildInputs = [ (lib.getDev launchd) libdispatch libxml2 objc4 icu ];
 
   patches = [

pkgs/stdenv/darwin/default.nix

@@ -245,7 +245,8 @@ in
       coreutils = bootstrapTools;
       gnugrep = bootstrapTools;
 
-      pbzx = bootstrapTools;
+      # Either pbzx or Libsystem is required from bootstrap tools (one is used building the other).
+      pbzx = if localSystem.isAarch64 then bootstrapTools else super.pbzx;
       cpio = self.stdenv.mkDerivation {
         name = "bootstrap-stage0-cpio";
         buildCommand = ''
@@ -255,7 +256,11 @@ in
         passthru.isFromBootstrapFiles = true;
       };
 
-      darwin = super.darwin.overrideScope (selfDarwin: _: {
+      darwin = super.darwin.overrideScope (selfDarwin: superDarwin: {
+        # Prevent CF from being propagated to the initial stdenv. Packages that require it
+        # will have to manually add it to their build inputs.
+        CF = null;
+
         binutils-unwrapped = bootstrapTools // {
           version = "boot";
         };
@@ -296,15 +301,6 @@ in
 
         sigtool = bootstrapTools;
       } // lib.optionalAttrs (! useAppleSDKLibs) {
-        CF = self.stdenv.mkDerivation {
-          name = "bootstrap-stage0-CF";
-          buildCommand = ''
-            mkdir -p $out/Library/Frameworks
-            ln -s ${bootstrapTools}/Library/Frameworks/CoreFoundation.framework $out/Library/Frameworks
-          '';
-          passthru.isFromBootstrapFiles = true;
-        };
-
         Libsystem = self.stdenv.mkDerivation {
           name = "bootstrap-stage0-Libsystem";
           buildCommand = ''
@@ -424,15 +420,18 @@ in
   # making sure both packages are present on x86_64-darwin and aarch64-darwin.
   (prevStage:
     # previous stage0 stdenv:
-    assert lib.all isFromBootstrapFiles (with prevStage; [ bash coreutils cpio gnugrep pbzx ]);
+    assert lib.all isFromBootstrapFiles (
+      with prevStage; [ bash coreutils cpio gnugrep ] ++ lib.optionals useAppleSDKLibs [ pbzx ]
+    );
 
     assert lib.all isFromBootstrapFiles (with prevStage.darwin; [
       binutils-unwrapped cctools print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isFromBootstrapFiles (with prevStage.darwin; [ CF Libsystem ]);
-    assert    useAppleSDKLibs  -> lib.all        isFromNixpkgs (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all isFromBootstrapFiles (with prevStage.darwin; [ Libsystem ]);
+    assert    useAppleSDKLibs  -> lib.all        isFromNixpkgs (with prevStage.darwin; [ Libsystem ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
+    assert (with prevStage.darwin; (! useAppleSDKLibs) -> CF == null);
 
     assert lib.all isFromBootstrapFiles (with prevStage.llvmPackages; [
       clang-unwrapped libclang libllvm llvm compiler-rt libcxx libcxxabi
@@ -445,7 +444,11 @@ in
       inherit (prevStage) ccWrapperStdenv
         coreutils gnugrep;
 
-      cmake = super.cmakeMinimal;
+      # Use this stage’s CF to build CMake. It’s required but can’t be included in the stdenv.
+      cmake = self.cmakeMinimal;
+      cmakeMinimal = super.cmakeMinimal.overrideAttrs (old: {
+        buildInputs = old.buildInputs ++ [ self.darwin.CF ];
+      });
 
       curl = super.curlMinimal;
 
@@ -457,9 +460,18 @@ in
 
       ninja = super.ninja.override { buildDocs = false; };
 
-      python3 = super.python3Minimal;
+      # Use this stage’s CF to build Python. It’s required but can’t be included in the stdenv.
+      python3 = self.python3Minimal;
+      python3Minimal = super.python3Minimal.overrideAttrs (old: {
+        buildInputs = old.buildInputs ++ [ self.darwin.CF ];
+      });
 
       darwin = super.darwin.overrideScope (selfDarwin: superDarwin: {
+        # Use this stage’s CF to build configd. It’s required but can’t be included in the stdenv.
+        configd = superDarwin.configd.overrideAttrs (old: {
+          buildInputs = old.buildInputs or [ ] ++ [ self.darwin.CF ];
+        });
+
         signingUtils = prevStage.darwin.signingUtils.override {
           inherit (selfDarwin) sigtool;
         };
@@ -529,7 +541,8 @@ in
     assert lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
 
@@ -628,7 +641,8 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd xnu ]);
 
@@ -725,7 +739,8 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -824,8 +839,9 @@ in
       binutils-unwrapped cctools locale libtapi print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ CF configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
     assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -951,7 +967,8 @@ in
     ]);
 
     assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
-    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -1031,7 +1048,8 @@ in
     ]);
 
     assert (! useAppleSDKLibs) -> lib.all isBuiltByBootstrapFilesCompiler (with prevStage.darwin; [ configd ]);
-    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all        isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem ]);
+    assert (! useAppleSDKLibs) -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all                   isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 
@@ -1197,7 +1215,8 @@ in
       binutils-unwrapped cctools libtapi locale print-reexports rewrite-tbd sigtool
     ]);
 
-    assert (! useAppleSDKLibs) -> lib.all isBuiltByNixpkgsCompiler (with prevStage.darwin; [ CF Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all isBuiltByNixpkgsCompiler (with prevStage.darwin; [ Libsystem configd ]);
+    assert (! useAppleSDKLibs) -> lib.all            isFromNixpkgs (with prevStage.darwin; [ CF ]);
     assert    useAppleSDKLibs  -> lib.all            isFromNixpkgs (with prevStage.darwin; [ CF Libsystem libobjc ]);
     assert lib.all isFromNixpkgs (with prevStage.darwin; [ dyld launchd libclosure libdispatch xnu ]);
 

pkgs/top-level/darwin-packages.nix

@@ -200,26 +200,29 @@ impure-cmds // appleSourcePackages // chooseLibs // {
 
   CoreSymbolication = callPackage ../os-specific/darwin/CoreSymbolication { };
 
-  # TODO: make swift-corefoundation build with apple_sdk_11_0.Libsystem
-  CF = if useAppleSDKLibs
-    then
-      # This attribute (CF) is included in extraBuildInputs in the stdenv. This
-      # is typically the open source project. When a project refers to
-      # "CoreFoundation" it has an extra setup hook to force impure system
-      # CoreFoundation into the link step.
-      #
-      # In this branch, we only have a single "CoreFoundation" to choose from.
-      # To be compatible with the existing convention, we define
-      # CoreFoundation with the setup hook, and CF as the same package but
-      # with the setup hook removed.
-      #
-      # This may seem unimportant, but without it packages (e.g., bacula) will
-      # fail with linker errors referring ___CFConstantStringClassReference.
-      # It's not clear to me why some packages need this extra setup.
-      lib.overrideDerivation apple_sdk.frameworks.CoreFoundation (drv: {
-        setupHook = null;
-      })
-    else callPackage ../os-specific/darwin/swift-corelibs/corefoundation.nix { };
+  # TODO: Remove the CF hook if a solution to the crashes is not found.
+  CF =
+    # CF used to refer to the open source version of CoreFoundation from the Swift
+    # project. As of macOS 14, the rpath-based approach allowing packages to choose
+    # which version to use no longer seems to work reliably. Sometimes they works,
+    # but sometimes they crash with the error (in the system crash logs):
+    # CF objects must have a non-zero isa.
+    # See https://developer.apple.com/forums/thread/739355 for more on that error.
+    #
+    # In this branch, we only have a single "CoreFoundation" to choose from.
+    # To be compatible with the existing convention, we define
+    # CoreFoundation with the setup hook, and CF as the same package but
+    # with the setup hook removed.
+    #
+    # This may seem unimportant, but without it packages (e.g., bacula) will
+    # fail with linker errors referring ___CFConstantStringClassReference.
+    # It's not clear to me why some packages need this extra setup.
+    lib.overrideDerivation apple_sdk.frameworks.CoreFoundation (drv: {
+      setupHook = null;
+    });
+
+  # Formerly the CF attribute. Use this is you need the open source release.
+  swift-corelibs-foundation = callPackage ../os-specific/darwin/swift-corelibs/corefoundation.nix { };
 
   # As the name says, this is broken, but I don't want to lose it since it's a direction we want to go in
   # libdispatch-broken = callPackage ../os-specific/darwin/swift-corelibs/libdispatch.nix { };