[llvmpy] fix py3.13 (#65)

.github/actions/setup_base/action.yml

@@ -168,12 +168,18 @@ runs:
         ccache -z
 
         echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
+        echo "CMAKE_MAKE_PROGRAM=Ninja" >> $GITHUB_ENV
         echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
         echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
         echo "Python3_EXECUTABLE=$(which $python3_command)" >> $GITHUB_ENV
         
-        if [[ "${{ inputs.arch }}" != "wasm32-wasi" ]]; then
+        if [[ "${{ inputs.os }}" == "ubuntu" ]] || [[ "${{ inputs.os }}" == "windows" ]]; then
           echo "CMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
           echo "CMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
           echo "CMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld" >> $GITHUB_ENV
         fi
+        
+        if [[ "${{ matrix.os }}" == "macos" ]]; then
+          echo MACOSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV
+          echo CMAKE_OSX_DEPLOYMENT_TARGET=11.0 >> $GITHUB_ENV
+        fi

.github/workflows/build_test_release_eudsl.yml

@@ -22,7 +22,9 @@ on:
     paths:
       - ".github/workflows/build_test_release_eudsl.yml"
       - "projects/eudsl-tblgen/**"
+      - "projects/eudsl-nbgen/**"
       - "projects/eudsl-py/**"
+      - "projects/eudsl-llvmpy/**"
       - "third_party/llvm-project"
   merge_group:
   push:
@@ -54,6 +56,11 @@ jobs:
           - name: "macos_arm64"
             runs-on: "macos-14"
             os: "macos"
+            arch: "aarch64"
+          - name: "macos_x86_64"
+            runs-on: "macos-13"
+            os: "macos"
+            arch: "x86_64"
 
     runs-on: ${{ matrix.runs-on }}
 
@@ -113,40 +120,44 @@ jobs:
             echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV
           fi
           
-          # since linux builds execute in the cibuildwheel almalinux container
-          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
-            echo CC=clang >> $GITHUB_ENV
-            echo CXX=clang++ >> $GITHUB_ENV
-          fi
-          
-          # these have to match the deployment targets of the LLVM build runners
-          # ld: warning: object file (libLLVMTableGen.a[3](Error.cpp.o)) was built for newer 'macOS' version (13.7) than being linked (10.13)
-          if [[ "${{ matrix.runs-on }}" == "macos-14" ]]; then
-            echo MACOSX_DEPLOYMENT_TARGET=14.0 >> $GITHUB_ENV
-          fi
-          
           $python3_command -m pip install cibuildwheel
+          
           if [[ "${{ matrix.os }}" == "ubuntu" ]] && [[ "${{ matrix.arch }}" == "aarch64" ]]; then
             sudo apt-get install libarchive-dev antlr libxml2-dev libxslt-dev libcurl4-openssl-dev
           fi
-          $python3_command -m pip install -r requirements-dev.txt
+          if [[ "${{ matrix.os }}" == "macos" ]]; then
+            brew install antlr boost libarchive
+          fi
+          if [[ "${{ matrix.runs-on }}" == "macos-13" ]]; then
+            sudo xcode-select -s /Library/Developer/CommandLineTools
+            echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV
+            # https://github.com/pthom/srcml_caller/blob/a6431edf9ae4ab2aa19b78b5a4682ae3510b199c/setup.py#L38
+            # very dumb
+            cibw_dir="$(dirname $($python3_command -c "import cibuildwheel; import inspect; print(inspect.getfile(cibuildwheel))"))"
+            sed -i.bak 's/CIBUILDWHEEL/__CIBUILDWHEEL__/g' "$cibw_dir/__main__.py"
+          fi
+          
+          $python3_command -m pip install -r requirements-dev.txt -v
 
       - name: "Build eudsl-tblgen"
         run: |
-
+          
           # this one is different from the rest as a smoketest re cibuildwheel
           if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
             export CCACHE_DIR=/host/$CCACHE_DIR
+            export CC=clang
+            export CXX=clang++
           fi
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
 
       - name: "Build eudsl-llvmpy"
-        if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
               export CCACHE_DIR=/host/$CCACHE_DIR
+              export CC=clang
+              export CXX=clang++
             fi
             $python3_command -m cibuildwheel "$PWD/projects/eudsl-llvmpy" --output-dir wheelhouse
           else
@@ -162,6 +173,8 @@ jobs:
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
               export CCACHE_DIR=/host/$CCACHE_DIR
+              export CC=clang
+              export CXX=clang++
             fi
             $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse
           else
@@ -227,21 +240,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: ["ubuntu-22.04", "ubuntu-22.04-arm", "macos-14", "windows-2019"]
+        runs-on: ["ubuntu-22.04", "ubuntu-22.04-arm", "macos-13", "macos-14", "windows-2019"]
         python-version: [
           # "3.9", "3.10", "3.11",
-          "3.12"
+          "3.12", "3.13"
         ]
         include: [
           {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
           {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
-          {runs-on: "macos-14", name: "macos_arm64", os: "macos"}
+          {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
+          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
 
     runs-on: ${{ matrix.runs-on }}
 
-    name: "Test eudsl-tblgen ${{ matrix.name }}"
+    name: "Test tblgen ${{ matrix.name }} ${{ matrix.python-version }}"
 
     defaults:
       run:
@@ -350,23 +364,24 @@ jobs:
           "ubuntu-22.04",
           "ubuntu-22.04-arm",
           "macos-14",
-          # "windows-2019"
+          "macos-13",
+          "windows-2019"
         ]
         python-version: [
           # "3.10", "3.11",
-          "3.12"
+          "3.12", "3.13"
         ]
         include: [
           {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
           {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
-          # TODO(max): enable on windows by statically linking
-          # {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
+          {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
           {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
+          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
         ]
 
     runs-on: ${{ matrix.runs-on }}
 
-    name: "Test eudsl-llvmpy ${{ matrix.name }}"
+    name: "Test llvmpy ${{ matrix.name }} ${{ matrix.python-version }}"
 
     defaults:
       run:
@@ -402,14 +417,13 @@ jobs:
 
   release-eudsl:
 
-    if: ${{ github.event_name == 'workflow_dispatch' }}
+    if: ${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' }}
     needs: [
       test-eudsl-tblgen,
       # test-eudsl-py,
       test-eudsl-llvmpy
     ]
     runs-on: "ubuntu-22.04"
-    name: "Release eudsl"
 
     permissions:
       id-token: write
@@ -422,9 +436,12 @@ jobs:
          "ubuntu_x86_64",
          "ubuntu_aarch64",
          "macos_arm64",
+         "macos_x86_64",
          "windows_x86_64"
         ]
 
+    name: "Release eudsl ${{ matrix.name }}"
+
     steps:
 
       - uses: actions/download-artifact@v4

projects/CMakeLists.txt

@@ -5,7 +5,7 @@
 
 if(NOT WIN32)
   add_subdirectory(eudsl-py)
-  add_subdirectory(eudsl-llvmpy)
 endif()
+add_subdirectory(eudsl-llvmpy)
 add_subdirectory(eudsl-nbgen)
 add_subdirectory(eudsl-tblgen)

projects/eudsl-llvmpy/CMakeLists.txt

@@ -120,29 +120,48 @@ include_directories("${EUDSLLLVM_BINARY_DIR}/generated")
 file(GLOB _gen_src CONFIGURE_DEPENDS "${EUDSLLLVM_BINARY_DIR}/generated/*.cpp")
 
 nanobind_add_module(eudslllvm_ext
-  NB_STATIC
+  NB_STATIC STABLE_ABI
   NB_DOMAIN eudslllvm
   src/eudslllvm_ext.cpp
   ${_gen_src}
 )
-target_link_libraries(eudslllvm_ext PRIVATE LLVM LTO)
+set(eudslllvm_ext_libs
+  PRIVATE
+  LLVMCore
+  LLVMExecutionEngine
+  LLVMOrcJIT
+  LLVMOrcDebugging
+  LLVMInterpreter
+  LLVMMCDisassembler
+  LLVMMCJIT)
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
+  list(APPEND eudslllvm_ext_libs
+    LLVMX86Info
+    LLVMX86CodeGen
+    LLVMX86AsmParser
+    LLVMX86Disassembler)
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
+  list(APPEND eudslllvm_ext_libs
+    LLVMAArch64Info
+    LLVMAArch64Desc
+    LLVMAArch64CodeGen
+    LLVMAArch64AsmParser
+    LLVMAArch64Disassembler)
+endif()
+target_link_libraries(eudslllvm_ext PRIVATE ${eudslllvm_ext_libs})
 set_target_properties(eudslllvm_ext
   PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY "${EUDSLLLVM_SRC_DIR}/llvm"
 )
 
-set(STANDALONE)
-if(EUDSLLLVM_STANDALONE_BUILD OR EUDSL_STANDALONE_BUILD)
-  set(STANDALONE STANDALONE)
-endif()
-patch_llvm_rpath(LLVM ${STANDALONE})
-patch_llvm_rpath(LTO ${STANDALONE})
-set_install_rpath_origin(eudslllvm_ext)
-# copy libLLVM into the ext dir for wheels
-install(IMPORTED_RUNTIME_ARTIFACTS LLVM LTO LIBRARY DESTINATION llvm)
-
 target_compile_options(eudslllvm_ext PRIVATE ${nanobind_options})
-target_compile_options(nanobind-static PRIVATE ${nanobind_options})
+set(_nanobind_tgt)
+if(TARGET nanobind-static)
+  set(_nanobind_tgt nanobind-static)
+elseif(TARGET nanobind-static-abi3)
+  set(_nanobind_tgt nanobind-static-abi3)
+endif()
+target_compile_options(${_nanobind_tgt} PRIVATE ${nanobind_options})
 
 # note WORKING_DIRECTORY
 set(NB_STUBGEN_CMD "${Python_EXECUTABLE}" "-m" "nanobind.stubgen"

projects/eudsl-llvmpy/eudsl-llvmpy-generate.py

@@ -1,4 +1,5 @@
 import argparse
+import platform
 import re
 from pathlib import Path
 from textwrap import dedent
@@ -36,7 +37,7 @@ def replacement(s):
     transformed_code = re.sub(pattern, replacement, transformed_code)
 
     pattern = r'#include "llvm-c/(\w+).h"'
-    replacement = rf'#include "{here}/\1.h"'
+    replacement = rf'#include "{here.as_posix()}/\1.h"'
     transformed_code = re.sub(pattern, replacement, transformed_code)
 
     transformed_code = transformed_code.replace(
@@ -82,7 +83,39 @@ def generate_header_bindings(cpp_code):
     options.python_strip_empty_comment_lines = True
     options.postprocess_pydef_function = postprocess
     # options.comments_exclude = True
-    options.fn_exclude_by_name__regex = "LLVMDisposeMessage|LLVMContextGetDiagnosticHandler|LLVMDisasmInstruction|LLVMDisposeErrorMessage|LLVMOrcCreateStaticLibrarySearchGeneratorForPath|LLVMRemarkVersion"
+    excludes = [
+        "LLVMContextGetDiagnosticHandler",
+        "LLVMDisasmInstruction",
+        "LLVMDisposeErrorMessage",
+        "LLVMDisposeMessage",
+        "LLVMOrcCreateStaticLibrarySearchGeneratorForPath",
+        "LLVMRemarkVersion",
+    ]
+
+    # APIs with callbacks that nanobind can't deduce/compile correctly
+    if platform.system() == "Windows":
+        excludes += [
+            "LLVMContextSetDiagnosticHandler",
+            "LLVMContextSetYieldCallback",
+            "LLVMCreateDisasm",
+            "LLVMCreateDisasmCPU",
+            "LLVMCreateDisasmCPUFeatures",
+            "LLVMCreateSimpleMCJITMemoryManager",
+            "LLVMInstallFatalErrorHandler",
+            "LLVMOrcCreateCustomCAPIDefinitionGenerator",
+            "LLVMOrcCreateCustomMaterializationUnit",
+            "LLVMOrcCreateDynamicLibrarySearchGeneratorForPath",
+            "LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess",
+            "LLVMOrcCreateRTDyldObjectLinkingLayerWithMCJITMemoryManagerLikeCallbacks",
+            "LLVMOrcExecutionSessionLookup",
+            "LLVMOrcExecutionSessionSetErrorReporter",
+            "LLVMOrcIRTransformLayerSetTransform",
+            "LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator",
+            "LLVMOrcObjectTransformLayerSetTransform",
+            "LLVMOrcThreadSafeModuleWithModuleDo",
+        ]
+
+    options.fn_exclude_by_name__regex = "|".join(excludes)
     generated_code = litgen.generate_code(options, cpp_code)
     return generated_code.pydef_code
 
@@ -341,6 +374,8 @@ def generate_nb_bindings(header_root: Path, output_root: Path):
     pp_dir = output_root / "pp"
     pp_dir.mkdir(parents=True, exist_ok=True)
     for header_f in header_root.rglob("*.h"):
+        if header_f.name == "lto.h":
+            continue
         with open(header_f) as ff:
             orig_code = ff.read()
         pp_header_f = pp_dir / header_f.name
@@ -354,7 +389,7 @@ def generate_nb_bindings(header_root: Path, output_root: Path):
             ff.write(
                 dedent(
                     f"""
-            #include "{pp_header_f}"
+            #include "{pp_header_f.as_posix()}"
             #include "types.h"
             #include <nanobind/nanobind.h>
             #include <nanobind/ndarray.h>