build_mac.yml

name: Mac

on:
  release:
    types: [created]
  pull_request:
    types: [opened, synchronize, reopened]
    paths-ignore:
      - "tools/**"
      - "docs/**"
      - ".vscode/**"
      - ".devcontainer/**"
      - ".github/**"
      - "!.github/workflows/build_mac.yml"
      - "**.md"

jobs:
  build-mac-arm64:
    concurrency:
      group: build-mac-arm64-${{ github.head_ref }}-${{ matrix.build_type }}
      cancel-in-progress: true
    runs-on: macos-latest
    strategy:
      matrix:
        build_type: [debug, release]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: false

      - name: Trust working directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Initialize and update submodules except docs/
        run: |
          # Retrieve all submodule paths, excluding `docs/`.
          submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "docs" { print $2 }')

          git submodule init

          for submodule in $submodules; do
            echo "Initializing submodule: $submodule"
            git submodule update --init --recursive "$submodule"
          done

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - uses: actions/setup-go@v5
        with:
          go-version: "stable"
          cache: false

      - name: Set ulimit and sysctl
        run: |
          # Because there are concurrent test cases which involves many
          # concurrent socket connections, we need to enlarge the maximum number
          # of the opened file descriptor.
          ulimit -n 102400

          # Because there are concurrent test cases which will create many
          # concurrent connections simutaneously, we increase the TCP listening
          # backlog value to 8192.
          sudo sysctl -w kern.ipc.somaxconn=8192
        shell: bash

      - name: Install tools and dependencies
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            # Since the TEN runtime and TEN manager are a combination of C and
            # Rust, to enable ASan, the ASan versions used by both the C and
            # Rust compilation environments must be consistent. Therefore, it is
            # necessary to lock the LLVM and Rust versions to enable ASan.
            brew install llvm@18
            rustup default nightly-2024-07-19
          else
            rustup default nightly
          fi

          brew install tree
          pip3 install --use-pep517 python-dotenv jinja2
          go install golang.org/dl/go1.20.12@latest && go1.20.12 download
          go env -w GOFLAGS="-buildvcs=false"
          cargo install --force cbindgen

      # - name: Install dependencies
      #   run: |
      #     # Install Homebrew if not already installed.
      #     if ! command -v brew &> /dev/null; then
      #       echo "Installing Homebrew..."
      #       /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
      #     fi

      #     # Update Homebrew and install x264 used by ffmpeg.
      #     brew update
      #     brew install x264

      - name: Build
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            export PATH="/opt/homebrew/opt/llvm@18/bin:$PATH"
          fi
          export PATH=$(pwd)/core/ten_gn:$PATH
          echo $PATH

          df -h .

          if [ "${{ matrix.build_type }}" = "debug" ]; then
            EXTRA_ARGS="ten_enable_python_binding=false"
          else
            EXTRA_ARGS="ten_enable_python_binding=true ten_enable_advanced_python_integration_tests=false"
          fi

          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false $EXTRA_ARGS

          tgn build mac arm64 ${{ matrix.build_type }}

          df -h .
          tree -I 'gen|obj' out

      - name: Upload ten_packages
        uses: actions/upload-artifact@v4
        with:
          name: ten_packages-mac-${{ matrix.build_type }}-arm64
          path: |
            out/mac/arm64/app/default_app_cpp
            out/mac/arm64/app/default_app_go
            out/mac/arm64/ten_packages/system/ten_runtime
            out/mac/arm64/ten_packages/system/ten_runtime_go
            out/mac/arm64/ten_packages/extension/default_extension_cpp
            out/mac/arm64/ten_packages/extension/default_extension_go

      - name: Package assets
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          cd out/mac/arm64
          zip -vr tman-mac-${{ matrix.build_type }}-arm64.zip ten_manager/bin/tman
          zip -vr ten_packages-mac-${{ matrix.build_type }}-arm64.zip \
            app/default_app_cpp \
            app/default_app_go \
            ten_packages/system/ten_runtime \
            ten_packages/system/ten_runtime_go \
            ten_packages/extension/default_extension_cpp \
            ten_packages/extension/default_extension_go \

          df -h .

      - name: Publish to release assets
        uses: softprops/action-gh-release@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
            out/mac/arm64/tman-mac-${{ matrix.build_type }}-arm64.zip
            out/mac/arm64/ten_packages-mac-${{ matrix.build_type }}-arm64.zip

      - name: Run Tests (ten_utils_unit_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/arm64/tests/standalone/ten_utils_unit_test
          out/mac/arm64/tests/standalone/ten_utils_unit_test

          df -h .

      - name: Run Tests (ten_runtime_unit_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/arm64/tests/standalone/ten_runtime_unit_test
          out/mac/arm64/tests/standalone/ten_runtime_unit_test

          df -h .

      - name: Run Tests (ten_runtime_smoke_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/arm64/tests/standalone/ten_runtime_smoke_test
          out/mac/arm64/tests/standalone/ten_runtime_smoke_test

          df -h .
      #   continue-on-error: true
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3

      - name: Run Tests (ten_rust standalone tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/arm64/tests/standalone/ten_rust

          chmod +x unit_test
          chmod +x integration_test

          ./unit_test || { echo "ten_rust unit test failed"; exit 1; }
          ./integration_test || { echo "ten_rust integration test failed"; exit 1; }

          df -h .

      - name: Run Tests (ten_manager standalone tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/arm64/tests/standalone/ten_manager

          chmod +x unit_test

          ./unit_test || { echo "ten_manager unit test failed"; exit 1; }

          df -h .

      - name: Install Python dependencies via script
        run: |
          python .github/tools/setup_pytest_dependencies.py

          df -h .

      - name: Run Tests (ten_manager pytest tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/arm64/
          pytest -s tests/ten_manager/

          df -h .

      - name: Run tests (ten_runtime integration tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            export PATH="/opt/homebrew/opt/llvm@18/bin:$PATH"
          fi
          export PATH=$(pwd)/core/ten_gn:$PATH
          cd out/mac/arm64/
          pytest -s tests/ten_runtime/integration/

          df -h .
      #   continue-on-error: true
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3

  build-mac-x64:
    concurrency:
      group: build-mac-x64-${{ github.head_ref }}-${{ matrix.build_type }}
      cancel-in-progress: true
    runs-on: macos-13
    strategy:
      matrix:
        build_type: [release]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: false

      - name: Trust working directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Initialize and update submodules except docs/
        run: |
          # Retrieve all submodule paths, excluding `docs/`.
          submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "docs" { print $2 }')

          git submodule init

          for submodule in $submodules; do
            echo "Initializing submodule: $submodule"
            git submodule update --init --recursive "$submodule"
          done

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - uses: actions/setup-go@v5
        with:
          go-version: "stable"
          cache: false

      - name: Set ulimit and sysctl
        run: |
          # Because there are concurrent test cases which involves many
          # concurrent socket connections, we need to enlarge the maximum number
          # of the opened file descriptor.
          ulimit -n 102400

          # Because there are concurrent test cases which will create many
          # concurrent connections simutaneously, we increase the TCP listening
          # backlog value to 8192.
          sudo sysctl -w kern.ipc.somaxconn=8192
        shell: bash

      - name: Install tools and dependencies
        run: |

          if [ ${{ matrix.build_type }} == "debug" ]; then
            # Since the TEN runtime and TEN manager are a combination of C and
            # Rust, to enable ASan, the ASan versions used by both the C and
            # Rust compilation environments must be consistent. Therefore, it is
            # necessary to lock the LLVM and Rust versions to enable ASan.
            brew install llvm@18
            rustup default nightly-2024-07-19
          else
            rustup default nightly
          fi

          brew install tree
          pip3 install --use-pep517 python-dotenv jinja2
          go install golang.org/dl/go1.20.12@latest && go1.20.12 download
          go env -w GOFLAGS="-buildvcs=false"
          cargo install --force cbindgen

      - name: Build
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            export PATH="/usr/local/opt/llvm@18/bin:$PATH"
          fi
          export PATH=$(pwd)/core/ten_gn:$PATH
          echo $PATH

          df -h .

          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_cargo_clean=true ten_enable_python_binding=true ten_enable_advanced_python_integration_tests=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false

          tgn build mac x64 ${{ matrix.build_type }}

          df -h .
          tree -I 'gen|obj' out
      #   continue-on-error: true
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3

      - name: Upload ten_packages
        uses: actions/upload-artifact@v4
        with:
          name: ten_packages-mac-${{ matrix.build_type }}-x64
          path: |
            out/mac/x64/app/default_app_cpp
            out/mac/x64/app/default_app_go
            out/mac/x64/ten_packages/system/ten_runtime
            out/mac/x64/ten_packages/system/ten_runtime_go
            out/mac/x64/ten_packages/extension/default_extension_cpp
            out/mac/x64/ten_packages/extension/default_extension_go

      - name: Package assets
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          cd out/mac/x64
          zip -vr tman-mac-${{ matrix.build_type }}-x64.zip ten_manager/bin/tman
          zip -vr ten_packages-mac-${{ matrix.build_type }}-x64.zip \
            app/default_app_cpp \
            app/default_app_go \
            ten_packages/system/ten_runtime \
            ten_packages/system/ten_runtime_go \
            ten_packages/extension/default_extension_cpp \
            ten_packages/extension/default_extension_go \

          df -h .

      - name: Publish to release assets
        uses: softprops/action-gh-release@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: |
            out/mac/x64/tman-mac-${{ matrix.build_type }}-x64.zip
            out/mac/x64/ten_packages-mac-${{ matrix.build_type }}-x64.zip

      - name: Run Tests (ten_utils_unit_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/x64/tests/standalone/ten_utils_unit_test
          out/mac/x64/tests/standalone/ten_utils_unit_test

          df -h .

      - name: Run Tests (ten_runtime_unit_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/x64/tests/standalone/ten_runtime_unit_test
          out/mac/x64/tests/standalone/ten_runtime_unit_test

          df -h .

      - name: Run Tests (ten_runtime_smoke_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/x64/tests/standalone/ten_runtime_smoke_test
          out/mac/x64/tests/standalone/ten_runtime_smoke_test

          df -h .

      - name: Run Tests (ten_rust standalone tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/x64/tests/standalone/ten_rust

          chmod +x unit_test
          chmod +x integration_test

          ./unit_test || { echo "ten_rust unit test failed"; exit 1; }
          ./integration_test || { echo "ten_rust integration test failed"; exit 1; }

          df -h .

      - name: Run Tests (ten_manager standalone tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/x64/tests/standalone/ten_manager

          chmod +x unit_test

          ./unit_test || { echo "ten_manager unit test failed"; exit 1; }

          df -h .

      - name: Install Python dependencies via script
        run: |
          python .github/tools/setup_pytest_dependencies.py

          df -h .

      - name: Run Tests (ten_manager pytest tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          cd out/mac/x64/
          pytest -s tests/ten_manager/

          df -h .

      - name: Run tests (ten_runtime integration tests)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            export PATH="/usr/local/opt/llvm@18/bin:$PATH"
          fi
          export PATH=$(pwd)/core/ten_gn:$PATH
          cd out/mac/x64/
          pytest -s tests/ten_runtime/integration/

          df -h .
      #   continue-on-error: true
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3

  # Building ten_runtime without the ten_rust library (i.e., ten_enable_ten_rust_apis=false).
  build-mac-arm64-without-ten-rust:
    concurrency:
      group: build-mac-arm64-without-ten-rust-${{ github.head_ref }}-${{ matrix.build_type }}
      cancel-in-progress: true
    runs-on: macos-latest
    strategy:
      matrix:
        build_type: [debug]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: false

      - name: Trust working directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Initialize and update submodules except docs/
        run: |
          # Retrieve all submodule paths, excluding `docs/`.
          submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "docs" { print $2 }')

          git submodule init

          for submodule in $submodules; do
            echo "Initializing submodule: $submodule"
            git submodule update --init --recursive "$submodule"
          done

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - uses: actions/setup-go@v5
        with:
          go-version: "stable"
          cache: false

      - name: Set ulimit and sysctl
        run: |
          # Because there are concurrent test cases which involves many
          # concurrent socket connections, we need to enlarge the maximum number
          # of the opened file descriptor.
          ulimit -n 102400

          # Because there are concurrent test cases which will create many
          # concurrent connections simutaneously, we increase the TCP listening
          # backlog value to 8192.
          sudo sysctl -w kern.ipc.somaxconn=8192
        shell: bash

      - name: Install tools and dependencies
        run: |

          if [ ${{ matrix.build_type }} == "debug" ]; then
            # Since the TEN runtime and TEN manager are a combination of C and
            # Rust, to enable ASan, the ASan versions used by both the C and
            # Rust compilation environments must be consistent. Therefore, it is
            # necessary to lock the LLVM and Rust versions to enable ASan.
            brew install llvm@18
            rustup default nightly-2024-07-19
          else
            rustup default nightly
          fi

          brew install tree
          pip3 install --use-pep517 python-dotenv jinja2
          go install golang.org/dl/go1.20.12@latest && go1.20.12 download
          go env -w GOFLAGS="-buildvcs=false"
          cargo install --force cbindgen

      - name: Build
        run: |
          if [ ${{ matrix.build_type }} == "debug" ]; then
            export PATH="/opt/homebrew/opt/llvm@18/bin:$PATH"
          fi
          export PATH=$(pwd)/core/ten_gn:$PATH
          echo $PATH

          df -h .

          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_serialized_rust_action=true ten_rust_enable_gen_cargo_config=false ten_enable_ten_rust_apis=false ten_enable_rust_incremental_build=false ten_manager_enable_frontend=false ten_enable_integration_tests_prebuilt=false ten_enable_nodejs_binding=false ten_enable_python_binding=false

          tgn build mac arm64 ${{ matrix.build_type }}

          df -h .
          tree -I 'gen|obj' out

      - name: Run Tests (ten_runtime_smoke_test)
        env:
          ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1
          MALLOC_CHECK_: 3
          TEN_ENABLE_MEMORY_TRACKING: "true"
          TEN_ENABLE_BACKTRACE_DUMP: "true"
        run: |
          chmod +x out/mac/arm64/tests/standalone/ten_runtime_smoke_test
          out/mac/arm64/tests/standalone/ten_runtime_smoke_test

          df -h .
      #   continue-on-error: true

      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3