contrib-test.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Contrib tests without LLMs

on:
  schedule:
    - cron: "0 0 * * *" # daily at midnight UTC
  workflow_dispatch: # allows manual triggering of the workflow
  pull_request:
    branches: ["main"]
    paths:
      - "autogen/**"
      - "test/agentchat/contrib/**"
      - "test/test_browser_utils.py"
      - "test/test_retrieve_utils.py"
      - ".github/workflows/contrib-test.yml"
      - "pyproject.toml"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: true
permissions:
  {}
  # actions: read
  # checks: read
  # contents: read
  # deployments: read
jobs:
  RetrieveChatTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for RetrieveChat
        run: |
          uv pip install --system -e .[test,retrievechat,retrievechat-qdrant]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/test_retrieve_utils.py test/agentchat/contrib/retrievechat/test_retrievechat.py test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py test/agentchat/contrib/vectordb
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: retrievechat, retrievechat-qdrant, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  RetrieveChatTest-Ubuntu:
    runs-on: ubuntu-latest
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
    services:
      pgvector:
        image: ankane/pgvector
        env:
          POSTGRES_DB: postgres
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
          POSTGRES_HOST_AUTH_METHOD: trust
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
      mongodb:
        image: mongodb/mongodb-atlas-local:latest
        ports:
          - 27017:27017
      couchbase:
        image: couchbase:enterprise-7.6.3
        ports:
          - "8091-8095:8091-8095"
          - "11210:11210"
          - "9102:9102"
        env:
          COUCHBASE_ADMINISTRATOR_USERNAME: Administrator
          COUCHBASE_ADMINISTRATOR_PASSWORD: ${{ secrets.COUCHBASE_PASSWORD }}
          COUCHBASE_BUCKET: autogen_test_bucket
          COUCHBASE_SCOPE: _default
        options: >-
          --health-cmd "curl -f http://localhost:8091/pools || exit 1"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install unstructured when python-version is 3.9 and on linux
        if: matrix.python-version == '3.9'
        run: |
          sudo apt-get update
          sudo apt-get install -y tesseract-ocr poppler-utils
          uv pip install --system --no-cache-dir unstructured[all-docs]==0.13.0
      - name: Install qdrant_client when python-version is not 3.9
        if: matrix.python-version != '3.9'
        run: |
          uv pip install --system -e .[retrievechat-qdrant]
      - name: Install pgvector when on linux
        run: |
          uv pip install --system -e .[test,retrievechat-pgvector,retrievechat-mongodb]
      - name: Install couchbase when on linux
        run: |
          uv pip install --system -e .[retrievechat-couchbase]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV

      - name: Wait for Couchbase to be ready
        run: |
          attempt_counter=0
          max_attempts=20
          until $(curl --output /dev/null --silent --head --fail http://localhost:8091); do
            if [ ${attempt_counter} -eq ${max_attempts} ];then
              echo "Max attempts reached. Couchbase failed to start."
              exit 1
            fi
            printf '.'
            attempt_counter=$(($attempt_counter+1))
            sleep 5
          done
      - name: Initialize Couchbase
        run: |
          docker exec ${{ job.services.couchbase.id }} couchbase-cli cluster-init \
            --cluster localhost:8091 \
            --cluster-name myCluster \
            --cluster-username Administrator \
            --cluster-password password \
            --services data,index,query,fts \
            --cluster-ramsize 512 \
            --cluster-index-ramsize 256 \
            --cluster-fts-ramsize 256
      - name: Create Couchbase bucket
        run: |
          docker exec ${{ job.services.couchbase.id }} couchbase-cli bucket-create \
            --cluster localhost:8091 \
            --username Administrator \
            --password password \
            --bucket autogen_test_bucket \
            --bucket-type couchbase \
            --bucket-ramsize 256
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/test_retrieve_utils.py test/agentchat/contrib/retrievechat test/agentchat/contrib/vectordb
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: retrievechat, retrievechat-qdrant, retrievechat-pgvector, retrievechat-mongodb, retrievechat-couchbase, ubuntu-latest, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  AgentEvalTest:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for AgentEval
        run: |
          uv pip install --system -e .[test]
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/agent_eval/
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: agent-eval, ${{ matrix.python-version }}, ${{ matrix.os }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  GPTAssistantAgent:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for GPTAssistantAgent
        run: |
          uv pip install --system -e .[test]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/test_gpt_assistant.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: gpt-assistant-agent, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  TeachableAgent:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Teachability
        run: |
          uv pip install --system -e .[test,teachable]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/capabilities/test_teachable_agent.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: teachable, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  WebSurfer:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for WebSurfer
        run: |
          uv pip install --system -e .[test,websurfer]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: websurfer, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  LMMTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.13"]
    steps:
      - uses: actions/checkout@v4
        with:
          lfs: true
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for LMM
        run: |
          uv pip install --system -e .[test,lmm]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py test/agentchat/contrib/capabilities/test_vision_capability.py
      - name: Image Gen Coverage
        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '3.13' }}
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/capabilities/test_image_generation_capability.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: lmm, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  GeminiTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Gemini
        run: |
          uv pip install --system -e .[test,gemini]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_gemini.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: gemini, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  TransformMessages:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Transform Messages
        run: |
          uv pip install --system -e '.[test,long-context]'
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/capabilities/test_transform_messages.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: long-context, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  LlamaIndexAgent:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for LlamaIndexConverableAgent
        run: |
          uv pip install --system -e .[test]
          uv pip install --system llama-index llama-index-llms-openai
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/test_llamaindex_conversable_agent.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: llama-index-agent, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  AnthropicTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]

    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Anthropic
        run: |
          uv pip install --system -e .[test,anthropic]

      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi

      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_anthropic.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: anthropic, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  CerebrasTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Cerebras
        run: |
          uv pip install --system -e .[test,cerebras]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_cerebras.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: cerebras, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  MistralTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Mistral
        run: |
          uv pip install --system -e .[test,mistral]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_mistral.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: mistral, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  TogetherTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Together
        run: |
          uv pip install --system -e .[test,together]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_together.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: together, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  GroqTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Groq
        run: |
          uv pip install --system -e .[test,groq]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_groq.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: groq, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  CohereTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Cohere
        run: |
          uv pip install --system -e .[test,cohere]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_cohere.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: cohere, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  OllamaTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Ollama
        run: |
          uv pip install --system -e .[test,ollama]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_ollama.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: ollama, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  BedrockTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Amazon Bedrock
        run: |
          uv pip install --system -e .[test,bedrock]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/oai/test_bedrock.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: bedrock, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  SwarmTest:
    runs-on: ${{ matrix.os }}
    environment: permit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os: macos-latest
            python-version: "3.9"
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies for Swarms
        run: |
          uv pip install --system -e .[test]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
          fi
      - name: Run tests
        run: |
          bash scripts/test-skip-llm.sh test/agentchat/contrib/test_swarm.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: swarm, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  ReasoningTest:
      runs-on: ${{ matrix.os }}
      environment: permit-tests
      strategy:
        fail-fast: false
        matrix:
          os: [ubuntu-latest, macos-latest, windows-latest]
          python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
          exclude:
            - os: macos-latest
              python-version: "3.9"
      steps:
        - uses: actions/checkout@v4
        - uses: astral-sh/setup-uv@v5
          with:
            version: "latest"
        - name: Set up Python ${{ matrix.python-version }}
          uses: actions/setup-python@v5
          with:
            python-version: ${{ matrix.python-version }}
        - name: Install packages and dependencies for Reasoning
          run: |
            uv pip install --system -e .[test]
        - name: Install Graphviz based on OS
          run: |
            if [[ ${{ matrix.os }} == 'ubuntu-latest' ]]; then
              sudo apt-get update
              sudo apt-get install -y graphviz
            elif [[ ${{ matrix.os }} == 'macos-latest' ]]; then
              brew install graphviz
            elif [[ ${{ matrix.os }} == 'windows-latest' ]]; then
              choco install graphviz
            fi
          shell: bash
        - name: Set AUTOGEN_USE_DOCKER based on OS
          shell: bash
          run: |
            if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
              echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
            fi
        - name: Run tests
          run: |
            bash scripts/test-skip-llm.sh test/agentchat/contrib/test_reasoning_agent.py
        - name: Show coverage report
          run: bash scripts/show-coverage-report.sh
        - name: Upload coverage to Codecov
          if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
          uses: codecov/codecov-action@v5
          with:
            files: ./coverage.xml
            flags: reasoning, ${{ matrix.os }}, ${{ matrix.python-version }}
            fail_ci_if_error: true
            token: ${{ secrets.CODECOV_TOKEN }}