task_runner_basic_e2e.yml

---
# Task Runner E2E tests for bare metal approach

name: Task_Runner_E2E  # Please do not modify the name as it is used in the composite action

on:
  schedule:
    - cron: "0 0 * * *" # Run every day at midnight
  push:
    branches: 
      - develop
  workflow_call:
  workflow_dispatch:
    inputs:
      num_rounds:
        description: "Number of rounds to train"
        required: false
        default: "5"
        type: string
      num_collaborators:
        description: "Number of collaborators"
        required: false
        default: "2"
        type: string
      model_name:
        description: "Model name"
        required: false
        default: "all"
        type: choice
        options:
          - all
          - torch/mnist
          - keras/mnist
          - keras/torch/mnist
          - keras/jax/mnist
      python_version:
        description: "Python version"
        required: false
        default: "all"
        type: choice
        options:
          - all
          - "3.10"
          - "3.11"
          - "3.12"
      jobs_to_run:
        description: "Jobs to run"
        type: choice
        default: "all"
        options:
          - all
          - test_with_tls
          - test_without_tls
          - test_without_client_auth
          - test_memory_logs
        required: false

permissions:
  contents: read

# Environment variables common for all the jobs
# DO NOT use double quotes for the values of the environment variables
env:
  NUM_ROUNDS: ${{ inputs.num_rounds || 5 }}
  NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }}
  MODEL_NAME: ${{ inputs.model_name || 'all' }}
  PYTHON_VERSION: ${{ inputs.python_version || 'all' }}
  JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }}

jobs:
  input_selection:
    if: |
      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
      (github.event_name == 'workflow_dispatch') ||
      (github.event.pull_request.draft == false)
    name: Input value selection
    runs-on: ubuntu-22.04
    outputs:
      # Output all the variables related to models and python versions to be used in the matrix strategy
      # for different jobs, however their usage depends on the selected job.
      selected_jobs: ${{ steps.input_selection.outputs.jobs_to_run }}
      selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }}
      selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }}
      selected_models_for_non_tls: ${{ steps.input_selection.outputs.models_for_non_tls }}
      selected_models_for_no_client_auth: ${{ steps.input_selection.outputs.models_for_no_client_auth }}
      selected_models_for_memory_logs: ${{ steps.input_selection.outputs.models_for_memory_logs }}
      selected_python_for_non_tls: ${{ steps.input_selection.outputs.python_for_non_tls }}
      selected_python_for_no_client_auth: ${{ steps.input_selection.outputs.python_for_no_client_auth }}
      selected_python_for_memory_logs: ${{ steps.input_selection.outputs.python_for_memory_logs }}
    steps:
      - name: Job to select input values
        id: input_selection
        run: |
          # ---------------------------------------------------------------
          # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run.
          # Thus these models are excluded from the matrix for now.
          # Default combination if no input is provided (i.e. 'all' is selected).
          # * TLS - models [keras/torch/mnist, keras/jax/mnist] and python versions [3.10, 3.11]
          # * Non-TLS - models [torch/mnist] and python version [3.12]
          # * No client auth - models [keras/mnist] and python version [3.10]
          # * Memory logs - models [torch/mnist] and python version [3.11]
          # ---------------------------------------------------------------
          echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT"

          if [ "${{ env.MODEL_NAME }}" == "all" ]; then
            echo "models_for_tls=[\"keras/torch/mnist\", \"keras/jax/mnist\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_non_tls=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_no_client_auth=[\"keras/mnist\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_memory_logs=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT"
          else
            echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
            echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
          fi
          if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then
            echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_non_tls=[\"3.12\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_memory_logs=[\"3.11\"]" >> "$GITHUB_OUTPUT"
          else
            echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
            echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
          fi

  test_with_tls:
    name: With TLS
    needs: input_selection
    if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_tls'
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    strategy:
      matrix:
        model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }}
        python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }}
        exclude: # Keras does not support Python 3.12
          - model_name: "keras/mnist"
            python_version: "3.12"
          - model_name: "keras/jax/mnist"
            python_version: "3.12"
          - model_name: "keras/torch/mnist" 
            python_version: "3.12"

      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests with TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "With_TLS"

  test_without_tls:
    name: Without TLS
    needs: input_selection
    if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_without_tls'
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    strategy:
      matrix:
        model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_non_tls) }}
        python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_non_tls) }}
        exclude: # Keras does not support Python 3.12
          - model_name: "keras/mnist"
            python_version: "3.12"
          - model_name: "keras/jax/mnist"
            python_version: "3.12"
          - model_name: "keras/torch/mnist" 
            python_version: "3.12"
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests without TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "Without_TLS"

  test_without_client_auth:
    name: Without Client Auth
    needs: input_selection
    if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_without_client_auth'
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    strategy:
      matrix:
        model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_no_client_auth) }}
        python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_no_client_auth) }}
        exclude: # Keras does not support Python 3.12
          - model_name: "keras/mnist"
            python_version: "3.12"
          - model_name: "keras/jax/mnist"
            python_version: "3.12"
          - model_name: "keras/torch/mnist" 
            python_version: "3.12"
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests without TLS
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth
          echo "Task runner end to end test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: 'Without_Client_Auth'

  test_memory_logs:
    name: With Memory Logs
    needs: input_selection
    if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs'
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    strategy:
      matrix:
        model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_memory_logs) }}
        python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_memory_logs) }}
        exclude: # Keras does not support Python 3.12
          - model_name: "keras/mnist"
            python_version: "3.12"
          - model_name: "keras/jax/mnist"
            python_version: "3.12"
          - model_name: "keras/torch/mnist" 
            python_version: "3.12"
      fail-fast: false # do not immediately fail if one of the combinations fail

    env:
      MODEL_NAME: ${{ matrix.model_name }}
      PYTHON_VERSION: ${{ matrix.python_version }}

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4
        with:
          fetch-depth: 2 # needed for detecting changes
          submodules: "true"
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Task Runner E2E tests with TLS and memory logs
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
          -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \
          --log_memory_usage
          echo "Task runner memory logs test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "With_Memory_Logs"

  test_straggler_check:
    name: Straggler Handling
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    if: |
      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
      (github.event_name == 'workflow_dispatch') ||
      (github.event.pull_request.draft == false)

    env:
      MODEL_NAME: "torch/mnist_straggler_check"
      PYTHON_VERSION: "3.10"

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Straggler Handling Interface Test
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
          echo "Straggler handling test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "Straggler_Handling"

  test_eden_compression:
    name: Eden Compression
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    if: |
      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
      (github.event_name == 'workflow_dispatch') ||
      (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression'))

    env:
      MODEL_NAME: "torch/mnist_eden_compression"
      PYTHON_VERSION: "3.10"

    steps:
      - name: Checkout OpenFL repository
        id: checkout_openfl
        uses: actions/checkout@v4

      - name: Pre test run
        uses: ./.github/actions/tr_pre_test_run
        if: ${{ always() }}

      - name: Run Eden Compression Test
        id: run_tests
        run: |
          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
          echo "Eden compression test run completed"

      - name: Post test run
        uses: ./.github/actions/tr_post_test_run
        if: ${{ always() }}
        with:
          test_type: "Eden_Compression"