diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index cdec69ba43..558c24d884 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -95,10 +95,10 @@ jobs: # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. # Default combination if no input is provided (i.e. 'all' is selected). - # * TLS - models [torch/mnist, keras/mnist] and python versions [3.10, 3.11, 3.12] - # * Non-TLS - models [torch/mnist] and python version [3.10] + # * TLS - models [keras/torch/mnist, keras/jax/mnist] and python versions [3.10, 3.11] + # * Non-TLS - models [torch/mnist] and python version [3.12] # * No client auth - models [keras/mnist] and python version [3.10] - # * Memory logs - models [torch/mnist] and python version [3.10] + # * Memory logs - models [torch/mnist] and python version [3.11] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" @@ -115,9 +115,9 @@ jobs: fi if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" - echo "python_for_non_tls=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_non_tls=[\"3.12\"]" >> "$GITHUB_OUTPUT" echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" - echo "python_for_memory_logs=[\"3.10\"]" >> "$GITHUB_OUTPUT" + echo "python_for_memory_logs=[\"3.11\"]" >> "$GITHUB_OUTPUT" else echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" @@ -135,6 +135,14 @@ jobs: matrix: model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }} python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }} + exclude: # Keras does not support Python 3.12 + - model_name: "keras/mnist" + python_version: "3.12" + - model_name: "keras/jax/mnist" + python_version: "3.12" + - model_name: "keras/torch/mnist" + python_version: "3.12" + fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -178,6 +186,13 @@ jobs: matrix: model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_non_tls) }} python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_non_tls) }} + exclude: # Keras does not support Python 3.12 + - model_name: "keras/mnist" + python_version: "3.12" + - model_name: "keras/jax/mnist" + python_version: "3.12" + - model_name: "keras/torch/mnist" + python_version: "3.12" fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -221,6 +236,13 @@ jobs: matrix: model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_no_client_auth) }} python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_no_client_auth) }} + exclude: # Keras does not support Python 3.12 + - model_name: "keras/mnist" + python_version: "3.12" + - model_name: "keras/jax/mnist" + python_version: "3.12" + - model_name: "keras/torch/mnist" + python_version: "3.12" fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -264,6 +286,13 @@ jobs: matrix: model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_memory_logs) }} python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_memory_logs) }} + exclude: # Keras does not support Python 3.12 + - model_name: "keras/mnist" + python_version: "3.12" + - model_name: "keras/jax/mnist" + python_version: "3.12" + - model_name: "keras/torch/mnist" + python_version: "3.12" fail-fast: false # do not immediately fail if one of the combinations fail env: diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index 55b3358471..8ff04ec989 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -59,7 +59,7 @@ jobs: matrix: # IMP: Models requiring higher config might fail on current GitHub runners due to resource constraints. model_name: ["keras/mnist", "keras/jax/mnist", "keras/torch/mnist"] - python_version: ["3.10", "3.11", "3.12"] + python_version: ["3.10", "3.11"] # Keras does not support Python 3.12 fail-fast: false # do not immediately fail if one of the combinations fail env: diff --git a/.github/workflows/task_runner_e2e_resiliency.yml b/.github/workflows/task_runner_e2e_resiliency.yml index 8b530f2542..1e145380c6 100644 --- a/.github/workflows/task_runner_e2e_resiliency.yml +++ b/.github/workflows/task_runner_e2e_resiliency.yml @@ -7,7 +7,7 @@ on: schedule: - cron: "0 5 * * *" # Run every day at 5 am UTC push: - branches: + branches: - develop workflow_call: workflow_dispatch: @@ -64,6 +64,7 @@ jobs: # Output all the variables related to models and python versions to be used in the matrix strategy # for different jobs, however their usage depends on the selected job. selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }} + selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }} steps: - name: Job to select input values id: input_selection @@ -73,6 +74,11 @@ jobs: else echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" fi + if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then + echo "python_for_tls=[\"3.10\", \"3.11\"]" >> "$GITHUB_OUTPUT" + else + echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" + fi resiliency_in_native: name: With TLS @@ -82,6 +88,10 @@ jobs: strategy: matrix: model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }} + python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }} + exclude: # Keras does not support Python 3.12 + - model_name: "keras/mnist" + python_version: "3.12" fail-fast: false # do not immediately fail if one of the combinations fail env: @@ -115,46 +125,4 @@ jobs: with: test_type: "Resiliency_Native" - # TODO - Once we have GitHub runners with higher configurations, we can enable this job. - # resiliency_in_dws: - # name: With TLS (Dockerized) - # needs: input_selection - # runs-on: ubuntu-22.04 - # timeout-minutes: 30 - # strategy: - # matrix: - # # Dockerized WS for other models require higher config runners. - # # Once the issue is fixed, we can enable the tests for other models as well. - # model_name: ["keras/mnist"] - # fail-fast: false # do not immediately fail if one of the combinations fail - - # env: - # MODEL_NAME: ${{ matrix.model_name }} - # PYTHON_VERSION: ${{ matrix.python_version }} - - # steps: - # - name: Checkout OpenFL repository - # id: checkout_openfl - # uses: actions/checkout@v4 - # with: - # fetch-depth: 2 # needed for detecting changes - # submodules: "true" - # token: ${{ secrets.GITHUB_TOKEN }} - - # - name: Pre test run - # uses: ./.github/actions/tr_pre_test_run - # if: ${{ always() }} - - # - name: Run Task Runner E2E tests with TLS - # id: run_tests - # run: | - # python -m pytest -s tests/end_to_end/test_suites/tr_resiliency_tests.py \ - # -m task_runner_dockerized_ws --model_name ${{ env.MODEL_NAME }} \ - # --num_collaborators ${{ env.NUM_COLLABORATORS }} --num_rounds ${{ env.NUM_ROUNDS }} - # echo "Task runner end to end test run completed" - - # - name: Post test run - # uses: ./.github/actions/tr_post_test_run - # if: ${{ always() }} - # with: - # test_type: "Resiliency_Dockerized" + # TODO - Add dockerized approach as well once we have GitHub runners with higher configurations.