Streamlining PR pipeline (#1378)

* WIP: Streamlining PR pipeline Signed-off-by: Chaurasiya, Payal <[email protected]> * Fix pipeline job Signed-off-by: Chaurasiya, Payal <[email protected]> * Fix pipeline job Signed-off-by: Chaurasiya, Payal <[email protected]> * trivy step Signed-off-by: Chaurasiya, Payal <[email protected]> * trivy step Signed-off-by: Chaurasiya, Payal <[email protected]> * fix eden compression Signed-off-by: Chaurasiya, Payal <[email protected]> * experiment Signed-off-by: Chaurasiya, Payal <[email protected]> * Revert dockerize test Signed-off-by: Chaurasiya, Payal <[email protected]> * Review comments fix Signed-off-by: Chaurasiya, Payal <[email protected]> * Revert trivy changes Signed-off-by: Chaurasiya, Payal <[email protected]> --------- Signed-off-by: Chaurasiya, Payal <[email protected]>
securefederatedai · Feb 14, 2025 · fb2094e · fb2094e
1 parent 8700542
commit fb2094e
Show file tree

Hide file tree

Showing 10 changed files with 131 additions and 249 deletions.
diff --git a/.github/workflows/double_ws_export.yml b/.github/workflows/double_ws_export.yml
diff --git a/.github/workflows/pr_pipeline.yml b/.github/workflows/pr_pipeline.yml
@@ -27,10 +27,6 @@ jobs:
     name: Docker Bench for Security
     uses: ./.github/workflows/docker-bench-security.yml
 
-  double_workspace_export:
-    name: Double workspace export
-    uses: ./.github/workflows/double_ws_export.yml
-
   fr_301_watermark_nb_run:
     name: Federated Runtime 301 MNIST Watermarking
     uses: ./.github/workflows/federated_runtime.yml
@@ -47,14 +43,10 @@ jobs:
     name: Pytest and code coverage
     uses: ./.github/workflows/pytest_coverage.yml
 
-  straggler_handling:
-    name: Straggler Handling Test
-    uses: ./.github/workflows/straggler-handling.yml
-
-  task_runner:
-    name: TaskRunner
-    uses: ./.github/workflows/taskrunner.yml
-
+  windows:
+    name: Windows pytest coverage and workspace runs
+    uses: ./.github/workflows/windows.yml
+
   task_runner_e2e:
     name: TaskRunner E2E
     uses: ./.github/workflows/task_runner_basic_e2e.yml
@@ -63,10 +55,6 @@ jobs:
     name: TaskRunner E2E Resiliency
     uses: ./.github/workflows/task_runner_e2e_resiliency.yml
 
-  taskrunner_eden_compression:
-    name: TaskRunner (Eden Compression)
-    uses: ./.github/workflows/taskrunner_eden_pipeline.yml
-
   tr_docker_gramine_direct:
     name: TaskRunner (docker/gramine-direct)
     uses: ./.github/workflows/tr_docker_gramine_direct.yml

diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml
@@ -72,7 +72,7 @@ jobs:
     if: |
       (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch') ||
-      (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'task_runner_e2e'))
+      (github.event.pull_request.draft == false)
     name: Input value selection
     runs-on: ubuntu-22.04
     outputs:
@@ -326,3 +326,75 @@ jobs:
         if: ${{ always() }}
         with:
           test_type: "With_Memory_Logs"
+
+  test_straggler_check:
+    name: Straggler Handling
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    if: |
+      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.pull_request.draft == false)
+
+    env:
+      MODEL_NAME: "torch/mnist_straggler_check"
+      PYTHON_VERSION: "3.10"
+
+    steps:
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/checkout@v4
+
+      - name: Pre test run
+        uses: ./.github/actions/tr_pre_test_run
+        if: ${{ always() }}
+
+      - name: Run Straggler Handling Interface Test
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
+          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
+          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
+          echo "Straggler handling test run completed"
+
+      - name: Post test run
+        uses: ./.github/actions/tr_post_test_run
+        if: ${{ always() }}
+        with:
+          test_type: "Straggler_Handling"
+
+  test_eden_compression:
+    name: Eden Compression
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    if: |
+      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression'))
+
+    env:
+      MODEL_NAME: "torch/mnist_eden_compression"
+      PYTHON_VERSION: "3.10"
+
+    steps:
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/checkout@v4
+
+      - name: Pre test run
+        uses: ./.github/actions/tr_pre_test_run
+        if: ${{ always() }}
+
+      - name: Run Eden Compression Test
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
+          -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
+          --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
+          echo "Eden compression test run completed"
+
+      - name: Post test run
+        uses: ./.github/actions/tr_post_test_run
+        if: ${{ always() }}
+        with:
+          test_type: "Eden_Compression"
diff --git a/.github/workflows/task_runner_e2e_resiliency.yml b/.github/workflows/task_runner_e2e_resiliency.yml
@@ -15,7 +15,7 @@ on:
       num_rounds:
         description: "Number of rounds to train"
         required: false
-        default: "50"
+        default: "30"
         type: string
       num_collaborators:
         description: "Number of collaborators"
@@ -25,12 +25,12 @@ on:
       model_name:
         description: "Model name"
         required: false
-        default: "all"
+        default: "torch/mnist"
         type: choice
         options:
-          - all
           - torch/mnist
           - keras/mnist
+          - all
       python_version:
         description: "Python version"
         required: false
@@ -47,17 +47,17 @@ permissions:
 # Environment variables common for all the jobs
 # DO NOT use double quotes for the values of the environment variables
 env:
-  NUM_ROUNDS: ${{ inputs.num_rounds || 50 }}
+  NUM_ROUNDS: ${{ inputs.num_rounds || 30 }}
   NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }}
-  MODEL_NAME: ${{ inputs.model_name || 'all' }}
+  MODEL_NAME: ${{ inputs.model_name || 'torch/mnist' }}
   PYTHON_VERSION: ${{ inputs.python_version || '3.10' }}
 
 jobs:
   input_selection:
     if: |
       (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
       (github.event_name == 'workflow_dispatch') ||
-      (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'task_runner_e2e'))
+      (github.event.pull_request.draft == false)
     name: Input value selection
     runs-on: ubuntu-22.04
     outputs:

diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -1,6 +1,7 @@
 name: Windows (latest)
 
 on:
+  workflow_call:
   schedule:
     - cron: '0 0 * * *'
 
@@ -36,8 +37,7 @@ jobs:
         coverage run -m pytest -rA
         coverage report
   
-  cli: # from taskrunner.yml
-    needs: [pytest-coverage]
+  keras_mnist: # from taskrunner.yml -  keras/mnist
     runs-on: windows-latest
     timeout-minutes: 15
     steps:
@@ -52,4 +52,42 @@ jobs:
           pip install .
       - name: Test TaskRunner API
         run: |
-          python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+          python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+
+  torch_mnist_eden_compression: # from taskrunner_eden_pipeline.yml - torch/mnist_eden_compression
+    runs-on: windows-latest
+    timeout-minutes: 15
+    if: |
+      (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression'))
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+      - name: Test TaskRunner API with Eden Compression
+        run: |
+          python -m tests.github.test_hello_federation --template torch/mnist_eden_compression --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3
+
+  torch_mnist_straggler_check: # from straggler-handling.yml - torch/mnist_straggler_check
+    runs-on: windows-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+      - name: Test Straggler Handling Interface
+        run: |
+          python -m tests.github.test_hello_federation --template torch/mnist_straggler_check --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3
diff --git a/.trivyignore b/.trivyignore
@@ -1,2 +1,3 @@
 # Accept the risk
 CVE-2025-0395
+CVE-2024-3596
diff --git a/tests/end_to_end/utils/constants.py b/tests/end_to_end/utils/constants.py
@@ -10,12 +10,15 @@ class ModelName(Enum):
     """
     # IMP - The model name must be same (and in uppercase) as the model value.
     # This is used to identify the model in the tests.
-    TORCH_MNIST = "torch/mnist"
+    KERAS_JAX_MNIST = "keras/jax/mnist"
     KERAS_MNIST = "keras/mnist"
+    KERAS_TORCH_MNIST = "keras/torch/mnist"
     TORCH_HISTOLOGY = "torch/histology"
+    TORCH_MNIST = "torch/mnist"
+    TORCH_MNIST_EDEN_COMPRESSION = "torch/mnist_eden_compression"
+    TORCH_MNIST_STRAGGLER_CHECK = "torch/mnist_straggler_check"
     XGB_HIGGS = "xgb_higgs"
-    KERAS_TORCH_MNIST = "keras/torch/mnist"
-    KERAS_JAX_MNIST = "keras/jax/mnist"
+
 
 NUM_COLLABORATORS = 2
 NUM_ROUNDS = 5