Adding slack notification check, moving e2e setup to K8s and time for run.

.github/workflows/e2e.yaml

@@ -10,17 +10,43 @@ on:
   workflow_dispatch:
     inputs:
       simplyBlockDeploy_branch:
-        description: ''
+        description: 'Branch for simplyBlockDeploy'
         required: true
         default: 'main'
+      spdk_csi_branch:
+        description: 'Branch for spdk-csi'
+        required: true
+        default: 'master'
       sbcli_cmd:
-        description: ''
+        description: 'Command for sbcli execution'
         required: true
         default: 'sbcli-dev'
       testname:
         description: 'Name of test to run. Empty to run all'
         required: false
         default: ''
+      send_slack_notification:
+        description: 'Send Slack notification?'
+        required: false
+        default: true
+        type: boolean
+      ndcs:
+        description: 'Value for NDCS'
+        required: false
+        default: 1
+      npcs:
+        description: 'Value for NPCS'
+        required: false
+        default: 1
+      bs:
+        description: 'Value for Block size'
+        required: false
+        default: 4096
+      chunk_bs:
+        description: 'Value for Chunk block size'
+        required: false
+        default: 4096
+
 jobs:
   e2e:
     runs-on: self-hosted
@@ -70,7 +96,7 @@ jobs:
               -backend-config="dynamodb_table=${TFSTATE_DYNAMODB_TABLE}" \
               -backend-config="encrypt=true"
 
-      - name: select or create workspace
+      - name: Select or Create Workspace
         run: |
           cd $GITHUB_WORKSPACE/simplyBlockDeploy
           terraform workspace select -or-create ghiaction-sbclie2e
@@ -87,10 +113,12 @@ jobs:
             -var "extra_nodes=1" -var "extra_nodes_instance_type=m6id.xlarge" \
             -var storage_nodes_ebs_size2=100 -var "region=us-east-2" \
             -var "sbcli_cmd=${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }}" -out=tfplan
+
       - name: Apply Terraform Changes
         run: |
           cd $GITHUB_WORKSPACE/simplyBlockDeploy
           terraform apply tfplan
+
       - name: Get Terraform Outputs
         id: terraform_outputs
         run: |
@@ -99,6 +127,7 @@ jobs:
           echo "::set-output name=bastion_public_ip::$output_bastion_public_ip"
           output_key_name=$(terraform output -raw key_name)
           echo "::set-output name=key_name::$output_key_name"
+
       - name: Bootstrap Cluster
         run: |
           cd $GITHUB_WORKSPACE/simplyBlockDeploy
@@ -107,6 +136,9 @@ jobs:
         env:
           SBCLI_CMD: ${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }}
 
+      - name: Record Test Start Time
+        run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV
+
       - name: Setup Tests & Run Tests
         timeout-minutes: 120
         run: |
@@ -131,6 +163,23 @@ jobs:
             TESTNAME="--testname ${{ github.event.inputs.testname }}"
           fi
           python3 e2e.py $TESTNAME
+
+      - name: Record Test End Time
+        if: always() 
+        run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV
+
+      - name: Calculate Total Time Taken
+        if: always()
+        run: |
+          TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME))
+          TEST_TIME_HOURS=$(($TEST_TIME / 3600))
+          TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60))
+          TEST_TIME_SECS=$(($TEST_TIME % 60))
+          echo "Test runtime: ${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s"
+          echo "TEST_TIME_HOURS=$TEST_TIME_HOURS" >> $GITHUB_ENV
+          echo "TEST_TIME_MINS=$TEST_TIME_MINS" >> $GITHUB_ENV
+          echo "TEST_TIME_SECS=$TEST_TIME_SECS" >> $GITHUB_ENV
+
       - name: Upload automation and docker logs to s3
         run: |
           cd $GITHUB_WORKSPACE/e2e/logs
@@ -146,7 +195,7 @@ jobs:
           RUN_ID: ${{ github.run_id }}
 
       - name: Parse test results
-        if: always() && github.ref == 'refs/heads/main' || github.ref == 'refs/heads/pre-release'
+        if: always() && (github.event.inputs.send_slack_notification == 'true')
         id: parse_results
         run: |
           cd $GITHUB_WORKSPACE/e2e/logs
@@ -209,7 +258,7 @@ jobs:
           echo "EOF" >> $GITHUB_ENV
 
       - name: Send Slack Notification
-        if: always() && github.ref == 'refs/heads/main' || github.ref == 'refs/heads/pre-release'
+        if: always() && (github.event.inputs.send_slack_notification == 'true')
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
           GITHUB_SERVER_URL: ${{ github.server_url }}
@@ -224,6 +273,13 @@ jobs:
           PASSED_CASES_BULLETS: ${{ env.PASSED_CASES_BULLETS }}
           FAILED_CASES_BULLETS: ${{ env.FAILED_CASES_BULLETS }}
           BRANCH_NAME: ${{ github.ref_name }}
+          TEST_TIME_HOURS: ${{ env.TEST_TIME_HOURS }}
+          TEST_TIME_MINS: ${{ env.TEST_TIME_MINS }}
+          TEST_TIME_SECS: ${{ env.TEST_TIME_SECS }}
+          NDCS: ${{ github.event.inputs.ndcs }}
+          NPCS: ${{ github.event.inputs.npcs }}
+          CHUNK_BS: ${{ github.event.inputs.chunk_bs }}
+          BS: ${{ github.event.inputs.bs }}
         run: |
           GITHUB_RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}"
           AWS_LOGS_URL="https://s3.console.aws.amazon.com/s3/buckets/${S3_BUCKET_NAME}?prefix=${RUN_ID}/&region=us-east-2"
@@ -232,17 +288,22 @@ jobs:
           else
             OVERALL_STATUS=":x: Overall Status: FAILURE"
           fi
-          MESSAGE="Python E2E tests run triggered on branch **${BRANCH_NAME}**. \n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\nTotal Tests: ${{ env.TOTAL_TESTS }}\nPassed Tests: ${{ env.PASSED_TESTS }}\nFailed Tests: ${{ env.FAILED_TESTS }}\n\n-- Test Cases Passed :white_check_mark:\n${{ env.PASSED_CASES_BULLETS }}\n\n-- Test Cases Failed :x:\n${{ env.FAILED_CASES_BULLETS }}"
+
+          TIME_TAKEN="${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s"
+
+          MESSAGE="Python E2E tests run triggered on branch *${BRANCH_NAME}*. \nTotal Time Taken to run the tests: ${TIME_TAKEN}. \n\n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\n*Configuration*: *NDCS: ${{ env.NDCS }}, NPCS: ${{ env.NPCS }}, Block Size: ${{ env.BS }}, Chunk Block Size: ${{ env.CHUNK_BS }}*\n\nTotal Tests: *${{ env.TOTAL_TESTS }}*, Passed Tests: *${{ env.PASSED_TESTS }}*, Failed Tests: *${{ env.FAILED_TESTS }}*\n\n-- Test Cases Passed :white_check_mark:\n${{ env.PASSED_CASES_BULLETS }}\n\n-- Test Cases Failed :x:\n${{ env.FAILED_CASES_BULLETS }}"
 
           curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"${MESSAGE}\"}" $SLACK_WEBHOOK_URL
+
       - name: Destroy Cluster
         if: always()
         run: |
           cd $GITHUB_WORKSPACE/simplyBlockDeploy
           terraform destroy --auto-approve
+
       - name: 'Cleanup build folder'
         run: |
           ls -la ./
           rm -rf ./* || true
           rm -rf ./.??* || true
-          ls -la ./
+          ls -la ./

.github/workflows/k8s-e2e.yaml

@@ -122,7 +122,7 @@ jobs:
       - name: Select or create workspace
         run: |
           cd $GITHUB_WORKSPACE/simplyBlockDeploy
-          terraform workspace select -or-create ghiaction-sbclie2e
+          terraform workspace select -or-create ghiaction-sbclie2ek8s
 
       - name: Validate Terraform Configuration
         run: |
@@ -286,7 +286,7 @@ jobs:
                           --npcs "${{ github.event.inputs.npcs }}" \
                           --bs "${{ github.event.inputs.bs }}" \
                           --chunk_bs "${{ github.event.inputs.chunk_bs }}" \
-                          --run_k8s
+                          --run_k8s True \
                           $TESTNAME
 
       - name: Record Test End Time
@@ -416,7 +416,7 @@ jobs:
 
           TIME_TAKEN="${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s"
 
-          MESSAGE="Python E2E K8s tests run triggered on branch *${BRANCH_NAME}*. \nTotal Time Taken to run the tests: ${TIME_TAKEN}. \n\n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\n*Configuration*: *NDCS: ${{ env.NDCS }}, NPCS: ${{ env.NPCS }}, Block Size: ${{ env.BS }}, Chunk Block Size: ${{ env.CHUNK_BS }}*\n\nTotal Tests: *${{ env.TOTAL_TESTS }}*, Passed Tests: *${{ env.PASSED_TESTS }}*, Failed Tests: *${{ env.FAILED_TESTS }}*\n\n-- Test Cases Passed :white_check_mark:\n${{ env.PASSED_CASES_BULLETS }}\n\n-- Test Cases Failed :x:\n${{ env.FAILED_CASES_BULLETS }}"
+          MESSAGE="Python E2E *K8s* tests run triggered on branch *${BRANCH_NAME}*. \nTotal Time Taken to run the tests: ${TIME_TAKEN}. \n\n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\n*Configuration*: *NDCS: ${{ env.NDCS }}, NPCS: ${{ env.NPCS }}, Block Size: ${{ env.BS }}, Chunk Block Size: ${{ env.CHUNK_BS }}*\n\nTotal Tests: *${{ env.TOTAL_TESTS }}*, Passed Tests: *${{ env.PASSED_TESTS }}*, Failed Tests: *${{ env.FAILED_TESTS }}*\n\n-- Test Cases Passed :white_check_mark:\n${{ env.PASSED_CASES_BULLETS }}\n\n-- Test Cases Failed :x:\n${{ env.FAILED_CASES_BULLETS }}"
 
           curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"${MESSAGE}\"}" $SLACK_WEBHOOK_URL
 

e2e/__init__.py

@@ -1,19 +1,28 @@
 from e2e_tests.single_node_outage import TestSingleNodeOutage
 from e2e_tests.single_node_failure import TestSingleNodeFailure
 from e2e_tests.single_node_multi_fio_perf import (
-    TestLvolFioNpcs0, TestLvolFioNpcs1, TestLvolFioNpcs2,
+    TestLvolFioNpcs0, TestLvolFioNpcs1, TestLvolFioNpcs2, TestLvolFioNpcsCustom
 )
 from e2e_tests.cloning_and_snapshot.multi_lvol_snapshot_fio import TestMultiLvolFio
 
 
 
-def get_all_tests():
+def get_all_tests(custom=True, k8s_test=False):
     tests = [
         TestSingleNodeOutage,
+        TestLvolFioNpcsCustom,
         TestLvolFioNpcs0,
         TestLvolFioNpcs1,
         TestLvolFioNpcs2,
         TestSingleNodeFailure,
         # TestMultiLvolFio, - Enable when testing
     ]
+    if not custom:
+        tests.remove(TestLvolFioNpcsCustom)
+    else:
+        tests.remove(TestLvolFioNpcs0)
+        tests.remove(TestLvolFioNpcs1)
+        tests.remove(TestLvolFioNpcs2)
+    if k8s_test:
+        tests.remove(TestSingleNodeFailure)
     return tests

e2e/e2e.py

@@ -13,16 +13,26 @@
 
 
 def main():
-    """Run complete test suite
-    """
+    """Run complete test suite"""
     parser = argparse.ArgumentParser(description="Run simplyBlock's E2E Test Framework")
     parser.add_argument('--testname', type=str, help="The name of the test to run", default=None)
     parser.add_argument('--fio_debug', type=bool, help="Add debug flag to fio", default=False)
+
+    # New arguments for ndcs, npcs, bs, chunk_bs with default values
+    parser.add_argument('--ndcs', type=int, help="Number of data chunks (ndcs)", default=1)
+    parser.add_argument('--npcs', type=int, help="Number of parity chunks (npcs)", default=1)
+    parser.add_argument('--bs', type=int, help="Block size (bs)", default=4096)
+    parser.add_argument('--chunk_bs', type=int, help="Chunk block size (chunk_bs)", default=4096)
+    parser.add_argument('--run_k8s', type=bool, help="Run K8s setup", default=False)
+
 
     args = parser.parse_args()
 
-    tests = get_all_tests()
-    # Find the test class based on the provided test name
+    if args.ndcs == 0 and args.npcs == 0:
+        tests = get_all_tests(custom=False, k8s_test=args.run_k8s)
+    else:
+        tests = get_all_tests(custom=True, k8s_test=args.run_k8s)
+
     test_class_run = []
     if args.testname is None or len(args.testname.strip()) == 0:
         test_class_run = tests
@@ -35,11 +45,16 @@ def main():
         available_tests = ', '.join(cls.__name__ for cls in tests)
         logger.info(f"Test '{args.testname}' not found. Available tests are: {available_tests}")
         raise TestNotFoundException(args.testname, available_tests)
-    
+
     errors = {}
     for test in test_class_run:
         logger.info(f"Running Test {test}")
-        test_obj = test(fio_debug=args.fio_debug)
+        test_obj = test(fio_debug=args.fio_debug,
+                        ndcs=args.ndcs,
+                        npcs=args.npcs,
+                        bs=args.bs,
+                        chunk_bs=args.chunk_bs,
+                        k8s_run=args.run_k8s)
         try:
             test_obj.setup()
             test_obj.run()
@@ -52,7 +67,6 @@ def main():
         except Exception as _:
             logger.error(f"Error During Teardown for test: {test.__name__}")
             logger.error(traceback.format_exc())
-            # errors[f"{test.__name__}"].append(exp)
         finally:
             if check_for_dumps():
                 logger.info("Found a core dump during test execution. "
@@ -63,18 +77,17 @@ def main():
     logger.info(f"Number of Total Cases: {len(test_class_run)}")
     logger.info(f"Number of Passed Cases: {len(test_class_run) - len(failed_cases)}")
     logger.info(f"Number of Failed Cases: {len(failed_cases)}")
-    
+
     logger.info("Test Wise run status:")
     for test in test_class_run:
         if test.__name__ not in failed_cases:
             logger.info(f"{test.__name__} PASSED CASE.")
         else:
             logger.info(f"{test.__name__} FAILED CASE.")
 
-
     if errors:
         raise MultipleExceptions(errors)
-    
+
 
 def check_for_dumps():
     """Validates whether core dumps present on machines
@@ -111,17 +124,5 @@ def check_for_dumps():
     return core_exist
 
 
-def generate_report():
-    """
-    If any of the above conditions are not true, the relevant outputs from logs or cli commands should be placed
-    automatically in a bug report; we may just create a shared folder and place a textfile bug report per run
-    there under the date of the run: No failure report file → everthing went ok.
-    failure report file for a particular date and time → contains relevant logs of the run
-    (fio output, output of sbcli sn list, sbcli sn list-devices, sbcli cluster status, sbcli cluster get-logs,
-    sbcli lvol get, sbcli lvol get-cluster-map, spdk log)
-    """
-    pass
-
-
 logger = setup_logger(__name__)
 main()

e2e/e2e_tests/cloning_and_snapshot/multi_lvol_snapshot_fio.py

@@ -58,8 +58,13 @@ def run(self):
             pool_name=self.pool_name
         )
         lvol_vs_disk = {}
+        if self.ndcs == 0 and self.npcs == 0:
+            lvol_config_list = ["1+0", "2+1", "4+1", "4+2", "8+1", "8+2"]
+        else:
+            lvol_config_list = [f"{self.ndcs}+{self.npcs}" ]
 
-        for config in ["1+0", "2+1", "4+1", "4+2", "8+1", "8+2"]:
+
+        for config in lvol_config_list:
             ndcs, npcs = config.split('+')
             lvol_name = f"lvol_{ndcs}_{npcs}"
 
@@ -96,12 +101,9 @@ def run(self):
 
         lvol_list = self.sbcli_utils.list_lvols()
 
-
-
         for fs_type in ["ext4", "xfs"]:
             self.logger.info(f"Processing filesystem type: {fs_type}")
-
-            for config in ["1+0", "2+1", "4+1", "4+2", "8+1", "8+2"]:
+            for config in lvol_config_list:
                 ndcs, npcs = config.split('+')
                 lvol_name = f"lvol_{ndcs}_{npcs}"
                 mount_point = f"{self.mount_path}/{lvol_name}"
@@ -114,7 +116,7 @@ def run(self):
                                         mount_path=mount_point)
 
             for size in ["5GiB", "10GiB", "20GiB", "40GiB"]:
-                for config in ["1+0", "2+1", "4+1", "4+2", "8+1", "8+2"]:
+                for config in lvol_config_list:
                     ndcs, npcs = config.split('+')
                     lvol_name = f"lvol_{ndcs}_{npcs}"
                     lvol_id = lvol_list[lvol_name]

e2e/e2e_tests/cluster_test_base.py

@@ -29,8 +29,13 @@ def __init__(self, **kwargs):
         self.common_utils = CommonUtils(self.sbcli_utils, self.ssh_obj)
         self.mgmt_nodes = None
         self.storage_nodes = None
+        self.ndcs = kwargs.get("ndcs", 1)
+        self.npcs = kwargs.get("npcs", 1)
+        self.bs = kwargs.get("bs", 4096)
+        self.chunk_bs = kwargs.get("chunk_bs", 4096)
+        self.k8s_test = kwargs.get("k8s_run", False)
         self.pool_name = "test_pool"
-        self.lvol_name = "test_lvol"
+        self.lvol_name = f"test_lvl_{self.ndcs}_{self.npcs}"
         self.mount_path = "/home/ec2-user/test_location"
         self.log_path = f"{os.path.dirname(self.mount_path)}/log_file.log"
         self.base_cmd = os.environ.get("SBCLI_CMD", "sbcli-dev")