E2E K8s Tests #272
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E K8s Tests | |
| on: | |
| push: | |
| branches: | |
| - pre-release | |
| - sbcli | |
| schedule: | |
| - cron: '0 9 * * *' # Runs every day at 9 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| simplyBlockDeploy_branch: | |
| description: 'Branch for simplyBlockDeploy' | |
| required: true | |
| default: 'main' | |
| simplyblock_csi_branch: | |
| description: 'Branch for simplyblock-csi' | |
| required: true | |
| default: 'master' | |
| sbcli_cmd: | |
| description: 'Command for sbcli execution' | |
| required: true | |
| default: 'sbcli-dev' | |
| testname: | |
| description: 'Name of test to run. Empty to run all' | |
| required: false | |
| default: '' | |
| send_slack_notification: | |
| description: 'Send Slack notification?' | |
| required: false | |
| default: true | |
| type: boolean | |
| ndcs: | |
| description: 'Number of data chunks' | |
| required: false | |
| default: 2 | |
| npcs: | |
| description: 'Number of parity chunks' | |
| required: false | |
| default: 1 | |
| bs: | |
| description: 'Block size' | |
| required: false | |
| default: 4096 | |
| chunk_bs: | |
| description: 'Chunk block size' | |
| required: false | |
| default: 4096 | |
| jobs: | |
| e2e: | |
| runs-on: self-hosted | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set send_slack_notification for scheduled and push events | |
| if: github.event_name == 'schedule' || github.event_name == 'push' | |
| run: echo "send_slack_notification=true" >> $GITHUB_ENV | |
| - name: Set send_slack_notification for manual workflow_dispatch | |
| if: github.event_name == 'workflow_dispatch' | |
| run: echo "send_slack_notification=${{ github.event.inputs.send_slack_notification }}" >> $GITHUB_ENV | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - run: go version | |
| - name: Install Helm v3.15.4 | |
| run: | | |
| curl -L https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o helm-v3.15.4-linux-amd64.tar.gz | |
| tar -zxvf helm-v3.15.4-linux-amd64.tar.gz | |
| sudo mv linux-amd64/helm /usr/local/bin/helm | |
| helm version | |
| - name: Install kubectl v1.31.0 | |
| run: | | |
| curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl" | |
| chmod +x kubectl | |
| sudo mv kubectl /usr/local/bin/ | |
| kubectl version --client | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - uses: actions/checkout@master | |
| name: Checkout code simplyBlockDeploy | |
| with: | |
| repository: simplyblock-io/simplyBlockDeploy | |
| ref: refs/heads/${{ github.event.inputs.simplyBlockDeploy_branch || 'main'}} | |
| path: 'simplyBlockDeploy' | |
| token: ${{ secrets.GH_ACCESS_KEY_ID_HAMDI }} | |
| - uses: actions/checkout@master | |
| name: Checkout code simplyblock_csi_branch | |
| with: | |
| repository: simplyblock-io/simplyblock-csi | |
| ref: refs/heads/${{ github.event.inputs.simplyblock_csi_branch || 'master'}} | |
| path: 'simplyblock-csi' | |
| token: ${{ secrets.GH_ACCESS_KEY_ID_HAMDI }} | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_wrapper: false | |
| - name: Initialize Terraform | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| export TFSTATE_BUCKET=simplyblock-terraform-state-bucket | |
| export TFSTATE_KEY=csi | |
| export TFSTATE_REGION=us-east-2 | |
| export TFSTATE_DYNAMODB_TABLE=terraform-up-and-running-locks | |
| terraform init -reconfigure \ | |
| -backend-config="bucket=${TFSTATE_BUCKET}" \ | |
| -backend-config="key=${TFSTATE_KEY}" \ | |
| -backend-config="region=${TFSTATE_REGION}" \ | |
| -backend-config="dynamodb_table=${TFSTATE_DYNAMODB_TABLE}" \ | |
| -backend-config="encrypt=true" | |
| - name: Select or create workspace | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| terraform workspace select -or-create ghiaction-sbclie2ek8s | |
| - name: Validate Terraform Configuration | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| terraform validate | |
| - name: Set parameters based on branch and input | |
| run: | | |
| if [[ "${GITHUB_REF##*/}" == *"pre-release"* ]]; then | |
| echo "Branch is pre-release. Setting SBCLI_CMD to sbcli-pre." | |
| echo "SBCLI_CMD=sbcli-pre" >> $GITHUB_ENV | |
| elif [[ "${GITHUB_REF##*/}" == "sbcli" ]]; then | |
| echo "Branch is sbcli. Setting SBCLI_CMD to sbcli." | |
| echo "SBCLI_CMD=sbcli" >> $GITHUB_ENV | |
| else | |
| echo "Setting SBCLI_CMD to input or default to sbcli-dev." | |
| echo "SBCLI_CMD=${{ github.event.inputs.sbcli_cmd || 'sbcli-dev' }}" >> $GITHUB_ENV | |
| fi | |
| echo "NDCS=${{ github.event.inputs.ndcs || 2 }}" >> $GITHUB_ENV | |
| echo "NPCS=${{ github.event.inputs.npcs || 1 }}" >> $GITHUB_ENV | |
| echo "BS=${{ github.event.inputs.bs || 4096 }}" >> $GITHUB_ENV | |
| echo "CHUNK_BS=${{ github.event.inputs.chunk_bs || 4096 }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Plan Terraform Changes | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| terraform plan \ | |
| -var "mgmt_nodes=1" -var "storage_nodes=4" -var "volumes_per_storage_nodes=1" \ | |
| -var storage_nodes_instance_type="m6g.2xlarge" \ | |
| -var "storage_nodes_arch=arm64" -var "snode_deploy_on_k8s=true" \ | |
| -var "sec_storage_nodes_instance_type=m6g.2xlarge" \ | |
| -var "extra_nodes=1" -var "extra_nodes_instance_type=m6gd.xlarge" \ | |
| -var storage_nodes_ebs_size2=100 -var "region=us-east-2" \ | |
| -var "extra_nodes_arch=arm64" -var "sec_storage_nodes=1" \ | |
| -var "sbcli_cmd=$SBCLI_CMD" -out=tfplan | |
| env: | |
| SBCLI_CMD: ${{ env.SBCLI_CMD }} | |
| - name: Apply Terraform Changes | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| terraform apply tfplan | |
| - name: Get Terraform Outputs | |
| id: terraform_outputs | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| output_bastion_public_ip=$(terraform output -raw bastion_public_ip) | |
| echo "::set-output name=bastion_public_ip::$output_bastion_public_ip" | |
| output_key_name=$(terraform output -raw key_name) | |
| echo "::set-output name=key_name::$output_key_name" | |
| output_extra_node_public_ip=$(terraform output -raw extra_nodes_public_ips) | |
| echo "::set-output name=extra_node_public_ip::$output_extra_node_public_ip" | |
| output_mgmt_private_ip=$(terraform output -raw mgmt_private_ips) | |
| echo "::set-output name=mgmt_private_ip::$output_mgmt_private_ip" | |
| - name: Bootstrap Cluster | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| ./bootstrap-cluster.sh --sbcli-cmd "$SBCLI_CMD" \ | |
| --max-lvol 10 --max-snap 10 --max-prov 400G --number-of-devices 1 \ | |
| --distr-ndcs $NDCS \ | |
| --distr-npcs $NPCS \ | |
| --distr-bs $BS \ | |
| --distr-chunk-bs $CHUNK_BS \ | |
| --k8s-snode --ha-type ha | |
| id: bootstrap_cluster | |
| env: | |
| SBCLI_CMD: ${{ env.SBCLI_CMD }} | |
| NDCS: ${{ env.NDCS }} | |
| NPCS: ${{ env.NPCS }} | |
| BS: ${{ env.BS }} | |
| CHUNK_BS: ${{ env.CHUNK_BS }} | |
| - name: Bootstrap k3s | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| ./bootstrap-k3s.sh --k8s-snode | |
| id: bootstrap_k3s | |
| - name: Configure Kubeconfig | |
| run: | | |
| mkdir -p $HOME/.kube | |
| scp -i "$HOME/.ssh/${{ steps.terraform_outputs.outputs.key_name }}" ec2-user@${{ steps.terraform_outputs.outputs.extra_node_public_ip }}:/etc/rancher/k3s/k3s.yaml $HOME/.kube/config | |
| sed -i "s/server: https:\/\/\(localhost\|127.0.0.1\)\(:[0-9]*\)/server: https:\/\/${{ steps.terraform_outputs.outputs.extra_node_public_ip }}\2/" $HOME/.kube/config | |
| - name: Install Helm Chart for simplyblock-csi | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyblock-csi/charts/latest/spdk-csi/ | |
| echo "Sleeping for 30 seconds before helm install" | |
| helm install spdk-csi ./ \ | |
| --namespace spdk-csi \ | |
| --create-namespace \ | |
| --set csiConfig.simplybk.uuid="${{ steps.bootstrap_cluster.outputs.cluster_id }}" \ | |
| --set csiConfig.simplybk.ip="${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }}" \ | |
| --set csiSecret.simplybk.secret="${{ steps.bootstrap_cluster.outputs.cluster_secret }}" \ | |
| --set logicalVolume.pool_name="testing1" \ | |
| --set logicalVolume.snapshot="True" \ | |
| --set image.spdkcsi.tag=latest \ | |
| --set image.simplyblock.tag=main \ | |
| --set logicalVolume.distr_ndcs="${NDCS}" \ | |
| --set logicalVolume.distr_npcs="${NPCS}" \ | |
| --set storagenode.create=true \ | |
| --set cachingnode.create=false \ | |
| --set logicalVolume.encryption=true | |
| env: | |
| NDCS: ${{ env.NDCS }} | |
| NPCS: ${{ env.NPCS }} | |
| - name: Check Cluster Status | |
| run: | | |
| CLUSTER_API_GATEWAY_ENDPOINT=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }} | |
| CLUSTER_UUID=${{ steps.bootstrap_cluster.outputs.cluster_id }} | |
| CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }} | |
| n=0 | |
| until [ "$n" -ge 60 ] | |
| do | |
| response=$(curl -s -X GET "$CLUSTER_API_GATEWAY_ENDPOINT/cluster/$CLUSTER_UUID" \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: $CLUSTER_UUID $CLUSTER_SECRET") | |
| status=$(echo $response | jq -r '.results[0].status') | |
| if [ "$status" != "active" ]; then | |
| echo "Cluster status is not active, current status: $status, retrying" | |
| n=$((n+1)) | |
| sleep 10 | |
| else | |
| echo "Cluster status is active" | |
| exit 0 | |
| fi | |
| done | |
| echo "Cluster status is not active" | |
| exit 1 | |
| - name: Record Test Start Time | |
| run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV | |
| - name: Setup Tests & Run Tests | |
| timeout-minutes: 120 | |
| run: | | |
| cd $GITHUB_WORKSPACE/e2e | |
| sudo apt-get install -y python3.12-venv | |
| python3 -m venv myenv | |
| source myenv/bin/activate | |
| python3 -m pip install -r requirements.txt | |
| echo "Running tests in namespace ${{ steps.get-namespace.outputs.namespace }}" | |
| export CLUSTER_ID=${{ steps.bootstrap_cluster.outputs.cluster_id }} | |
| export CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }} | |
| export CLUSTER_IP=${{ steps.bootstrap_cluster.outputs.cluster_ip }} | |
| export API_BASE_URL=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }} | |
| export BASTION_SERVER=${{ steps.terraform_outputs.outputs.bastion_public_ip }} | |
| export KEY_NAME=${{ steps.terraform_outputs.outputs.key_name }} | |
| export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} | |
| export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| export AWS_REGION=${{ secrets.AWS_REGION }} | |
| export SBCLI_CMD=${SBCLI_CMD} | |
| TESTNAME="" | |
| if [ -n "${{ github.event.inputs.testname }}" ]; then | |
| TESTNAME="--testname ${{ github.event.inputs.testname }}" | |
| fi | |
| python3 e2e.py $TESTNAME \ | |
| --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \ | |
| --run_k8s True | |
| env: | |
| SBCLI_CMD: ${{ env.SBCLI_CMD }} | |
| NDCS: ${{ env.NDCS }} | |
| NPCS: ${{ env.NPCS }} | |
| BS: ${{ env.BS }} | |
| CHUNK_BS: ${{ env.CHUNK_BS }} | |
| - name: Record Test End Time | |
| if: always() | |
| run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV | |
| - name: Calculate Total Time Taken | |
| if: always() | |
| run: | | |
| TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME)) | |
| TEST_TIME_HOURS=$(($TEST_TIME / 3600)) | |
| TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60)) | |
| TEST_TIME_SECS=$(($TEST_TIME % 60)) | |
| echo "Test runtime: ${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s" | |
| echo "TEST_TIME_HOURS=$TEST_TIME_HOURS" >> $GITHUB_ENV | |
| echo "TEST_TIME_MINS=$TEST_TIME_MINS" >> $GITHUB_ENV | |
| echo "TEST_TIME_SECS=$TEST_TIME_SECS" >> $GITHUB_ENV | |
| - name: Upload automation and docker logs to s3 | |
| run: | | |
| cd $GITHUB_WORKSPACE/e2e/logs | |
| ./upload_logs.sh | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| ./upload_docker_logs_to_s3.sh --k8s --namespace "spdk-csi" | |
| if: always() | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| S3_BUCKET_NAME: "simplyblock-e2e-test-logs" | |
| RUN_ID: ${{ github.run_id }} | |
| - name: Parse test results | |
| if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true') | |
| id: parse_results | |
| run: | | |
| cd $GITHUB_WORKSPACE/e2e/logs | |
| echo "Looking for the latest non-empty log file..." | |
| # Find the latest non-empty log file | |
| COUNTER=0 | |
| MAX_ATTEMPTS=10 | |
| while [ $COUNTER -lt $MAX_ATTEMPTS ]; do | |
| LATEST_LOG=$(ls -t *.log | head -n 1) | |
| if [ -s "$LATEST_LOG" ]; then | |
| echo "Found non-empty log file: $LATEST_LOG" | |
| break | |
| fi | |
| echo "Attempt $((COUNTER + 1)): No non-empty log file found. Retrying..." | |
| COUNTER=$((COUNTER + 1)) | |
| sleep 1 # Add a small delay to avoid rapid looping | |
| done | |
| if [ ! -s "$LATEST_LOG" ]; then | |
| echo "No non-empty log file found after $MAX_ATTEMPTS attempts" | |
| exit 1 | |
| fi | |
| echo "Parsing the identified log file: $LATEST_LOG" | |
| # Parse the identified log file | |
| echo "Total tests" | |
| TOTAL_TESTS=$(grep -i "Number of Total Cases" "$LATEST_LOG" | awk '{print $NF}') | |
| echo "number Passed tests" | |
| PASSED_TESTS=$(grep -i "Number of Passed Cases" "$LATEST_LOG" | awk '{print $NF}') | |
| echo "number Failed tests" | |
| FAILED_TESTS=$(grep -i "Number of Failed Cases" "$LATEST_LOG" | awk '{print $NF}') | |
| echo "List Passed tests" | |
| PASSED_CASES=$(grep "PASSED CASE" "$LATEST_LOG" | awk -F 'INFO - | FAILED CASE' '{print $2}') | |
| echo "List Failed tests" | |
| FAILED_CASES=$(grep "FAILED CASE" "$LATEST_LOG" | awk -F 'INFO - | FAILED CASE' '{print $2}') | |
| # Format passed and failed cases as bullet points | |
| echo "Adding PASSED cases with bullets: $PASSED_CASES" | |
| echo "Adding FAILED cases with bullets: $PASSED_CASES" | |
| PASSED_CASES_BULLETS=$(echo "$PASSED_CASES" | awk '{printf " • %s\n", $0}') | |
| FAILED_CASES_BULLETS=$(echo "$FAILED_CASES" | awk '{printf " • %s\n", $0}') | |
| echo "PASSED cases with bullets: $PASSED_CASES_BULLETS" | |
| echo "FAILED cases with bullets: $FAILED_CASES_BULLETS" | |
| echo "TOTAL_TESTS=${TOTAL_TESTS}" | |
| echo "PASSED_TESTS=${PASSED_TESTS}" | |
| echo "FAILED_TESTS=${FAILED_TESTS}" | |
| echo "PASSED_CASES=${PASSED_CASES}" | |
| echo "FAILED_CASES=${FAILED_CASES}" | |
| echo "PASSED_TESTS=${PASSED_TESTS}" >> $GITHUB_ENV | |
| echo "FAILED_TESTS=${FAILED_TESTS}" >> $GITHUB_ENV | |
| echo "TOTAL_TESTS=${TOTAL_TESTS}" >> $GITHUB_ENV | |
| echo "PASSED_CASES<<EOF" >> $GITHUB_ENV | |
| echo "${PASSED_CASES}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "FAILED_CASES<<EOF" >> $GITHUB_ENV | |
| echo "${FAILED_CASES}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "PASSED_CASES_BULLETS<<EOF" >> $GITHUB_ENV | |
| echo "${PASSED_CASES_BULLETS}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| echo "FAILED_CASES_BULLETS<<EOF" >> $GITHUB_ENV | |
| echo "${FAILED_CASES_BULLETS}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Send Slack Notification | |
| if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true') | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| GITHUB_SERVER_URL: ${{ github.server_url }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| S3_BUCKET_NAME: "simplyblock-e2e-test-logs" | |
| RUN_ID: ${{ github.run_id }} | |
| PASSED_TESTS: ${{ env.PASSED_TESTS }} | |
| FAILED_TESTS: ${{ env.FAILED_TESTS }} | |
| TOTAL_TESTS: ${{ env.TOTAL_TESTS }} | |
| PASSED_CASES: ${{ env.PASSED_CASES }} | |
| FAILED_CASES: ${{ env.FAILED_CASES }} | |
| PASSED_CASES_BULLETS: ${{ env.PASSED_CASES_BULLETS }} | |
| FAILED_CASES_BULLETS: ${{ env.FAILED_CASES_BULLETS }} | |
| BRANCH_NAME: ${{ github.ref_name }} | |
| TEST_TIME_HOURS: ${{ env.TEST_TIME_HOURS }} | |
| TEST_TIME_MINS: ${{ env.TEST_TIME_MINS }} | |
| TEST_TIME_SECS: ${{ env.TEST_TIME_SECS }} | |
| NDCS: ${{ env.NDCS }} | |
| NPCS: ${{ env.NPCS }} | |
| CHUNK_BS: ${{ env.CHUNK_BS }} | |
| BS: ${{ env.BS }} | |
| run: | | |
| GITHUB_RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" | |
| AWS_LOGS_URL="https://s3.console.aws.amazon.com/s3/buckets/${S3_BUCKET_NAME}?prefix=${RUN_ID}/®ion=us-east-2" | |
| if [[ ${{ job.status }} == 'success' ]]; then | |
| OVERALL_STATUS=":white_check_mark: Overall Status: SUCCESS" | |
| else | |
| OVERALL_STATUS=":x: Overall Status: FAILURE <!channel>" | |
| fi | |
| TIME_TAKEN="${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s" | |
| MESSAGE="Python E2E *K8s* tests run triggered on branch *${BRANCH_NAME}*. \nTotal Time Taken to run the tests: ${TIME_TAKEN}. \n\n${OVERALL_STATUS}\nGitHub Run: ${GITHUB_RUN_URL}\nAWS Logs: ${AWS_LOGS_URL}\n\n*Configuration*: *NDCS: ${NDCS}, NPCS: ${NPCS}, Block Size: ${BS}, Chunk Block Size: ${CHUNK_BS}*\n\nTotal Tests: *${TOTAL_TESTS}*, Passed Tests: *${PASSED_TESTS}*, Failed Tests: *${FAILED_TESTS}*\n\n-- Test Cases Passed :white_check_mark:\n${PASSED_CASES_BULLETS}\n\n-- Test Cases Failed :x:\n${FAILED_CASES_BULLETS}" | |
| curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"${MESSAGE}\"}" $SLACK_WEBHOOK_URL | |
| - name: Destroy Cluster | |
| if: always() | |
| run: | | |
| cd $GITHUB_WORKSPACE/simplyBlockDeploy | |
| terraform destroy --auto-approve | |
| - name: Cleanup build folder | |
| run: | | |
| ls -la ./ | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| ls -la ./ | |
| rm -rf $HOME/.kube |