integration-tests-command.yml

name: Appsmith External Integration Test Workflow

on:
  # This workflow is only triggered by the ok to test command dispatch
  repository_dispatch:
    types: [ok-to-test-command]

jobs:
  tag-check:
    runs-on: ubuntu-latest
    outputs:
      tags: ${{ steps.checkAll.outputs.tags }}
      matrix: ${{ steps.checkAll.outputs.matrix }}
    steps:
      - name: Check if tags arg is present in the command
        id: checkTags
        run: |
          checkArg=`echo '${{toJSON(github.event.client_payload.slash_command.args.named)}}' | jq 'has("tags")'`
          echo "tags=$checkArg" >> $GITHUB_OUTPUT

      - name: Add a comment on the PR with tags documentation link
        if: steps.checkTags.outputs.tags != 'true'
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.client_payload.pull_request.number }}
          body: |
            The provided command lacks any tags. Please execute '/ok-to-test' again, specifying the tags you want to include or use `@tag.All` to run all specs.
            Explore the tags documentation [here](https://www.notion.so/appsmith/Ok-to-test-With-Tags-7c0fc64d4efb4afebf53348cd6252918)
      
      - name: Stop the workflow run if tags are not present
        if: steps.checkTags.outputs.tags != 'true'
        run: exit 1

      - name: Check if @tag.All is present in tags
        if: steps.checkTags.outputs.tags == 'true'
        id: checkAll
        run: |
          tags="${{ github.event.client_payload.slash_command.args.named.tags }}"
          if [[ $tags == *"@tag.All"* ]]; then
            echo "tags=" >> $GITHUB_OUTPUT
            echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT
          else
            echo "tags=$tags" >> $GITHUB_OUTPUT
            echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT
          fi

      # This step creates a comment on the PR with a link to this workflow run.
      - name: Add a comment on the PR with link to workflow run
        if: steps.checkTags.outputs.tags == 'true'
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.client_payload.pull_request.number }}
          body: |
            Tests running at: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
            Workflow: `${{ github.workflow }}`.
            Tags: `${{ steps.checkAll.outputs.tags }}`.

  server-build:
    needs: [tag-check]
    if: success()
    name: server-build
    uses: ./.github/workflows/server-build.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}
      skip-tests: true

  client-build:
    needs: [tag-check]
    if: success()
    name: client-build
    uses: ./.github/workflows/client-build.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}
      check-test-files: "true"

  rts-build:
    needs: [tag-check]
    if: success()
    name: rts-build
    uses: ./.github/workflows/rts-build.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}

  test-appsmithctl:
    needs: [tag-check]
    if: success()
    name: appsmithctl
    uses: ./.github/workflows/appsmithctl.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}

  build-docker-image:
    needs: [tag-check, client-build, server-build, rts-build]
    # Only run if the build step is successful
    if: success()
    name: build-docker-image
    uses: ./.github/workflows/build-docker-image.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}

  ci-test:
    needs: [tag-check, build-docker-image]
    # Only run if the build step is successful
    if: success()
    name: ci-test
    uses: ./.github/workflows/ci-test-custom-script.yml
    secrets: inherit
    with:
      pr: ${{ github.event.client_payload.pull_request.number }}
      tags: ${{ needs.tag-check.outputs.tags }}
      matrix: ${{ needs.tag-check.outputs.matrix }}


  ci-test-result:
    needs: [ci-test]
    # Only run if the ci-test with matrices step is successful
    if: always()
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Dump the client payload context
        env:
          PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }}
        run: echo "$PAYLOAD_CONTEXT"

      - name: Setup node
        if: needs.ci-test.result != 'success'
        uses: actions/setup-node@v3
        with:
          node-version: 18

      - name: install pg
        if: needs.ci-test.result != 'success'
        run: npm install pg

      - name: Fetch the failed specs
        if: needs.ci-test.result != 'success'
        id: failed_specs
        env:
          DB_HOST: ${{ secrets.CYPRESS_DB_HOST }}
          DB_NAME: ${{ secrets.CYPRESS_DB_NAME }}
          DB_USER: ${{ secrets.CYPRESS_DB_USER }}
          DB_PWD: ${{ secrets.CYPRESS_DB_PWD }}
          RUN_ID: ${{ github.run_id }}
          ATTEMPT_NUMBER: ${{ github.run_attempt }}
        uses: actions/github-script@v6
        with:
          script: |
            const { Pool } = require("pg");
            const { DB_HOST, DB_NAME, DB_USER, DB_PWD, RUN_ID, ATTEMPT_NUMBER } = process.env

            const client = await new Pool({
              user: DB_USER,
              host: DB_HOST,
              database: DB_NAME,
              password: DB_PWD,
              port: 5432,
              connectionTimeoutMillis: 60000,
            }).connect();

            const result = await client.query(
              `SELECT DISTINCT name FROM public."specs"
                WHERE "matrixId" IN
                (SELECT id FROM public."matrix"
                WHERE "attemptId" = (
                  SELECT id FROM public."attempt" WHERE "workflowId" = $1 and "attempt" = $2
                  )
                ) AND status = 'fail'`,
              [RUN_ID, ATTEMPT_NUMBER],
            );
            client.release();
            return result.rows.map((spec) => spec.name);

      # In case for any ci job failure, create combined failed spec
      - name: combine all specs for CI
        id: combine_ci
        if: needs.ci-test.result != 'success'
        run: |
          failed_specs=$(echo ${{steps.failed_specs.outputs.result}} | sed 's/\[\|\]//g' | tr -d ' ' | tr ',' '\n')
          while read -r line; do
            echo "$line" >> ~/combined_failed_spec_ci
          done <<< "$failed_specs"
          if [[ -z $(grep '[^[:space:]]' ~/combined_failed_spec_ci) ]] ; then
            echo "specs_failed=0" >> $GITHUB_OUTPUT
          else
            echo "specs_failed=1" >> $GITHUB_OUTPUT
          fi

      # Upload combined failed CI spec list to a file
      # This is done for debugging.
      - name: upload combined failed spec
        if: needs.ci-test.result != 'success'
        uses: actions/upload-artifact@v3
        with:
          name: combined_failed_spec_ci
          path: ~/combined_failed_spec_ci

      - name: Get Latest flaky Tests
        shell: bash
        run: |
          curl --request POST --url https://yatin-s-workspace-jk8ru5.us-east-1.xata.sh/db/CypressKnownFailures:main/tables/CypressKnownFailuires/query --header 'Authorization: Bearer ${{ secrets.XATA_TOKEN }}' --header 'Content-Type: application/json'|jq -r |grep Spec|cut -d ':' -f 2 2> /dev/null|sed 's/"//g'|sed 's/,//g' >  ~/knownfailures

      # Verify CI test failures against known failures
      - name: Verify CI test failures against known failures
        if: needs.ci-test.result != 'success'
        shell: bash
        run: |
          new_failed_spec_env="<ol>$(comm -1 -3 <(sort ~/knownfailures) <(sort -u ~/combined_failed_spec_ci) | sed 's/|cypress|cypress/\n/g' | sed 's/^/<li>/')</ol>"
          echo "$new_failed_spec_env"
          echo "new_failed_spec_env<<EOF" >> $GITHUB_ENV
          echo "$new_failed_spec_env" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Add a comment on the PR with new CI failures
        if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '1'
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.client_payload.pull_request.number }}
          body: |
            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
            Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
            Cypress dashboard: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank"> Click here!</a>
            The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}}
            To know the list of identified flaky tests - <a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">Refer here</a>

      - name: Add a comment on the PR when ci-test is failed but no specs found
        if: needs.ci-test.result != 'success' && steps.combine_ci.outputs.specs_failed == '0'
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.client_payload.pull_request.number }}
          body: |
            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
            Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
            Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a>
            It seems like there are some failures 😔. We are not able to recognize it, please check this manually <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">here.</a>

      - name: Add a comment on the PR when ci-test is success
        if: needs.ci-test.result == 'success' && steps.combine_ci.outputs.specs_failed == '0'
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.client_payload.pull_request.number }}
          body: |
            Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>.
            Commit: `${{ github.event.client_payload.slash_command.args.named.sha }}`.
            Cypress dashboard url: <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Click here!</a>
            All cypress tests have passed 🎉🎉🎉

      # Update check run called "ci-test-result"
      - name: Mark ci-test-result job as complete
        uses: actions/github-script@v6
        id: update-check-run
        if: ${{ always() }}
        env:
          run_id: ${{ github.run_id }}
          repository: ${{ github.repository }}
          number: ${{ github.event.client_payload.pull_request.number }}
          job: ${{ github.job }}
          # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
          conclusion: ${{ job.status }}
          matrix_result: ${{ toJson(needs.ci-test) }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { data: pull } = await github.rest.pulls.get({
              ...context.repo,
              pull_number: process.env.number
            });
            const ref = pull.head.sha;

            const { data: checks } = await github.rest.checks.listForRef({
              ...context.repo,
              ref
            });

            const check = checks.check_runs.filter(c => c.name === process.env.job);

            if(check.length == 0) {
              try {
                console.log("Going to create the check run: ci-test-result");
                const head_sha = pull.head.sha;
                const { data: completed_at } = await github.rest.checks.create({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  head_sha: head_sha,
                  name: process.env.job,
                  status: 'completed',
                  conclusion: JSON.parse(process.env.matrix_result).result,
                  output: {
                    title: "Integration tests result for ok to test",
                    summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
                  }
                });
                console.log({ completed_at });
                return completed_at;  
              } catch(e) {
                console.error("Error while creating the check run: ci-test-result");
                console.error({ error: e.message });
              }
            } else {
              try {
                console.log("Going to update the check run ci-test-result with id: " + check[0].id);
                const { data: result } = await github.rest.checks.update({
                  ...context.repo,
                  check_run_id: check[0].id,
                  status: 'completed',
                  conclusion: JSON.parse(process.env.matrix_result).result,
                    output: {
                      title: "Integration tests result for ok to test",
                      summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
                    }
                });
                console.log({ result });
                return result;
              } catch(e) {
                console.error("Error while updating the check run: ci-test-result for check_id: " + check[0].id);
                console.error({ error: e.message });
              }
            }

      - name: Dump the client payload context
        env:
          PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }}
        run: echo "$PAYLOAD_CONTEXT"

      - name: Check ci-test set status
        if: needs.ci-test.result != 'success'
        run: exit 1

  package:
    needs: [ci-test]
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: app/client
    # Run this job only if all the previous steps are a success and the reference if the release or master branch
    if: success() && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/master')

    steps:
      # Update check run called "package"
      - name: Mark package job as complete
        uses: actions/github-script@v6
        id: update-check-run
        if: ${{ always() }}
        env:
          run_id: ${{ github.run_id }}
          repository: ${{ github.repository }}
          number: ${{ github.event.client_payload.pull_request.number }}
          job: ${{ github.job }}
          # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run
          conclusion: ${{ job.status }}
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const { data: pull } = await github.rest.pulls.get({
              ...context.repo,
              pull_number: process.env.number
            });
            const ref = pull.head.sha;

            const { data: checks } = await github.rest.checks.listForRef({
              ...context.repo,
              ref
            });

            const check = checks.check_runs.filter(c => c.name === process.env.job);

            if(check.length == 0) {
              const head_sha = pull.head.sha;
              const { data: completed_at } = await github.rest.checks.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                head_sha: head_sha,
                name: process.env.job,
                status: 'completed',
                conclusion: process.env.conclusion,
                output: {
                  title: "Package result for ok to test",
                  summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
                }
              });

              return completed_at;
            } else {
              const { data: result } = await github.rest.checks.update({
              ...context.repo,
              check_run_id: check[0].id,
              status: 'completed',
              conclusion: process.env.conclusion,
                output: {
                  title: "Package result for ok to test",
                  summary: "https://github.com/" + process.env.repository + "/actions/runs/" + process.env.run_id
                }
            });

            return result;
            }