ci(workflow): upload daily next.js test trace (#5254)

kwonoj · web-flow · commit b0649f66d688 · 2023-06-22T10:20:19.000-07:00
### Description WEB-1194. This PR revises daily test workflow, and upload traces to datadog. It skips one test affected by #5294, once resolved will be reenabled. Signed-off-by: OJ Kwon <1210596+kwonoj@users.noreply.github.com>
diff --git a/.github/workflows/daily-nextjs-integration-test.yml b/.github/workflows/daily-nextjs-integration-test.yml
@@ -24,6 +24,7 @@ jobs:
     permissions:
       pull-requests: write
     uses: ./.github/workflows/nextjs-integration-test.yml
+    secrets: inherit
     with:
       diff_base: "none"
       version: ${{ inputs.version || 'canary' }}
diff --git a/.github/workflows/nextjs-integration-test.yml b/.github/workflows/nextjs-integration-test.yml
@@ -28,16 +28,19 @@ env:
   # Enabling backtrace will makes snapshot tests fail
   RUST_BACKTRACE: 0
   NEXT_TELEMETRY_DISABLED: 1
-  # Path to the next-dev binary located in **docker container** image.
-  # To use this, set __INTERNAL_CUSTOM_TURBOPACK_BINARY to the path. Not being used currently.
-  NEXT_DEV_BIN: /work/next-dev
-  # Path to the custom next-swc bindings located in **docker container** image.
-  NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
-  FAILED_TEST_LIST_PATH: /work/integration-test-data/test-results/main/failed-test-path-list.json
-  # Glob pattern to run specific tests with --turbo.
-  NEXT_DEV_TEST_GLOB: "*"
+  TEST_CONCURRENCY: 6
   # pnpm version should match to what upstream next.js uses
   PNPM_VERSION: 7.24.3
+  DATADOG_API_KEY: ${{ secrets.DD_KEY_TURBOPACK }}
+  DATADOG_TRACE_NEXTJS_TEST: 'true'
+  DD_ENV: 'ci'
+  # Turbopack specific customization for the test runner
+  __INTERNAL_NEXT_DEV_TEST_TURBO_DEV: TRUE
+  __INTERNAL_CUSTOM_TURBOPACK_BINDINGS: ${{ github.workspace }}/packages/next-swc/native/next-swc.linux-x64-gnu.node
+  NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ github.workspace }}/integration-test-data/test-results/main/failed-test-path-list.json
+  NEXT_TEST_CONTINUE_ON_ERROR: TRUE
+  NEXT_E2E_TEST_TIMEOUT: 240000
+  NEXT_TEST_JOB: 1
 
 jobs:
   # First, build next-dev and Next.js both to execute across tests.
@@ -49,50 +52,57 @@ jobs:
 
   # Actual test scheduling. These jobs mimic the same jobs in Next.js repo,
   # which we do allow some of duplications to make it easier to update if upstream changes.
-  # Refer build_test_deploy.yml in the Next.js repo for more details.
-  test_dev:
+  # Refer build_and_test.yml in the Next.js repo for more details.
+  test-dev:
     # This job name is being used in github action to collect test results. Do not change it, or should update
     # ./.github/actions/next-integration-stat to match the new name.
     name: Next.js integration test (Development)
-    runs-on: ubuntu-latest-8-core-oss
+    timeout-minutes: 180
+    runs-on: ubuntu-latest-16-core-oss
     needs: [setup_nextjs]
     strategy:
+      fail-fast: false
       matrix:
-        node: [16, 18]
-        group: [1, 2, 3, 4, 5]
+        group: [1, 2, 3]
 
     steps:
       - uses: actions/cache/restore@v3
         id: restore-build
         with:
           path: ./*
-          key: ${{ inputs.version }}-${{ github.sha }}
+          key: ${{ inputs.version }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt}}-${{ github.run_number }}
           fail-on-cache-miss: true
-
-      # This step is quite same as upstream next.js integration execution, but with some differences:
-      # Sets `__INTERNAL_CUSTOM_TURBOPACK_BINARY`, `__INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH` to setup custom turbopack.
-      # Sets `NEXT_TEST_SKIP_RETRY_MANIFEST`, `NEXT_TEST_CONTINUE_ON_ERROR` to continue on error but do not retry on the known failed tests.
-      # Do not set --timings flag
-      - run: |
-          docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_SKIP_RETRY_MANIFEST=${FAILED_TEST_LIST_PATH} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev xvfb-run node run-tests.js --type development -g ${{ matrix.group }}/5 -c 1 >> /proc/1/fd/1"
-        name: Run test/development
+      - name: Run test/development
+        run: |
+          ls $NEXT_TEST_SKIP_RETRY_MANIFEST
+          npm i -g pnpm@$PNPM_VERSION && pnpm --version
+          npm i -g yarn
+          sudo npx playwright install-deps && pnpm playwright install
+          NEXT_TEST_MODE=dev node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --test-pattern '^(development|e2e|unit)/.*\.test\.(js|jsx|ts|tsx)$'
+          ls test
+          ls test/turbopack-test-junit-report
         # It is currently expected to fail some of next.js integration test, do not fail CI check.
         continue-on-error: true
         env:
-          RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDev / Group ${{ matrix.group }}
           # marker to parse log output, do not delete / change.
           NEXT_INTEGRATION_TEST: true
+      - name: Upload test reports artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test trace reports
+          path: |
+            test/turbopack-test-junit-report
+          if-no-files-found: ignore
 
-  test_dev_e2e:
-    # This job name is being used in github action to collect test results. Do not change it, or should update
-    # ./.github/actions/next-integration-stat to match the new name.
-    name: Next.js integration test (e2e/Development)
-    runs-on: ubuntu-latest-8-core-oss
+  test-prod:
+    name: Next.js integration test (Production)
     needs: [setup_nextjs]
+    runs-on: ubuntu-latest-16-core-oss
+    timeout-minutes: 180
     strategy:
+      fail-fast: false
       matrix:
-        node: [16, 18]
-        group: [1, 2, 3, 4, 5, 6, 7]
+        group: [1, 2, 3]
 
     steps:
       - uses: actions/cache/restore@v3
@@ -101,121 +111,32 @@ jobs:
           path: ./*
           key: ${{ inputs.version }}-${{ github.sha }}
           fail-on-cache-miss: true
-
-      - run: |
-          docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_SKIP_RETRY_MANIFEST=${FAILED_TEST_LIST_PATH} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev xvfb-run node run-tests.js --type e2e -g ${{ matrix.group }}/7 -c 1 >> /proc/1/fd/1"
-        name: Run test/e2e (dev)
+      - name: Run test/production
+        run: |
+          npm i -g pnpm@$PNPM_VERSION && pnpm --version
+          npm i -g yarn
+          sudo npx playwright install-deps && pnpm playwright install
+          NEXT_TEST_MODE=start node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --test-pattern '^(production|e2e)/.*\.test\.(js|jsx|ts|tsx)$'
         continue-on-error: true
         env:
-          RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDevE2E / Group ${{ matrix.group }} / Node ${{ matrix.node }}
-          NEXT_TEST_MODE: dev
-          RECORD_REPLAY_TEST_METRICS: 1
           NEXT_INTEGRATION_TEST: true
-
-  test_cna:
-    # This job name is being used in github action to collect test results. Do not change it, or should update
-    # ./.github/actions/next-integration-stat to match the new name.
-    name: Next.js integration test (Create Next App) (No group)
-    runs-on: ubuntu-latest-8-core-oss
-    needs: [setup_nextjs]
-    steps:
-      - uses: actions/cache/restore@v3
-        id: restore-build
-        with:
-          path: ./*
-          key: ${{ inputs.version }}-${{ github.sha }}
-          fail-on-cache-miss: true
-
-      # TODO: This test currently seems to load wasm/swc and does not load the next-dev binary.
-      # Temporary disabled until figure out details.
-      #- run: |
-      #    docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v16 | FORCE=1 bash && npm i -g pnpm@${PNPM_VERSION} > /dev/null && __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_SKIP_RETRY_MANIFEST=${FAILED_TEST_LIST_PATH} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_CNA=1 xvfb-run node run-tests.js test/integration/create-next-app/index.test.ts test/integration/create-next-app/templates.test.ts -c 1 >> /proc/1/fd/1"
-      #  name: Run test/e2e (create-next-app)
-      #  continue-on-error: true
-      #  env:
-      #    RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDevE2E / Group ${{ matrix.group }} / Node ${{ matrix.node }}
-      #    NEXT_TEST_MODE: dev
-      #    RECORD_REPLAY_TEST_METRICS: 1
-      #    NEXT_INTEGRATION_TEST: true
-
-  test_integration:
-    # This job name is being used in github action to collect test results. Do not change it, or should update
-    # ./.github/actions/next-integration-stat to match the new name.
-    name: Next.js integration test (integration)
-    runs-on: ubuntu-latest-8-core-oss
-    needs: [setup_nextjs]
-    strategy:
-      fail-fast: false
-      matrix:
-        group:
-          [
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12,
-            13,
-            14,
-            15,
-            16,
-            17,
-            18,
-            19,
-            20,
-            21,
-            22,
-            23,
-            24,
-            25,
-            26,
-            27,
-            28,
-          ]
-
-    steps:
-      - uses: actions/cache/restore@v3
-        id: restore-build
+      - name: Upload test reports artifact
+        uses: actions/upload-artifact@v3
         with:
+          name: Test trace reports
           path: |
-            ./*
-          key: ${{ inputs.version }}-${{ github.sha }}
-          fail-on-cache-miss: true
+            test/turbopack-test-junit-report
+          if-no-files-found: ignore
 
-      - run: |
-          docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v16 | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_SKIP_RETRY_MANIFEST=${FAILED_TEST_LIST_PATH} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 xvfb-run node run-tests.js -g ${{ matrix.group }}/28 -c 1 >> /proc/1/fd/1"
-        name: Test Integration
-        continue-on-error: true
-        env:
-          RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testIntegration / Group ${{ matrix.group }}
-          NEXT_INTEGRATION_TEST: true
-
-  test_flaky_subset:
-    # This job name is being used in github action to collect test results. Do not change it, or should update
-    # ./.github/actions/next-integration-stat to match the new name.
-    name: Next.js integration test (FLAKY_SUBSET)
-    runs-on: ubuntu-latest
+  test-integration:
+    name: Next.js integration test (Integration)
     needs: [setup_nextjs]
-    env:
-      # List of test files to run that expected to pass always. Whole test suite should pass.
-      TEST_FILES_LIST: |
-        test/development/acceptance-app/dynamic-error.test.ts \
-        test/development/acceptance-app/unsupported-app-features.test.ts \
-        test/development/acceptance-app/ReactRefresh.test.ts \
-        test/development/basic/legacy-decorators.test.ts \
-        test/integration/plugin-mdx-rs/test/index.test.js \
-        test/e2e/app-dir/underscore-ignore-app-paths/underscore-ignore-app-paths.test.ts \
-        test/e2e/app-dir/app-alias/app-alias.test.ts
-
+    runs-on: ubuntu-latest-16-core-oss
+    timeout-minutes: 180
     strategy:
+      fail-fast: false
       matrix:
-        node: [16]
+        group: [1, 2, 3, 4, 5, 6]
 
     steps:
       - uses: actions/cache/restore@v3
@@ -224,21 +145,27 @@ jobs:
           path: ./*
           key: ${{ inputs.version }}-${{ github.sha }}
           fail-on-cache-miss: true
-
-      - run: |
-          docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && ls packages/next-swc/native && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && node scripts/install-native.mjs && __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_TEST_CONTINUE_ON_ERROR=TRUE NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev xvfb-run node run-tests.js --type development -c 1 $TEST_FILES_LIST >> /proc/1/fd/1"
-        name: Run test/flaky_subset
-        # This should not fail, but if fails collect result to update dashboard.
+      - name: Run test/integration
+        run: |
+          npm i -g pnpm@$PNPM_VERSION && pnpm --version
+          npm i -g yarn
+          sudo npx playwright install-deps && pnpm playwright install
+          node run-tests.js -g ${{ matrix.group }}/6 -c ${TEST_CONCURRENCY} --test-pattern '^(integration)/.*\.test\.(js|jsx|ts|tsx)$'
         continue-on-error: true
         env:
-          # marker to parse log output, do not delete / change.
           NEXT_INTEGRATION_TEST: true
+      - name: Upload test reports artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test trace reports
+          path: |
+            test/turbopack-test-junit-report
+          if-no-files-found: ignore
 
   # Collect integration test results from execute_tests,
   # Store it as github artifact for next step to consume.
   collect_nextjs_integration_stat:
-    needs:
-      [test_dev, test_dev_e2e, test_cna, test_integration, test_flaky_subset]
+    needs: [test-dev, test-prod, test-integration]
     name: Next.js integration test status report
     runs-on: ubuntu-latest-8-core-oss
     permissions:
@@ -263,3 +190,26 @@ jobs:
             nextjs-test-results.json
             failed-test-path-list.json
             slack-payload.json
+
+  upload_test_trace:
+    needs: [test-dev, test-prod, test-integration]
+    name: Upload test trace to datadog
+    runs-on: ubuntu-latest-8-core-oss
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Download test reports artifact
+        id: download-test-reports
+        uses: actions/download-artifact@v3
+        continue-on-error: true
+        with:
+          name: Test trace reports
+          path: test
+      - name: Upload test trace to datadog
+        continue-on-error: true
+        run: |
+          npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0
+          jrm ./nextjs-test-result-junit.xml "test/**/*.xml"
+          ls -al ./*.xml
+          # We'll tag this to the Turbopack service, not the next.js
+          DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack.daily --service Turbopack ./nextjs-test-result-junit.xml
diff --git a/.github/workflows/setup-nextjs-build.yml b/.github/workflows/setup-nextjs-build.yml
@@ -121,6 +121,10 @@ jobs:
           # This is being used in github action to collect test results. Do not change it, or should update ./.github/actions/next-integration-test to match.
           docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c 'curl https://install-node.vercel.app/v16 | FORCE=1 bash && cd /work && echo RUNNING NEXTJS VERSION: $(packages/next/dist/bin/next --version) && ls -al packages/next-swc/native && node -e "console.log(\"Container\", require(\"os\").arch(), require(\"os\").platform())"'
 
+      - name: Temporary test skip
+        run: |
+          rm -rf test/integration/jsconfig-paths/test/index.test.js
+
       # Once build completes, creates a cache of the build output
       # so subsequent job to actually execute tests can reuse it.
       # Note that we do not use upload / download artifacts for this -
@@ -132,4 +136,4 @@ jobs:
         with:
           path: |
             ./*
-          key: ${{ inputs.version }}-${{ github.sha }}
+          key: ${{ inputs.version }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt}}-${{ github.run_number }}
