versioned-report.yml

# This workflow is a separate workflow from `report.yml` for regenerating the versioned report data needed for Bowtie's UI and `bowtie trend`.
# It retests all of Bowtie's supported implementations for all their multiple versions listed in `matrix-versions.json` file if one exists for them.
name: Collect New Versioned Test Results

on:
  workflow_dispatch:
    inputs:
      implementation:
        description: The name of an implementation.
        required: false
        type: string

jobs:
  list-implementations:
    runs-on: ubuntu-latest
    outputs:
      implementations: ${{ steps.implementations-matrix.outputs.implementations }}
    steps:
      - uses: actions/checkout@v4

      - name: Install Bowtie
        uses: ./

      - name: List all Bowtie Supported Implementations having a "matrix-versions.json" file
        id: implementations-matrix
        run: |
          implementation=${{ inputs.implementation }}
          IMPLEMENTATIONS=$(bowtie filter-implementations --format json)
          MATRIX="[]"
          if [ -n "$implementation" ]; then
            if echo "$IMPLEMENTATIONS" | jq -e --arg impl "$implementation" 'index($impl) != null' > /dev/null; then
              if [ -f "implementations/$implementation/matrix-versions.json" ]; then
                MATRIX=$(echo $MATRIX | jq --arg impl "$implementation" '. + [$impl]')
              else
                echo "No \`matrix-versions.json\` file found for implementation ('$implementation')."
                exit 1
              fi
            else
              echo "No such implementation ('$implementation') found. Please provide a correct implementation name."
              echo "To see a list of all Bowtie supported implementations, run \`bowtie filter-implementations\`."
              exit 1
            fi
          else
            for impl in $(echo "$IMPLEMENTATIONS" | jq -r '.[]'); do
              if [ -f "implementations/$impl/matrix-versions.json" ]; then
                MATRIX=$(echo $MATRIX | jq --arg impl "$impl" '. + [$impl]')
              fi
            done
          fi
          echo "implementations=$(echo $MATRIX | jq -c .)" >> $GITHUB_OUTPUT

  regenerate-versioned-reports:
    needs: list-implementations
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        implementation: ${{ fromJson(needs.list-implementations.outputs.implementations) }}

    steps:
      - uses: actions/checkout@v4

      - name: Install Bowtie
        uses: ./

        # just so that we don't get rate limited (429)
      - name: Checkout JSON Schema Test Suite
        uses: actions/checkout@v4
        with:
          repository: json-schema-org/JSON-Schema-Test-Suite
          path: json-schema-test-suite

      - name: Make a ${{ matrix.implementation }} directory to keep all its version reports
        run: mkdir ${{ matrix.implementation }}

      - name: Generate New Versioned Reports for all Supported Dialects of ${{ matrix.implementation }}
        id: generate-new-versioned-report
        run: |
          impl=${{ matrix.implementation }}
          MATRIX_VERSIONS_FILE="implementations/$impl/matrix-versions.json"
          cp $MATRIX_VERSIONS_FILE $impl
          for version in $(jq -r '.[]' $MATRIX_VERSIONS_FILE); do
            SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json)
            if [ -n "$SUPPORTED_DIALECTS" ]; then
              mkdir "$impl/v$version"
              for dialect in $SUPPORTED_DIALECTS; do
                bowtie suite -i image:$impl:$version ./json-schema-test-suite/tests/$dialect > "$impl/v$version/$dialect.json"
              done
            fi
          done

      # This is useful to debug whether Bowtie accidentally fetched some huge
      # number of container images.
      - name: Show what images we fetched
        run: docker images
        if: always()

      # FIXME: We ignore for now as now this exits unsuccessfully if there
      #        are *test* failures
      - name: Check that all Generated Versioned Reports of ${{ matrix.implementation }} are Valid
        run: |
          impl=${{ matrix.implementation }}
          MATRIX_VERSIONS=$(jq -r '.[]' "implementations/$impl/matrix-versions.json")
          for version in $MATRIX_VERSIONS; do
            SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json)
            if [ -n "$SUPPORTED_DIALECTS" ]; then
              for dialect in $SUPPORTED_DIALECTS; do
                bowtie summary --show failures "$impl/v$version/$dialect.json" --format markdown >> $GITHUB_STEP_SUMMARY
              done
            fi
          done
        continue-on-error: true

      - uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.implementation }}
          path: |
            ${{ matrix.implementation }}

  combine-all-versioned-reports:
    runs-on: ubuntu-latest
    needs: regenerate-versioned-reports
    steps:
      - name: Download all uploaded artifacts
        uses: actions/download-artifact@v4
        with:
          path: implementations

      - name: Download latest versioned test reports
        uses: dawidd6/action-download-artifact@v9
        with:
          workflow: versioned-report.yml
          branch: main
          name: implementations
          path: latest-implementations
        if: ${{ inputs.implementation != null }}

      - name: Prepare artifact for upload
        run: |
          rm -rf "latest-implementations/${{ inputs.implementation }}"
          cp -r "implementations/${{ inputs.implementation }}" "latest-implementations/"
        if: ${{ inputs.implementation != null }}

      - uses: actions/upload-artifact@v4
        with:
          name: implementations
          path: ${{ inputs.implementation != null && 'latest-implementations' || 'implementations' }}