ci-post-land.yml

name: ci-post-land

on:
  create:
    branches: [main, release-*]
  push:
    branches: [main, release-*] # gha-test-*

defaults:
  run:
    shell: bash

env:
  max_threads: 16
  pre_command: cd /opt/git/diem/

jobs:
  prepare:
    runs-on: ubuntu-latest
    outputs:
      changes-target-branch: ${{ steps.changes.outputs.changes-target-branch }}
      changes-base-git-rev: ${{ steps.changes.outputs.changes-base-git-rev }}
      changes-base-githash: ${{ steps.changes.outputs.changes-base-githash }}
      changes-pull-request-number: ${{ steps.changes.outputs.changes-pull-request-number }}
      rust-changes: ${{ steps.rust-changes.outputs.changes-found }}
      base-image-changes: ${{ steps.base-image-changes.outputs.changes-found }}
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Git Hooks and Checks
        run: ./scripts/git-checks.sh
      - id: changes
        name: determine changes
        uses: diem/actions/changes@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          workflow-file: ci-post-land.yml
          github-token: ${{secrets.GITHUB_TOKEN}}
      - id: rust-changes
        name: find rust/cargo changes.
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '^documentation\|^docker\|^scripts\|^developers.diem.com'
          invert: "true"
      - id: base-image-changes
        name: find dev-setup.sh/base docker image changes
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '.github/workflows/ci-post-land.yml\|.github/actions/dockerhub_login/action.yml\|docker/ci/github/Dockerfile\|scripts/dev_setup.sh\|rust-toolchain'

  # NOTE: update-sscache-osx removed as cost savings effort. It strongly resembled update-sccache-ubuntu
  # except for a 'runs-on: macos-11', and no container.  Restore when there are more osx contributors.

  update-sccache-ubuntu:
    needs: prepare
    if: ${{ needs.prepare.outputs.rust-changes == 'true' }}
    environment:
      name: Sccache
    runs-on: ubuntu-latest-xl
    container:
      image: diem/build_environment:${{ needs.prepare.outputs.changes-target-branch }}
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
      - uses: actions/cache@v2.1.6
        with:
          path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache"
          key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
          restore-keys: "crates-${{ runner.os }}"
      - name: build all unit test code.
        run: |
          $pre_command && cargo x test --no-run --jobs ${max_threads} --unit
        env:
          TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }}
          SCCACHE_AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          SCCACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
      - uses: ./.github/actions/build-teardown

  # In the future when the script can differentiate between target branches for each auto run then we'll
  # update this report to differentiate branches.
  unit-test-allure-report:
    name: Unit Test Reports
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/main' }}
    environment:
      name: Sccache
    container:
      image: diem/build_environment:main
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
        # build setup is called to ensure the latest docker image tooling is in place and the ~/.profile is called.
      - name: Publish unit test results
        uses: diem/actions/process-test-results@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          #used to read artifacts, annoying to need one.
          token: ${{ secrets.GITHUB_TOKEN }}
          artifacts: "unit-test-results codegen-unit-test-results"
          workflow_file: ci-test.yml
          branch: auto
          history: 20
          recreate: false
          allure_configuration: ./allure-configuration
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.ENV_DIEM_S3_AWS_REGION }}

  build_ci_base_docker_image:
    needs: prepare
    runs-on: ubuntu-latest-xl
    if: ${{ needs.prepare.outputs.base-image-changes == 'true' || github.event_name == 'create' }}
    continue-on-error: false
    env:
      DOCKERHUB_ORG: diem
    environment:
      name: Docker
      url: https://hub.docker.com/u/diem
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Git Hooks and Checks
        run: ./scripts/git-checks.sh
      - name: build image
        run: docker build -f docker/ci/github/Dockerfile -t ${{ env.DOCKERHUB_ORG }}/build_environment:${{ needs.prepare.outputs.changes-target-branch }} .
      - name: Sign in to dockerhub, install image signing cert.
        uses: ./.github/actions/dockerhub_login
        with:
          username: ${{ secrets.ENV_DOCKERHUB_USERNAME }}
          password: ${{ secrets.ENV_DOCKERHUB_PASSWORD }}
          key_material: ${{ secrets.ENV_DOCKERHUB_KEY_MATERIAL }}
          key_name: ${{ secrets.ENV_DOCKERHUB_KEY_NAME }}
          key_password: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}
      - name: Push to dockerhub.
        run: |
          if [[ "$DOCKERHUB_LOGGED_IN" == true ]]; then
            disable_content_trust=true
            if [[ "$DOCKERHUB_CAN_SIGN" == true ]]; then
              disable_content_trust=false
            fi
            docker push --disable-content-trust=${disable_content_trust} ${{ env.DOCKERHUB_ORG }}/build_environment:${{ needs.prepare.outputs.changes-target-branch }}
          fi
        env:
          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}

  run-cluster-test-pre-release-suite:
    needs: prepare
    name: Run the pre-release suite of Cluster Test
    runs-on: self-hosted
    if: ${{  startsWith(needs.prepare.outputs.changes-target-branch, 'release') }}
    # The pre-release suite run time varies 1~1.5 hr.
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 0 #get all the history!!!
      - name: set_env
        id: set_env
        run: |
          HEAD_GIT_REV=$(git rev-parse --short=8 HEAD)
          echo "HEAD_GIT_REV=$HEAD_GIT_REV" >> $GITHUB_ENV
          IMAGE_TAG=$(echo ${GITHUB_REF#refs/heads/})_$HEAD_GIT_REV
          echo $IMAGE_TAG
          echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV
      - name: poll_images
        # Poll until images are ready
        env:
          AWS_REGION: us-west-2
          RETRIES: 30
        run: |
          set +e
          retry=0
          status=1
          while [[ $status != 0 && $retry -lt $RETRIES ]]; do
            status=0
            for image in diem/validator diem/validator_tcb diem/init diem/cluster_test; do
              aws ecr describe-images --region $AWS_REGION --repository-name $image --image-ids=imageTag=$IMAGE_TAG
              status=$((status + $?))
            done
            retry=$((retry + 1))
            if [[ $status != 0 ]] ; then
              echo "CI has not pushed all images to ECR."
              echo "Wait $((1*$retry)) of $((1*$RETRIES)) minutes before retry."
              sleep 1m
            fi
          done
          exit $status
      - name: Run Cluster Test
        run: |
          date
          BASE_GIT_REV=$(git rev-parse $HEAD_GIT_REV^)
          ./scripts/cti --tag $IMAGE_TAG --timeout-secs 7200 \
            --env SLACK_CHANGELOG_URL=${{ secrets.WEBHOOK_CHANGELOG }} \
            --changelog $BASE_GIT_REV $HEAD_GIT_REV \
            --suite pre_release
      - name: Push alert
        if: ${{ failure() }}
        run: |
          jq -n \
            --arg msg "*${{ github.job }}* job in ${{ github.workflow }} workflow failed with $IMAGE_TAG." \
            --arg url "https://github.com/${{ github.repository }}/actions/runs/${{github.run_id}}" \
            '{
              "attachments": [
                {
                  "text": $msg,
                  "actions": [
                    {
                      "type": "button",
                      "text": "Visit Job",
                      "url": $url
                    }
                  ]
                }
              ]
            }' > /tmp/payload
          curl -X POST -H 'Content-type: application/json' -d @/tmp/payload ${{ secrets.WEBHOOK_PUSH }}

  build-release-images:
    needs: prepare
    runs-on: ubuntu-latest-xl
    continue-on-error: false
    env:
      TAG: github-1
      CHANGES_TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }}
    environment:
      name: Docker
      url: https://hub.docker.com/u/diem
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          fetch-depth: 0 #get all the history!!!
      - name: setup_aws_ecr_login
        run: |
          echo 'AWS_ECR_ACCOUNT_URL=${{ secrets.ENV_NOVI_ECR_AWS_ACCOUNT_NUM }}.dkr.ecr.${{ secrets.ENV_NOVI_ECR_AWS_REGION }}.amazonaws.com' >> $GITHUB_ENV
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.ENV_NOVI_ECR_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.ENV_NOVI_ECR_AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.ENV_NOVI_ECR_AWS_REGION }}
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1.3.3
      - name: Sign in to dockerhub, install image signing cert.
        uses: ./.github/actions/dockerhub_login
        with:
          username: ${{ secrets.ENV_DOCKERHUB_USERNAME }}
          password: ${{ secrets.ENV_DOCKERHUB_PASSWORD }}
          key_material: ${{ secrets.ENV_DOCKERHUB_KEY_MATERIAL }}
          key_name: ${{ secrets.ENV_DOCKERHUB_KEY_NAME }}
          key_password: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}
      - name: should pre build docker images (targeting a release branch)?
        if: ${{ github.ref == 'refs/heads/auto' }}
        run: |
          if  [[ ! "$CHANGES_TARGET_BRANCH" =~ "^release-[0-9|.]+$" ]] && [[ ! "$CHANGES_TARGET_BRANCH" =~ "^gha-test-[0-9|.]+$" ]] ; then
            echo Targeting branch $CHANGES_TARGET_BRANCH will not pre-publish docker images.
          fi
      - name: pre-release docker images
        if: ${{ github.ref == 'refs/heads/auto' }}
        run: |
          BRANCH="$CHANGES_TARGET_BRANCH"
          success=0
          tmpfile=$(mktemp)
          echo "Failed to push:" > "${tmpfile}"
          docker/build_push.sh -u -p -b ${BRANCH} -n client || success=$(echo "client" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n init || success=$(echo "init" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n faucet || success=$(echo "faucet" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n tools || success=$(echo "tools" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n validator || success=$(echo "validator" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n validator-tcb || success=$(echo "validator-tcb" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -p -b ${BRANCH} -n cluster-test || success=$(echo "cluster-test" >> "${tmpfile}"; echo 1)
          if [[ "$success" == "1" ]]; then
            cat "${tmpfile}"
          fi
          exit $success
        env:
          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}
      - name: pull pre images (or build if not pullable) and push release docker images if not on auto branch.
        if: ${{ github.ref != 'refs/heads/auto' }}
        run: |
          set -x
          BRANCH=$(echo "$GITHUB_REF" | sed 's|.*/||' )
          success=0
          tmpfile=$(mktemp)
          echo "Failed to push:" >> "${tmpfile}"
          docker/build_push.sh -u -b ${BRANCH} -n client || success=$(echo "client" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n init ||  success=$(echo "init" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n faucet ||  success=$(echo "faucet" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n tools ||  success=$(echo "tools" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n validator || success=$(echo "validator" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n validator-tcb || success=$(echo "validator-tcb" >> "${tmpfile}"; echo 1)
          docker/build_push.sh -u -b ${BRANCH} -n cluster-test || success=$(echo "cluster-test" >> "${tmpfile}"; echo 1)
          if [[ "$success" == "1" ]]; then
            cat "${tmpfile}"
          fi
          exit $success
        env:
          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}
      - name: push to novi ecr
        if: ${{ always() && github.ref != 'refs/heads/auto' }}
        run: |
          #push to novi ecr with standard names
          BRANCH=$(echo "$GITHUB_REF" | sed 's|.*/||' )
          GIT_REV=$(git rev-parse --short=8 HEAD)
          aws ecr get-login-password --region ${{ secrets.ENV_NOVI_ECR_AWS_REGION }} | \
          docker login --username AWS --password-stdin "${AWS_ECR_ACCOUNT_URL}"
          docker/docker_republish.sh -t ${BRANCH}_${GIT_REV} -r ${AWS_ECR_ACCOUNT_URL} -d

  rustdoc:
    needs: prepare
    if: ${{ needs.prepare.outputs.changes-target-branch == 'main' && needs.prepare.outputs.rust-changes == 'true'}}
    runs-on: ubuntu-latest
    container:
      image: diem/build_environment:${{ needs.prepare.outputs.changes-target-branch }}
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2.3.4
      - uses: ./.github/actions/build-setup
      - uses: actions/cache@v2.1.6
        with:
          path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache"
          key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
          restore-keys: "crates-${{ runner.os }}"
      - name: Build Documentation
        # Build the rust crate docs
        # Use `RUSTC_BOOTSTRAP` in order to use the `--enable-index-page` flag of rustdoc
        # This is needed in order to generate a landing page `index.html` for workspaces
        run: cargo doc --no-deps --workspace --lib
        env:
          RUSTC_BOOTSTRAP: 1
          RUSTDOCFLAGS: "-Z unstable-options --enable-index-page"
          CARGO_INCREMENTAL: 0
          RUSTFLAGS: -D warnings
      - name: Deploy Docs
        uses: peaceiris/actions-gh-pages@v3.8.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: gh-pages
          publish_dir: ./target/doc
          force_orphan: true