diff --git a/.github/workflows/dependabot_changelog.yml b/.github/workflows/dependabot_changelog.yml new file mode 100644 index 000000000000..858acb18403b --- /dev/null +++ b/.github/workflows/dependabot_changelog.yml @@ -0,0 +1,52 @@ +name: Write changelog for dependabot PR +on: + # /!\ pull_request_target is a security concern as it grants a read-write GITHUB_TOKEN + # within this workflow. We mitigate the risk by: + # - ensuring this workflow does nothing when NOT triggered by dependabot + # - downscoping permissions to the bare minimum. + # See https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions + # and in particular the section "Responding to events" for advice. (That page + # recommends an untrusted/trusted workflow pair, but that is overkill for this + # workflow.) + pull_request_target: + types: + - opened + +permissions: + # The list of permission scopes is defined here: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions + # and here: + # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token + # The best reference I could find for their meanings is: + # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps + actions: none + checks: none + contents: write + deployments: none + id-token: none + issues: none + discussions: none + packages: none + pages: none + pull-requests: none + repository-projects: none + security-events: none + statuses: none + +jobs: + add-changelog: + runs-on: 'ubuntu-latest' + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + - name: Write, commit and push changelog + run: | + echo "${{ github.event.pull_request.title }}." > "changelog.d/${{ github.event.pull_request.number }}".docker + git add changelog.d + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "GitHub Actions" + git commit -m "Changelog" + git push + shell: bash diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml deleted file mode 100644 index 0708d631cd36..000000000000 --- a/.github/workflows/release-artifacts.yml +++ /dev/null @@ -1,180 +0,0 @@ -# GitHub actions workflow which builds the release artifacts. - -name: Build release artifacts - -on: - # we build on PRs and develop to (hopefully) get early warning - # of things breaking (but only build one set of debs) - pull_request: - push: - branches: ["develop", "release-*"] - - # we do the full build on tags. - tags: ["v*"] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: write - -jobs: - get-distros: - name: "Calculate list of debian distros" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - id: set-distros - run: | - # if we're running from a tag, get the full list of distros; otherwise just use debian:sid - dists='["debian:sid"]' - if [[ $GITHUB_REF == refs/tags/* ]]; then - dists=$(scripts-dev/build_debian_packages.py --show-dists-json) - fi - echo "::set-output name=distros::$dists" - # map the step outputs to job outputs - outputs: - distros: ${{ steps.set-distros.outputs.distros }} - - # now build the packages with a matrix build. - build-debs: - needs: get-distros - name: "Build .deb packages" - runs-on: ubuntu-latest - strategy: - matrix: - distro: ${{ fromJson(needs.get-distros.outputs.distros) }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - path: src - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - with: - install: true - - - name: Set up docker layer caching - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Set up python - uses: actions/setup-python@v2 - - - name: Build the packages - # see https://github.com/docker/build-push-action/issues/252 - # for the cache magic here - run: | - ./src/scripts-dev/build_debian_packages.py \ - --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \ - --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \ - --docker-build-arg=--progress=plain \ - --docker-build-arg=--load \ - "${{ matrix.distro }}" - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - - name: Upload debs as artifacts - uses: actions/upload-artifact@v2 - with: - name: debs - path: debs/* - - build-wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, macos-10.15] - is_pr: - - ${{ startsWith(github.ref, 'refs/pull/') }} - - exclude: - # Don't build macos wheels on PR CI. - - is_pr: true - os: "macos-10.15" - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v3 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0 - - # Only build a single wheel in CI. - - name: Set env vars. - run: | - echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/pull/') - - - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse - env: - # Skip testing for platforms which various libraries don't have wheels - # for, and so need extra build deps. - CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx* - - - uses: actions/upload-artifact@v3 - with: - name: Wheel - path: ./wheelhouse/*.whl - - build-sdist: - name: Build sdist - runs-on: ubuntu-latest - if: ${{ !startsWith(github.ref, 'refs/pull/') }} - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - run: pip install build - - - name: Build sdist - run: python -m build --sdist - - - uses: actions/upload-artifact@v2 - with: - name: Sdist - path: dist/*.tar.gz - - - # if it's a tag, create a release and attach the artifacts to it - attach-assets: - name: "Attach assets to release" - if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} - needs: - - build-debs - - build-wheels - - build-sdist - runs-on: ubuntu-latest - steps: - - name: Download all workflow run artifacts - uses: actions/download-artifact@v2 - - name: Build a tarball for the debs - run: tar -cvJf debs.tar.xz debs - - name: Attach to release - uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a # PR#109 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - files: | - Sdist/* - Wheel/* - debs.tar.xz - # if it's not already published, keep the release as a draft. - draft: true - # mark it as a prerelease if the tag contains 'rc'. - prerelease: ${{ contains(github.ref, 'rc') }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 9fe61930a55a..000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,508 +0,0 @@ -name: Tests - -on: - push: - branches: ["develop", "release-*"] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Job to detect what has changed so we don't run e.g. Rust checks on PRs that - # don't modify Rust code. - changes: - runs-on: ubuntu-latest - outputs: - rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} - steps: - - uses: dorny/paths-filter@v2 - id: filter - # We only check on PRs - if: startsWith(github.ref, 'refs/pull/') - with: - filters: | - rust: - - 'rust/**' - - 'Cargo.toml' - - check-sampleconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "all" - - run: poetry run scripts-dev/generate_sample_config.sh --check - - run: poetry run scripts-dev/config-lint.sh - - check-schema-delta: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" - - run: scripts-dev/check_schema_delta.py --force-colors - - lint: - uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1" - with: - typechecking-extras: "all" - - lint-crlf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Check line endings - run: scripts-dev/check_line_terminators.sh - - lint-newsfile: - if: ${{ github.base_ref == 'develop' || contains(github.base_ref, 'release-') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - uses: actions/setup-python@v2 - - run: "pip install 'towncrier>=18.6.0rc1'" - - run: scripts-dev/check-newsfragment.sh - env: - PULL_REQUEST_NUMBER: ${{ github.event.number }} - - lint-pydantic: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "all" - - run: poetry run scripts-dev/check_pydantic_models.py - - lint-clippy: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - components: clippy - - uses: Swatinem/rust-cache@v2 - - - run: cargo clippy - - lint-rustfmt: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - - run: cargo fmt --check - - # Dummy step to gate other tests on without repeating the whole list - linting-done: - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - needs: - - lint - - lint-crlf - - lint-newsfile - - lint-pydantic - - check-sampleconfig - - check-schema-delta - - lint-clippy - - lint-rustfmt - runs-on: ubuntu-latest - steps: - - run: "true" - - calculate-test-jobs: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - id: get-matrix - run: .ci/scripts/calculate_jobs.py - outputs: - trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} - sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} - - trial: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: calculate-test-jobs - runs-on: ubuntu-latest - strategy: - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} - if: ${{ matrix.job.postgres-version }} - run: | - docker run -d -p 5432:5432 \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.job.postgres-version }} - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.job.python-version }} - extras: ${{ matrix.job.extras }} - - name: Await PostgreSQL - if: ${{ matrix.job.postgres-version }} - timeout-minutes: 2 - run: until pg_isready -h localhost; do sleep 1; done - - run: poetry run trial --jobs=2 tests - env: - SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} - SYNAPSE_POSTGRES_HOST: localhost - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-olddeps: - # Note: sqlite only; no postgres - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - # There aren't wheels for some of the older deps, so we need to install - # their build dependencies - - run: | - sudo apt-get -qq install build-essential libffi-dev python-dev \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - # Calculating the old-deps actually takes a bunch of time, so we cache the - # pyproject.toml / poetry.lock. We need to cache pyproject.toml as - # otherwise the `poetry install` step will error due to the poetry.lock - # file being outdated. - # - # This caches the output of `Prepare old deps`, which should generate the - # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. - - uses: actions/cache@v3 - id: cache-poetry-old-deps - name: Cache poetry.lock - with: - path: | - poetry.lock - pyproject.toml - key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} - - name: Prepare old deps - if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' - run: .ci/scripts/prepare_old_deps.sh - - # We only now install poetry so that `setup-python-poetry` caches the - # right poetry.lock's dependencies. - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: '3.7' - extras: "all test" - - - run: poetry run trial -j2 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - trial-pypy: - # Very slow; only run if the branch name includes 'pypy' - # Note: sqlite only; no postgres. Completely untested since poetry move. - if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} - needs: linting-done - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["pypy-3.7"] - extras: ["all"] - - steps: - - uses: actions/checkout@v2 - # Install libs necessary for PyPy to build binary wheels for dependencies - - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - extras: ${{ matrix.extras }} - - run: poetry run trial --jobs=2 tests - - name: Dump logs - # Logs are most useful when the command fails, always include them. - if: ${{ always() }} - # Note: Dumps to workflow logs instead of using actions/upload-artifact - # This keeps logs colocated with failing jobs - # It also ignores find's exit code; this is a best effort affair - run: >- - find _trial_temp -name '*.log' - -exec echo "::group::{}" \; - -exec cat {} \; - -exec echo "::endgroup::" \; - || true - - sytest: - if: ${{ !failure() && !cancelled() }} - needs: calculate-test-jobs - runs-on: ubuntu-latest - container: - image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} - volumes: - - ${{ github.workspace }}:/src - env: - SYTEST_BRANCH: ${{ github.head_ref }} - POSTGRES: ${{ matrix.job.postgres && 1}} - MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} - WORKERS: ${{ matrix.job.workers && 1 }} - BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} - TOP: ${{ github.workspace }} - - strategy: - fail-fast: false - matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} - - steps: - - uses: actions/checkout@v2 - - name: Prepare test blacklist - run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - name: Run SyTest - run: /bootstrap.sh synapse - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Upload SyTest logs - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - export-data: - if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail - needs: [linting-done, portdb] - runs-on: ubuntu-latest - env: - TOP: ${{ github.workspace }} - - services: - postgres: - image: postgres - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - extras: "postgres" - - run: .ci/scripts/test_export_data_command.sh - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - - portdb: - if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail - needs: linting-done - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python-version: "3.7" - postgres-version: "10" - - - python-version: "3.10" - postgres-version: "14" - - services: - postgres: - image: postgres:${{ matrix.postgres-version }} - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: "postgres" - POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - - run: sudo apt-get -qq install xmlsec1 postgresql-client - - uses: matrix-org/setup-python-poetry@v1 - with: - python-version: ${{ matrix.python-version }} - extras: "postgres" - - run: .ci/scripts/test_synapse_port_db.sh - id: run_tester_script - env: - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - PGDATABASE: postgres - - name: "Upload schema differences" - uses: actions/upload-artifact@v3 - if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} - with: - name: Schema dumps - path: | - unported.sql - ported.sql - schema_diff - - complement: - if: "${{ !failure() && !cancelled() }}" - needs: linting-done - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v2 for synapse - uses: actions/checkout@v2 - with: - path: synapse - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - run: | - set -o pipefail - POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt - shell: bash - name: Run Complement Tests - - cargo-test: - if: ${{ needs.changes.outputs.rust == 'true' }} - runs-on: ubuntu-latest - needs: - - linting-done - - changes - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.58.1 - override: true - - uses: Swatinem/rust-cache@v2 - - - run: cargo test - - # a job which marks all the other jobs as complete, thus allowing PRs to be merged. - tests-done: - if: ${{ always() }} - needs: - - trial - - trial-olddeps - - sytest - - export-data - - portdb - - complement - - cargo-test - runs-on: ubuntu-latest - steps: - - uses: matrix-org/done-action@v2 - with: - needs: ${{ toJSON(needs) }} - - # The newsfile lint may be skipped on non PR builds - # Cargo test is skipped if there is no changes on Rust code - skippable: | - lint-newsfile - cargo-test diff --git a/changelog.d/.docker b/changelog.d/.docker new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/changelog.d/.docker @@ -0,0 +1 @@ + diff --git a/changelog.d/13995.docker b/changelog.d/13995.docker new file mode 100644 index 000000000000..355a95b7b800 --- /dev/null +++ b/changelog.d/13995.docker @@ -0,0 +1 @@ +WIP prototyping an action to add changelogs for dependabot 2