-
Notifications
You must be signed in to change notification settings - Fork 4.3k
112 lines (108 loc) · 5.34 KB
/
publish_connectors.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
name: Connector Ops CI - Publish Connectors
on:
push:
branches:
- master
paths:
- "airbyte-integrations/connectors/**/metadata.yaml"
workflow_dispatch:
inputs:
connectors-options:
description: "Options to pass to the 'airbyte-ci connectors' command group."
default: "--name=source-pokeapi"
publish-options:
description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
default: "--pre-release"
airbyte_ci_binary_url:
description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci."
default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci"
jobs:
publish_connectors:
name: Publish connectors
runs-on: connector-publish-large
steps:
- name: Checkout Airbyte
uses: actions/checkout@v4
- name: Publish modified connectors [On merge to master]
id: publish-modified-connectors
if: github.event_name == 'push'
uses: ./.github/actions/run-airbyte-ci
with:
context: "master"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
python_registry_token: ${{ secrets.PYPI_TOKEN }}
max_attempts: 2
- name: Publish connectors [manual]
id: publish-connectors
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/run-airbyte-ci
with:
context: "manual"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
python_registry_token: ${{ secrets.PYPI_TOKEN }}
airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
max_attempts: 2
set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ failure() && github.ref == 'refs/heads/master' }}
steps:
- name: Call Instatus Webhook
uses: joelwmale/webhook-action@master
with:
url: ${{ secrets.INSTATUS_CONNECTOR_PUBLISH_PIPELINE_WEBHOOK_URL }}
body: '{ "trigger": "down", "status": "HASISSUES" }'
notify-failure-slack-channel:
name: "Notify Slack Channel on Publish Failures"
runs-on: ubuntu-latest
needs:
- publish_connectors
if: ${{ always() && contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v4
- name: Match GitHub User to Slack User
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Send publish failures to connector-publish-failures channel
id: slack
uses: slackapi/[email protected]
with:
# This data can be any valid JSON from a previous step in the GitHub Action
payload: |
{
"channel": "#connector-publish-failures",
"username": "Connectors CI/CD Bot",
"text": "🚨 Publish workflow failed:\n ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}