From c35362369b22e35b707e59bbdf47ddcc033f5329 Mon Sep 17 00:00:00 2001 From: default Date: Fri, 9 Aug 2024 13:02:28 +0000 Subject: [PATCH] chore(ci): Import Firefox schema github workflow --- .github/workflows/firefox-schema-import.yml | 140 ++++++++++ scripts/download-import-schema-from-gecko-dev | 2 + scripts/workflow-github-script-helpers.mjs | 260 ++++++++++++++++++ 3 files changed, 402 insertions(+) create mode 100644 .github/workflows/firefox-schema-import.yml create mode 100644 scripts/workflow-github-script-helpers.mjs diff --git a/.github/workflows/firefox-schema-import.yml b/.github/workflows/firefox-schema-import.yml new file mode 100644 index 00000000000..dbc0055d0a3 --- /dev/null +++ b/.github/workflows/firefox-schema-import.yml @@ -0,0 +1,140 @@ +name: Import Firefox API Schema + +run-name: | + ref: ${{ github.ref_name }} + github-dev-branch: ${{ inputs.gecko-dev-branch }} + +concurrency: + group: firefox-schema-import-${{ inputs.gecko-dev-branch }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + gecko-dev-branch: + type: string + description: Which gecko-dev branch to import API schema from. + default: beta + required: true + run-tests: + type: boolean + default: false + required: false + description: | + run-tests - Whether addons-linter tests should be executed to + check imported schema for test failures. + create-issue: + type: boolean + required: false + default: false + description: | + create-issue - Whether an issue should be created + on detected changes or detected failures. + create-pull: + type: boolean + required: false + default: false + description: | + create-pull - Whether a pull request should be created + on detected changes. + workflow_dispatch: + inputs: + gecko-dev-branch: + type: choice + description: Which gecko-dev branch to import API schema from. + default: beta + required: true + options: + - beta + - master + run-tests: + type: boolean + default: false + required: false + description: | + run-tests - Whether addons-linter tests should be executed to + check imported schema for test failures. + create-issue: + type: boolean + required: false + default: false + description: | + create-issue - Whether an issue should be created + on detected changes or detected failures. + create-pull: + type: boolean + required: false + default: false + description: | + create-pull - Whether a pull request should be created + on detected changes. + +jobs: + import-firefox-schema: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + ref: master + - name: Setup NodeJS + uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm ci + - name: Import Firefox API Schema from gecko-dev branch "${{ inputs.gecko-dev-branch }}" + run: ./scripts/download-import-schema-from-gecko-dev "${{ inputs.gecko-dev-branch }}" + - name: Handle Firefox API Schema import result + uses: actions/github-script@v7 + with: + result-encoding: string + retries: 1 + # API docs: https://github.com/actions/github-script and https://octokit.github.io/rest.js/v20 + script: | + const workflowInputs = ${{ toJson(inputs) }}; + + const { + getImportResultState, + createBranchAndCommit, + findExistingIssue, + createIssue, + findExistingPull, + createPull, + } = await import('${{ github.workspace }}/scripts/workflow-github-script-helpers.mjs'); + + const importState = await getImportResultState({ github, context, workflowInputs }); + if (!importState.has_pending_changes) { + console.log("Import completed. No changes detected."); + return; + } + + if (importState.has_remote_branch_changes) { + console.log("Import completed. An existing branch with changes has been detected."); + return; + } + + const commitDiff = createBranchAndCommit(importState); + + let issue = null; + if (workflowInputs['create-issue']) { + issue = await findExistingIssue({ ...importState, github, context }); + if (!issue) { + issue = await createIssue({ importState, github, context }); + console.log("Created new tracking issue:", issue.html_url); + } else { + console.log("Not creating new issue, existing issue found:", issue.html_url); + } + } + + let pull = null; + if (workflowInputs['create-pull']) { + pull = await findExistingPull({ ...importState, github, context }); + if (!pull) { + pull = await createPull({ importState, issue, github, context }); + if (pull) { + console.log("Created new pull request:", pull.html_url); + } + } else { + console.log("Not creating new pullrequest, existing found:", pull.html_url); + } + } diff --git a/scripts/download-import-schema-from-gecko-dev b/scripts/download-import-schema-from-gecko-dev index 60ef8e4adc4..708a01c7f23 100755 --- a/scripts/download-import-schema-from-gecko-dev +++ b/scripts/download-import-schema-from-gecko-dev @@ -95,6 +95,8 @@ function importSchemaFromPartialClone() { `Importing WebExtensions API JSONSchema data from Gecko ${version_display}` ); + shell.cp(version_file_path, 'src/schema/imported/version_display.txt'); + shell.exec(`./scripts/firefox-schema-import ${tmpDir}`); shell.echo('Schema changes diff:\n'); diff --git a/scripts/workflow-github-script-helpers.mjs b/scripts/workflow-github-script-helpers.mjs new file mode 100644 index 00000000000..b6319587ed6 --- /dev/null +++ b/scripts/workflow-github-script-helpers.mjs @@ -0,0 +1,260 @@ +/** + * This ES module exports helper functions used by the `firefox-schema-import` + * Gihub Actions Workflow. + */ +import shell from 'shelljs'; + +const BASE_ISSUE_SUBJECT = 'Import Firefox API Schema from Firefox'; +const BASE_COMMIT_MESSAGE = 'feat: Imported Firefox API Schema from Firefox'; +const MAIN_BRANCH = 'master'; + +// Fail as soon as a shelljs command fails. +shell.config.fatal = true; + +export async function getImportResultState({ + github, + context, + workflowInputs, +}) { + const workflow_run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}/`; + const version_display_file_path = 'src/schema/imported/version_display.txt'; + const version_display = shell.cat(version_display_file_path).trim(); + const version = version_display.split('.')[0]; + const beta_build_number = version_display.split('b')[1]; + const nightly_build_number = version_display.split('a')[1]; + const branch_name = `feat/api-schema-import-fx${version}`; + const current_date = new Date().toISOString().slice(0, 10); + const has_pending_changes = hasPendingSchemaChanges(version_display); + const has_remote_branch_changes = await hasRemoteBranchChanges({ + github, + context, + branch_name, + firefox_version: version, + }); + const tests = workflowInputs['run-tests'] ? runTests() : undefined; + + const resultState = { + firefox_version: version, + firefox_version_display: version_display, + beta_build_number: beta_build_number + ? parseInt(beta_build_number, 10) + : undefined, + nightly_build_number: nightly_build_number + ? parseInt(nightly_build_number, 10) + : undefined, + branch_name, + current_date, + has_pending_changes, + has_remote_branch_changes, + tests, + workflow_run_url, + }; + + shell.echo( + `Firefox API Schema import result state: ${JSON.stringify( + resultState, + null, + 2 + )}` + ); + return resultState; +} + +export async function findExistingIssue({ github, context, firefox_version }) { + const issues = await github.request('GET /repos/{owner}/{repo}/issues', { + owner: context.repo.owner, + repo: context.repo.repo, + q: `${BASE_ISSUE_SUBJECT} ${firefox_version}`, + }); + if (issues.data.length) { + shell.echo( + `Found existing issues: ${issues.data + .map((issue) => { + return `\n\t#${issue.number} - ${issue.title} (created by ${issue.user.login})`; + }) + .join('\n')}` + ); + return issues.data[0]; + } + return null; +} + +export async function createIssue({ github, context, importState }) { + const { firefox_version, workflow_run_url } = importState; + + const title = `${BASE_ISSUE_SUBJECT} ${firefox_version}`; + const body = [ + `This issue is tracking importing Firefox API Schemas from Firefox ${firefox_version}.`, + '', + `Workflow run: ${workflow_run_url}`, + '', + 'Automated Import Firefox API Schema workflow results:', + '```', + JSON.stringify(importState, null, 2), + '```', + ].join('\n'); + const issue = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + return issue.data; +} + +export async function findExistingPull({ github, context, branch_name }) { + const pulls = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: branch_name, + }); + if (pulls.status !== 200 || pulls.data.length > 1) { + console.error('findExistingPull got unexpected result', pulls); + throw new Error('findExistingPull got unexpected result'); + } + return pulls.data[0]; +} + +export async function createPull({ github, context, importState, issue }) { + const { + firefox_version, + firefox_version_display, + beta_build_number, + has_remote_branch_changes, + branch_name, + } = importState; + if (!beta_build_number) { + shell.echo( + 'No pull request created. Import not related to a mozilla-central beta branch' + ); + return null; + } + if (has_remote_branch_changes) { + shell.echo( + `No pull request created. Found changes applied to the remote ${branch_name}` + ); + return null; + } + + // Force push until the import is running on a beta build number >= 8. + if (beta_build_number < 8) { + shell.exec(`git push origin ${branch_name} -f`); + } else { + shell.exec(`git push origin ${branch_name}`); + } + + const title = `feat: Import Firefox API Schema for ${firefox_version}`; + const body = [ + `This pull request is importing Firefox API Schemas for Firefox ${firefox_version} (${firefox_version_display}).`, + '', + 'TODO: create a summary of the bugzilla issues related to the imported changes', + '', + `Fixes #${issue.number}`, + ].join('\n'); + + const pull = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head: importState.branch_name, + base: MAIN_BRANCH, + title, + body, + }); + return pull.data; +} + +function runWithShellJSConfig({ fatal }, cb) { + try { + shell.config.fatal = fatal; + return cb(); + } finally { + shell.config.fatal = true; + } +} + +function hasPendingSchemaChanges(firefox_version_display) { + shell.echo( + `Check for changes from Importing Firefox ${firefox_version_display} API Schema...` + ); + const gitStatus = shell.exec(`git status`, { silent: true }); + if (!gitStatus.stdout.trim().split('\n').length) { + shell.echo('No changes to the schema data'); + return false; + } + return true; +} + +async function hasRemoteBranchChanges({ + github, + context, + branch_name, + firefox_version, +}) { + shell.echo( + `Check for existing remote changes in the branch ${branch_name}...` + ); + + const githubRemoteBranchResult = await github.rest.repos + .getBranch({ + owner: context.repo.owner, + repo: context.repo.repo, + branch: branch_name, + }) + .catch((err) => err); + + if (githubRemoteBranchResult.status === 404) { + shell.echo(`No remote branch named ${branch_name} has been found`); + return false; + } + + if (githubRemoteBranchResult.status !== 200) { + // Throw the octokit request error. + throw githubRemoteBranchResult; + } + + const commitMessage = githubRemoteBranchResult.data.commit.commit.message; + shell.echo( + `Found commit message from remote branch ${branch_name}:\n ${commitMessage}` + ); + return commitMessage.startsWith( + `${BASE_COMMIT_MESSAGE} ${firefox_version} (workflow ` + ); +} + +function runTests() { + return runWithShellJSConfig({ fatal: false }, () => { + shell.echo(`Execute repository build script ("npm run build")...`); + let result = shell.exec('npm run build', { silent: true }); + if (result.code !== 0) { + return { + has_failures: true, + stderr: result.stderr.trim(), + }; + } + shell.echo(`Execute repository tests ("npm run test-once")...`); + result = shell.exec('npm run test-once', { silent: true }); + const has_failures = result.code !== 0; + return { + has_failures, + stderr: has_failures ? result.stderr.trim() : undefined, + }; + }); +} + +export function createBranchAndCommit({ + branch_name, + firefox_version, + current_date, +}) { + shell.echo(`Create new commit in branch ${branch_name}`); + shell.exec('git config user.email "$GITHUB_ACTOR@users.noreply.github.com"'); + shell.exec('git config user.name "$GITHUB_ACTOR"'); + shell.exec('git add src/schema/imported'); + shell.exec(`git checkout -b ${branch_name}`); + const automatedCommitMarker = `(workflow $GITHUB_JOB ${current_date})`; + shell.exec( + `git commit -m "${BASE_COMMIT_MESSAGE} ${firefox_version} ${automatedCommitMarker}"` + ); + const commitDiff = shell.exec('git log -p -r HEAD^..HEAD'); + return commitDiff.stdout.trim(); +}