diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 53b9efb6ba462..168904b098475 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -44,6 +44,8 @@ env: # changed if we want to normalize results against a different baseline. BASE_TAG: '6.1.1' LOCAL_DIR: build + TARGET_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || '' }} + TARGET_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} jobs: # Runs the performance test suite. @@ -65,9 +67,17 @@ jobs: # - Install MU plugin. # - Run performance tests (current commit). # - Print performance tests results. + # - Check out target commit (target branch or previous commit). + # - Install npm dependencies. + # - Build WordPress. + # - Run performance tests (previous/target commit). + # - Print target performance tests results. + # - Reset to original commit. # - Set the environment to the baseline version. # - Run baseline performance tests. - # - Print base line performance tests results. + # - Print baseline performance tests results. + # - Compare results with base. + # - Add workflow summary. # - Set the base sha. # - Set commit details. # - Publish performance results. @@ -152,7 +162,31 @@ jobs: run: npm run test:performance - name: Print performance tests results - run: "node ./tests/performance/results.js" + run: node ./tests/performance/results.js + + - name: Check out target commit (target branch or previous commit) + run: | + if [[ -z "$TARGET_REF" ]]; then + git fetch -n origin $TARGET_SHA + else + git fetch -n origin $TARGET_REF + fi + git reset --hard $TARGET_SHA + + - name: Install npm dependencies + run: npm ci + + - name: Build WordPress + run: npm run build + + - name: Run target performance tests (base/previous commit) + run: npm run test:performance -- --prefix=before + + - name: Print target performance tests results + run: node ./tests/performance/results.js --prefix=before + + - name: Reset to original commit + run: git reset --hard $GITHUB_SHA - name: Set the environment to the baseline version run: | @@ -162,8 +196,14 @@ jobs: - name: Run baseline performance tests run: npm run test:performance -- --prefix=base - - name: Print base line performance tests results - run: "node ./tests/performance/results.js --prefix=base" + - name: Print baseline performance tests results + run: node ./tests/performance/results.js --prefix=base + + - name: Compare results with base + run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md + + - name: Add workflow summary + run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY - name: Set the base sha # Only needed when publishing results. diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js new file mode 100644 index 0000000000000..e722552aeda04 --- /dev/null +++ b/tests/performance/compare-results.js @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +/** + * External dependencies. + */ +const fs = require( 'fs' ); +const path = require( 'path' ); +const { median } = require( './utils' ); + +/** + * Parse test files into JSON objects. + * + * @param {string} fileName The name of the file. + * @returns An array of parsed objects from each file. + */ +const parseFile = ( fileName ) => + JSON.parse( + fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' ) + ); + +// The list of test suites to log. +const testSuites = [ 'home-block-theme', 'home-classic-theme' ]; + +// The current commit's results. +const testResults = Object.fromEntries( + testSuites.map( ( key ) => [ + key, + parseFile( `${ key }.test.results.json` ), + ] ) +); + +// The previous commit's results. +const prevResults = Object.fromEntries( + testSuites.map( ( key ) => [ + key, + parseFile( `before-${ key }.test.results.json` ), + ] ) +); + +const args = process.argv.slice( 2 ); + +const summaryFile = args[ 0 ]; + +/** + * Formats an array of objects as a Markdown table. + * + * For example, this array: + * + * [ + * { + * foo: 123, + * bar: 456, + * baz: 'Yes', + * }, + * { + * foo: 777, + * bar: 999, + * baz: 'No', + * } + * ] + * + * Will result in the following table: + * + * | foo | bar | baz | + * |-----|-----|-----| + * | 123 | 456 | Yes | + * | 777 | 999 | No | + * + * @param {Array} rows Table rows. + * @returns {string} Markdown table content. + */ +function formatAsMarkdownTable( rows ) { + let result = ''; + const headers = Object.keys( rows[ 0 ] ); + for ( const header of headers ) { + result += `| ${ header } `; + } + result += '|\n'; + for ( const header of headers ) { + result += '| ------ '; + } + result += '|\n'; + + for ( const row of rows ) { + for ( const value of Object.values( row ) ) { + result += `| ${ value } `; + } + result += '|\n'; + } + + return result; +} + +/** + * Returns a Markdown link to a Git commit on the current GitHub repository. + * + * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e` + * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a]. + * + * @param {string} sha Commit SHA. + * @return string Link + */ +function linkToSha(sha) { + const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop'; + + return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`; +} + +let summaryMarkdown = `# Performance Test Results\n\n`; + +if ( process.env.GITHUB_SHA ) { + summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`; +} else { + summaryMarkdown += `🛎️ Performance test results are in!\n\n`; +} + +if ( process.env.TARGET_SHA ) { + summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`; +} + +if ( process.env.GITHUB_SHA ) { + summaryMarkdown += `**Note:** Due to the nature of how GitHub Actions work, some variance in the results is expected.\n\n`; +} + +console.log( 'Performance Test Results\n' ); + +console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' ); + +for ( const key of testSuites ) { + const current = testResults[ key ]; + const prev = prevResults[ key ]; + + const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace( + /-+/g, + ' ' + ); + + const rows = []; + + for ( const [ metric, values ] of Object.entries( current ) ) { + const value = median( values ); + const prevValue = median( prev[ metric ] ); + + const delta = value - prevValue; + const percentage = Math.round( ( delta / value ) * 100 ); + rows.push( { + Metric: metric, + Before: `${ prevValue.toFixed( 2 ) } ms`, + After: `${ value.toFixed( 2 ) } ms`, + 'Diff abs.': `${ delta.toFixed( 2 ) } ms`, + 'Diff %': `${ percentage.toFixed( 2 ) } %`, + } ); + } + + summaryMarkdown += `## ${ title }\n\n`; + summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`; + + console.log( title ); + console.table( rows ); +} + +fs.writeFileSync( + summaryFile, + summaryMarkdown +);