diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 561abfd5a9..cf8d6db17c 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -8,18 +8,22 @@ on: pull_request: branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: EXPERIMENT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} CONTROL_BRANCH_NAME: 'main' FIDELITY: 100 - THROTTLE: 2 + THROTTLE: 4 FORK_NAME: ${{ github.event.pull_request.head.repo.full_name }} jobs: master-krausest-comparison: name: Glimmer Krausest Benchmark runs-on: ubuntu-latest - timeout-minutes: 35 + timeout-minutes: 70 steps: - uses: actions/checkout@v4 with: diff --git a/benchmark/benchmarks/krausest/lib/components/Application.hbs b/benchmark/benchmarks/krausest/lib/components/Application.hbs index 5f60e94954..ef116217ff 100644 --- a/benchmark/benchmarks/krausest/lib/components/Application.hbs +++ b/benchmark/benchmarks/krausest/lib/components/Application.hbs @@ -13,7 +13,7 @@
- Create 10,000 rows + Create 5,000 rows
@@ -44,7 +44,12 @@ {{#each this.items as |item|}} - + {{/each}}
diff --git a/benchmark/benchmarks/krausest/lib/components/Application.ts b/benchmark/benchmarks/krausest/lib/components/Application.ts index abcd0fcc6b..e841942489 100644 --- a/benchmark/benchmarks/krausest/lib/components/Application.ts +++ b/benchmark/benchmarks/krausest/lib/components/Application.ts @@ -1,10 +1,22 @@ import { swapRows, type Item, updateData, buildData } from '@/utils/data'; import { createCell } from '@glimmer-workspace/benchmark-env'; +import { fn } from '@glimmer/runtime'; export default class Application { cell!: ReturnType; - lastSelected: Item | null = null; + selectedItemCell!: ReturnType; constructor() { this.cell = createCell(this, 'cell', []); + this.selectedItemCell = createCell(this, 'selectedItem', null); + } + fn = fn; + eq = (a: Item | null, b: Item | null) => { + return a === b; + }; + get selectedItem() { + return this.selectedItemCell.get() as Item | null; + } + set selectedItem(value: Item | null) { + this.selectedItemCell.set(value); } get items() { return this.cell.get() as Item[]; @@ -13,17 +25,13 @@ export default class Application { this.cell.set(value); } select = (item: Item) => { - if (this.lastSelected !== item && this.lastSelected !== null) { - this.lastSelected.selected = false; - } - this.lastSelected = item; - item.selected = true; + this.selectedItem = item; }; create = () => { this.items = buildData(1000); }; runLots = () => { - this.items = buildData(10000); + this.items = buildData(5000); }; add = () => { this.items = this.items.concat(buildData(1000)); @@ -33,11 +41,15 @@ export default class Application { }; clear = () => { this.items = []; + this.selectedItem = null; }; swapRows = () => { this.items = swapRows(this.items); }; remove = (item: Item) => { this.items = this.items.filter((el) => el !== item); + if (this.selectedItem === item) { + this.selectedItem = null; + } }; } diff --git a/benchmark/benchmarks/krausest/lib/components/Row.hbs b/benchmark/benchmarks/krausest/lib/components/Row.hbs index 05a870981f..83317551cc 100644 --- a/benchmark/benchmarks/krausest/lib/components/Row.hbs +++ b/benchmark/benchmarks/krausest/lib/components/Row.hbs @@ -1,4 +1,4 @@ - + {{@item.id}} {{@item.label}} diff --git a/benchmark/benchmarks/krausest/lib/components/Row.ts b/benchmark/benchmarks/krausest/lib/components/Row.ts index 2f72d7f6c0..a49b1cc67a 100644 --- a/benchmark/benchmarks/krausest/lib/components/Row.ts +++ b/benchmark/benchmarks/krausest/lib/components/Row.ts @@ -2,7 +2,7 @@ import type { Item } from '@/utils/data'; type RowArgs = { item: Item; - select: (item: Item) => void; + select: () => void; remove: (item: Item) => void; }; @@ -15,6 +15,6 @@ export default class Row { this.args.remove(this.args.item); }; onSelect = () => { - this.args.select(this.args.item); + this.args.select(); }; } diff --git a/benchmark/benchmarks/krausest/lib/index.ts b/benchmark/benchmarks/krausest/lib/index.ts index 5d3f6c3d60..f7ac21134f 100644 --- a/benchmark/benchmarks/krausest/lib/index.ts +++ b/benchmark/benchmarks/krausest/lib/index.ts @@ -5,7 +5,7 @@ import ApplicationTemplate from '@/components/Application.hbs'; import Row from '@/components/Row'; import RowTemplate from '@/components/Row.hbs'; import ButtonTemplate from '@/components/BsButton.hbs'; -import { enforcePaintEvent, ButtonSelectors, emitDomClickEvent } from '@/utils/compat'; +import { enforcePaintEvent, ButtonSelectors, emitDomClickEvent, waitForIdle } from '@/utils/compat'; export default async function render(element: HTMLElement, isInteractive: boolean) { const benchmark = createBenchmark(); @@ -16,64 +16,126 @@ export default async function render(element: HTMLElement, isInteractive: boolea // starting app + await waitForIdle(); + const app = await benchmark.render('Application', {}, element, isInteractive); + await waitForIdle(); + await app('render1000Items1', () => { emitDomClickEvent(ButtonSelectors.Create1000); }); + await waitForIdle(); + await app('clearItems1', () => { emitDomClickEvent(ButtonSelectors.Clear); }); + await waitForIdle(); + await app('render1000Items2', () => { emitDomClickEvent(ButtonSelectors.Create1000); }); + await waitForIdle(); + await app('clearItems2', () => { emitDomClickEvent(ButtonSelectors.Clear); }); - await app('render10000Items1', () => { - emitDomClickEvent(ButtonSelectors.Create10000); + await waitForIdle(); + + await app('render5000Items1', () => { + emitDomClickEvent(ButtonSelectors.Create5000); + }); + + await waitForIdle(); + + await app('clearManyItems1', () => { + emitDomClickEvent(ButtonSelectors.Clear); + }); + + await waitForIdle(); + + await app('render5000Items2', () => { + emitDomClickEvent(ButtonSelectors.Create5000); }); - await app('clearItems3', () => { + await waitForIdle(); + + await app('clearManyItems2', () => { emitDomClickEvent(ButtonSelectors.Clear); }); + await waitForIdle(); + await app('render1000Items3', () => { emitDomClickEvent(ButtonSelectors.Create1000); }); + await waitForIdle(); + await app('append1000Items1', () => { emitDomClickEvent(ButtonSelectors.Append1000); }); + await waitForIdle(); + + await app('append1000Items2', () => { + emitDomClickEvent(ButtonSelectors.Append1000); + }); + + await waitForIdle(); + await app('updateEvery10thItem1', () => { emitDomClickEvent(ButtonSelectors.UpdateEvery10th); }); + await waitForIdle(); + + await app('updateEvery10thItem2', () => { + emitDomClickEvent(ButtonSelectors.UpdateEvery10th); + }); + + await waitForIdle(); + await app('selectFirstRow1', () => { emitDomClickEvent(ButtonSelectors.SelectFirstRow); }); + await waitForIdle(); + await app('selectSecondRow1', () => { emitDomClickEvent(ButtonSelectors.SelectSecondRow); }); + await waitForIdle(); + await app('removeFirstRow1', () => { emitDomClickEvent(ButtonSelectors.RemoveFirstRow); }); + await waitForIdle(); + await app('removeSecondRow1', () => { emitDomClickEvent(ButtonSelectors.RemoveSecondRow); }); + await waitForIdle(); + await app('swapRows1', () => { emitDomClickEvent(ButtonSelectors.SwapRows); }); + await waitForIdle(); + + await app('swapRows2', () => { + emitDomClickEvent(ButtonSelectors.SwapRows); + }); + + await waitForIdle(); + await app('clearItems4', () => { emitDomClickEvent(ButtonSelectors.Clear); }); diff --git a/benchmark/benchmarks/krausest/lib/utils/compat.ts b/benchmark/benchmarks/krausest/lib/utils/compat.ts index 946cd0656c..a58656769f 100644 --- a/benchmark/benchmarks/krausest/lib/utils/compat.ts +++ b/benchmark/benchmarks/krausest/lib/utils/compat.ts @@ -1,6 +1,6 @@ export enum ButtonSelectors { Create1000 = '#run', - Create10000 = '#runlots', + Create5000 = '#runlots', Append1000 = '#add', UpdateEvery10th = '#update', SelectFirstRow = 'tr:nth-child(1) a[data-test-select]', @@ -26,6 +26,12 @@ export function emitDomClickEvent(selector: ButtonSelectors) { } } +export function waitForIdle() { + return new Promise((resolve) => { + requestIdleCallback(resolve); + }); +} + export function enforcePaintEvent() { const docElem = document.documentElement; const refNode = docElem.firstElementChild || docElem.firstChild; diff --git a/benchmark/benchmarks/krausest/lib/utils/data.ts b/benchmark/benchmarks/krausest/lib/utils/data.ts index b9fca05032..e163a842b7 100644 --- a/benchmark/benchmarks/krausest/lib/utils/data.ts +++ b/benchmark/benchmarks/krausest/lib/utils/data.ts @@ -7,8 +7,6 @@ export class Item { /** @type {string} */ _label = createCell(this, 'label', ''); - _selected = createCell(this, 'selected', false); - constructor(id: number, label: string) { this.id = id; this.label = label; @@ -19,13 +17,6 @@ export class Item { set label(value: string) { this._label.set(value); } - get selected() { - return this._selected.get(); - } - - set selected(value) { - this._selected.set(value); - } } function _random(max: number) { diff --git a/benchmark/benchmarks/krausest/tsconfig.json b/benchmark/benchmarks/krausest/tsconfig.json index 4cea3cb636..e0b2a52cb4 100644 --- a/benchmark/benchmarks/krausest/tsconfig.json +++ b/benchmark/benchmarks/krausest/tsconfig.json @@ -23,7 +23,8 @@ "paths": { "@/components/*": ["./lib/components/*"], "@/utils/*": ["./lib/utils/*"], - "@glimmer-workspace/benchmark-env": ["../../../packages/@glimmer-workspace/benchmark-env"] + "@glimmer-workspace/benchmark-env": ["../../../packages/@glimmer-workspace/benchmark-env"], + "@glimmer/runtime": ["../../../packages/@glimmer/runtime"] } }, "include": ["./lib/", "./browser.js"], diff --git a/benchmark/benchmarks/krausest/vite.config.mts b/benchmark/benchmarks/krausest/vite.config.mts index 3ae3f4a92d..ae34fe4b7b 100644 --- a/benchmark/benchmarks/krausest/vite.config.mts +++ b/benchmark/benchmarks/krausest/vite.config.mts @@ -20,6 +20,7 @@ export default defineConfig({ alias: { '@glimmer-workspace/benchmark-env': '@glimmer-workspace/benchmark-env/index.ts', '@glimmer/debug': packagePath('@glimmer/debug'), + '@glimmer/runtime': packagePath('@glimmer/runtime'), '@/components': path.join(currentPath, 'lib', 'components'), '@/utils': path.join(currentPath, 'lib', 'utils'), }, diff --git a/bin/setup-bench.mjs b/bin/setup-bench.mjs index 7431dbe694..eee3ffd1d8 100644 --- a/bin/setup-bench.mjs +++ b/bin/setup-bench.mjs @@ -21,20 +21,26 @@ const controlBranchName = process.env['CONTROL_BRANCH_NAME'] || 'main'; // same order as in benchmark/benchmarks/krausest/lib/index.ts const appMarkers = [ + 'render', 'render1000Items1', 'clearItems1', 'render1000Items2', 'clearItems2', - 'render10000Items1', - 'clearItems3', + 'render5000Items1', + 'clearManyItems1', + 'render5000Items2', + 'clearManyItems2', 'render1000Items3', 'append1000Items1', + 'append1000Items2', 'updateEvery10thItem1', + 'updateEvery10thItem2', 'selectFirstRow1', 'selectSecondRow1', 'removeFirstRow1', 'removeSecondRow1', 'swapRows1', + 'swapRows2', 'clearItems4', ].reduce((acc, marker) => { return acc + ',' + marker + 'Start,' + marker + 'End'; @@ -166,12 +172,21 @@ await new Promise((resolve) => { setTimeout(resolve, 5000); }); -const output = - await $`./node_modules/.bin/tracerbench compare --regressionThreshold 25 --fidelity ${fidelity} --markers ${markers} --controlURL ${CONTROL_URL} --experimentURL ${EXPERIMENT_URL} --report --headless --cpuThrottleRate ${throttleRate}`; - -fs.writeFileSync( - 'tracerbench-results/msg.txt', - output.stdout.split('Benchmark Results Summary').pop() ?? '' -); +try { + const output = + await $`./node_modules/.bin/tracerbench compare --regressionThreshold 25 --sampleTimeout 60 --fidelity ${fidelity} --markers ${markers} --controlURL ${CONTROL_URL} --experimentURL ${EXPERIMENT_URL} --report --headless --cpuThrottleRate ${throttleRate}`; + + try { + fs.writeFileSync( + 'tracerbench-results/msg.txt', + output.stdout.split('Benchmark Results Summary').pop() ?? '' + ); + } catch (e) { + // fine + } +} catch (p) { + console.error(p); + process.exit(1); +} process.exit(0);