adjust benchmark close to real-life usage

.github/workflows/perf.yml

@@ -8,18 +8,22 @@ on:
   pull_request:
     branches: [main]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   EXPERIMENT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   CONTROL_BRANCH_NAME: 'main'
   FIDELITY: 100
-  THROTTLE: 2
+  THROTTLE: 4
   FORK_NAME: ${{ github.event.pull_request.head.repo.full_name }}
 
 jobs:
   master-krausest-comparison:
     name: Glimmer Krausest Benchmark
     runs-on: ubuntu-latest
-    timeout-minutes: 35
+    timeout-minutes: 70
     steps:
       - uses: actions/checkout@v4
         with:

benchmark/benchmarks/krausest/lib/components/Application.hbs

@@ -13,7 +13,7 @@
           </div>
           <div class="col-sm-6 smallpad">
             <BsButton id="runlots" {{on "click" this.runLots}}>
-              Create 10,000 rows
+              Create 5,000 rows
             </BsButton>
           </div>
           <div class="col-sm-6 smallpad">
@@ -44,7 +44,12 @@
   <table class="table table-hover table-striped test-data">
     <tbody>
       {{#each this.items as |item|}}
-        <Row @select={{this.select}} @remove={{this.remove}} @item={{item}} />
+        <Row
+          @select={{this.fn this.select item}}
+          @remove={{this.remove}}
+          @item={{item}}
+          class="{{if (this.eq item this.selectedItem) 'danger'}}"
+        />
       {{/each}}
     </tbody>
   </table>

benchmark/benchmarks/krausest/lib/components/Application.ts

@@ -1,10 +1,22 @@
 import { swapRows, type Item, updateData, buildData } from '@/utils/data';
 import { createCell } from '@glimmer-workspace/benchmark-env';
+import { fn } from '@glimmer/runtime';
 export default class Application {
   cell!: ReturnType<typeof createCell>;
-  lastSelected: Item | null = null;
+  selectedItemCell!: ReturnType<typeof createCell>;
   constructor() {
     this.cell = createCell(this, 'cell', []);
+    this.selectedItemCell = createCell(this, 'selectedItem', null);
+  }
+  fn = fn;
+  eq = (a: Item | null, b: Item | null) => {
+    return a === b;
+  };
+  get selectedItem() {
+    return this.selectedItemCell.get() as Item | null;
+  }
+  set selectedItem(value: Item | null) {
+    this.selectedItemCell.set(value);
   }
   get items() {
     return this.cell.get() as Item[];
@@ -13,17 +25,13 @@ export default class Application {
     this.cell.set(value);
   }
   select = (item: Item) => {
-    if (this.lastSelected !== item && this.lastSelected !== null) {
-      this.lastSelected.selected = false;
-    }
-    this.lastSelected = item;
-    item.selected = true;
+    this.selectedItem = item;
   };
   create = () => {
     this.items = buildData(1000);
   };
   runLots = () => {
-    this.items = buildData(10000);
+    this.items = buildData(5000);
   };
   add = () => {
     this.items = this.items.concat(buildData(1000));
@@ -33,11 +41,15 @@ export default class Application {
   };
   clear = () => {
     this.items = [];
+    this.selectedItem = null;
   };
   swapRows = () => {
     this.items = swapRows(this.items);
   };
   remove = (item: Item) => {
     this.items = this.items.filter((el) => el !== item);
+    if (this.selectedItem === item) {
+      this.selectedItem = null;
+    }
   };
 }

benchmark/benchmarks/krausest/lib/components/Row.hbs

@@ -1,4 +1,4 @@
-<tr class={{if @item.selected "danger"}}>
+<tr ...attributes>
   <td class="col-md-1">{{@item.id}}</td>
   <td class="col-md-4"><a data-test-select {{on "click" this.onSelect}}>{{@item.label}}</a></td>
   <td class="col-md-1">

benchmark/benchmarks/krausest/lib/components/Row.ts

@@ -2,7 +2,7 @@ import type { Item } from '@/utils/data';
 
 type RowArgs = {
   item: Item;
-  select: (item: Item) => void;
+  select: () => void;
   remove: (item: Item) => void;
 };
 
@@ -15,6 +15,6 @@ export default class Row {
     this.args.remove(this.args.item);
   };
   onSelect = () => {
-    this.args.select(this.args.item);
+    this.args.select();
   };
 }

benchmark/benchmarks/krausest/lib/index.ts

@@ -5,7 +5,7 @@ import ApplicationTemplate from '@/components/Application.hbs';
 import Row from '@/components/Row';
 import RowTemplate from '@/components/Row.hbs';
 import ButtonTemplate from '@/components/BsButton.hbs';
-import { enforcePaintEvent, ButtonSelectors, emitDomClickEvent } from '@/utils/compat';
+import { enforcePaintEvent, ButtonSelectors, emitDomClickEvent, waitForIdle } from '@/utils/compat';
 
 export default async function render(element: HTMLElement, isInteractive: boolean) {
   const benchmark = createBenchmark();
@@ -16,64 +16,126 @@ export default async function render(element: HTMLElement, isInteractive: boolea
 
   // starting app
 
+  await waitForIdle();
+
   const app = await benchmark.render('Application', {}, element, isInteractive);
 
+  await waitForIdle();
+
   await app('render1000Items1', () => {
     emitDomClickEvent(ButtonSelectors.Create1000);
   });
 
+  await waitForIdle();
+
   await app('clearItems1', () => {
     emitDomClickEvent(ButtonSelectors.Clear);
   });
 
+  await waitForIdle();
+
   await app('render1000Items2', () => {
     emitDomClickEvent(ButtonSelectors.Create1000);
   });
 
+  await waitForIdle();
+
   await app('clearItems2', () => {
     emitDomClickEvent(ButtonSelectors.Clear);
   });
 
-  await app('render10000Items1', () => {
-    emitDomClickEvent(ButtonSelectors.Create10000);
+  await waitForIdle();
+
+  await app('render5000Items1', () => {
+    emitDomClickEvent(ButtonSelectors.Create5000);
+  });
+
+  await waitForIdle();
+
+  await app('clearManyItems1', () => {
+    emitDomClickEvent(ButtonSelectors.Clear);
+  });
+
+  await waitForIdle();
+
+  await app('render5000Items2', () => {
+    emitDomClickEvent(ButtonSelectors.Create5000);
   });
 
-  await app('clearItems3', () => {
+  await waitForIdle();
+
+  await app('clearManyItems2', () => {
     emitDomClickEvent(ButtonSelectors.Clear);
   });
 
+  await waitForIdle();
+
   await app('render1000Items3', () => {
     emitDomClickEvent(ButtonSelectors.Create1000);
   });
 
+  await waitForIdle();
+
   await app('append1000Items1', () => {
     emitDomClickEvent(ButtonSelectors.Append1000);
   });
 
+  await waitForIdle();
+
+  await app('append1000Items2', () => {
+    emitDomClickEvent(ButtonSelectors.Append1000);
+  });
+
+  await waitForIdle();
+
   await app('updateEvery10thItem1', () => {
     emitDomClickEvent(ButtonSelectors.UpdateEvery10th);
   });
 
+  await waitForIdle();
+
+  await app('updateEvery10thItem2', () => {
+    emitDomClickEvent(ButtonSelectors.UpdateEvery10th);
+  });
+
+  await waitForIdle();
+
   await app('selectFirstRow1', () => {
     emitDomClickEvent(ButtonSelectors.SelectFirstRow);
   });
 
+  await waitForIdle();
+
   await app('selectSecondRow1', () => {
     emitDomClickEvent(ButtonSelectors.SelectSecondRow);
   });
 
+  await waitForIdle();
+
   await app('removeFirstRow1', () => {
     emitDomClickEvent(ButtonSelectors.RemoveFirstRow);
   });
 
+  await waitForIdle();
+
   await app('removeSecondRow1', () => {
     emitDomClickEvent(ButtonSelectors.RemoveSecondRow);
   });
 
+  await waitForIdle();
+
   await app('swapRows1', () => {
     emitDomClickEvent(ButtonSelectors.SwapRows);
   });
 
+  await waitForIdle();
+
+  await app('swapRows2', () => {
+    emitDomClickEvent(ButtonSelectors.SwapRows);
+  });
+
+  await waitForIdle();
+
   await app('clearItems4', () => {
     emitDomClickEvent(ButtonSelectors.Clear);
   });

benchmark/benchmarks/krausest/lib/utils/compat.ts

@@ -1,6 +1,6 @@
 export enum ButtonSelectors {
   Create1000 = '#run',
-  Create10000 = '#runlots',
+  Create5000 = '#runlots',
   Append1000 = '#add',
   UpdateEvery10th = '#update',
   SelectFirstRow = 'tr:nth-child(1) a[data-test-select]',
@@ -26,6 +26,12 @@ export function emitDomClickEvent(selector: ButtonSelectors) {
   }
 }
 
+export function waitForIdle() {
+  return new Promise((resolve) => {
+    requestIdleCallback(resolve);
+  });
+}
+
 export function enforcePaintEvent() {
   const docElem = document.documentElement;
   const refNode = docElem.firstElementChild || docElem.firstChild;

benchmark/benchmarks/krausest/lib/utils/data.ts

@@ -7,8 +7,6 @@ export class Item {
   /** @type {string} */
   _label = createCell(this, 'label', '');
 
-  _selected = createCell(this, 'selected', false);
-
   constructor(id: number, label: string) {
     this.id = id;
     this.label = label;
@@ -19,13 +17,6 @@ export class Item {
   set label(value: string) {
     this._label.set(value);
   }
-  get selected() {
-    return this._selected.get();
-  }
-
-  set selected(value) {
-    this._selected.set(value);
-  }
 }
 
 function _random(max: number) {

benchmark/benchmarks/krausest/tsconfig.json

@@ -23,7 +23,8 @@
     "paths": {
       "@/components/*": ["./lib/components/*"],
       "@/utils/*": ["./lib/utils/*"],
-      "@glimmer-workspace/benchmark-env": ["../../../packages/@glimmer-workspace/benchmark-env"]
+      "@glimmer-workspace/benchmark-env": ["../../../packages/@glimmer-workspace/benchmark-env"],
+      "@glimmer/runtime": ["../../../packages/@glimmer/runtime"]
     }
   },
   "include": ["./lib/", "./browser.js"],

benchmark/benchmarks/krausest/vite.config.mts

@@ -20,6 +20,7 @@ export default defineConfig({
     alias: {
       '@glimmer-workspace/benchmark-env': '@glimmer-workspace/benchmark-env/index.ts',
       '@glimmer/debug': packagePath('@glimmer/debug'),
+      '@glimmer/runtime': packagePath('@glimmer/runtime'),
       '@/components': path.join(currentPath, 'lib', 'components'),
       '@/utils': path.join(currentPath, 'lib', 'utils'),
     },

bin/setup-bench.mjs

@@ -21,20 +21,26 @@ const controlBranchName = process.env['CONTROL_BRANCH_NAME'] || 'main';
 
 // same order as in benchmark/benchmarks/krausest/lib/index.ts
 const appMarkers = [
+  'render',
   'render1000Items1',
   'clearItems1',
   'render1000Items2',
   'clearItems2',
-  'render10000Items1',
-  'clearItems3',
+  'render5000Items1',
+  'clearManyItems1',
+  'render5000Items2',
+  'clearManyItems2',
   'render1000Items3',
   'append1000Items1',
+  'append1000Items2',
   'updateEvery10thItem1',
+  'updateEvery10thItem2',
   'selectFirstRow1',
   'selectSecondRow1',
   'removeFirstRow1',
   'removeSecondRow1',
   'swapRows1',
+  'swapRows2',
   'clearItems4',
 ].reduce((acc, marker) => {
   return acc + ',' + marker + 'Start,' + marker + 'End';
@@ -166,12 +172,21 @@ await new Promise((resolve) => {
   setTimeout(resolve, 5000);
 });
 
-const output =
-  await $`./node_modules/.bin/tracerbench compare --regressionThreshold 25 --fidelity ${fidelity} --markers ${markers} --controlURL ${CONTROL_URL} --experimentURL ${EXPERIMENT_URL} --report --headless --cpuThrottleRate ${throttleRate}`;
-
-fs.writeFileSync(
-  'tracerbench-results/msg.txt',
-  output.stdout.split('Benchmark Results Summary').pop() ?? ''
-);
+try {
+  const output =
+    await $`./node_modules/.bin/tracerbench compare --regressionThreshold 25 --sampleTimeout 60 --fidelity ${fidelity} --markers ${markers} --controlURL ${CONTROL_URL} --experimentURL ${EXPERIMENT_URL} --report --headless --cpuThrottleRate ${throttleRate}`;
+
+  try {
+    fs.writeFileSync(
+      'tracerbench-results/msg.txt',
+      output.stdout.split('Benchmark Results Summary').pop() ?? ''
+    );
+  } catch (e) {
+    // fine
+  }
+} catch (p) {
+  console.error(p);
+  process.exit(1);
+}
 
 process.exit(0);