[kbn/optimizer] include used dll refs in cache key (#129928)

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -8,6 +8,7 @@
 
 import { Bundle, BundleSpec, parseBundles } from './bundle';
 import { Hashes } from './hashes';
+import { parseDllManifest } from './dll_manifest';
 
 jest.mock('fs');
 
@@ -31,13 +32,31 @@ it('creates cache keys', () => {
   ].sort(() => (Math.random() > 0.5 ? 1 : -1));
 
   const hashes = new Hashes(new Map(hashEntries));
+  const dllManifest = parseDllManifest({
+    name: 'manifest-name',
+    content: {
+      './some-foo.ts': {
+        id: 1,
+        buildMeta: {
+          a: 'b',
+        },
+        unknownField: 'hi',
+      },
+    },
+  });
+  const dllRefKeys = ['./some-foo.ts'];
 
-  expect(bundle.createCacheKey(['/foo/bar/a', '/foo/bar/c'], hashes)).toMatchInlineSnapshot(`
+  expect(bundle.createCacheKey(['/foo/bar/a', '/foo/bar/c'], hashes, dllManifest, dllRefKeys))
+    .toMatchInlineSnapshot(`
     Object {
       "checksums": Object {
         "/foo/bar/a": "123",
         "/foo/bar/c": "789",
       },
+      "dllName": "manifest-name",
+      "dllRefs": Object {
+        "./some-foo.ts": "1:ku/53aRMuAA+4TmQeCWA/w:GtuPW9agF2yecW0xAIHtUQ",
+      },
       "spec": Object {
         "banner": undefined,
         "contextDir": "/foo/bar",

packages/kbn-optimizer/src/common/bundle.ts

@@ -14,6 +14,7 @@ import { UnknownVals } from './ts_helpers';
 import { omit } from './obj_helpers';
 import { includes } from './array_helpers';
 import type { Hashes } from './hashes';
+import { ParsedDllManifest } from './dll_manifest';
 
 const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
@@ -88,12 +89,19 @@ export class Bundle {
 
   /**
    * Calculate the cache key for this bundle based from current
-   * mtime values.
+   * state determined by looking at files on disk.
    */
-  createCacheKey(paths: string[], hashes: Hashes): unknown {
+  createCacheKey(
+    paths: string[],
+    hashes: Hashes,
+    dllManifest: ParsedDllManifest,
+    dllRefKeys: string[]
+  ): unknown {
     return {
       spec: omit(this.toSpec(), ['pageLoadAssetSizeLimit']),
       checksums: Object.fromEntries(paths.map((p) => [p, hashes.getCached(p)] as const)),
+      dllName: dllManifest.name,
+      dllRefs: Object.fromEntries(dllRefKeys.map((k) => [k, dllManifest.content[k]] as const)),
     };
   }
 

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -19,6 +19,7 @@ export interface State {
   workUnits?: number;
   referencedPaths?: string[];
   bundleRefExportIds?: string[];
+  dllRefKeys?: string[];
 }
 
 const DEFAULT_STATE: State = {};
@@ -90,6 +91,10 @@ export class BundleCache {
     return this.get().bundleRefExportIds;
   }
 
+  public getDllRefKeys() {
+    return this.get().dllRefKeys;
+  }
+
   public getCacheKey() {
     return this.get().cacheKey;
   }

packages/kbn-optimizer/src/common/dll_manifest.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Crypto from 'crypto';
+
+export interface DllManifest {
+  name: string;
+  content: Record<string, any>;
+}
+
+export interface ParsedDllManifest {
+  name: string;
+  content: Record<string, any>;
+}
+
+const hash = (s: string) => {
+  return Crypto.createHash('md5').update(s).digest('base64').replace(/=+$/, '');
+};
+
+export function parseDllManifest(manifest: DllManifest): ParsedDllManifest {
+  return {
+    name: manifest.name,
+    content: Object.fromEntries(
+      Object.entries(manifest.content).map(([k, v]) => {
+        const { id, buildMeta, ...other } = v;
+        const metaJson = JSON.stringify(buildMeta) || '{}';
+        const otherJson = JSON.stringify(other) || '{}';
+
+        return [
+          k,
+          [
+            v.id,
+            ...(metaJson !== '{}' ? [hash(metaJson)] : []),
+            ...(otherJson !== '{}' ? [hash(otherJson)] : []),
+          ].join(':'),
+        ];
+      })
+    ),
+  };
+}

packages/kbn-optimizer/src/common/index.ts

@@ -20,3 +20,4 @@ export * from './parse_path';
 export * from './theme_tags';
 export * from './obj_helpers';
 export * from './hashes';
+export * from './dll_manifest';

packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts

@@ -10,20 +10,31 @@ import Path from 'path';
 
 import cpy from 'cpy';
 import del from 'del';
-import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+import { createAbsolutePathSerializer, createStripAnsiSerializer } from '@kbn/dev-utils';
 
 import { OptimizerConfig } from '../optimizer/optimizer_config';
-import { allValuesFrom, Bundle, Hashes } from '../common';
+import { allValuesFrom, Bundle, Hashes, ParsedDllManifest } from '../common';
 import { getBundleCacheEvent$ } from '../optimizer/bundle_cache';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
 const MOCK_REPO_DIR = Path.resolve(TMP_DIR, 'mock_repo');
 
+jest.mock('../common/dll_manifest', () => ({
+  parseDllManifest: jest.fn(),
+}));
+
+const EMPTY_DLL_MANIFEST: ParsedDllManifest = {
+  name: 'foo',
+  content: {},
+};
+jest.requireMock('../common/dll_manifest').parseDllManifest.mockReturnValue(EMPTY_DLL_MANIFEST);
+
 expect.addSnapshotSerializer({
   print: () => '<Bundle>',
   test: (v) => v instanceof Bundle,
 });
+expect.addSnapshotSerializer(createStripAnsiSerializer());
 expect.addSnapshotSerializer(createAbsolutePathSerializer(MOCK_REPO_DIR));
 
 beforeEach(async () => {
@@ -55,14 +66,15 @@ it('emits "bundle cached" event when everything is updated', async () => {
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -94,14 +106,15 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -133,14 +146,15 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async ()
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey: undefined,
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -172,14 +186,15 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey: 'old',
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -188,11 +203,11 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "[32m- Expected[39m
-    [31m+ Received[39m
+        "diff": "- Expected
+    + Received
 
-    [32m- \\"old\\"[39m
-    [31m+ \\"optimizerCacheKey\\"[39m",
+    - \\"old\\"
+    + \\"optimizerCacheKey\\"",
         "reason": "optimizer cache key mismatch",
         "type": "bundle not cached",
       },
@@ -216,14 +231,15 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
     Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
   ];
   const hashes = await Hashes.ofFiles(referencedPaths);
-  const cacheKey = bundle.createCacheKey(referencedPaths, hashes);
+  const cacheKey = bundle.createCacheKey(referencedPaths, hashes, EMPTY_DLL_MANIFEST, []);
 
   bundle.cache.set({
     cacheKey,
     optimizerCacheKey,
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: ['plugin/bar/public'],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -232,12 +248,12 @@ it('emits "bundle not cached" event when bundleRefExportIds is outdated, include
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "[32m- Expected[39m
-    [31m+ Received[39m
+        "diff": "- Expected
+    + Received
 
-    [2m  [[22m
-    [31m+   \\"plugin/bar/public\\"[39m
-    [2m  ][22m",
+      [
+    +   \\"plugin/bar/public\\"
+      ]",
         "reason": "bundle references outdated",
         "type": "bundle not cached",
       },
@@ -267,6 +283,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => {
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
@@ -304,6 +321,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
     referencedPaths,
     moduleCount: referencedPaths.length,
     bundleRefExportIds: [],
+    dllRefKeys: [],
   });
 
   jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'new');
@@ -314,14 +332,53 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
     Array [
       Object {
         "bundle": <Bundle>,
-        "diff": "[32m- Expected[39m
-    [31m+ Received[39m
+        "diff": "- Expected
+    + Received
 
-    [32m- \\"old\\"[39m
-    [31m+ \\"new\\"[39m",
+    - \\"old\\"
+    + \\"new\\"",
         "reason": "cache key mismatch",
         "type": "bundle not cached",
       },
     ]
   `);
 });
+
+it('emits "dll references missing" when cacheKey has no dllRefs', async () => {
+  const config = OptimizerConfig.create({
+    repoRoot: MOCK_REPO_DIR,
+    pluginScanDirs: [],
+    pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
+    maxWorkerCount: 1,
+  });
+  const [bundle] = config.bundles;
+
+  const optimizerCacheKey = 'optimizerCacheKey';
+  const referencedPaths = [
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'),
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'),
+    Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'),
+  ];
+
+  bundle.cache.set({
+    cacheKey: 'correct',
+    optimizerCacheKey,
+    referencedPaths,
+    moduleCount: referencedPaths.length,
+    bundleRefExportIds: [],
+  });
+
+  jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'correct');
+
+  const cacheEvents = await allValuesFrom(getBundleCacheEvent$(config, optimizerCacheKey));
+
+  expect(cacheEvents).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "bundle": <Bundle>,
+        "reason": "dll references missing",
+        "type": "bundle not cached",
+      },
+    ]
+  `);
+});

packages/kbn-optimizer/src/integration_tests/optimizer_built_paths.test.ts

@@ -22,6 +22,7 @@ it(`finds all the optimizer files relative to it's path`, async () => {
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle_refs.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/bundle.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/compiler_messages.js,
+      <absolute path>/node_modules/@kbn/optimizer/target_node/common/dll_manifest.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/event_stream_helpers.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/hashes.js,
       <absolute path>/node_modules/@kbn/optimizer/target_node/common/index.js,