From 19325e42345f8bdcc96bbc1320d7b545cdbaa2eb Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 25 May 2023 13:47:19 +0700 Subject: [PATCH 01/11] chore(web): minor esbuild version bump to get 'alias' option, adds tslib --- package-lock.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package-lock.json b/package-lock.json index b0ccb18f595..aea4d14f432 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11057,6 +11057,7 @@ "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^10.9.1", + "tslib": "^2.5.2", "typescript": "^4.9.5" } } From b66c173fc2cde52b30650d81e2f20c8778088294 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Jun 2023 12:22:28 +0700 Subject: [PATCH 02/11] feat(web): prototype manual treeshake for tslib (in web/browser build-bundler) --- web/src/app/browser/build-bundler.js | 122 ++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index 93b1b01c06e..bc0ea616958 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -29,6 +29,104 @@ if(process.argv.length > 2) { } } +// Component #1: Detect all `tslib` helpers we actually want to use. +const tslibHelperNames = [ + "__extends", + "__assign", + "__rest", + "__decorate", + "__param", + "__metadata", + "__awaiter", + "__generator", + "__exportStar", + "__createBinding", + "__values", + "__read", + "__spread", + "__spreadArrays", + "__await", + "__asyncGenerator", + "__asyncDelegator", + "__asyncValues", + "__makeTemplateObject", + "__importStar", + "__importDefault", + "__classPrivateFieldGet", + "__classPrivateFieldSet" +]; + +const detectedHelpers = []; + +let tslibHelperDetectionPlugin = { + name: 'tslib helper use detection', + setup(build) { + build.onLoad({filter: /\.js$/}, async (args) => { + // + if(/tslib.js$/.test(args.path)) { + return; + } + + let source = await fs.promises.readFile(args.path, 'utf8'); + + for(let helper of tslibHelperNames) { + if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { + detectedHelpers.push(helper); + } + } + + return; + }); + } +} +// + +// Component #2: when we've actually determined which ones are safe to remove, this plugin +// can remove their code. +let tslibForcedTreeshakingPlugin = { + name: 'tslib helpers - forced treeshaking', + setup(build) { + build.onLoad({filter: /tslib.js$/}, async (args) => { + let source = await fs.promises.readFile(args.path, 'utf8'); + + // TODO: transformations to eliminate the stuff we don't want. + for(let unusedHelper of unusedHelpers) { + // Removes the 'exporter' line used to actually export it from the tslib source. + source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + + // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. + let definitionStart = source.indexOf(`${unusedHelper} = function`); + if(definitionStart == -1) { + console.error("tslib has likely been updated recently; could not erase definition for helper " + unusedHelper); + continue; + } + let scopeDepth = 0; + let i = definitionStart; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0) { + if(char == '{') { + scopeDepth++; + } + i++; + char = source.charAt(i); + } + i++; // we want to erase it, too. + + source = source.replace(source.substring(definitionStart, i), ''); + + // The top-level var declaration is auto-removed by esbuild when no references to it remain. + + } + + return { + contents: source, + loader: 'js' + }; + }); + } +} +// + /* * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - * the `@class` emit comment-annotation is designed to facilitate tree-shaking for ES5-targeted @@ -62,12 +160,34 @@ const commonConfig = { 'index': '../../../build/app/browser/obj/debug-main.js', }, outfile: '../../../build/app/browser/debug/keymanweb.js', - plugins: [ es5ClassAnnotationAsPurePlugin ], + plugins: [ tslibForcedTreeshakingPlugin, es5ClassAnnotationAsPurePlugin ], target: "es5", treeShaking: true, tsconfig: './tsconfig.json' }; +// tslib tree-shake phase 1 - detecting which helpers are safe to remove. +await esbuild.build({ + ...commonConfig, + plugins: [ tslibHelperDetectionPlugin ], + write: false +}); + +// Logs on the tree-shaking decisions +console.log("Detected helpers from tslib: "); +console.log(detectedHelpers.sort()); + +console.log(); +console.log("Unused helpers: "); +const unusedHelpers = []; +tslibHelperNames.forEach((entry) => { + if(!detectedHelpers.find((detected) => detected == entry)) { + unusedHelpers.push(entry); + } +}); +console.log(unusedHelpers); + +// From here, the builds are configured to do phase 2 from the preprocessing data done 'til now. await esbuild.build(commonConfig); let result = await esbuild.build({ From 8deddfbab2b092b7693d4bbf50325afbc52e1a1d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Jun 2023 16:11:58 +0700 Subject: [PATCH 03/11] refactor(web): refactored unused helper detection aspect --- web/src/app/browser/build-bundler.js | 222 +++++++++++++++------------ 1 file changed, 120 insertions(+), 102 deletions(-) diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index bc0ea616958..06024965b47 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -29,103 +29,87 @@ if(process.argv.length > 2) { } } -// Component #1: Detect all `tslib` helpers we actually want to use. -const tslibHelperNames = [ - "__extends", - "__assign", - "__rest", - "__decorate", - "__param", - "__metadata", - "__awaiter", - "__generator", - "__exportStar", - "__createBinding", - "__values", - "__read", - "__spread", - "__spreadArrays", - "__await", - "__asyncGenerator", - "__asyncDelegator", - "__asyncValues", - "__makeTemplateObject", - "__importStar", - "__importDefault", - "__classPrivateFieldGet", - "__classPrivateFieldSet" -]; - -const detectedHelpers = []; - -let tslibHelperDetectionPlugin = { - name: 'tslib helper use detection', - setup(build) { - build.onLoad({filter: /\.js$/}, async (args) => { - // - if(/tslib.js$/.test(args.path)) { - return; - } +async function determineNeededDowncompileHelpers(config, log) { + // Component #1: Detect all `tslib` helpers we actually want to use. + const tslibHelperNames = [ + "__extends", + "__assign", + "__rest", + "__decorate", + "__param", + "__metadata", + "__awaiter", + "__generator", + "__exportStar", + "__createBinding", + "__values", + "__read", + "__spread", + "__spreadArrays", + "__await", + "__asyncGenerator", + "__asyncDelegator", + "__asyncValues", + "__makeTemplateObject", + "__importStar", + "__importDefault", + "__classPrivateFieldGet", + "__classPrivateFieldSet" + ]; + + const detectedHelpers = []; + + let tslibHelperDetectionPlugin = { + name: 'tslib helper use detection', + setup(build) { + build.onLoad({filter: /\.js$/}, async (args) => { + // + if(/tslib.js$/.test(args.path)) { + return; + } - let source = await fs.promises.readFile(args.path, 'utf8'); + let source = await fs.promises.readFile(args.path, 'utf8'); - for(let helper of tslibHelperNames) { - if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { - detectedHelpers.push(helper); + for(let helper of tslibHelperNames) { + if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { + detectedHelpers.push(helper); + } } - } - return; - }); + return; + }); + } } -} -// -// Component #2: when we've actually determined which ones are safe to remove, this plugin -// can remove their code. -let tslibForcedTreeshakingPlugin = { - name: 'tslib helpers - forced treeshaking', - setup(build) { - build.onLoad({filter: /tslib.js$/}, async (args) => { - let source = await fs.promises.readFile(args.path, 'utf8'); + // tslib tree-shake phase 1 - detecting which helpers are safe to remove. + await esbuild.build({ + ...config, + plugins: [ tslibHelperDetectionPlugin, ...config.plugins ], + write: false + }); - // TODO: transformations to eliminate the stuff we don't want. - for(let unusedHelper of unusedHelpers) { - // Removes the 'exporter' line used to actually export it from the tslib source. - source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + // At this point, we can determine what's unused. + detectedHelpers.sort(); - // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. - let definitionStart = source.indexOf(`${unusedHelper} = function`); - if(definitionStart == -1) { - console.error("tslib has likely been updated recently; could not erase definition for helper " + unusedHelper); - continue; - } - let scopeDepth = 0; - let i = definitionStart; - let char = source.charAt(i); - while(char != '}' || --scopeDepth != 0) { - if(char == '{') { - scopeDepth++; - } - i++; - char = source.charAt(i); - } - i++; // we want to erase it, too. - - source = source.replace(source.substring(definitionStart, i), ''); - - // The top-level var declaration is auto-removed by esbuild when no references to it remain. + const unusedHelpers = []; + tslibHelperNames.forEach((entry) => { + if(!detectedHelpers.find((detected) => detected == entry)) { + unusedHelpers.push(entry); + } + }); - } + if(log) { + // Logs on the tree-shaking decisions + console.log("Detected helpers from tslib: "); + console.log(); - return { - contents: source, - loader: 'js' - }; - }); + console.log(); + console.log("Unused helpers: "); + console.log(unusedHelpers); } + + return unusedHelpers; } -// /* * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - @@ -160,32 +144,66 @@ const commonConfig = { 'index': '../../../build/app/browser/obj/debug-main.js', }, outfile: '../../../build/app/browser/debug/keymanweb.js', - plugins: [ tslibForcedTreeshakingPlugin, es5ClassAnnotationAsPurePlugin ], + plugins: [ es5ClassAnnotationAsPurePlugin ], target: "es5", treeShaking: true, tsconfig: './tsconfig.json' }; // tslib tree-shake phase 1 - detecting which helpers are safe to remove. -await esbuild.build({ - ...commonConfig, - plugins: [ tslibHelperDetectionPlugin ], - write: false -}); +const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig); -// Logs on the tree-shaking decisions -console.log("Detected helpers from tslib: "); -console.log(detectedHelpers.sort()); +// Component #2: when we've actually determined which ones are safe to remove, this plugin +// can remove their code. +let tslibForcedTreeshakingPlugin = { + name: 'tslib helpers - forced treeshaking', + setup(build) { + build.onLoad({filter: /tslib.js$/}, async (args) => { + let source = await fs.promises.readFile(args.path, 'utf8'); -console.log(); -console.log("Unused helpers: "); -const unusedHelpers = []; -tslibHelperNames.forEach((entry) => { - if(!detectedHelpers.find((detected) => detected == entry)) { - unusedHelpers.push(entry); + // TODO: transformations to eliminate the stuff we don't want. + for(let unusedHelper of unusedHelpers) { + // Removes the 'exporter' line used to actually export it from the tslib source. + source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + + // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. + let definitionStart = source.indexOf(`${unusedHelper} = function`); + if(definitionStart == -1) { + if(unusedHelper == '__createBinding') { + console.warn("Currently unable to force-treeshake the __createBinding tslib helper"); + } else { + console.error("tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper); + } + continue; + } + + let scopeDepth = 0; + let i = definitionStart; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0) { + if(char == '{') { + scopeDepth++; + } + i++; + char = source.charAt(i); + } + i++; // we want to erase it, too. + + source = source.replace(source.substring(definitionStart, i), ''); + + // The top-level var declaration is auto-removed by esbuild when no references to it remain. + + } + + return { + contents: source, + loader: 'js' + }; + }); } -}); -console.log(unusedHelpers); +} + +commonConfig.plugins = [tslibForcedTreeshakingPlugin, ...commonConfig.plugins]; // From here, the builds are configured to do phase 2 from the preprocessing data done 'til now. await esbuild.build(commonConfig); From 06803300280853a113d0ab68ec59208a0d22039e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 08:59:51 +0700 Subject: [PATCH 04/11] chore(web): prep to formalize the tslib treeshaker, integration with esbuild logs --- web/src/app/browser/build-bundler.js | 166 ++++++++++++++++++--------- 1 file changed, 110 insertions(+), 56 deletions(-) diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index 06024965b47..98ac54e2c54 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -6,7 +6,6 @@ */ import esbuild from 'esbuild'; -import { spawn } from 'child_process'; import fs from 'fs'; let EMIT_FILESIZE_PROFILE = false; @@ -29,8 +28,8 @@ if(process.argv.length > 2) { } } +// Component #1: Detect all `tslib` helpers we actually want to use. async function determineNeededDowncompileHelpers(config, log) { - // Component #1: Detect all `tslib` helpers we actually want to use. const tslibHelperNames = [ "__extends", "__assign", @@ -65,6 +64,8 @@ async function determineNeededDowncompileHelpers(config, log) { build.onLoad({filter: /\.js$/}, async (args) => { // if(/tslib.js$/.test(args.path)) { + // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. return; } @@ -76,6 +77,8 @@ async function determineNeededDowncompileHelpers(config, log) { } } + // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. return; }); } @@ -84,6 +87,9 @@ async function determineNeededDowncompileHelpers(config, log) { // tslib tree-shake phase 1 - detecting which helpers are safe to remove. await esbuild.build({ ...config, + // `tslibHelperDetectionPlugin` is pass-through, and so has no net effect on + // the build. We just use this run to scan for any utilized `tslib` helper funcs, + // not to manipulate the actual source in any way. plugins: [ tslibHelperDetectionPlugin, ...config.plugins ], write: false }); @@ -111,6 +117,105 @@ async function determineNeededDowncompileHelpers(config, log) { return unusedHelpers; } +// Component #2: when we've actually determined which ones are safe to remove, this plugin +// can remove their code. +function configuredDowncompileTreeshakePlugin(unusedHelpers) { + function indexToSourcePosition(source, index) { + let priorText = source.substring(0, index); + let lineNum = priorText.split('\n').length; + let lastLineBreakIndex = priorText.lastIndexOf('\n'); + let colNum = index - lastLineBreakIndex; + let nextLineBreakIndex = source.indexOf('\n', lastLineBreakIndex+1); + + return { + line: lineNum, + column: colNum, + lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex) + } + } + + return { + name: 'tslib forced treeshaking', + setup(build) { + build.onLoad({filter: /tslib.js$/}, async (args) => { + const trueSource = await fs.promises.readFile(args.path, 'utf8'); + let source = trueSource; + + let warnings = []; + let errors = []; + + for(let unusedHelper of unusedHelpers) { + // Removes the 'exporter' line used to actually export it from the tslib source. + source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + + // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. + let definitionStart = source.indexOf(`${unusedHelper} = function`); + + // Emission of warnings & errors + if(definitionStart == -1) { + let matchString = `${unusedHelper} =` + let bestGuessIndex = trueSource.indexOf(matchString); + if(bestGuessIndex == -1) { + matchString = `var ${unusedHelper}` + bestGuessIndex = trueSource.indexOf(matchString); + } + + if(bestGuessIndex == -1) { + matchString = ''; + } + + let logLocation = indexToSourcePosition(trueSource, bestGuessIndex); + + let location = { + file: args.path, + line: logLocation.line, + column: logLocation.column, + length: matchString.length, + lineText: logLocation.lineText + } + + if(unusedHelper == '__createBinding') { + warnings.push({ + text: "Currently unable to force-treeshake the __createBinding tslib helper", + location: location + }); + } else { + warnings.push({ + text: "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, + location: location + }); + } + continue; + } + + let scopeDepth = 0; + let i = definitionStart; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0) { + if(char == '{') { + scopeDepth++; + } + i++; + char = source.charAt(i); + } + i++; // we want to erase it, too. + + source = source.replace(source.substring(definitionStart, i), ''); + + // The top-level var declaration is auto-removed by esbuild when no references to it remain. + } + + return { + contents: source, + loader: 'js', + warnings: warnings, + errors: errors + }; + }); + } + }; +}; + /* * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - * the `@class` emit comment-annotation is designed to facilitate tree-shaking for ES5-targeted @@ -150,62 +255,11 @@ const commonConfig = { tsconfig: './tsconfig.json' }; -// tslib tree-shake phase 1 - detecting which helpers are safe to remove. +// Prepare the needed setup for `tslib` treeshaking. const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig); +commonConfig.plugins = [configuredDowncompileTreeshakePlugin(unusedHelpers), ...commonConfig.plugins]; -// Component #2: when we've actually determined which ones are safe to remove, this plugin -// can remove their code. -let tslibForcedTreeshakingPlugin = { - name: 'tslib helpers - forced treeshaking', - setup(build) { - build.onLoad({filter: /tslib.js$/}, async (args) => { - let source = await fs.promises.readFile(args.path, 'utf8'); - - // TODO: transformations to eliminate the stuff we don't want. - for(let unusedHelper of unusedHelpers) { - // Removes the 'exporter' line used to actually export it from the tslib source. - source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); - - // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. - let definitionStart = source.indexOf(`${unusedHelper} = function`); - if(definitionStart == -1) { - if(unusedHelper == '__createBinding') { - console.warn("Currently unable to force-treeshake the __createBinding tslib helper"); - } else { - console.error("tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper); - } - continue; - } - - let scopeDepth = 0; - let i = definitionStart; - let char = source.charAt(i); - while(char != '}' || --scopeDepth != 0) { - if(char == '{') { - scopeDepth++; - } - i++; - char = source.charAt(i); - } - i++; // we want to erase it, too. - - source = source.replace(source.substring(definitionStart, i), ''); - - // The top-level var declaration is auto-removed by esbuild when no references to it remain. - - } - - return { - contents: source, - loader: 'js' - }; - }); - } -} - -commonConfig.plugins = [tslibForcedTreeshakingPlugin, ...commonConfig.plugins]; - -// From here, the builds are configured to do phase 2 from the preprocessing data done 'til now. +// And now... do the actual builds. await esbuild.build(commonConfig); let result = await esbuild.build({ From a37d3236ac769dd143253ac2ea4dba083827fbd7 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 09:47:13 +0700 Subject: [PATCH 05/11] refactor(web): extracts treeshaker into tslib helper-project --- common/web/tslib/package.json | 13 ++ common/web/tslib/src/esbuild-tools.ts | 219 ++++++++++++++++++++++++++ common/web/tslib/tsconfig.json | 5 +- package-lock.json | 3 + web/src/app/browser/build-bundler.js | 191 +--------------------- 5 files changed, 240 insertions(+), 191 deletions(-) create mode 100644 common/web/tslib/src/esbuild-tools.ts diff --git a/common/web/tslib/package.json b/common/web/tslib/package.json index 164a5257ec4..84eed17970e 100644 --- a/common/web/tslib/package.json +++ b/common/web/tslib/package.json @@ -2,6 +2,16 @@ "name": "@keymanapp/tslib", "description": "An ES5 + esbuild-compatible wrapper for the 'tslib' library", "main": "./build/index.js", + "exports": { + ".": { + "types": "./build/index.d.ts", + "import": "./build/index.js" + }, + "./esbuild-tools": { + "types": "./build/esbuild-tools.d.ts", + "import": "./build/esbuild-tools.js" + } + }, "scripts": { "build": "gosh ./build.sh", "clean": "tsc -b --clean", @@ -11,5 +21,8 @@ "tslib": "^2.5.2", "typescript": "^4.9.5" }, + "devDependencies": { + "esbuild": "^0.15.16" + }, "type": "module" } diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts new file mode 100644 index 00000000000..81856d09e3c --- /dev/null +++ b/common/web/tslib/src/esbuild-tools.ts @@ -0,0 +1,219 @@ +import esbuild from 'esbuild'; +import fs from 'fs'; + +// Note: this package is intended 100% as a dev-tool, hence why esbuild is just a dev-dependency. + +// Component #1: Detect all `tslib` helpers we actually want to use. +export async function determineNeededDowncompileHelpers(config: esbuild.BuildOptions, log: boolean) { + const tslibHelperNames = [ + "__extends", + "__assign", + "__rest", + "__decorate", + "__param", + "__metadata", + "__awaiter", + "__generator", + "__exportStar", + "__createBinding", + "__values", + "__read", + "__spread", + "__spreadArrays", + "__await", + "__asyncGenerator", + "__asyncDelegator", + "__asyncValues", + "__makeTemplateObject", + "__importStar", + "__importDefault", + "__classPrivateFieldGet", + "__classPrivateFieldSet" + ]; + + const detectedHelpers: string[] = []; + + let tslibHelperDetectionPlugin = { + name: 'tslib helper use detection', + setup(build) { + build.onLoad({filter: /\.js$/}, async (args) => { + // + if(/tslib.js$/.test(args.path)) { + // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. + return; + } + + let source = await fs.promises.readFile(args.path, 'utf8'); + + for(let helper of tslibHelperNames) { + if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { + detectedHelpers.push(helper); + } + } + + // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + // configured plugins from working. + return; + }); + } + } + + // tslib tree-shake phase 1 - detecting which helpers are safe to remove. + await esbuild.build({ + ...config, + // `tslibHelperDetectionPlugin` is pass-through, and so has no net effect on + // the build. We just use this run to scan for any utilized `tslib` helper funcs, + // not to manipulate the actual source in any way. + plugins: [ tslibHelperDetectionPlugin, ...(config.plugins ?? []) ], + write: false + }); + + // At this point, we can determine what's unused. + detectedHelpers.sort(); + + const unusedHelpers: string[] = []; + tslibHelperNames.forEach((entry) => { + if(!detectedHelpers.find((detected) => detected == entry)) { + unusedHelpers.push(entry); + } + }); + + if(log) { + // Logs on the tree-shaking decisions + console.log("Detected helpers from tslib: "); + console.log(); + + console.log(); + console.log("Unused helpers: "); + console.log(unusedHelpers); + } + + return unusedHelpers; +} + +function indexToSourcePosition(source: string, index: number) { + let priorText = source.substring(0, index); + let lineNum = priorText.split('\n').length; + let lastLineBreakIndex = priorText.lastIndexOf('\n'); + let colNum = index - lastLineBreakIndex; + let nextLineBreakIndex = source.indexOf('\n', lastLineBreakIndex+1); + + return { + line: lineNum, + column: colNum, + lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex) + } +} + +// Ugh. https://github.com/evanw/esbuild/issues/2656#issuecomment-1304013941 +function wrapClassPlugin(instancePlugin: esbuild.Plugin) { + return { + name: instancePlugin.name, + setup: instancePlugin.setup.bind(instancePlugin) + }; +} + +// Component #2: when we've actually determined which ones are safe to remove, this plugin +// can remove their code. +class TslibTreeshaker implements esbuild.Plugin { + public readonly name = 'tslib forced treeshaking'; + + private unusedHelpers: string[]; + + constructor(unusedHelpers: string[]) { + this.unusedHelpers = unusedHelpers; + } + + setup(build: esbuild.PluginBuild) { + build.onLoad({filter: /tslib.js$/}, async (args) => { + const trueSource = await fs.promises.readFile(args.path, 'utf8'); + let source = trueSource; + + let warnings: esbuild.Message[] = []; + let errors: esbuild.Message[] = []; + + for(let unusedHelper of this.unusedHelpers) { + // Removes the 'exporter' line used to actually export it from the tslib source. + source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); + + // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. + let definitionStart = source.indexOf(`${unusedHelper} = function`); + + // Emission of warnings & errors + if(definitionStart == -1) { + let matchString = `${unusedHelper} =` + let bestGuessIndex = trueSource.indexOf(matchString); + if(bestGuessIndex == -1) { + matchString = `var ${unusedHelper}` + bestGuessIndex = trueSource.indexOf(matchString); + } + + if(bestGuessIndex == -1) { + matchString = ''; + } + + let logLocation = indexToSourcePosition(trueSource, bestGuessIndex); + + let location: esbuild.Location = { + file: args.path, + line: logLocation.line, + column: logLocation.column, + length: matchString.length, + lineText: logLocation.lineText, + namespace: '', + suggestion: '' + } + + if(unusedHelper == '__createBinding') { + warnings.push({ + id: '', + notes: [], + detail: '', + pluginName: this.name, + text: "Currently unable to force-treeshake the __createBinding tslib helper", + location: location + }); + } else { + warnings.push({ + id: '', + notes: [], + detail: '', + pluginName: this.name, + text: "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, + location: location + }); + } + continue; + } + + let scopeDepth = 0; + let i = definitionStart; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0) { + if(char == '{') { + scopeDepth++; + } + i++; + char = source.charAt(i); + } + i++; // we want to erase it, too. + + source = source.replace(source.substring(definitionStart, i), ''); + + // The top-level var declaration is auto-removed by esbuild when no references to it remain. + } + + return { + contents: source, + loader: 'js', + warnings: warnings, + errors: errors + }; + }); + } +}; + +export function buildTslibTreeshaker(unusedHelpers: string[]) { + return wrapClassPlugin(new TslibTreeshaker(unusedHelpers)); +}; \ No newline at end of file diff --git a/common/web/tslib/tsconfig.json b/common/web/tslib/tsconfig.json index 459f336baa9..c141af3d3c5 100644 --- a/common/web/tslib/tsconfig.json +++ b/common/web/tslib/tsconfig.json @@ -4,18 +4,19 @@ "allowJs": true, "module": "es6", "moduleResolution": "Node", + "allowSyntheticDefaultImports": true, "inlineSources": true, "sourceMap": true, "declaration": true, "target": "es5", "tsBuildInfoFile": "./build/tsconfig.tsbuildinfo", "types": ["node"], - "lib": ["es6"], + "lib": ["es6", "DOM"], // The latter is b/c esbuild expects the type. "baseUrl": "./src", "outDir": "./build/", "rootDir": "./src" }, "include": [ - "./src/index.ts" + "./src/*.ts" ] } diff --git a/package-lock.json b/package-lock.json index aea4d14f432..5ffd15651ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -355,6 +355,9 @@ "dependencies": { "tslib": "^2.5.2", "typescript": "^4.9.5" + }, + "devDependencies": { + "esbuild": "^0.15.16" } }, "common/web/types": { diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index 98ac54e2c54..743f80c4512 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -7,6 +7,7 @@ import esbuild from 'esbuild'; import fs from 'fs'; +import { determineNeededDowncompileHelpers, buildTslibTreeshaker } from '@keymanapp/tslib/esbuild-tools'; let EMIT_FILESIZE_PROFILE = false; @@ -28,194 +29,6 @@ if(process.argv.length > 2) { } } -// Component #1: Detect all `tslib` helpers we actually want to use. -async function determineNeededDowncompileHelpers(config, log) { - const tslibHelperNames = [ - "__extends", - "__assign", - "__rest", - "__decorate", - "__param", - "__metadata", - "__awaiter", - "__generator", - "__exportStar", - "__createBinding", - "__values", - "__read", - "__spread", - "__spreadArrays", - "__await", - "__asyncGenerator", - "__asyncDelegator", - "__asyncValues", - "__makeTemplateObject", - "__importStar", - "__importDefault", - "__classPrivateFieldGet", - "__classPrivateFieldSet" - ]; - - const detectedHelpers = []; - - let tslibHelperDetectionPlugin = { - name: 'tslib helper use detection', - setup(build) { - build.onLoad({filter: /\.js$/}, async (args) => { - // - if(/tslib.js$/.test(args.path)) { - // Returning `undefined` makes this 'pass-through' - it doesn't prevent other - // configured plugins from working. - return; - } - - let source = await fs.promises.readFile(args.path, 'utf8'); - - for(let helper of tslibHelperNames) { - if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { - detectedHelpers.push(helper); - } - } - - // Returning `undefined` makes this 'pass-through' - it doesn't prevent other - // configured plugins from working. - return; - }); - } - } - - // tslib tree-shake phase 1 - detecting which helpers are safe to remove. - await esbuild.build({ - ...config, - // `tslibHelperDetectionPlugin` is pass-through, and so has no net effect on - // the build. We just use this run to scan for any utilized `tslib` helper funcs, - // not to manipulate the actual source in any way. - plugins: [ tslibHelperDetectionPlugin, ...config.plugins ], - write: false - }); - - // At this point, we can determine what's unused. - detectedHelpers.sort(); - - const unusedHelpers = []; - tslibHelperNames.forEach((entry) => { - if(!detectedHelpers.find((detected) => detected == entry)) { - unusedHelpers.push(entry); - } - }); - - if(log) { - // Logs on the tree-shaking decisions - console.log("Detected helpers from tslib: "); - console.log(); - - console.log(); - console.log("Unused helpers: "); - console.log(unusedHelpers); - } - - return unusedHelpers; -} - -// Component #2: when we've actually determined which ones are safe to remove, this plugin -// can remove their code. -function configuredDowncompileTreeshakePlugin(unusedHelpers) { - function indexToSourcePosition(source, index) { - let priorText = source.substring(0, index); - let lineNum = priorText.split('\n').length; - let lastLineBreakIndex = priorText.lastIndexOf('\n'); - let colNum = index - lastLineBreakIndex; - let nextLineBreakIndex = source.indexOf('\n', lastLineBreakIndex+1); - - return { - line: lineNum, - column: colNum, - lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex) - } - } - - return { - name: 'tslib forced treeshaking', - setup(build) { - build.onLoad({filter: /tslib.js$/}, async (args) => { - const trueSource = await fs.promises.readFile(args.path, 'utf8'); - let source = trueSource; - - let warnings = []; - let errors = []; - - for(let unusedHelper of unusedHelpers) { - // Removes the 'exporter' line used to actually export it from the tslib source. - source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); - - // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. - let definitionStart = source.indexOf(`${unusedHelper} = function`); - - // Emission of warnings & errors - if(definitionStart == -1) { - let matchString = `${unusedHelper} =` - let bestGuessIndex = trueSource.indexOf(matchString); - if(bestGuessIndex == -1) { - matchString = `var ${unusedHelper}` - bestGuessIndex = trueSource.indexOf(matchString); - } - - if(bestGuessIndex == -1) { - matchString = ''; - } - - let logLocation = indexToSourcePosition(trueSource, bestGuessIndex); - - let location = { - file: args.path, - line: logLocation.line, - column: logLocation.column, - length: matchString.length, - lineText: logLocation.lineText - } - - if(unusedHelper == '__createBinding') { - warnings.push({ - text: "Currently unable to force-treeshake the __createBinding tslib helper", - location: location - }); - } else { - warnings.push({ - text: "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, - location: location - }); - } - continue; - } - - let scopeDepth = 0; - let i = definitionStart; - let char = source.charAt(i); - while(char != '}' || --scopeDepth != 0) { - if(char == '{') { - scopeDepth++; - } - i++; - char = source.charAt(i); - } - i++; // we want to erase it, too. - - source = source.replace(source.substring(definitionStart, i), ''); - - // The top-level var declaration is auto-removed by esbuild when no references to it remain. - } - - return { - contents: source, - loader: 'js', - warnings: warnings, - errors: errors - }; - }); - } - }; -}; - /* * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - * the `@class` emit comment-annotation is designed to facilitate tree-shaking for ES5-targeted @@ -257,7 +70,7 @@ const commonConfig = { // Prepare the needed setup for `tslib` treeshaking. const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig); -commonConfig.plugins = [configuredDowncompileTreeshakePlugin(unusedHelpers), ...commonConfig.plugins]; +commonConfig.plugins = [buildTslibTreeshaker(unusedHelpers), ...commonConfig.plugins]; // And now... do the actual builds. await esbuild.build(commonConfig); From 9a6a2f88bf20779a9d3ac99828e3ab53f28d0b78 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 11:50:04 +0700 Subject: [PATCH 06/11] feat(web): extends capabilities of the tslib treeshaker components, finalizes coverage --- common/web/tslib/src/esbuild-tools.ts | 225 +++++++++++++++++++------- web/src/app/browser/build-bundler.js | 2 +- 2 files changed, 166 insertions(+), 61 deletions(-) diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts index 81856d09e3c..7eccc64d654 100644 --- a/common/web/tslib/src/esbuild-tools.ts +++ b/common/web/tslib/src/esbuild-tools.ts @@ -4,7 +4,7 @@ import fs from 'fs'; // Note: this package is intended 100% as a dev-tool, hence why esbuild is just a dev-dependency. // Component #1: Detect all `tslib` helpers we actually want to use. -export async function determineNeededDowncompileHelpers(config: esbuild.BuildOptions, log: boolean) { +export async function determineNeededDowncompileHelpers(config: esbuild.BuildOptions, ignoreFilePattern?: RegExp, log?: boolean) { const tslibHelperNames = [ "__extends", "__assign", @@ -19,6 +19,7 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt "__values", "__read", "__spread", + "__spreadArray", "__spreadArrays", "__await", "__asyncGenerator", @@ -26,9 +27,15 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt "__asyncValues", "__makeTemplateObject", "__importStar", + "__setModuleDefault", // only used by the previous entry! "__importDefault", "__classPrivateFieldGet", - "__classPrivateFieldSet" + "__classPrivateFieldSet", + "__classPrivateFieldIn", + "__runInitializers", + "__setFunctionName", + "__propKey", + "__esDecorate" ]; const detectedHelpers: string[] = []; @@ -37,18 +44,47 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt name: 'tslib helper use detection', setup(build) { build.onLoad({filter: /\.js$/}, async (args) => { - // - if(/tslib.js$/.test(args.path)) { - // Returning `undefined` makes this 'pass-through' - it doesn't prevent other + let source = await fs.promises.readFile(args.path, 'utf8'); + + let warnings: esbuild.Message[] = []; + + if(/tslib.js$/.test(args.path) || ignoreFilePattern?.test(args.path)) { + let declarationRegex = /var (__[a-zA-Z0-9]+)/g; + let results = source.match(declarationRegex); + + for(let result of results) { + let capture = result.substring(4); + + if(!tslibHelperNames.find((entry) => entry == capture)) { + // TODO: integrate as esbuild log message. + console.log("Missing? " + capture); + + // Match: `result` itself. Grab index, build location, build message. + let index = source.indexOf(result); + + warnings.push(buildMessage( + 'tslib helper use detection', + 'Probable `tslib` helper `' + capture + '` has not been validated for tree-shaking potential', + indexToSourceLocation(source, index, result.length, args.path) + )); + } + } + + // Returning `undefined` for `content` makes this 'pass-through' - it doesn't prevent other // configured plugins from working. - return; + return { + warnings: warnings + }; } - let source = await fs.promises.readFile(args.path, 'utf8'); for(let helper of tslibHelperNames) { if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { detectedHelpers.push(helper); + + if(helper == '__importStar') { + detectedHelpers.push('__setModuleDefault'); + } } } @@ -92,7 +128,7 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt return unusedHelpers; } -function indexToSourcePosition(source: string, index: number) { +function indexToSourceLocation(source: string, index: number, matchLength: number, file: string): esbuild.Location { let priorText = source.substring(0, index); let lineNum = priorText.split('\n').length; let lastLineBreakIndex = priorText.lastIndexOf('\n'); @@ -100,9 +136,24 @@ function indexToSourcePosition(source: string, index: number) { let nextLineBreakIndex = source.indexOf('\n', lastLineBreakIndex+1); return { + file: file, line: lineNum, column: colNum, - lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex) + lineText: source.substring(lastLineBreakIndex+1, nextLineBreakIndex), + namespace: '', + suggestion: '', + length: matchLength + } +} + +function buildMessage(pluginName: string, message: string, location: esbuild.Location): esbuild.Message { + return { + id: '', + notes: [], + detail: '', + pluginName: pluginName, + text: message, + location: location } } @@ -125,6 +176,67 @@ class TslibTreeshaker implements esbuild.Plugin { this.unusedHelpers = unusedHelpers; } + treeshakeDefinition(source: string, start: number, expectTernary: boolean, name: string, location: esbuild.Location) { + let scopeDepth = 0; + let parenDepth = 0; + let topLevelTernaryActive = 0; + let i = start; + let char = source.charAt(i); + while(char != '}' || --scopeDepth != 0 || topLevelTernaryActive) { + if(char == '{') { + scopeDepth++; + } else if(char == '(') { + parenDepth++; + } else if(char == ')') { + parenDepth--; + } else if(char == '?' && scopeDepth == 0) { + if(!expectTernary) { + return { + source: source, + error: buildMessage(this.name, `Unexpected conditional for definition of ${name}`, location) + } + } + topLevelTernaryActive++; + } else if(char == ':' && topLevelTernaryActive && scopeDepth == 0) { + topLevelTernaryActive--; + } + i++; + + if(i > source.length) { + return { + source: source, + error: buildMessage(this.name, `Failed to determine end of definition for ${name}`, location) + }; + } + + char = source.charAt(i); + } + i++; // we want to erase the final '}', too. + + // The functions may be wrapped with parens. + if(parenDepth == 1) { + let nextOpen = source.indexOf('(', i); + let nextClosed = source.indexOf(')', i); + + if(nextOpen < nextClosed) { + return { + source: source, + error: buildMessage(this.name, `Failed to determine end of definition for ${name}`, location) + }; + } else { + i = nextClosed + 1; + } + } + + if(source.charAt(i) == ';') { + i++; + } + + return { + source: source.replace(source.substring(start, i), '') + }; + } + setup(build: esbuild.PluginBuild) { build.onLoad({filter: /tslib.js$/}, async (args) => { const trueSource = await fs.promises.readFile(args.path, 'utf8'); @@ -138,70 +250,63 @@ class TslibTreeshaker implements esbuild.Plugin { source = source.replace(`exporter\(\"${unusedHelper}\", ${unusedHelper}\);`, ''); // Removes the actual helper function definition - obviously, the biggest filesize savings to be had here. - let definitionStart = source.indexOf(`${unusedHelper} = function`); + let matchString = `${unusedHelper} = function`; + let expectTernary = false; + + // A special case - it has two different versions depending on if Object.create exists or not. + // This is established via a ternary conditional. Just adds a bit to the parsing. + if(unusedHelper == '__createBinding') { + matchString = `${unusedHelper} = `; + expectTernary = true; + } else if(unusedHelper == '__setModuleDefault') { + matchString = `var ${unusedHelper} = `; // is inlined, not declared at top! + expectTernary = true; + } + let definitionStart = source.indexOf(matchString); + + if(definitionStart > -1) { + const result = this.treeshakeDefinition( + source, + definitionStart, + expectTernary, + unusedHelper, + indexToSourceLocation(trueSource, trueSource.indexOf(matchString), matchString.length, args.path) + ); + + if(result.error) { + errors.push(result.error); + } else { + source = result.source; + } + continue; + } - // Emission of warnings & errors + // If we reached this point, we couldn't treeshake the unused helper appropriately. if(definitionStart == -1) { - let matchString = `${unusedHelper} =` + // Matches the standard definition pattern's left-hand assignment component. + matchString = `${unusedHelper} =` let bestGuessIndex = trueSource.indexOf(matchString); + if(bestGuessIndex == -1) { + // Failing the above, we match the declaration higher up within the file. matchString = `var ${unusedHelper}` bestGuessIndex = trueSource.indexOf(matchString); } + // Failing THAT, we just give up and go start-of-file. if(bestGuessIndex == -1) { matchString = ''; + bestGuessIndex = 0; } - let logLocation = indexToSourcePosition(trueSource, bestGuessIndex); - - let location: esbuild.Location = { - file: args.path, - line: logLocation.line, - column: logLocation.column, - length: matchString.length, - lineText: logLocation.lineText, - namespace: '', - suggestion: '' - } - - if(unusedHelper == '__createBinding') { - warnings.push({ - id: '', - notes: [], - detail: '', - pluginName: this.name, - text: "Currently unable to force-treeshake the __createBinding tslib helper", - location: location - }); - } else { - warnings.push({ - id: '', - notes: [], - detail: '', - pluginName: this.name, - text: "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, - location: location - }); - } - continue; + warnings.push( + buildMessage( + this.name, + "tslib has likely been updated recently; could not force-treeshake tslib helper " + unusedHelper, + indexToSourceLocation(trueSource, bestGuessIndex, matchString.length, args.path) + ) + ); } - - let scopeDepth = 0; - let i = definitionStart; - let char = source.charAt(i); - while(char != '}' || --scopeDepth != 0) { - if(char == '{') { - scopeDepth++; - } - i++; - char = source.charAt(i); - } - i++; // we want to erase it, too. - - source = source.replace(source.substring(definitionStart, i), ''); - - // The top-level var declaration is auto-removed by esbuild when no references to it remain. } return { diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index 743f80c4512..6c21fed2556 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -69,7 +69,7 @@ const commonConfig = { }; // Prepare the needed setup for `tslib` treeshaking. -const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig); +const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig, /worker-main\.wrapped\.(?:min\.).js?/); commonConfig.plugins = [buildTslibTreeshaker(unusedHelpers), ...commonConfig.plugins]; // And now... do the actual builds. From fed4618d027afca1916ca2dd081b7e89a5836808 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 14:39:55 +0700 Subject: [PATCH 07/11] feat(common/models): tslib use, treeshaking in lm-worker --- common/models/templates/build.sh | 5 +++-- common/models/templates/tsconfig.json | 1 + common/models/wordbreakers/src/default/index.ts | 8 +++----- common/models/wordbreakers/tsconfig.json | 1 + common/web/lm-worker/build-bundler.js | 11 ++++++++++- common/web/lm-worker/build.sh | 1 + common/web/lm-worker/tsconfig.json | 3 ++- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/common/models/templates/build.sh b/common/models/templates/build.sh index b5ae2a8c236..0fe305ca5ad 100755 --- a/common/models/templates/build.sh +++ b/common/models/templates/build.sh @@ -18,8 +18,9 @@ cd "$(dirname "$THIS_SCRIPT")" ################################ Main script ################################ builder_describe "Builds the predictive-text model template implementation module" \ - "@../../web/keyman-version" \ - "@../wordbreakers" \ + "@/common/web/keyman-version" \ + "@/common/web/tslib" \ + "@/common/models/wordbreakers" \ "clean" \ "configure" \ "build" \ diff --git a/common/models/templates/tsconfig.json b/common/models/templates/tsconfig.json index 9284bddd209..7ff603ef29d 100644 --- a/common/models/templates/tsconfig.json +++ b/common/models/templates/tsconfig.json @@ -7,6 +7,7 @@ "moduleResolution": "node16", "sourceMap": true, "sourceRoot": "/common/models/templates/src", + "importHelpers": true, "inlineSources": true, "strict": true, "lib": ["es6"], diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 65dbd0faf00..20e159ed198 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -431,7 +431,7 @@ function findBoundaries(text: string, options?: DefaultWordBreakerOptions): numb } // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter].concat(SET_MIDNUMLETQ); if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { continue; } @@ -474,7 +474,7 @@ function findBoundaries(text: string, options?: DefaultWordBreakerOptions): numb } // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum].concat(SET_MIDNUMLETQ); if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { continue; } @@ -488,9 +488,7 @@ function findBoundaries(text: string, options?: DefaultWordBreakerOptions): numb } // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, - WordBreakProperty.Numeric, - ...SET_AHLETTER]; + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, WordBreakProperty.Numeric].concat(SET_AHLETTER); if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { continue; } diff --git a/common/models/wordbreakers/tsconfig.json b/common/models/wordbreakers/tsconfig.json index 6c029da40af..549b7410d6e 100644 --- a/common/models/wordbreakers/tsconfig.json +++ b/common/models/wordbreakers/tsconfig.json @@ -8,6 +8,7 @@ "moduleResolution": "Node16", "declaration": true, "sourceMap": true, + "importHelpers": true, "inlineSources": true, "sourceRoot": "/common/models/wordbreakers/src", "strict": true, diff --git a/common/web/lm-worker/build-bundler.js b/common/web/lm-worker/build-bundler.js index 605ff3cd400..eb7bb5ea415 100644 --- a/common/web/lm-worker/build-bundler.js +++ b/common/web/lm-worker/build-bundler.js @@ -6,8 +6,8 @@ */ import esbuild from 'esbuild'; -import { spawn } from 'child_process'; import fs from 'fs'; +import { determineNeededDowncompileHelpers, buildTslibTreeshaker } from '@keymanapp/tslib/esbuild-tools'; /* * Refer to https://github.com/microsoft/TypeScript/issues/13721#issuecomment-307259227 - @@ -92,7 +92,12 @@ await esbuild.build({ target: "es5" }); +// The one that's actually a component of our releases. + const embeddedWorkerBuildOptions = { + alias: { + 'tslib': '@keymanapp/tslib' + }, bundle: true, sourcemap: true, format: "iife", @@ -106,6 +111,10 @@ const embeddedWorkerBuildOptions = { target: "es5" } +// Prepare the needed setup for `tslib` treeshaking. +const unusedHelpers = await determineNeededDowncompileHelpers(embeddedWorkerBuildOptions); +embeddedWorkerBuildOptions.plugins = [buildTslibTreeshaker(unusedHelpers), ...embeddedWorkerBuildOptions.plugins]; + // Direct-use version await esbuild.build(embeddedWorkerBuildOptions); diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 75a72df12df..82fba9bb39b 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -28,6 +28,7 @@ WORKER_OUTPUT_FILENAME=build/lib/worker-main.js builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ "@/common/web/keyman-version" \ + "@/common/web/tslib" \ "@/common/models/wordbreakers" \ "@/common/models/templates" \ "@/common/tools/sourcemap-path-remapper" \ diff --git a/common/web/lm-worker/tsconfig.json b/common/web/lm-worker/tsconfig.json index 551f1645a95..2fdda0135a1 100644 --- a/common/web/lm-worker/tsconfig.json +++ b/common/web/lm-worker/tsconfig.json @@ -5,7 +5,8 @@ "allowJs": false, "declaration": true, "module": "es6", - "moduleResolution": "node", + "moduleResolution": "node16", + "importHelpers": true, "inlineSourceMap": true, "inlineSources": true, "sourceRoot": "/common/web/lm-worker/src", From 734ebee46cb7cd6b06775fe424ec4a46b79f1a5b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 14:54:48 +0700 Subject: [PATCH 08/11] chore(web): cleans up console msg --- common/web/tslib/src/esbuild-tools.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts index 7eccc64d654..7e9c0dd191e 100644 --- a/common/web/tslib/src/esbuild-tools.ts +++ b/common/web/tslib/src/esbuild-tools.ts @@ -56,9 +56,6 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt let capture = result.substring(4); if(!tslibHelperNames.find((entry) => entry == capture)) { - // TODO: integrate as esbuild log message. - console.log("Missing? " + capture); - // Match: `result` itself. Grab index, build location, build message. let index = source.indexOf(result); From db4f0a3664f310d8a53ce4349fadad70eff5ae81 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 15:35:34 +0700 Subject: [PATCH 09/11] fix(web): fixed wrapped-worker helper-detection ignore --- common/web/tslib/src/esbuild-tools.ts | 5 ++++- web/src/app/browser/build-bundler.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts index 7e9c0dd191e..1bbc4f2a639 100644 --- a/common/web/tslib/src/esbuild-tools.ts +++ b/common/web/tslib/src/esbuild-tools.ts @@ -48,7 +48,7 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt let warnings: esbuild.Message[] = []; - if(/tslib.js$/.test(args.path) || ignoreFilePattern?.test(args.path)) { + if(/tslib.js$/.test(args.path)) { let declarationRegex = /var (__[a-zA-Z0-9]+)/g; let results = source.match(declarationRegex); @@ -74,6 +74,9 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt }; } + if(ignoreFilePattern?.test(args.path)) { + return; + } for(let helper of tslibHelperNames) { if(source.indexOf(helper) > -1 && !detectedHelpers.find((entry) => entry == helper)) { diff --git a/web/src/app/browser/build-bundler.js b/web/src/app/browser/build-bundler.js index 6c21fed2556..cc81f7c402c 100644 --- a/web/src/app/browser/build-bundler.js +++ b/web/src/app/browser/build-bundler.js @@ -69,7 +69,7 @@ const commonConfig = { }; // Prepare the needed setup for `tslib` treeshaking. -const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig, /worker-main\.wrapped\.(?:min\.).js?/); +const unusedHelpers = await determineNeededDowncompileHelpers(commonConfig, /worker-main\.wrapped(?:\.min)?\.js/); commonConfig.plugins = [buildTslibTreeshaker(unusedHelpers), ...commonConfig.plugins]; // And now... do the actual builds. From d5b2e278a9cafac3b2452b4d26e30d281331547a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Jun 2023 15:51:08 +0700 Subject: [PATCH 10/11] fix(web): noticed an extra dep or two among the helpers --- common/web/tslib/src/esbuild-tools.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/web/tslib/src/esbuild-tools.ts b/common/web/tslib/src/esbuild-tools.ts index 1bbc4f2a639..9e258992535 100644 --- a/common/web/tslib/src/esbuild-tools.ts +++ b/common/web/tslib/src/esbuild-tools.ts @@ -14,7 +14,7 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt "__metadata", "__awaiter", "__generator", - "__exportStar", + "__exportStar", // uses __createBinding "__createBinding", "__values", "__read", @@ -84,6 +84,9 @@ export async function determineNeededDowncompileHelpers(config: esbuild.BuildOpt if(helper == '__importStar') { detectedHelpers.push('__setModuleDefault'); + detectedHelpers.push('__createBinding'); + } else if(helper == '__exportStar') { + detectedHelpers.push('__createBinding'); } } } From a5358d1f8da274d57e857fa37fd64ab7658486d1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Jun 2023 08:11:20 +0700 Subject: [PATCH 11/11] chore(common/web): cleanup per PR review --- common/web/tslib/build.sh | 2 +- common/web/tslib/package.json | 6 +++--- package-lock.json | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/common/web/tslib/build.sh b/common/web/tslib/build.sh index 92435e57040..411e32d8bbb 100755 --- a/common/web/tslib/build.sh +++ b/common/web/tslib/build.sh @@ -24,4 +24,4 @@ builder_parse "$@" builder_run_action configure verify_npm_setup builder_run_action clean rm -rf build/ -builder_run_action build tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" \ No newline at end of file +builder_run_action build tsc --build \ No newline at end of file diff --git a/common/web/tslib/package.json b/common/web/tslib/package.json index 84eed17970e..ceec8c64099 100644 --- a/common/web/tslib/package.json +++ b/common/web/tslib/package.json @@ -13,15 +13,15 @@ } }, "scripts": { - "build": "gosh ./build.sh", - "clean": "tsc -b --clean", - "tsc": "tsc" + "build": "gosh ./build.sh build", + "clean": "gosh ./build.sh clean" }, "dependencies": { "tslib": "^2.5.2", "typescript": "^4.9.5" }, "devDependencies": { + "@keymanapp/resources-gosh": "*", "esbuild": "^0.15.16" }, "type": "module" diff --git a/package-lock.json b/package-lock.json index 5ffd15651ba..960e9a30479 100644 --- a/package-lock.json +++ b/package-lock.json @@ -351,12 +351,12 @@ }, "common/web/tslib": { "name": "@keymanapp/tslib", - "license": "MIT", "dependencies": { "tslib": "^2.5.2", "typescript": "^4.9.5" }, "devDependencies": { + "@keymanapp/resources-gosh": "*", "esbuild": "^0.15.16" } }, @@ -11060,7 +11060,6 @@ "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^10.9.1", - "tslib": "^2.5.2", "typescript": "^4.9.5" } }