diff --git a/ui/.build/src/build.ts b/ui/.build/src/build.ts index 8ce532f57458a..08e007cf82df8 100644 --- a/ui/.build/src/build.ts +++ b/ui/.build/src/build.ts @@ -7,8 +7,9 @@ import { tsc, stopTscWatch } from './tsc.ts'; import { sass, stopSass } from './sass.ts'; import { esbuild, stopEsbuildWatch } from './esbuild.ts'; import { sync, stopSync } from './sync.ts'; +import { hash } from './hash.ts'; import { stopManifest } from './manifest.ts'; -import { env, errorMark, colors as c } from './env.ts'; +import { env, errorMark, c } from './env.ts'; import { i18n, stopI18nWatch } from './i18n.ts'; import { unique } from './algo.ts'; import { clean } from './clean.ts'; @@ -37,7 +38,7 @@ export async function build(pkgs: string[]): Promise { fs.promises.mkdir(env.buildTempDir), ]); - await Promise.all([sass(), sync(), i18n()]); + await Promise.all([sass(), sync().then(hash), i18n()]); await Promise.all([tsc(), esbuild(), monitor(pkgs)]); } diff --git a/ui/.build/src/clean.ts b/ui/.build/src/clean.ts index a582b265ca675..1133823367ed5 100644 --- a/ui/.build/src/clean.ts +++ b/ui/.build/src/clean.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'fs'; import fg from 'fast-glob'; -import { env, colors as c } from './env.ts'; +import { env, c } from './env.ts'; const globOpts: fg.Options = { absolute: true, diff --git a/ui/.build/src/console.ts b/ui/.build/src/console.ts index 14cc931dfbbe2..41c2298a5216e 100644 --- a/ui/.build/src/console.ts +++ b/ui/.build/src/console.ts @@ -1,5 +1,5 @@ import { createServer, IncomingMessage, ServerResponse } from 'node:http'; -import { env, errorMark, warnMark, colors as c } from './env.ts'; +import { env, errorMark, warnMark, c } from './env.ts'; export async function startConsole() { if (!env.remoteLog || !env.watch) return; diff --git a/ui/.build/src/env.ts b/ui/.build/src/env.ts index 7c5cb22a26489..52a1dbd9c8c42 100644 --- a/ui/.build/src/env.ts +++ b/ui/.build/src/env.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import type { Package } from './parse.ts'; import { unique, isEquivalent } from './algo.ts'; -import { updateManifest } from './manifest.ts'; +import { type Manifest, updateManifest } from './manifest.ts'; // state, logging, and exit code logic @@ -25,10 +25,6 @@ export const env = new (class { readonly i18nDestDir = path.join(this.rootDir, 'translation', 'dest'); readonly i18nJsDir = path.join(this.rootDir, 'translation', 'js'); - packages: Map = new Map(); - workspaceDeps: Map = new Map(); - building: Package[] = []; - watch = false; clean = false; prod = false; @@ -43,11 +39,17 @@ export const env = new (class { startTime: number | undefined = Date.now(); logTime = true; logContext = true; - color: any = { - build: 'green', - sass: 'magenta', - tsc: 'yellow', - esbuild: 'blue', + logColor = true; + + packages: Map = new Map(); + workspaceDeps: Map = new Map(); + building: Package[] = []; + manifest: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = { + i18n: {}, + js: {}, + css: {}, + hashed: {}, + dirty: false, }; get sass(): boolean { @@ -97,7 +99,7 @@ export const env = new (class { } good(ctx = 'build'): void { - this.log(colors.good('No errors') + this.watch ? ` - ${colors.grey('Watching')}...` : '', { ctx: ctx }); + this.log(c.good('No errors') + this.watch ? ` - ${c.grey('Watching')}...` : '', { ctx: ctx }); } log(d: any, { ctx = 'build', error = false, warn = false }: any = {}): void { @@ -108,19 +110,17 @@ export const env = new (class { ? d.join('\n') : JSON.stringify(d); - const esc = this.color ? escape : (text: string, _: any) => text; - - if (!this.color) text = stripColorEscapes(text); + if (!this.logColor) text = stripColorEscapes(text); const prefix = ( (this.logTime === false ? '' : prettyTime()) + - (!ctx || !this.logContext ? '' : `[${esc(ctx, colorForCtx(ctx, this.color))}] `) + (!ctx || !this.logContext ? '' : `[${escape(ctx, colorForCtx(ctx))}] `) ).trim(); lines(text).forEach(line => console.log( `${prefix ? prefix + ' - ' : ''}${ - error ? esc(line, codes.error) : warn ? esc(line, codes.warn) : line + error ? escape(line, codes.error) : warn ? escape(line, codes.warn) : line }`, ), ); @@ -132,13 +132,11 @@ export const env = new (class { const allDone = this.exitCode.size === 3; if (ctx !== 'tsc' || code === 0) this.log( - `${code === 0 ? 'Done' : colors.red('Failed')}` + - (this.watch ? ` - ${colors.grey('Watching')}...` : ''), + `${code === 0 ? 'Done' : c.red('Failed')}` + (this.watch ? ` - ${c.grey('Watching')}...` : ''), { ctx }, ); if (allDone) { - if (this.startTime && !err) - this.log(`Done in ${colors.green((Date.now() - this.startTime) / 1000 + '')}s`); + if (this.startTime && !err) this.log(`Done in ${c.green((Date.now() - this.startTime) / 1000 + '')}s`); this.startTime = undefined; // it's pointless to time subsequent builds, they are too fast } if (!this.watch && err) process.exitCode = err; @@ -148,11 +146,12 @@ export const env = new (class { export const lines = (s: string): string[] => s.split(/[\n\r\f]+/).filter(x => x.trim()); -const escape = (text: string, code: string): string => `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m`; +const escape = (text: string, code: string): string => + env.logColor ? `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m` : text; const colorLines = (text: string, code: string) => lines(text) - .map(t => (env.color ? escape(t, code) : t)) + .map(t => escape(t, code)) .join('\n'); const codes: Record = { @@ -168,7 +167,7 @@ const codes: Record = { warn: '33', }; -export const colors: Record string> = { +export const c: Record string> = { red: (text: string): string => colorLines(text, codes.red), green: (text: string): string => colorLines(text, codes.green), yellow: (text: string): string => colorLines(text, codes.yellow), @@ -183,11 +182,16 @@ export const colors: Record string> = { cyanBold: (text: string): string => colorLines(text, codes.cyan + ';1'), }; -export const errorMark: string = colors.red('✘ ') + colors.error('[ERROR]'); -export const warnMark: string = colors.yellow('⚠ ') + colors.warn('[WARNING]'); +export const errorMark: string = c.red('✘ ') + c.error('[ERROR]'); +export const warnMark: string = c.yellow('⚠ ') + c.warn('[WARNING]'); -const colorForCtx = (ctx: string, color: any): string => - color && ctx in color && color[ctx] in codes ? codes[color[ctx]] : codes.grey; +const colorForCtx = (ctx: string): string => + ({ + build: codes.green, + sass: codes.magenta, + tsc: codes.yellow, + esbuild: codes.blue, + })[ctx] ?? codes.grey; const pad2 = (n: number) => (n < 10 ? `0${n}` : `${n}`); diff --git a/ui/.build/src/esbuild.ts b/ui/.build/src/esbuild.ts index 56f7442a2cc21..0cba5f7a6dc8b 100644 --- a/ui/.build/src/esbuild.ts +++ b/ui/.build/src/esbuild.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import es from 'esbuild'; import fs from 'node:fs'; -import { env, errorMark, colors as c } from './env.ts'; +import { env, errorMark, c } from './env.ts'; import { type Manifest, updateManifest } from './manifest.ts'; import { trimAndConsolidateWhitespace, readable } from './parse.ts'; diff --git a/ui/.build/src/hash.ts b/ui/.build/src/hash.ts new file mode 100644 index 0000000000000..3899b55cff1fd --- /dev/null +++ b/ui/.build/src/hash.ts @@ -0,0 +1,88 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { globArray } from './parse.ts'; +import { updateManifest } from './manifest.ts'; +import { env } from './env.ts'; + +export async function hash(): Promise { + const newHashLinks = new Map(); + const alreadyHashed = new Map(); + const hashed = ( + await Promise.all( + env.building.flatMap(pkg => + pkg.hash.map(async hash => + (await globArray(hash.glob, { cwd: env.outDir })).map(path => ({ + path, + replace: hash.replace, + root: pkg.root, + })), + ), + ), + ) + ).flat(); + + const sourceStats = await Promise.all(hashed.map(hash => fs.promises.stat(hash.path))); + + for (const [i, stat] of sourceStats.entries()) { + const name = hashed[i].path.slice(env.outDir.length + 1); + if (stat.mtimeMs === env.manifest.hashed[name]?.mtime) + alreadyHashed.set(name, env.manifest.hashed[name].hash!); + else newHashLinks.set(name, stat.mtimeMs); + } + await Promise.allSettled([...alreadyHashed].map(([name, hash]) => link(name, hash))); + + for await (const { name, hash } of [...newHashLinks.keys()].map(hashLink)) { + env.manifest.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) }); + } + if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(env.manifest.hashed).length) return; + + for (const key of Object.keys(env.manifest.hashed)) { + if (!hashed.some(x => x.path.endsWith(key))) delete env.manifest.hashed[key]; + } + // TODO find a better home for all of this + const replaceMany: Map }> = new Map(); + for (const { root, path, replace } of hashed) { + if (!replace) continue; + const replaceInOne = replaceMany.get(replace) ?? { root, mapping: {} }; + const from = path.slice(env.outDir.length + 1); + replaceInOne.mapping[from] = asHashed(from, env.manifest.hashed[from].hash!); + replaceMany.set(replace, replaceInOne); + } + for await (const { name, hash } of [...replaceMany].map(([n, r]) => replaceAllIn(n, r.root, r.mapping))) { + env.manifest.hashed[name] = { hash }; + } + updateManifest({ dirty: true }); +} + +async function replaceAllIn(name: string, root: string, files: Record) { + const result = Object.entries(files).reduce( + (data, [from, to]) => data.replaceAll(from, to), + await fs.promises.readFile(path.join(root, name), 'utf8'), + ); + const hash = crypto.createHash('sha256').update(result).digest('hex').slice(0, 8); + await fs.promises.writeFile(path.join(env.hashOutDir, asHashed(name, hash)), result); + return { name, hash }; +} + +async function hashLink(name: string) { + const src = path.join(env.outDir, name); + const hash = crypto + .createHash('sha256') + .update(await fs.promises.readFile(src)) + .digest('hex') + .slice(0, 8); + await link(name, hash); + return { name, hash }; +} + +async function link(name: string, hash: string) { + const link = path.join(env.hashOutDir, asHashed(name, hash)); + return fs.promises.symlink(path.join('..', name), link).catch(() => {}); +} + +function asHashed(path: string, hash: string) { + const name = path.slice(path.lastIndexOf('/') + 1); + const extPos = name.indexOf('.'); + return extPos < 0 ? `${name}.${hash}` : `${name.slice(0, extPos)}.${hash}${name.slice(extPos)}`; +} diff --git a/ui/.build/src/i18n.ts b/ui/.build/src/i18n.ts index bb2768f571daa..90d960a1c2b05 100644 --- a/ui/.build/src/i18n.ts +++ b/ui/.build/src/i18n.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import crypto from 'node:crypto'; import fs from 'node:fs'; import { XMLParser } from 'fast-xml-parser'; -import { env, colors as c } from './env.ts'; +import { env, c } from './env.ts'; import { globArray, readable } from './parse.ts'; import { type Manifest, updateManifest } from './manifest.ts'; import { quantize, zip } from './algo.ts'; diff --git a/ui/.build/src/manifest.ts b/ui/.build/src/manifest.ts index 5d4d4ac02e265..2eed58d381bdb 100644 --- a/ui/.build/src/manifest.ts +++ b/ui/.build/src/manifest.ts @@ -2,7 +2,7 @@ import cps from 'node:child_process'; import path from 'node:path'; import fs from 'node:fs'; import crypto from 'node:crypto'; -import { env, colors as c, warnMark } from './env.ts'; +import { env, c, warnMark } from './env.ts'; import { allSources as allCssSources } from './sass.ts'; import { jsLogger } from './console.ts'; import { shallowSort, isEquivalent } from './algo.ts'; @@ -10,27 +10,20 @@ import { shallowSort, isEquivalent } from './algo.ts'; type SplitAsset = { hash?: string; path?: string; imports?: string[]; inline?: string; mtime?: number }; export type Manifest = { [key: string]: SplitAsset }; -export const manifests: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = { - i18n: {}, - js: {}, - css: {}, - hashed: {}, - dirty: false, -}; let writeTimer: NodeJS.Timeout; export function stopManifest(): void { clearTimeout(writeTimer); } -export function updateManifest(update: Partial = {}): void { - if (update?.dirty) manifests.dirty = true; - for (const key of Object.keys(update ?? {}) as (keyof typeof manifests)[]) { - if (key === 'dirty' || isEquivalent(manifests[key], update?.[key])) continue; - manifests[key] = shallowSort({ ...manifests[key], ...update?.[key] }); - manifests.dirty = true; +export function updateManifest(update: Partial = {}): void { + if (update?.dirty) env.manifest.dirty = true; + for (const key of Object.keys(update ?? {}) as (keyof typeof env.manifest)[]) { + if (key === 'dirty' || isEquivalent(env.manifest[key], update?.[key])) continue; + env.manifest[key] = shallowSort({ ...env.manifest[key], ...update?.[key] }); + env.manifest.dirty = true; } - if (!manifests.dirty) return; + if (!env.manifest.dirty) return; clearTimeout(writeTimer); writeTimer = setTimeout(writeManifest, 500); } @@ -54,12 +47,12 @@ async function writeManifest() { if (env.remoteLog) clientJs.push(jsLogger()); const pairLine = ([name, info]: [string, SplitAsset]) => `'${name.replaceAll("'", "\\'")}':'${info.hash}'`; - const jsLines = Object.entries(manifests.js) + const jsLines = Object.entries(env.manifest.js) .filter(([name, _]) => !/common\.[A-Z0-9]{8}/.test(name)) .map(pairLine) .join(','); - const cssLines = Object.entries(manifests.css).map(pairLine).join(','); - const hashedLines = Object.entries(manifests.hashed).map(pairLine).join(','); + const cssLines = Object.entries(env.manifest.css).map(pairLine).join(','); + const hashedLines = Object.entries(env.manifest.hashed).map(pairLine).join(','); clientJs.push(`window.site.manifest={\ncss:{${cssLines}},\njs:{${jsLines}},\nhashed:{${hashedLines}}\n};`); @@ -72,9 +65,9 @@ async function writeManifest() { new Date(new Date().toUTCString()).toISOString().split('.')[0] + '+00:00' }';\n`; const serverManifest = { - js: { manifest: { hash }, ...manifests.js, ...manifests.i18n }, - css: { ...manifests.css }, - hashed: { ...manifests.hashed }, + js: { manifest: { hash }, ...env.manifest.js, ...env.manifest.i18n }, + css: { ...env.manifest.css }, + hashed: { ...env.manifest.hashed }, }; await Promise.all([ @@ -84,7 +77,7 @@ async function writeManifest() { JSON.stringify(serverManifest, null, env.prod ? undefined : 2), ), ]); - manifests.dirty = false; + env.manifest.dirty = false; env.log( `Manifest '${c.cyan(`public/compiled/manifest.${env.prod ? 'prod' : 'dev'}.json`)}' -> '${c.cyan( `public/compiled/manifest.${hash}.js`, @@ -96,17 +89,17 @@ async function isComplete() { for (const bundle of [...env.packages.values()].map(x => x.bundle ?? []).flat()) { if (!bundle.module) continue; const name = path.basename(bundle.module, '.ts'); - if (!manifests.js[name]) { + if (!env.manifest.js[name]) { env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.ts')}'`); return false; } } for (const css of await allCssSources()) { const name = path.basename(css, '.scss'); - if (!manifests.css[name]) { + if (!env.manifest.css[name]) { env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.scss')}'`); return false; } } - return Object.keys(manifests.i18n).length > 0; + return Object.keys(env.manifest.i18n).length > 0; } diff --git a/ui/.build/src/parse.ts b/ui/.build/src/parse.ts index bf63c5bcbdd08..47c411c59ee8a 100644 --- a/ui/.build/src/parse.ts +++ b/ui/.build/src/parse.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import fg from 'fast-glob'; -import { env, errorMark, colors as c } from './env.ts'; +import { env, errorMark, c } from './env.ts'; export type Bundle = { module?: string; inline?: string }; diff --git a/ui/.build/src/sass.ts b/ui/.build/src/sass.ts index 40d7352741b56..9a21d3b05300f 100644 --- a/ui/.build/src/sass.ts +++ b/ui/.build/src/sass.ts @@ -4,7 +4,7 @@ import ps from 'node:process'; import path from 'node:path'; import crypto from 'node:crypto'; import clr from 'tinycolor2'; -import { env, colors as c, lines, errorMark } from './env.ts'; +import { env, c, lines, errorMark } from './env.ts'; import { globArray, readable } from './parse.ts'; import { updateManifest } from './manifest.ts'; import { clamp } from './algo.ts'; diff --git a/ui/.build/src/sync.ts b/ui/.build/src/sync.ts index 3355b2097c2a8..4e067d86b8164 100644 --- a/ui/.build/src/sync.ts +++ b/ui/.build/src/sync.ts @@ -1,9 +1,8 @@ import fs from 'node:fs'; import path from 'node:path'; -import crypto from 'node:crypto'; import { type Sync, globArray, globArrays } from './parse.ts'; -import { updateManifest, manifests } from './manifest.ts'; -import { env, errorMark, colors as c } from './env.ts'; +import { hash } from './hash.ts'; +import { env, errorMark, c } from './env.ts'; import { quantize } from './algo.ts'; const syncWatch: fs.FSWatcher[] = []; @@ -22,7 +21,7 @@ export async function sync(): Promise { for (const pkg of env.building) { for (const sync of pkg.sync) { - for (const src of await globSync(sync)) { + for (const src of await syncGlob(sync)) { if (env.watch) watched.set(src, [...(watched.get(src) ?? []), sync]); } } @@ -36,7 +35,6 @@ export async function sync(): Promise { if (!watched.has(path.dirname(src))) watched.set(path.dirname(src), []); } } - hashedManifest(); if (env.watch) for (const dir of watched.keys()) { const watcher = fs.watch(dir); @@ -45,9 +43,7 @@ export async function sync(): Promise { updated.add(dir); clearTimeout(watchTimeout); watchTimeout = setTimeout(() => { - Promise.all([...updated].flatMap(d => (watched.get(d) ?? []).map(x => globSync(x)))).then( - hashedManifest, - ); + Promise.all([...updated].flatMap(d => (watched.get(d) ?? []).map(x => syncGlob(x)))).then(hash); updated.clear(); }, 2000); }); @@ -63,7 +59,7 @@ export function isUnmanagedAsset(absfile: string): boolean { return true; } -async function globSync(cp: Sync): Promise> { +async function syncGlob(cp: Sync): Promise> { const watchDirs = new Set(); const dest = path.join(env.rootDir, cp.dest) + path.sep; @@ -106,84 +102,3 @@ async function syncOne(absSrc: string, absDest: string, pkgName: string) { env.log(`[${c.grey(pkgName)}] - ${errorMark} - failed sync '${c.cyan(absSrc)}' to '${c.cyan(absDest)}'`); } } - -async function hashedManifest(): Promise { - const newHashLinks = new Map(); - const alreadyHashed = new Map(); - const hashed = ( - await Promise.all( - env.building.flatMap(pkg => - pkg.hash.map(async hash => - (await globArray(hash.glob, { cwd: env.outDir })).map(path => ({ - path, - replace: hash.replace, - root: pkg.root, - })), - ), - ), - ) - ).flat(); - - const sourceStats = await Promise.all(hashed.map(hash => fs.promises.stat(hash.path))); - - for (const [i, stat] of sourceStats.entries()) { - const name = hashed[i].path.slice(env.outDir.length + 1); - if (stat.mtimeMs === manifests.hashed[name]?.mtime) alreadyHashed.set(name, manifests.hashed[name].hash!); - else newHashLinks.set(name, stat.mtimeMs); - } - await Promise.allSettled([...alreadyHashed].map(([name, hash]) => link(name, hash))); - - for await (const { name, hash } of [...newHashLinks.keys()].map(hashLink)) { - manifests.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) }); - } - if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(manifests.hashed).length) return; - - for (const key of Object.keys(manifests.hashed)) { - if (!hashed.some(x => x.path.endsWith(key))) delete manifests.hashed[key]; - } - // TODO find a better home for all of this - const replaceMany: Map }> = new Map(); - for (const { root, path, replace } of hashed) { - if (!replace) continue; - const replaceInOne = replaceMany.get(replace) ?? { root, mapping: {} }; - const from = path.slice(env.outDir.length + 1); - replaceInOne.mapping[from] = asHashed(from, manifests.hashed[from].hash!); - replaceMany.set(replace, replaceInOne); - } - for await (const { name, hash } of [...replaceMany].map(([n, r]) => replaceAllIn(n, r.root, r.mapping))) { - manifests.hashed[name] = { hash }; - } - updateManifest({ dirty: true }); -} - -async function replaceAllIn(name: string, root: string, files: Record) { - const result = Object.entries(files).reduce( - (data, [from, to]) => data.replaceAll(from, to), - await fs.promises.readFile(path.join(root, name), 'utf8'), - ); - const hash = crypto.createHash('sha256').update(result).digest('hex').slice(0, 8); - await fs.promises.writeFile(path.join(env.hashOutDir, asHashed(name, hash)), result); - return { name, hash }; -} - -async function hashLink(name: string) { - const src = path.join(env.outDir, name); - const hash = crypto - .createHash('sha256') - .update(await fs.promises.readFile(src)) - .digest('hex') - .slice(0, 8); - await link(name, hash); - return { name, hash }; -} - -async function link(name: string, hash: string) { - const link = path.join(env.hashOutDir, asHashed(name, hash)); - return fs.promises.symlink(path.join('..', name), link).catch(() => {}); -} - -function asHashed(path: string, hash: string) { - const name = path.slice(path.lastIndexOf('/') + 1); - const extPos = name.indexOf('.'); - return extPos < 0 ? `${name}.${hash}` : `${name.slice(0, extPos)}.${hash}${name.slice(extPos)}`; -} diff --git a/ui/.build/src/tsc.ts b/ui/.build/src/tsc.ts index 8955489280fc8..64c00cbf4fd05 100644 --- a/ui/.build/src/tsc.ts +++ b/ui/.build/src/tsc.ts @@ -3,7 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import ts from 'typescript'; import { Worker } from 'node:worker_threads'; -import { env, colors as c, errorMark } from './env.ts'; +import { env, c, errorMark } from './env.ts'; import { globArray, folderSize, readable } from './parse.ts'; import { clamp } from './algo.ts'; import type { WorkerData, Message } from './tscWorker.ts';