Skip to content

Commit

Permalink
keep globals in env and move hash out of sync
Browse files Browse the repository at this point in the history
  • Loading branch information
schlawg committed Jan 19, 2025
1 parent 205554f commit 675028b
Show file tree
Hide file tree
Showing 12 changed files with 152 additions and 151 deletions.
5 changes: 3 additions & 2 deletions ui/.build/src/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import { tsc, stopTscWatch } from './tsc.ts';
import { sass, stopSass } from './sass.ts';
import { esbuild, stopEsbuildWatch } from './esbuild.ts';
import { sync, stopSync } from './sync.ts';
import { hash } from './hash.ts';
import { stopManifest } from './manifest.ts';
import { env, errorMark, colors as c } from './env.ts';
import { env, errorMark, c } from './env.ts';
import { i18n, stopI18nWatch } from './i18n.ts';
import { unique } from './algo.ts';
import { clean } from './clean.ts';
Expand Down Expand Up @@ -37,7 +38,7 @@ export async function build(pkgs: string[]): Promise<void> {
fs.promises.mkdir(env.buildTempDir),
]);

await Promise.all([sass(), sync(), i18n()]);
await Promise.all([sass(), sync().then(hash), i18n()]);
await Promise.all([tsc(), esbuild(), monitor(pkgs)]);
}

Expand Down
2 changes: 1 addition & 1 deletion ui/.build/src/clean.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { promises as fs } from 'fs';
import fg from 'fast-glob';
import { env, colors as c } from './env.ts';
import { env, c } from './env.ts';

const globOpts: fg.Options = {
absolute: true,
Expand Down
2 changes: 1 addition & 1 deletion ui/.build/src/console.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createServer, IncomingMessage, ServerResponse } from 'node:http';
import { env, errorMark, warnMark, colors as c } from './env.ts';
import { env, errorMark, warnMark, c } from './env.ts';

export async function startConsole() {
if (!env.remoteLog || !env.watch) return;
Expand Down
58 changes: 31 additions & 27 deletions ui/.build/src/env.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import path from 'node:path';
import type { Package } from './parse.ts';
import { unique, isEquivalent } from './algo.ts';
import { updateManifest } from './manifest.ts';
import { type Manifest, updateManifest } from './manifest.ts';

// state, logging, and exit code logic

Expand All @@ -25,10 +25,6 @@ export const env = new (class {
readonly i18nDestDir = path.join(this.rootDir, 'translation', 'dest');
readonly i18nJsDir = path.join(this.rootDir, 'translation', 'js');

packages: Map<string, Package> = new Map();
workspaceDeps: Map<string, string[]> = new Map();
building: Package[] = [];

watch = false;
clean = false;
prod = false;
Expand All @@ -43,11 +39,17 @@ export const env = new (class {
startTime: number | undefined = Date.now();
logTime = true;
logContext = true;
color: any = {
build: 'green',
sass: 'magenta',
tsc: 'yellow',
esbuild: 'blue',
logColor = true;

packages: Map<string, Package> = new Map();
workspaceDeps: Map<string, string[]> = new Map();
building: Package[] = [];
manifest: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = {
i18n: {},
js: {},
css: {},
hashed: {},
dirty: false,
};

get sass(): boolean {
Expand Down Expand Up @@ -97,7 +99,7 @@ export const env = new (class {
}

good(ctx = 'build'): void {
this.log(colors.good('No errors') + this.watch ? ` - ${colors.grey('Watching')}...` : '', { ctx: ctx });
this.log(c.good('No errors') + this.watch ? ` - ${c.grey('Watching')}...` : '', { ctx: ctx });
}

log(d: any, { ctx = 'build', error = false, warn = false }: any = {}): void {
Expand All @@ -108,19 +110,17 @@ export const env = new (class {
? d.join('\n')
: JSON.stringify(d);

const esc = this.color ? escape : (text: string, _: any) => text;

if (!this.color) text = stripColorEscapes(text);
if (!this.logColor) text = stripColorEscapes(text);

const prefix = (
(this.logTime === false ? '' : prettyTime()) +
(!ctx || !this.logContext ? '' : `[${esc(ctx, colorForCtx(ctx, this.color))}] `)
(!ctx || !this.logContext ? '' : `[${escape(ctx, colorForCtx(ctx))}] `)
).trim();

lines(text).forEach(line =>
console.log(
`${prefix ? prefix + ' - ' : ''}${
error ? esc(line, codes.error) : warn ? esc(line, codes.warn) : line
error ? escape(line, codes.error) : warn ? escape(line, codes.warn) : line
}`,
),
);
Expand All @@ -132,13 +132,11 @@ export const env = new (class {
const allDone = this.exitCode.size === 3;
if (ctx !== 'tsc' || code === 0)
this.log(
`${code === 0 ? 'Done' : colors.red('Failed')}` +
(this.watch ? ` - ${colors.grey('Watching')}...` : ''),
`${code === 0 ? 'Done' : c.red('Failed')}` + (this.watch ? ` - ${c.grey('Watching')}...` : ''),
{ ctx },
);
if (allDone) {
if (this.startTime && !err)
this.log(`Done in ${colors.green((Date.now() - this.startTime) / 1000 + '')}s`);
if (this.startTime && !err) this.log(`Done in ${c.green((Date.now() - this.startTime) / 1000 + '')}s`);
this.startTime = undefined; // it's pointless to time subsequent builds, they are too fast
}
if (!this.watch && err) process.exitCode = err;
Expand All @@ -148,11 +146,12 @@ export const env = new (class {

export const lines = (s: string): string[] => s.split(/[\n\r\f]+/).filter(x => x.trim());

const escape = (text: string, code: string): string => `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m`;
const escape = (text: string, code: string): string =>
env.logColor ? `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m` : text;

const colorLines = (text: string, code: string) =>
lines(text)
.map(t => (env.color ? escape(t, code) : t))
.map(t => escape(t, code))
.join('\n');

const codes: Record<string, string> = {
Expand All @@ -168,7 +167,7 @@ const codes: Record<string, string> = {
warn: '33',
};

export const colors: Record<string, (text: string) => string> = {
export const c: Record<string, (text: string) => string> = {
red: (text: string): string => colorLines(text, codes.red),
green: (text: string): string => colorLines(text, codes.green),
yellow: (text: string): string => colorLines(text, codes.yellow),
Expand All @@ -183,11 +182,16 @@ export const colors: Record<string, (text: string) => string> = {
cyanBold: (text: string): string => colorLines(text, codes.cyan + ';1'),
};

export const errorMark: string = colors.red('✘ ') + colors.error('[ERROR]');
export const warnMark: string = colors.yellow('⚠ ') + colors.warn('[WARNING]');
export const errorMark: string = c.red('✘ ') + c.error('[ERROR]');
export const warnMark: string = c.yellow('⚠ ') + c.warn('[WARNING]');

const colorForCtx = (ctx: string, color: any): string =>
color && ctx in color && color[ctx] in codes ? codes[color[ctx]] : codes.grey;
const colorForCtx = (ctx: string): string =>
({
build: codes.green,
sass: codes.magenta,
tsc: codes.yellow,
esbuild: codes.blue,
})[ctx] ?? codes.grey;

const pad2 = (n: number) => (n < 10 ? `0${n}` : `${n}`);

Expand Down
2 changes: 1 addition & 1 deletion ui/.build/src/esbuild.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import path from 'node:path';
import es from 'esbuild';
import fs from 'node:fs';
import { env, errorMark, colors as c } from './env.ts';
import { env, errorMark, c } from './env.ts';
import { type Manifest, updateManifest } from './manifest.ts';
import { trimAndConsolidateWhitespace, readable } from './parse.ts';

Expand Down
88 changes: 88 additions & 0 deletions ui/.build/src/hash.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { globArray } from './parse.ts';
import { updateManifest } from './manifest.ts';
import { env } from './env.ts';

export async function hash(): Promise<void> {
const newHashLinks = new Map<string, number>();
const alreadyHashed = new Map<string, string>();
const hashed = (
await Promise.all(
env.building.flatMap(pkg =>
pkg.hash.map(async hash =>
(await globArray(hash.glob, { cwd: env.outDir })).map(path => ({
path,
replace: hash.replace,
root: pkg.root,
})),
),
),
)
).flat();

const sourceStats = await Promise.all(hashed.map(hash => fs.promises.stat(hash.path)));

for (const [i, stat] of sourceStats.entries()) {
const name = hashed[i].path.slice(env.outDir.length + 1);
if (stat.mtimeMs === env.manifest.hashed[name]?.mtime)
alreadyHashed.set(name, env.manifest.hashed[name].hash!);
else newHashLinks.set(name, stat.mtimeMs);
}
await Promise.allSettled([...alreadyHashed].map(([name, hash]) => link(name, hash)));

for await (const { name, hash } of [...newHashLinks.keys()].map(hashLink)) {
env.manifest.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) });
}
if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(env.manifest.hashed).length) return;

for (const key of Object.keys(env.manifest.hashed)) {
if (!hashed.some(x => x.path.endsWith(key))) delete env.manifest.hashed[key];
}
// TODO find a better home for all of this
const replaceMany: Map<string, { root: string; mapping: Record<string, string> }> = new Map();
for (const { root, path, replace } of hashed) {
if (!replace) continue;
const replaceInOne = replaceMany.get(replace) ?? { root, mapping: {} };
const from = path.slice(env.outDir.length + 1);
replaceInOne.mapping[from] = asHashed(from, env.manifest.hashed[from].hash!);
replaceMany.set(replace, replaceInOne);
}
for await (const { name, hash } of [...replaceMany].map(([n, r]) => replaceAllIn(n, r.root, r.mapping))) {
env.manifest.hashed[name] = { hash };
}
updateManifest({ dirty: true });
}

async function replaceAllIn(name: string, root: string, files: Record<string, string>) {
const result = Object.entries(files).reduce(
(data, [from, to]) => data.replaceAll(from, to),
await fs.promises.readFile(path.join(root, name), 'utf8'),
);
const hash = crypto.createHash('sha256').update(result).digest('hex').slice(0, 8);
await fs.promises.writeFile(path.join(env.hashOutDir, asHashed(name, hash)), result);
return { name, hash };
}

async function hashLink(name: string) {
const src = path.join(env.outDir, name);
const hash = crypto
.createHash('sha256')
.update(await fs.promises.readFile(src))
.digest('hex')
.slice(0, 8);
await link(name, hash);
return { name, hash };
}

async function link(name: string, hash: string) {
const link = path.join(env.hashOutDir, asHashed(name, hash));
return fs.promises.symlink(path.join('..', name), link).catch(() => {});
}

function asHashed(path: string, hash: string) {
const name = path.slice(path.lastIndexOf('/') + 1);
const extPos = name.indexOf('.');
return extPos < 0 ? `${name}.${hash}` : `${name.slice(0, extPos)}.${hash}${name.slice(extPos)}`;
}
2 changes: 1 addition & 1 deletion ui/.build/src/i18n.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import path from 'node:path';
import crypto from 'node:crypto';
import fs from 'node:fs';
import { XMLParser } from 'fast-xml-parser';
import { env, colors as c } from './env.ts';
import { env, c } from './env.ts';
import { globArray, readable } from './parse.ts';
import { type Manifest, updateManifest } from './manifest.ts';
import { quantize, zip } from './algo.ts';
Expand Down
43 changes: 18 additions & 25 deletions ui/.build/src/manifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,28 @@ import cps from 'node:child_process';
import path from 'node:path';
import fs from 'node:fs';
import crypto from 'node:crypto';
import { env, colors as c, warnMark } from './env.ts';
import { env, c, warnMark } from './env.ts';
import { allSources as allCssSources } from './sass.ts';
import { jsLogger } from './console.ts';
import { shallowSort, isEquivalent } from './algo.ts';

type SplitAsset = { hash?: string; path?: string; imports?: string[]; inline?: string; mtime?: number };
export type Manifest = { [key: string]: SplitAsset };

export const manifests: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = {
i18n: {},
js: {},
css: {},
hashed: {},
dirty: false,
};
let writeTimer: NodeJS.Timeout;

export function stopManifest(): void {
clearTimeout(writeTimer);
}

export function updateManifest(update: Partial<typeof manifests> = {}): void {
if (update?.dirty) manifests.dirty = true;
for (const key of Object.keys(update ?? {}) as (keyof typeof manifests)[]) {
if (key === 'dirty' || isEquivalent(manifests[key], update?.[key])) continue;
manifests[key] = shallowSort({ ...manifests[key], ...update?.[key] });
manifests.dirty = true;
export function updateManifest(update: Partial<typeof env.manifest> = {}): void {
if (update?.dirty) env.manifest.dirty = true;
for (const key of Object.keys(update ?? {}) as (keyof typeof env.manifest)[]) {
if (key === 'dirty' || isEquivalent(env.manifest[key], update?.[key])) continue;
env.manifest[key] = shallowSort({ ...env.manifest[key], ...update?.[key] });
env.manifest.dirty = true;
}
if (!manifests.dirty) return;
if (!env.manifest.dirty) return;
clearTimeout(writeTimer);
writeTimer = setTimeout(writeManifest, 500);
}
Expand All @@ -54,12 +47,12 @@ async function writeManifest() {
if (env.remoteLog) clientJs.push(jsLogger());

const pairLine = ([name, info]: [string, SplitAsset]) => `'${name.replaceAll("'", "\\'")}':'${info.hash}'`;
const jsLines = Object.entries(manifests.js)
const jsLines = Object.entries(env.manifest.js)
.filter(([name, _]) => !/common\.[A-Z0-9]{8}/.test(name))
.map(pairLine)
.join(',');
const cssLines = Object.entries(manifests.css).map(pairLine).join(',');
const hashedLines = Object.entries(manifests.hashed).map(pairLine).join(',');
const cssLines = Object.entries(env.manifest.css).map(pairLine).join(',');
const hashedLines = Object.entries(env.manifest.hashed).map(pairLine).join(',');

clientJs.push(`window.site.manifest={\ncss:{${cssLines}},\njs:{${jsLines}},\nhashed:{${hashedLines}}\n};`);

Expand All @@ -72,9 +65,9 @@ async function writeManifest() {
new Date(new Date().toUTCString()).toISOString().split('.')[0] + '+00:00'
}';\n`;
const serverManifest = {
js: { manifest: { hash }, ...manifests.js, ...manifests.i18n },
css: { ...manifests.css },
hashed: { ...manifests.hashed },
js: { manifest: { hash }, ...env.manifest.js, ...env.manifest.i18n },
css: { ...env.manifest.css },
hashed: { ...env.manifest.hashed },
};

await Promise.all([
Expand All @@ -84,7 +77,7 @@ async function writeManifest() {
JSON.stringify(serverManifest, null, env.prod ? undefined : 2),
),
]);
manifests.dirty = false;
env.manifest.dirty = false;
env.log(
`Manifest '${c.cyan(`public/compiled/manifest.${env.prod ? 'prod' : 'dev'}.json`)}' -> '${c.cyan(
`public/compiled/manifest.${hash}.js`,
Expand All @@ -96,17 +89,17 @@ async function isComplete() {
for (const bundle of [...env.packages.values()].map(x => x.bundle ?? []).flat()) {
if (!bundle.module) continue;
const name = path.basename(bundle.module, '.ts');
if (!manifests.js[name]) {
if (!env.manifest.js[name]) {
env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.ts')}'`);
return false;
}
}
for (const css of await allCssSources()) {
const name = path.basename(css, '.scss');
if (!manifests.css[name]) {
if (!env.manifest.css[name]) {
env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.scss')}'`);
return false;
}
}
return Object.keys(manifests.i18n).length > 0;
return Object.keys(env.manifest.i18n).length > 0;
}
2 changes: 1 addition & 1 deletion ui/.build/src/parse.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import fg from 'fast-glob';
import { env, errorMark, colors as c } from './env.ts';
import { env, errorMark, c } from './env.ts';

export type Bundle = { module?: string; inline?: string };

Expand Down
2 changes: 1 addition & 1 deletion ui/.build/src/sass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import ps from 'node:process';
import path from 'node:path';
import crypto from 'node:crypto';
import clr from 'tinycolor2';
import { env, colors as c, lines, errorMark } from './env.ts';
import { env, c, lines, errorMark } from './env.ts';
import { globArray, readable } from './parse.ts';
import { updateManifest } from './manifest.ts';
import { clamp } from './algo.ts';
Expand Down
Loading

0 comments on commit 675028b

Please sign in to comment.