Skip to content

Commit b4f0345

Browse files
mcdurdinmcdurdin
committed
feat(common): TEMP COMMIT -- precompile ajv schemas
ajv seems to have some bugs around precompiling and formats, particularly when using ESM: * ajv-validator/ajv#1837 * ajv-validator/ajv-cli#200 Workaround for non-esm works for cloudevents/sdk-javascript#471 So if we want to work with this, we should look at how that is done, but it is taking too many cycles to pursue for now.
1 parent e0c2778 commit b4f0345

23 files changed

+297
-111
lines changed

‎common/web/types/.eslintrc.cjs

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ module.exports = {
88
"coverage/*",
99
"node_modules/*",
1010
"test/fixtures/*",
11+
"tools/*",
12+
"src/schemas/*"
1113
],
1214
overrides: [
1315
{

‎common/web/types/build.sh

+9-2
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,16 @@ function compile_schemas() {
4646

4747
# TODO: use https://github.com/tc39/proposal-json-modules instead of this once it stablises
4848
for schema in "${schemas[@]}"; do
49-
local fn="$THIS_SCRIPT_PATH/src/schemas/$(basename "$schema" .json)"
49+
local schema_base="$(basename "$schema" .json)"
50+
local fn="$THIS_SCRIPT_PATH/src/schemas/$schema_base"
51+
52+
builder_echo "Compiling schema $schema_base.json"
5053
echo 'export default ' > "$fn.ts"
5154
cat "$fn.json" >> "$fn.ts"
55+
# echo "declare module '$schema_base';" > "$fn.validator.d.ts"
56+
# node tools/compile-schema.js "$fn.json" "$fn.validator.js"
57+
ajv compile -c "./tools/formats.cjs" -s "$fn.json" -o "$fn.validator.cjs"
58+
#--code-esm true
5259
done
5360
}
5461

@@ -69,7 +76,7 @@ function copy_cldr_imports() {
6976
function do_configure() {
7077
compile_schemas
7178
copy_cldr_imports
72-
verify_npm_setup
79+
# verify_npm_setup
7380
}
7481

7582
function do_test() {

‎common/web/types/package.json

+4-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
},
2828
"dependencies": {
2929
"@keymanapp/keyman-version": "*",
30-
"ajv": "^8.11.0",
3130
"restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
3231
"semver": "^7.5.2",
3332
"xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"
@@ -39,6 +38,9 @@
3938
"@types/node": "^20.4.1",
4039
"@types/semver": "^7.3.12",
4140
"@types/xml2js": "^0.4.5",
41+
"ajv": "^8.12.0",
42+
"ajv-cli": "^5.0.0",
43+
"ajv-formats": "^2.1.1",
4244
"c8": "^7.12.0",
4345
"chai": "^4.3.4",
4446
"chalk": "^2.4.2",
@@ -74,6 +76,7 @@
7476
"src/ldml-keyboard/unicodeset-parser-api.ts",
7577
"src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
7678
"src/osk/osk.ts",
79+
"src/schemas/*",
7780
"test/"
7881
]
7982
}

‎common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-reader.ts

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
import { default as AjvModule } from 'ajv';
2-
const Ajv = AjvModule.default; // The actual expected Ajv type.
31
import { TouchLayoutFile } from "./keyman-touch-layout-file.js";
4-
import Schemas from '../../src/schemas.js';
2+
import SchemaValidators from '../schema-validators.js';
53

64
export class TouchLayoutFileReader {
75
public read(source: Uint8Array): TouchLayoutFile {
@@ -69,11 +67,10 @@ export class TouchLayoutFileReader {
6967
}
7068

7169
public validate(source: TouchLayoutFile): void {
72-
const ajv = new Ajv();
73-
if(!ajv.validate(Schemas.touchLayoutClean, source))
70+
if(!SchemaValidators.touchLayoutClean(source))
7471
/* c8 ignore next 3 */
7572
{
76-
throw new Error(ajv.errorsText());
73+
throw new Error((<any>SchemaValidators.touchLayoutClean).errors);
7774
}
7875
}
7976

‎common/web/types/src/kpj/kpj-file-reader.ts

+4-8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
import * as xml2js from 'xml2js';
22
import { KPJFile, KPJFileProject } from './kpj-file.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { KeymanDeveloperProject, KeymanDeveloperProjectFile10, KeymanDeveloperProjectType } from './keyman-developer-project.js';
75
import { CompilerCallbacks } from '../util/compiler-interfaces.js';
8-
import Schemas from '../schemas.js';
6+
import SchemaValidators from '../schema-validators.js';
97

108
export class KPJFileReader {
119
constructor(private callbacks: CompilerCallbacks) {
@@ -35,13 +33,11 @@ export class KPJFileReader {
3533
}
3634

3735
public validate(source: KPJFile): void {
38-
const ajv = new Ajv();
39-
if(!ajv.validate(Schemas.kpj, source)) {
40-
const ajvLegacy = new Ajv();
41-
if(!ajvLegacy.validate(Schemas.kpj90, source)) {
36+
if(!SchemaValidators.kpj(source)) {
37+
if(!SchemaValidators.kpj90(source)) {
4238
// If the legacy schema also does not validate, then we will only report
4339
// the errors against the modern schema
44-
throw new Error(ajv.errorsText());
40+
throw new Error((<any>SchemaValidators.kpj).errors);
4541
}
4642
}
4743
}

‎common/web/types/src/kvk/kvks-file-reader.ts

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
import * as xml2js from 'xml2js';
22
import KVKSourceFile from './kvks-file.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { DEFAULT_KVK_FONT, VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js';
75
import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js';
86
import { BUILDER_KVK_HEADER_VERSION, KVK_HEADER_IDENTIFIER_BYTES } from './kvk-file.js';
9-
import Schemas from '../schemas.js';
7+
import SchemaValidators from '../schema-validators.js';
108

119

1210
export default class KVKSFileReader {
@@ -85,9 +83,8 @@ export default class KVKSFileReader {
8583
}
8684

8785
public validate(source: KVKSourceFile): void {
88-
const ajv = new Ajv();
89-
if(!ajv.validate(Schemas.kvks, source)) {
90-
throw new Error(ajv.errorsText());
86+
if(!SchemaValidators.kvks(source)) {
87+
throw new Error((<any>SchemaValidators.kvks).errorsText());
9188
}
9289
}
9390

‎common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
import * as xml2js from 'xml2js';
22
import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js';
3-
import { default as AjvModule } from 'ajv';
4-
const Ajv = AjvModule.default; // The actual expected Ajv type.
53
import { boxXmlArray } from '../util/util.js';
64
import { CompilerCallbacks } from '../util/compiler-interfaces.js';
75
import { constants } from '@keymanapp/ldml-keyboard-constants';
86
import { CommonTypesMessages } from '../util/common-events.js';
97
import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js';
10-
import Schemas from '../schemas.js';
8+
import SchemaValidators from '../schema-validators.js';
119

1210
interface NameAndProps {
1311
'$'?: any; // content
@@ -207,9 +205,8 @@ export class LDMLKeyboardXMLSourceFileReader {
207205
* @returns true if valid, false if invalid
208206
*/
209207
public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile): boolean {
210-
const ajv = new Ajv();
211-
if(!ajv.validate(Schemas.ldmlKeyboard3, source)) {
212-
for (let err of ajv.errors) {
208+
if(!SchemaValidators.ldmlKeyboard3(source)) {
209+
for (let err of (<any>SchemaValidators.ldmlKeyboard3).errors) {
213210
this.callbacks.reportMessage(CommonTypesMessages.Error_SchemaValidationError({
214211
instancePath: err.instancePath,
215212
keyword: err.keyword,

‎common/web/types/src/main.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,5 @@ export * as KeymanFileTypes from './util/file-types.js';
4747

4848
export * as Osk from './osk/osk.js';
4949

50-
export * as Schemas from './schemas.js';
50+
export * as Schemas from './schemas.js';
51+
export * as SchemaValidators from './schema-validators.js';

‎common/web/types/src/osk/osk.ts

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import { TouchLayoutFile, TouchLayoutFlick, TouchLayoutKey, TouchLayoutPlatform, TouchLayoutSubKey } from "src/keyman-touch-layout/keyman-touch-layout-file.js";
22
import { VisualKeyboard } from "../kvk/visual-keyboard.js";
3-
import { default as AjvModule } from 'ajv';
4-
import Schemas from "../schemas.js";
5-
const Ajv = AjvModule.default; // The actual expected Ajv type.
3+
import SchemaValidators from "../schema-validators.js";
64

75
export interface StringRefUsage {
86
filename: string;
@@ -24,11 +22,10 @@ export interface StringResult {
2422
export type PuaMap = {[index:string]: string};
2523

2624
export function parseMapping(mapping: any) {
27-
const ajv = new Ajv();
28-
if(!ajv.validate(Schemas.displayMap, <any>mapping))
25+
if(!SchemaValidators.displayMap(<any>mapping))
2926
/* c8 ignore next 3 */
3027
{
31-
throw new Error(ajv.errorsText());
28+
throw new Error((<any>SchemaValidators.displayMap).errorsText());
3229
}
3330

3431
let map: PuaMap = {};

‎common/web/types/src/schema-validators.ts

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import kpj from './schemas/kpj.schema.validator.cjs';
2+
import kpj90 from './schemas/kpj-9.0.schema.validator.cjs';
3+
import kvks from './schemas/kvks.schema.validator.cjs';
4+
import ldmlKeyboard3 from './schemas/ldml-keyboard3.schema.validator.cjs';
5+
import ldmlKeyboardTest3 from './schemas/ldml-keyboardtest3.schema.validator.cjs';
6+
import displayMap from './schemas/displaymap.schema.validator.cjs';
7+
import touchLayoutClean from './schemas/keyman-touch-layout.clean.spec.validator.cjs';
8+
import touchLayout from './schemas/keyman-touch-layout.spec.validator.cjs';
9+
import keyboard_info from './schemas/keyboard_info.schema.validator.cjs';
10+
11+
// import AjvFormatsModule from 'ajv-formats';
12+
// const ajvFormats = AjvFormatsModule.default;
13+
// ajvFormats.default(keyboard_info);
14+
15+
const SchemaValidators = {
16+
kpj,
17+
kpj90,
18+
kvks,
19+
ldmlKeyboard3,
20+
ldmlKeyboardTest3,
21+
displayMap,
22+
touchLayoutClean,
23+
touchLayout,
24+
keyboard_info,
25+
};
26+
27+
export default SchemaValidators;

‎common/web/types/test/helpers/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import path from "path";
2-
import fs from "fs";
1+
import * as path from "path";
2+
import * as fs from "fs";
33
import { fileURLToPath } from "url";
44

55
/**

‎common/web/types/test/tsconfig.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
"outDir": "../build/test",
88
"baseUrl": ".",
99
"strictNullChecks": false, // TODO: get rid of this as some point
10-
"allowSyntheticDefaultImports": true // for ajv
10+
"allowJs": true,
11+
"allowSyntheticDefaultImports": true
1112
},
1213
"include": [
1314
"**/test-*.ts",

‎common/web/types/tools/compile-schema.cjs

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
const fs = require('fs');
2+
// import * as fs from 'fs';
3+
// import * as path from 'path';
4+
const Ajv = require('ajv');
5+
// import AjvModule from 'ajv';
6+
// const AjvFormats = require('ajv-formats');
7+
// import AjvFormatsModule from 'ajv-formats';
8+
const standaloneCode = require('ajv/dist/standalone');
9+
// import standaloneCode from 'ajv/dist/standalone/index.js';
10+
// import formats from 'ajv-formats/dist/formats.js';
11+
// const Ajv = AjvModule.default; // The actual expected Ajv type.
12+
// const ajvFormats = AjvFormatsModule.default;
13+
const ajvFormats = require('ajv-formats');
14+
// import formats from './formats.cjs';
15+
16+
// import { createRequire } from 'module';
17+
// const require = createRequire;
18+
19+
const schema = JSON.parse(fs.readFileSync(process.argv[2], 'utf-8'));
20+
21+
// ajvFormats['date-time'] = {validate, compile}
22+
23+
// The generated code will have a default export:
24+
// `module.exports = <validateFunctionCode>;module.exports.default = <validateFunctionCode>;`
25+
const ajv = new Ajv({code: {source: true}})
26+
// let formats = require('ajv-formats/dist/formats');
27+
ajvFormats(ajv);//.default(ajv);
28+
//ajv.opts.code.formats = formats.fullFormats;
29+
// formats(ajv);
30+
// ajv.opts.code.formats = require('ajv-formats').fullFormats;
31+
// ajv.opts.code.formats = formats(); //{validate:ajv.validate, compile:ajv.compile};
32+
// ajv.opts.code.formats = { "date-time": { validate: ajv.formats['date-time'].validate.toString(), compare: ajv.formats['date-time'].compare.toString() } };
33+
34+
const validate = ajv.compile(schema)
35+
// ajvFormats.default(validate)
36+
const moduleCode = standaloneCode(ajv, validate)
37+
38+
fs.writeFileSync(process.argv[3], moduleCode, 'utf-8');
39+
40+
// Now you can write the module code to file
41+
// fs.writeFileSync(path.join(__dirname, "../consume/validate-cjs.js"), moduleCode)

‎common/web/types/tools/formats.cjs

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// import * as ajvFormats from 'ajv-formats';
2+
3+
// export default function formats(ajv) {
4+
// ajvFormats.default(ajv);
5+
// }
6+
7+
function formats(ajv) {
8+
require("ajv-formats")(ajv);
9+
}
10+
11+
module.exports = formats;

‎common/web/types/tsconfig.json

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
"outDir": "build/src/",
66
"rootDir": "src/",
77
"baseUrl": ".",
8-
"allowSyntheticDefaultImports": true, // for ajv
9-
"resolveJsonModule": true,
8+
"allowJs": true,
9+
"allowSyntheticDefaultImports": true,
1010
},
1111
"include": [
12-
"src/**/*.ts"
12+
"src/**/*.ts",
13+
"src/schemas/*.cjs" // Import the validators
1314
],
1415
"references": [
1516
{ "path": "../keyman-version" },

‎common/web/utils/package.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,20 @@
2323
},
2424
"homepage": "https://github.com/keymanapp/keyman#readme",
2525
"devDependencies": {
26-
"@keymanapp/resources-gosh": "*",
2726
"@keymanapp/keyman-version": "*",
27+
"@keymanapp/resources-gosh": "*",
28+
"@types/node": "^14.0.5",
2829
"c8": "^7.12.0",
2930
"chai": "^4.3.4",
3031
"mocha": "^10.0.0",
3132
"mocha-teamcity-reporter": "^4.0.0",
32-
"@types/node": "^14.0.5",
3333
"typescript": "^4.9.5"
3434
},
3535
"type": "module",
3636
"paths": {
3737
"@keymanapp/keyman-version": "*"
38+
},
39+
"dependencies": {
40+
"ajv-formats": "^2.1.1"
3841
}
3942
}

‎developer/src/kmc-keyboard-info/build.sh

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ cd "$THIS_SCRIPT_PATH"
1111

1212
builder_describe "Build Keyman kmc keyboard-info Compiler module" \
1313
"@/common/web/types" \
14+
"@/developer/src/common/web/utils" \
1415
"clean" \
1516
"configure" \
1617
"build" \

‎developer/src/kmc-keyboard-info/package.json

+1-2
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,8 @@
2525
},
2626
"dependencies": {
2727
"@keymanapp/common-types": "*",
28-
"@keymanapp/kmc-package": "*",
2928
"@keymanapp/developer-utils": "*",
30-
"ajv": "^8.11.0",
29+
"@keymanapp/kmc-package": "*",
3130
"ajv-formats": "^2.1.1"
3231
},
3332
"bundleDependencies": [

‎developer/src/kmc-keyboard-info/src/index.ts

+3-15
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,7 @@ import langtags from "./imports/langtags.js";
1111
import { validateMITLicense } from "@keymanapp/developer-utils";
1212
import { KmpCompiler } from "@keymanapp/kmc-package";
1313

14-
import AjvModule from 'ajv';
15-
import AjvFormatsModule from 'ajv-formats';
16-
const Ajv = AjvModule.default; // The actual expected Ajv type.
17-
const ajvFormats = AjvFormatsModule.default;
18-
19-
import { Schemas } from "@keymanapp/common-types";
14+
import { SchemaValidators } from "@keymanapp/common-types";
2015
import { packageKeysExamplesToKeyboardInfo } from "./example-keys.js";
2116

2217
const regionNames = new Intl.DisplayNames(['en'], { type: "region" });
@@ -290,17 +285,10 @@ export class KeyboardInfoCompiler {
290285

291286
const jsonOutput = JSON.stringify(keyboard_info, null, 2);
292287

293-
// TODO: look at performance improvements by precompiling Ajv schemas on first use
294-
const ajv = new Ajv({ logger: {
295-
log: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_OutputValidation({message})),
296-
warn: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Warn_OutputValidation({message})),
297-
error: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_OutputValidation({message})),
298-
}});
299-
ajvFormats.default(ajv);
300-
if(!ajv.validate(Schemas.default.keyboard_info, keyboard_info)) {
288+
if(!SchemaValidators.default.keyboard_info(keyboard_info)) {
301289
// This is an internal fatal error; we should not be capable of producing
302290
// invalid output, so it is best to throw and die
303-
throw new Error(ajv.errorsText());
291+
throw new Error((<any>SchemaValidators.default.keyboard_info).errorsText());
304292
}
305293

306294
return new TextEncoder().encode(jsonOutput);

0 commit comments

Comments
 (0)