diff --git a/tests/README.md b/tests/README.md index 31c79785..fc2382fe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,7 +5,7 @@ To run the tests follow these steps: 1. Install [node and npm](https://nodejs.org) - should run with any recent version 2. Run `npm install` in this folder to install the dependencies 3. Run the tests with `npm test`. This will also lint the files and verify it follows best practices. -4. To show the files nicely formatted in a web browser, run `npm run render`. It starts a server and opens the corresponding page in a web browser. +4. To show the files nicely formatted in a web browser, run `npm start`. It starts a server and opens the corresponding page in a web browser. ## Development processes @@ -28,8 +28,3 @@ Sometimes it is useful to define a new "data type" on top of the JSON types (num For example, a client could make a select box with all collections available by adding a subtype `collection-id` to the JSON type `string`. If you think a new subype should be added, you need to add it to the `meta/subtype-schemas.json` file. It must be a valid JSON Schema. The tests mentioned above will also verify to a certain degree that the subtypes are defined correctly. - -## Examples - -To get out of proposal state, at least two examples must be provided. -The examples are located in the `examples` folder and will also be validated to some extent in the tests. \ No newline at end of file diff --git a/tests/package.json b/tests/package.json index be51806f..500d063b 100644 --- a/tests/package.json +++ b/tests/package.json @@ -1,6 +1,6 @@ { - "name": "@openeo/processes-validator", - "version": "0.2.0", + "name": "@openeo/processes", + "version": "1.2.0", "author": "openEO Consortium", "contributors": [ { @@ -18,19 +18,13 @@ "url": "git+https://github.com/Open-EO/openeo-processes.git" }, "devDependencies": { - "@apidevtools/json-schema-ref-parser": "^9.0.6", - "@openeo/js-processgraphs": "^1.0.0", - "ajv": "^6.12.4", + "@openeo/processes-lint": "^0.1.3", "concat-json-files": "^1.1.0", - "glob": "^7.1.6", - "http-server": "^14.1.1", - "jest": "^26.4.2", - "markdown-spellcheck": "^1.3.1", - "markdownlint": "^0.26.0" + "http-server": "^14.1.1" }, "scripts": { - "test": "jest", + "test": "openeo-processes-lint testConfig.json", "generate": "concat-json-files \"../{*,proposals/*}.json\" -t \"processes.json\"", - "render": "npm run generate && http-server -p 9876 -o docs.html -c-1" + "start": "npm run generate && http-server -p 9876 -o docs.html -c-1" } } diff --git a/tests/processes.test.js b/tests/processes.test.js deleted file mode 100644 index 1fcf5f02..00000000 --- a/tests/processes.test.js +++ /dev/null @@ -1,298 +0,0 @@ -const glob = require('glob'); -const fs = require('fs'); -const path = require('path'); -const { normalizeString, checkDescription, checkSpelling, checkJsonSchema, getAjv, prepareSchema, isObject } = require('./testHelpers'); - -const anyOfRequired = [ - "quantiles", - "array_element" -]; - -var jsv = null; -beforeAll(async () => { - jsv = await getAjv(); -}); - -var loader = (file, proposal = false) => { - try { - var fileContent = fs.readFileSync(file); - // Check JSON structure for faults - var p = JSON.parse(fileContent); - - // Prepare for tests - processes.push([file, p, fileContent.toString(), proposal]); - processIds.push(p.id); - } catch(err) { - processes.push([file, {}, "", proposal]); - console.error(err); - expect(err).toBeUndefined(); - } -}; - -var processes = []; -var processIds = []; - -const files = glob.sync("../*.json", {realpath: true}); -files.forEach(file => loader(file)); - -const proposals = glob.sync("../proposals/*.json", {realpath: true}); -proposals.forEach(file => loader(file, true)); - -test("Check for duplicate process ids", () => { - const duplicates = processIds.filter((id, index) => processIds.indexOf(id) !== index); - expect(duplicates).toEqual([]); -}); - -describe.each(processes)("%s", (file, p, fileContent, proposal) => { - - test("File / JSON", () => { - const ext = path.extname(file); - // Check that the process file has a lower-case json extension - expect(ext).toEqual(".json"); - // Check that the process name is also the file name - expect(path.basename(file, ext)).toEqual(p.id); - // lint: Check whether the file is correctly JSON formatted - expect(normalizeString(JSON.stringify(p, null, 4))).toEqual(normalizeString(fileContent)); - }); - - test("ID", () => { - expect(typeof p.id).toBe('string'); - expect(/^\w+$/.test(p.id)).toBeTruthy(); - }); - - test("Summary", () => { - expect(typeof p.summary === 'undefined' || typeof p.summary === 'string').toBeTruthy(); - // lint: Summary should be short - expect(p.summary.length).toBeLessThan(60); - // lint: Summary should not end with a dot - expect(/[^\.]$/.test(p.summary)).toBeTruthy(); - checkSpelling(p.summary, p); - }); - - test("Description", () => { - // description - expect(typeof p.description).toBe('string'); - // lint: Description should be longer than a summary - expect(p.description.length).toBeGreaterThan(60); - checkDescription(p.description, p, processIds); - }); - - test("Categories", () => { - // categories - expect(Array.isArray(p.categories)).toBeTruthy(); - // lint: There should be at least one category assigned - expect(p.categories.length).toBeGreaterThan(0); - if (Array.isArray(p.categories)) { - for(let i in p.categories) { - expect(typeof p.categories[i]).toBe('string'); - } - } - }); - - test("Flags", () => { - checkFlags(p, proposal); - }); - - test("Parameters", () => { - expect(Array.isArray(p.parameters)).toBeTruthy(); - }); - - var params = o2a(p.parameters); - if (params.length > 0) { - test.each(params)("Parameters > %s", (key, param) => { - checkParam(param, p); - }); - } - - test("Return Value", () => { - expect(isObject(p.returns)).toBeTruthy(); - expect(p.returns).not.toBeNull(); - - // return value description - expect(typeof p.returns.description).toBe('string'); - // lint: Description should not be empty - expect(p.returns.description.length).toBeGreaterThan(0); - checkDescription(p.returns.description, p, processIds); - - // return value schema - expect(p.returns.schema).not.toBeNull(); - expect(typeof p.returns.schema).toBe('object'); - // lint: Description should not be empty - checkJsonSchema(jsv, p.returns.schema); - }); - - test("Exceptions", () => { - expect(typeof p.exceptions === 'undefined' || isObject(p.exceptions)).toBeTruthy(); - }); - - var exceptions = o2a(p.exceptions); - if (exceptions.length > 0) { - test.each(exceptions)("Exceptions > %s", (key, e) => { - expect(/^\w+$/.test(key)).toBeTruthy(); - - // exception message - expect(typeof e.message).toBe('string'); - checkSpelling(e.message, p); - - // exception description - expect(typeof e.description === 'undefined' || typeof e.description === 'boolean').toBeTruthy(); - checkDescription(e.description, p, processIds); - - // exception http code - if (typeof e.http !== 'undefined') { - expect(e.http).toBeGreaterThanOrEqual(100); - expect(e.http).toBeLessThan(600); - } - }); - } - - test("Examples", () => { - expect(typeof p.examples === 'undefined' || Array.isArray(p.examples)).toBeTruthy(); - }); - - if (Array.isArray(p.examples) && p.examples.length > 0) { - - test.each(p.examples)("Examples > %#", (example) => { - // Make an object for easier access later - var parametersObj = {}; - for(var i in p.parameters) { - parametersObj[p.parameters[i].name] = p.parameters[i]; - } - var paramKeys = Object.keys(parametersObj); - - expect(isObject(example)).toBeTruthy(); - expect(example).not.toBeNull(); - - // example title - expect(typeof example.title === 'undefined' || typeof example.title === 'string').toBeTruthy(); - checkSpelling(example.title, p); - - // example description - expect(typeof example.description === 'undefined' || typeof example.description === 'string').toBeTruthy(); - checkDescription(example.description, p, processIds); - - // example process graph - expect(example.process_graph).toBeUndefined(); - - // example arguments - expect(typeof example.arguments).toBe('object'); - expect(example.arguments).not.toBeNull(); - // Check argument values - for(let argName in example.arguments) { - // Does parameter with this name exist? - - expect(paramKeys).toContain(argName); - checkJsonSchemaValue(parametersObj[argName].schema, example.arguments[argName]); - } - // Check whether all required parameters are set - for(let key in parametersObj) { - if (!parametersObj[key].optional) { - expect(example.arguments[key]).toBeDefined(); - } - } - - // example returns: Nothing to validate, everything is allowed - }); - } - - test("Links", () => { - expect(typeof p.links === 'undefined' || Array.isArray(p.links)).toBeTruthy(); - }); - - if (Array.isArray(p.links)) { - test.each(p.links)("Links > %#", (link) => { - expect(isObject(link)).toBeTruthy(); - - // link href - expect(typeof link.href).toBe('string'); - - // link rel - expect(typeof link.rel === 'undefined' || typeof link.rel === 'string').toBeTruthy(); - - // link title - expect(typeof link.title === 'undefined' || typeof link.title === 'string').toBeTruthy(); - checkSpelling(link.title, p); - - // link type - expect(typeof link.type === 'undefined' || typeof link.type === 'string').toBeTruthy(); - }); - } -}); - -function checkFlags(p, proposal = false) { - // deprecated - expect(typeof p.deprecated === 'undefined' || typeof p.deprecated === 'boolean').toBeTruthy(); - // lint: don't specify defaults - expect(typeof p.deprecated === 'undefined' || p.deprecated === true).toBeTruthy(); - if (proposal) { - // experimental must be true for proposals - expect(p.experimental).toBe(true); - } - else { - // experimental must not be false for stable - // lint: don't specify defaults, so false should not be set explicitly - expect(p.experimental).toBeUndefined(); - } -} - -function checkParam(param, p, checkCbParams = true) { - // parameter name - expect(typeof param.name).toBe('string'); - expect(/^\w+$/.test(param.name)).toBeTruthy(); - - // parameter description - expect(typeof param.description).toBe('string'); - // lint: Description should not be empty - expect(param.description.length).toBeGreaterThan(0); - checkDescription(param.description, p, processIds); - - // Parameter flags - expect(typeof param.optional === 'undefined' || typeof param.optional === 'boolean').toBeTruthy(); - // lint: don't specify default value "false" for optional - expect(typeof param.optional === 'undefined' || param.optional === true).toBeTruthy(); - // lint: make sure there's no old required flag - expect(typeof param.required === 'undefined').toBeTruthy(); - // lint: require a default value if the parameter is optional - if (param.optional === true && !anyOfRequired.includes(p.id)) { - expect(param.default).toBeDefined(); - } - // Check flags (recommended / experimental) - checkFlags(param); - - // Parameter schema - expect(param.schema).not.toBeNull(); - expect(typeof param.schema).toBe('object'); - checkJsonSchema(jsv, param.schema); - - if (checkCbParams) { - // Checking that callbacks (process-graphs) define their parameters - if (typeof param.schema === 'object' && param.schema.subtype === 'process-graph') { - // lint: A callback without parameters is not very useful - expect(Array.isArray(param.schema.parameters) && param.schema.parameters.length > 0).toBeTruthy(); - - // Check all callback params - for(var i in param.schema.parameters) { - checkParam(param.schema.parameters[i], p, false); - } - } - } -} - -function checkJsonSchemaValue(schema, value) { - jsv.validate(prepareSchema(schema), value); - expect(jsv.errors).toBeNull(); -} - -function o2a(o) { - if (!o) { - return []; - } - var a = []; - for(var k in o) { - a.push([ - o[k].name ? o[k].name : k, // name - o[k] // obj - ]); - } - return a; -} \ No newline at end of file diff --git a/tests/subtypes-file.test.js b/tests/subtypes-file.test.js deleted file mode 100644 index e70f7e8f..00000000 --- a/tests/subtypes-file.test.js +++ /dev/null @@ -1,29 +0,0 @@ -const fs = require('fs'); -const $RefParser = require("@apidevtools/json-schema-ref-parser"); -const { checkJsonSchema, getAjv, isObject, normalizeString } = require('./testHelpers'); - -test("File subtype-schemas.json", async () => { - let schema; - let fileContent; - try { - fileContent = fs.readFileSync('../meta/subtype-schemas.json'); - schema = JSON.parse(fileContent); - } catch(err) { - console.error("The file for subtypes is invalid and can't be read:"); - console.error(err); - expect(err).toBeUndefined(); - } - - expect(isObject(schema)).toBeTruthy(); - expect(isObject(schema.definitions)).toBeTruthy(); - - // lint: Check whether the file is correctly JSON formatted - expect(normalizeString(JSON.stringify(schema, null, 4))).toEqual(normalizeString(fileContent.toString())); - - // Is JSON Schema valid? - checkJsonSchema(await getAjv(), schema); - - // is everything dereferencable? - let subtypes = await $RefParser.dereference(schema, { dereference: { circular: "ignore" } }); - expect(isObject(subtypes)).toBeTruthy(); -}); \ No newline at end of file diff --git a/tests/subtypes-schemas.test.js b/tests/subtypes-schemas.test.js deleted file mode 100644 index ff1b72bd..00000000 --- a/tests/subtypes-schemas.test.js +++ /dev/null @@ -1,54 +0,0 @@ -const $RefParser = require("@apidevtools/json-schema-ref-parser"); -const { checkDescription, checkSpelling, isObject } = require('./testHelpers'); - -// I'd like to run the tests for each subtype individually instead of in a loop, -// but jest doesn't support that, so you need to figure out yourself what is broken. -// The console.log in afterAll ensures we have a hint of which process was checked last - -// Load and dereference schemas -let subtypes = {}; -let lastTest = null; -let testsCompleted = 0; -beforeAll(async () => { - subtypes = await $RefParser.dereference('../meta/subtype-schemas.json', { dereference: { circular: "ignore" } }); - return subtypes; -}); - -afterAll(async () => { - if (testsCompleted != Object.keys(subtypes.definitions).length) { - console.log('The schema the test has likely failed for: ' + lastTest); - } -}); - -test("Schemas in subtype-schemas.json", () => { - // Each schema must contain at least a type, subtype, title and description - for(let name in subtypes.definitions) { - let schema = subtypes.definitions[name]; - lastTest = name; - - // Schema is object - expect(isObject(schema)).toBeTruthy(); - - // Type is array with an element or a stirng - expect((Array.isArray(schema.type) && schema.type.length > 0) || typeof schema.type === 'string').toBeTruthy(); - - // Subtype is a string - expect(typeof schema.subtype === 'string').toBeTruthy(); - - // Check title - expect(typeof schema.title === 'string').toBeTruthy(); - // lint: Summary should be short - expect(schema.title.length).toBeLessThan(60); - // lint: Summary should not end with a dot - expect(/[^\.]$/.test(schema.title)).toBeTruthy(); - checkSpelling(schema.title, schema); - - // Check description - expect(typeof schema.description).toBe('string'); - // lint: Description should be longer than a summary - expect(schema.description.length).toBeGreaterThan(60); - checkDescription(schema.description, schema); - - testsCompleted++; - } -}); \ No newline at end of file diff --git a/tests/testConfig.json b/tests/testConfig.json new file mode 100644 index 00000000..60d8b893 --- /dev/null +++ b/tests/testConfig.json @@ -0,0 +1,13 @@ +{ + "folder": "../", + "proposalsFolder": "../proposals/", + "ignoredWords": ".words", + "anyOfRequired": [ + "array_element", + "quantiles" + ], + "subtypeSchemas": "../meta/subtype-schemas.json", + "checkSubtypeSchemas": true, + "forbidDeprecatedTypes": false, + "verbose": false +} diff --git a/tests/testHelpers.js b/tests/testHelpers.js deleted file mode 100644 index 418fd830..00000000 --- a/tests/testHelpers.js +++ /dev/null @@ -1,231 +0,0 @@ -const glob = require('glob'); -const fs = require('fs'); -const path = require('path'); -const ajv = require('ajv'); -const $RefParser = require("@apidevtools/json-schema-ref-parser"); -const markdownlint = require('markdownlint'); -const spellcheck = require('markdown-spellcheck').default; - -const ajvOptions = { - schemaId: 'auto', - format: 'full' -}; - -const spellcheckOptions = { - ignoreAcronyms: true, - ignoreNumbers: true, - suggestions: false, - relativeSpellingFiles: true, - dictionary: { - language: "en-us" - } -}; - -// Read custom dictionary for spell check -const words = fs.readFileSync('.words').toString().split(/\r\n|\n|\r/); -for(let i in words) { - spellcheck.spellcheck.addWord(words[i]); -} -// Add the process IDs to the word list -const files = glob.sync("../{*,proposals/*}.json", {realpath: true}); -for(let i in files) { - spellcheck.spellcheck.addWord(path.basename(files[i], path.extname(files[i]))); -} - - -async function getAjv() { - let subtypes = await $RefParser.dereference( - require('../meta/subtype-schemas.json'), - { - dereference: { circular: "ignore" } - } - ); - - let jsv = new ajv(ajvOptions); - jsv.addKeyword("parameters", { - dependencies: [ - "type", - "subtype" - ], - metaSchema: { - type: "array", - items: { - type: "object", - required: [ - "name", - "description", - "schema" - ], - properties: { - name: { - type: "string", - pattern: "^[A-Za-z0-9_]+$" - }, - description: { - type: "string" - }, - optional: { - type: "boolean" - }, - deprecated: { - type: "boolean" - }, - experimental: { - type: "boolean" - }, - default: { - // Any type - }, - schema: { - oneOf: [ - { - type: "object", - // ToDo: Check Schema - }, - { - type: "array", - items: { - type: "object" - // ToDo: Check Schema - } - } - ] - } - } - } - }, - valid: true - }); - jsv.addKeyword("subtype", { - dependencies: [ - "type" - ], - metaSchema: { - type: "string", - enum: Object.keys(subtypes.definitions) - }, - compile: function (subtype, schema) { - if (schema.type != subtypes.definitions[subtype].type) { - throw "Subtype '"+subtype+"' not allowed for type '"+schema.type+"'." - } - return () => true; - }, - errors: false - }); - - return jsv; -} - -function isObject(obj) { - return (typeof obj === 'object' && obj === Object(obj) && !Array.isArray(obj)); -} - -function normalizeString(str) { - return str.replace(/\r\n|\r|\n/g, "\n").trim(); -} - -function checkDescription(text, p = null, processIds = [], commonmark = true) { - if (!text) { - return; - } - - // Check markdown - if (commonmark) { - const options = { - strings: { - description: text - }, - config: { - "line-length": false, // Nobody cares in JSON files anyway - "first-line-h1": false, // Usually no headings in descriptions - "fenced-code-language": false, // Usually no languages available anyway - "single-trailing-newline": false, // New lines at end of a JSON string doesn't make sense. We don't have files here. - } - }; - const result = markdownlint.sync(options); - expect(result).toEqual({description: []}); - } - - // Check spelling - checkSpelling(text, p); - - // Check whether process references are referencing valid processes - if (Array.isArray(processIds) && processIds.length > 0) { - let matches = text.matchAll(/(?:^|[^\w`])``(\w+)\(\)``(?![\w`])/g); - for(match of matches) { - expect(processIds).toContain(match[1]); - } - } -} - -function checkSpelling(text, p = null) { - if (!text) { - return; - } - - const errors = spellcheck.spell(text, spellcheckOptions); - if (errors.length > 0) { - let pre = "Misspelled word"; - if (p && p.id) { - pre += " in " + p.id; - } - console.warn(pre + ": " + JSON.stringify(errors)); - } -} - -function prepareSchema(schema) { - if (Array.isArray(schema)) { - schema = { - anyOf: schema - }; - } - if (typeof schema["$schema"] === 'undefined') { - // Set applicable JSON SChema draft version if not already set - schema["$schema"] = "http://json-schema.org/draft-07/schema#"; - } - return schema; -} - -function checkJsonSchema(jsv, schema, checkFormat = true) { - if (Array.isArray(schema)) { - // lint: For array schemas there should be more than one schema specified, otherwise use directly the schema object - expect(schema.length).toBeGreaterThan(1); - } - - let result = jsv.compile(prepareSchema(schema)); - expect(result.errors).toBeNull(); - - checkSchemaRecursive(schema, checkFormat); -} - -function checkSchemaRecursive(schema, checkFormat = true) { - for(var i in schema) { - var val = schema[i]; - if (typeof val === 'object' && val !== null) { - checkSchemaRecursive(val, checkFormat); - } - - switch(i) { - case 'title': - case 'description': - checkSpelling(val); - break; - case 'format': - if (checkFormat && schema.subtype !== val) { - throw "format '"+val+"' has no corresponding subtype."; - } - break; - } - } -} - -module.exports = { - getAjv, - normalizeString, - checkDescription, - checkSpelling, - checkJsonSchema, - checkSchemaRecursive, - prepareSchema, - isObject -}; \ No newline at end of file