From 8dffb7babe05387f2335991ba1a0af4f4ddef18c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 2 Nov 2021 12:29:14 +0000 Subject: [PATCH 1/2] tests: Move metaschema to this repository We want to make more changes to it soon, and we may as well not be dependent on a external repository --- tests/schema/meta-schema.json | 267 ++++++++++++++++++++++++++++++++++ tests/test_data.py | 1 + tests/test_json.py | 17 +-- 3 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 tests/schema/meta-schema.json diff --git a/tests/schema/meta-schema.json b/tests/schema/meta-schema.json new file mode 100644 index 00000000..0e377fb9 --- /dev/null +++ b/tests/schema/meta-schema.json @@ -0,0 +1,267 @@ +{ + "id": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "positiveInteger": { + "type": "integer", + "minimum": 0 + }, + "positiveIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/positiveInteger" + }, + { + "default": 0 + } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "$schema": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": {}, + "multipleOf": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "boolean", + "default": false + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "boolean", + "default": false + }, + "maxLength": { + "$ref": "#/definitions/positiveInteger" + }, + "minLength": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "items": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/schemaArray" + } + ], + "default": {} + }, + "maxItems": { + "$ref": "#/definitions/positiveInteger" + }, + "minItems": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxProperties": { + "$ref": "#/definitions/positiveInteger" + }, + "minProperties": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "required": { + "$ref": "#/definitions/stringArray" + }, + "additionalProperties": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "definitions": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/stringArray" + } + ] + } + }, + "enum": { + "type": "array", + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/simpleTypes" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/simpleTypes" + }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { + "type": "string" + }, + "allOf": { + "$ref": "#/definitions/schemaArray" + }, + "anyOf": { + "$ref": "#/definitions/schemaArray" + }, + "oneOf": { + "$ref": "#/definitions/schemaArray" + }, + "not": { + "$ref": "#" + }, + "$ref": { + "type": "string" + }, + "omitWhenMerged": { + "type": "boolean", + "default": false + }, + "deprecated": { + "type": "object", + "additionalProperties": false, + "required": [ + "deprecatedVersion", + "description" + ], + "properties": { + "deprecatedVersion": { + "type": "string" + }, + "description": { + "type": "string" + } + } + }, + "codelist": { + "type": "string" + }, + "openCodelist": { + "type": "boolean", + "default": false + }, + "wholeListMerge": { + "type": "boolean", + "default": false + }, + "versionId": { + "type": "boolean", + "default": false + }, + "version": { + "type": "string" + }, + "propertyOrder": { + "type": "integer" + } + }, + "dependencies": { + "exclusiveMaximum": [ + "maximum" + ], + "exclusiveMinimum": [ + "minimum" + ] + }, + "default": {}, + "additionalProperties": false +} diff --git a/tests/test_data.py b/tests/test_data.py index 153c14db..a3cb3269 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -439,6 +439,7 @@ def test_all_examples_and_data_files_are_used(): files_used.extend(test_valid_statement_json_parametrize_data) files_used.extend(test_valid_package_json_parametrize_data) files_used.extend([a[0] for a in test_invalid_statement_json_parametrize_data]) + files_used.append('schema/meta-schema.json') for data in test_invalid_package_json_parametrize_data: if data[0]: files_used.append(data[0]) diff --git a/tests/test_json.py b/tests/test_json.py index c32abbe9..6a0869c1 100755 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1,4 +1,4 @@ -from bods_validate import absolute_path_to_source_schema_dir +from bods_validate import absolute_path_to_source_schema_dir, this_dir from warnings import warn from collections import Counter @@ -7,9 +7,10 @@ from jscc.testing.filesystem import walk_json_data, walk_csv_data from jscc.schema import is_json_schema from jscc.testing.checks import validate_items_type, validate_letter_case, validate_schema -from jscc.testing.util import http_get import pytest +import os +import json def test_empty(): @@ -32,15 +33,9 @@ def test_invalid_json(): ) -schemas = [(path, name, data) for path, name, _, data in walk_json_data() if is_json_schema(data)] -metaschema = http_get("https://standard.open-contracting.org/schema/1__1__4/meta-schema.json").json() - -metaschema["properties"]["version"] = { - "type": "string", -} -metaschema["properties"]["propertyOrder"] = { - "type": "integer", -} +schemas = [(path, name, data) for path, name, _, data in walk_json_data() if is_json_schema(data) and not path.endswith('tests/schema/meta-schema.json')] +with open(os.path.join(this_dir, 'schema', 'meta-schema.json')) as fp: + metaschema = json.load(fp) @pytest.mark.parametrize("path,name,data", schemas) From a7bbddeab810b1db37d2f4936c319d9be149804a Mon Sep 17 00:00:00 2001 From: James Date: Tue, 2 Nov 2021 12:01:47 +0000 Subject: [PATCH 2/2] schema: Annotations to oneOf and add oneOfEnumSelectorField https://github.com/openownership/lib-cove-bods/issues/64 This involves changes to build_schemas_for_testing that are required by the change from anyOf to oneOf. But actually, this code should have been required before but the compile tool was not processing anyOf properly. https://github.com/OpenDataServices/compile-to-json-schema/issues/28 https://github.com/openownership/data-standard/pull/376 --- docs/schema/changelog.rst | 2 ++ schema/components.json | 3 ++- tests/bods_validate.py | 11 +++++++++++ tests/schema/meta-schema.json | 3 +++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/schema/changelog.rst b/docs/schema/changelog.rst index d3de47a2..b168d54d 100644 --- a/docs/schema/changelog.rst +++ b/docs/schema/changelog.rst @@ -21,6 +21,7 @@ Added - ``Country.name`` is now a required field (previously it was defined as "MUST" in the description). - ``Jurisdiction.name`` is now a required field (previously it was defined as "MUST" in the description). - ``SecuritiesListing.stockExchangeJurisdiction`` has minimum and maximum lengths to match the two lists that values could be from. +- Annotations have ``oneOfEnumSelectorField`` added to provide hints to validation code which will produce better error messages. Changed ------- @@ -31,6 +32,7 @@ Changed - Clarified ``Country.code`` is from the ISO 3166-1 list (previously it was unclear which ISO list was meant and used "digit" when it meant "letter"). - Clarified ``Jurisdiction.code`` is from the ISO 3166-1 or ISO 3166-2 list (previously it was unclear which ISO list was meant and used "digit" when it meant "letter"). - Clarified ``SecuritiesListing.stockExchangeJurisdiction`` is from the ISO 3166-1 or ISO 3166-2 list (previously it was unclear which ISO list was meant and used "digit" when it meant "letter"). +- Annotations changes from a ``anyOf`` to a ``oneOf``. This is technically correct and also is needed to improve validation messages. [0.2] - 2019-06-30 diff --git a/schema/components.json b/schema/components.json index 8a069059..be848dc5 100644 --- a/schema/components.json +++ b/schema/components.json @@ -395,7 +395,8 @@ "statementPointerTarget", "motivation" ], - "anyOf": [ + "oneOfEnumSelectorField": "motivation", + "oneOf": [ { "properties": { "motivation": { diff --git a/tests/bods_validate.py b/tests/bods_validate.py index 2ce0bf9d..85fd01b8 100755 --- a/tests/bods_validate.py +++ b/tests/bods_validate.py @@ -39,6 +39,17 @@ def build_schemas_for_testing(): ) with open(os.path.join(absolute_path_to_schema_dir, source_file), "w") as fp: created_json = json.loads(ctjs.get_as_string()) + # The items in the annotations objects must not have additionalProperties set + # They factor out common elements of the oneOf bits as allowed in + # https://json-schema.org/understanding-json-schema/reference/combining.html#factoring-schemas + # But additionalProperties is applied to the oneOf schema only, not the full item and that causes issues + if source_file.endswith('entity-statement.json') or source_file.endswith('ownership-or-control-statement.json') or source_file.endswith('person-statement.json'): + for j in range(0, 2): + created_json['properties']['annotations']['items']['oneOf'][j]['additionalProperties'] = True + if source_file.endswith('bods-package.json'): + for i in range(0, 3): + for j in range(0, 2): + created_json['items']['oneOf'][i]['properties']['annotations']['items']['oneOf'][j]['additionalProperties'] = True # write with correct indentation fp.write(json.dumps(created_json, ensure_ascii=False, indent=2, separators=(',', ': ')) + '\n') diff --git a/tests/schema/meta-schema.json b/tests/schema/meta-schema.json index 0e377fb9..8fd05a9d 100644 --- a/tests/schema/meta-schema.json +++ b/tests/schema/meta-schema.json @@ -252,6 +252,9 @@ }, "propertyOrder": { "type": "integer" + }, + "oneOfEnumSelectorField": { + "type": "string" } }, "dependencies": {