diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 4c5b12303a9..4616aa11272 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -119,6 +119,7 @@ def __len__(self): class ColumnInfo(JsonSchemaMixin, Replaceable): name: str description: str = '' + meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None @@ -180,6 +181,7 @@ def patch(self, patch): self.description = patch.description self.columns = patch.columns self.docrefs = patch.docrefs + self.meta = patch.meta if dbt.flags.STRICT_MODE: self.to_dict(validate=True) @@ -215,6 +217,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory): docrefs: List[Docref] = field(default_factory=list) description: str = field(default='') columns: Dict[str, ColumnInfo] = field(default_factory=dict) + meta: Dict[str, Any] = field(default_factory=dict) patch_path: Optional[str] = None build_path: Optional[str] = None @@ -455,6 +458,7 @@ class ParsedNodePatch(JsonSchemaMixin, Replaceable): original_file_path: str columns: Dict[str, ColumnInfo] docrefs: List[Docref] + meta: Dict[str, Any] @dataclass @@ -507,6 +511,8 @@ class ParsedSourceDefinition( docrefs: List[Docref] = field(default_factory=list) description: str = '' columns: Dict[str, ColumnInfo] = field(default_factory=dict) + meta: Dict[str, Any] = field(default_factory=dict) + source_meta: Dict[str, Any] = field(default_factory=dict) @property def is_ephemeral_model(self): diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 9ab03aabd1f..855b7190a5f 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -58,6 +58,7 @@ class UnparsedRunHook(UnparsedNode): class NamedTested(JsonSchemaMixin, Replaceable): name: str description: str = '' + meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None tests: Optional[List[Union[Dict[str, Any], str]]] = None @@ -159,6 +160,7 @@ class ExternalPartition(AdditionalPropertiesAllowed, Replaceable): name: str = '' description: str = '' data_type: str = '' + meta: Dict[str, Any] = field(default_factory=dict) def __post_init__(self): if self.name == '' or self.data_type == '': @@ -206,6 +208,7 @@ def __post_init__(self): class UnparsedSourceDefinition(JsonSchemaMixin, Replaceable): name: str description: str = '' + meta: Dict[str, Any] = field(default_factory=dict) database: Optional[str] = None schema: Optional[str] = None loader: str = '' diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 8bea6eb46cf..6117c79f8d8 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -66,10 +66,11 @@ def __init__(self): self.column_info: Dict[str, ColumnInfo] = {} self.docrefs: List[Docref] = [] - def add(self, column_name, description, data_type): + def add(self, column_name, description, data_type, meta): self.column_info[column_name] = ColumnInfo(name=column_name, description=description, - data_type=data_type) + data_type=data_type, + meta=meta) def collect_docrefs( @@ -222,9 +223,10 @@ def parse_column( column_name = column.name description = column.description data_type = column.data_type + meta = column.meta collect_docrefs(block.target, refs, column_name, description) - refs.add(column_name, description, data_type) + refs.add(column_name, description, data_type, meta) if not column.tests: return @@ -340,6 +342,7 @@ def generate_source_node( NodeType.Source, self.project.project_name, source.name, table.name ]) description = table.description or '' + meta = table.meta or {} source_description = source.description or '' collect_docrefs(source, refs, None, description, source_description) @@ -348,6 +351,7 @@ def generate_source_node( freshness = self._calculate_freshness(source, table) quoting = source.quoting.merged(table.quoting) path = block.path.original_file_path + source_meta = source.meta or {} return ParsedSourceDefinition( package_name=self.project.project_name, @@ -364,6 +368,8 @@ def generate_source_node( external=table.external, source_name=source.name, source_description=source_description, + source_meta=source_meta, + meta=meta, loader=source.loader, docrefs=refs.docrefs, loaded_at_field=loaded_at_field, @@ -385,7 +391,8 @@ def generate_node_patch( original_file_path=block.path.original_file_path, description=description, columns=refs.column_info, - docrefs=refs.docrefs + docrefs=refs.docrefs, + meta=block.target.meta, ) def parse_target_model( diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 8080bf9dad3..f49734ce043 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -852,6 +852,7 @@ def expected_seeded_manifest(self, model_database=None): 'unique_id': 'model.test.model', 'fqn': ['test', 'model'], 'tags': [], + 'meta': {}, 'config': model_config, 'schema': my_schema_name, 'database': model_database, @@ -862,26 +863,31 @@ def expected_seeded_manifest(self, model_database=None): 'name': 'id', 'description': 'The user ID number', 'data_type': None, + 'meta': {}, }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, + 'meta': {}, }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, + 'meta': {}, }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, + 'meta': {}, }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, + 'meta': {}, }, }, 'patch_path': schema_yml_path, @@ -925,6 +931,7 @@ def expected_seeded_manifest(self, model_database=None): 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], + 'meta': {}, 'schema': my_schema_name, 'database': self.default_database, 'alias': 'seed', @@ -971,6 +978,7 @@ def expected_seeded_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], + 'meta': {}, 'unique_id': 'test.test.not_null_model_id', 'docrefs': [], 'compiled': True, @@ -1018,6 +1026,7 @@ def expected_seeded_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], + 'meta': {}, 'unique_id': 'test.test.test_nothing_model_', 'docrefs': [], 'compiled': True, @@ -1065,6 +1074,7 @@ def expected_seeded_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], + 'meta': {}, 'unique_id': 'test.test.unique_model_id', 'docrefs': [], 'compiled': True, @@ -1224,6 +1234,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.ephemeral_copy', 'compiled': True, 'compiled_sql': ANY, @@ -1239,12 +1250,14 @@ def expected_postgres_references_manifest(self, model_database=None): 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {} }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', - 'data_type': None + 'data_type': None, + 'meta': {} }, }, 'config': { @@ -1299,6 +1312,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.ephemeral_summary', 'compiled': True, 'compiled_sql': ANY, @@ -1314,12 +1328,14 @@ def expected_postgres_references_manifest(self, model_database=None): 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {} }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', - 'data_type': None + 'data_type': None, + 'meta': {} }, }, 'config': { @@ -1373,6 +1389,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'schema': my_schema_name, 'sources': [], 'tags': [], + 'meta': {}, 'unique_id': 'model.test.view_summary', 'compiled': True, 'compiled_sql': ANY, @@ -1415,6 +1432,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'seed.test.seed', 'compiled': True, 'compiled_sql': '', @@ -1428,7 +1446,8 @@ def expected_postgres_references_manifest(self, model_database=None): 'id': { 'description': 'An ID field', 'name': 'id', - 'data_type': None + 'data_type': None, + 'meta': {} } }, 'quoting': { @@ -1463,6 +1482,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'identifier': 'seed', 'loaded_at_field': None, 'loader': 'a_loader', + 'meta': {}, 'name': 'my_table', 'original_file_path': self.dir('ref_models/schema.yml'), 'package_name': 'test', @@ -1472,6 +1492,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'schema': my_schema_name, 'source_description': 'My source', 'source_name': 'my_source', + 'source_meta': {}, 'unique_id': 'source.test.my_source.my_table', 'fqn': ['test', 'my_source', 'my_table'], } @@ -1692,32 +1713,38 @@ def expected_bigquery_complex_manifest(self): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.clustered', 'columns': { 'email': { 'description': "The user's email", 'name': 'email', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'first_name': { 'description': "The user's name", 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'id': { 'description': 'The user id', 'name': 'id', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'updated_at': { 'description': 'When the user was updated', 'name': 'updated_at', - 'data_type': None + 'data_type': None, + 'meta': {}, }, }, 'description': 'A clustered and partitioned copy of the test model', @@ -1760,32 +1787,38 @@ def expected_bigquery_complex_manifest(self): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.multi_clustered', 'columns': { 'email': { 'description': "The user's email", 'name': 'email', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'first_name': { 'description': "The user's name", 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'id': { 'description': 'The user id', 'name': 'id', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'updated_at': { 'description': 'When the user was updated', 'name': 'updated_at', - 'data_type': None + 'data_type': None, + 'meta': {}, }, }, 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', @@ -1829,32 +1862,38 @@ def expected_bigquery_complex_manifest(self): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.nested_view', 'columns': { 'field_1': { 'name': 'field_1', 'description': 'The first field', 'data_type': None, + 'meta': {}, }, 'field_2': { 'name': 'field_2', 'description': 'The second field', 'data_type': None, + 'meta': {}, }, 'field_3': { 'name': 'field_3', 'description': 'The third field', 'data_type': None, + 'meta': {}, }, 'nested_field.field_4': { 'name': 'nested_field.field_4', 'description': 'The first nested field', 'data_type': None, + 'meta': {}, }, 'nested_field.field_5': { 'name': 'nested_field.field_5', 'description': 'The second nested field', 'data_type': None, + 'meta': {}, }, }, 'description': 'The test model', @@ -1899,6 +1938,7 @@ def expected_bigquery_complex_manifest(self): 'schema': my_schema_name, 'database': self.default_database, 'tags': [], + 'meta': {}, 'unique_id': 'model.test.nested_table', 'columns': {}, 'description': '', @@ -1930,6 +1970,7 @@ def expected_bigquery_complex_manifest(self): 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], + 'meta': {}, 'config': { 'enabled': True, 'materialized': 'seed', @@ -2099,6 +2140,7 @@ def expected_redshift_incremental_view_manifest(self): 'unique_id': 'model.test.model', 'fqn': ['test', 'model'], 'tags': [], + 'meta': {}, 'config': { 'bind': False, 'column_types': {}, @@ -2120,26 +2162,31 @@ def expected_redshift_incremental_view_manifest(self): 'name': 'id', 'description': 'The user ID number', 'data_type': None, + 'meta': {}, }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, + 'meta': {}, }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, + 'meta': {}, }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, + 'meta': {}, }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, + 'meta': {}, }, }, 'patch_path': self.dir('rs_models/schema.yml'), @@ -2171,6 +2218,7 @@ def expected_redshift_incremental_view_manifest(self): 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], + 'meta': {}, 'config': { 'column_types': {}, 'enabled': True, @@ -2354,12 +2402,36 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'target/compiled/test/model.sql' ), 'columns': { - 'id': {'description': 'The user ID number', 'name': 'id', 'data_type': None}, - 'first_name': {'description': "The user's first name", 'name': 'first_name', 'data_type': None}, - 'email': {'description': "The user's email", 'name': 'email', 'data_type': None}, - 'ip_address': {'description': "The user's IP address", 'name': 'ip_address', 'data_type': None}, - 'updated_at': {'description': "The last time this user's email was updated", - 'name': 'updated_at', 'data_type': None} + 'id': { + 'description': 'The user ID number', + 'name': 'id', + 'data_type': None, + 'meta': {} + }, + 'first_name': { + 'description': "The user's first name", + 'name': 'first_name', + 'data_type': None, + 'meta': {} + }, + 'email': { + 'description': "The user's email", + 'name': 'email', + 'data_type': None, + 'meta': {} + }, + 'ip_address': { + 'description': "The user's IP address", + 'name': 'ip_address', + 'data_type': None, + 'meta': {} + }, + 'updated_at': { + 'description': "The last time this user's email was updated", + 'name': 'updated_at', + 'data_type': None, + 'meta': {} + } }, 'compiled': True, 'compiled_sql': compiled_sql, @@ -2375,6 +2447,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes_injected': True, 'fqn': ['test', 'model'], 'injected_sql': compiled_sql, + 'meta': {}, 'name': 'model', 'original_file_path': model_sql_path, 'package_name': 'test', @@ -2426,6 +2499,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], 'injected_sql': '', + 'meta': {}, 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', @@ -2479,6 +2553,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'not_null_model_id'], 'injected_sql': AnyStringWith('id is null'), + 'meta': {}, 'name': 'not_null_model_id', 'original_file_path': schema_yml_path, 'package_name': 'test', @@ -2536,6 +2611,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'test_nothing_model_'], 'injected_sql': AnyStringWith('select 0'), + 'meta': {}, 'name': 'test_nothing_model_', 'original_file_path': schema_yml_path, 'package_name': 'test', @@ -2593,6 +2669,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'unique_model_id'], 'injected_sql': AnyStringWith('count(*)'), + 'meta': {}, 'name': 'unique_model_id', 'original_file_path': schema_yml_path, 'package_name': 'test', @@ -2659,12 +2736,14 @@ def expected_postgres_references_run_results(self): 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', - 'data_type': None + 'data_type': None, + 'meta': {}, }, }, 'compiled': True, @@ -2711,6 +2790,7 @@ def expected_postgres_references_run_results(self): 'extra_ctes_injected': True, 'fqn': ['test', 'ephemeral_summary'], 'injected_sql': ephemeral_injected_sql, + 'meta': {}, 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), 'package_name': 'test', @@ -2751,12 +2831,14 @@ def expected_postgres_references_run_results(self): 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', - 'data_type': None + 'data_type': None, + 'meta': {}, }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', - 'data_type': None + 'data_type': None, + 'meta': {}, }, }, 'compiled': True, @@ -2802,6 +2884,7 @@ def expected_postgres_references_run_results(self): 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], 'injected_sql': view_compiled_sql, + 'meta': {}, 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), 'package_name': 'test', @@ -2857,6 +2940,7 @@ def expected_postgres_references_run_results(self): 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], 'injected_sql': '', + 'meta': {}, 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', diff --git a/test/integration/035_docs_blocks/test_docs_blocks.py b/test/integration/035_docs_blocks/test_docs_blocks.py index d6d7b4c5631..12df291dae3 100644 --- a/test/integration/035_docs_blocks/test_docs_blocks.py +++ b/test/integration/035_docs_blocks/test_docs_blocks.py @@ -38,6 +38,7 @@ def test_postgres_valid_doc_ref(self): 'name': 'id', 'description': 'The user ID number', 'data_type': None, + 'meta': {}, }, model_data['columns']['id'] ) @@ -46,6 +47,7 @@ def test_postgres_valid_doc_ref(self): 'name': 'first_name', 'description': "The user's first name", 'data_type': None, + 'meta': {}, }, model_data['columns']['first_name'] ) @@ -55,6 +57,7 @@ def test_postgres_valid_doc_ref(self): 'name': 'last_name', 'description': "The user's last name", 'data_type': None, + 'meta': {}, }, model_data['columns']['last_name'] ) @@ -80,6 +83,7 @@ def test_postgres_alternative_docs_path(self): 'name': 'id', 'description': 'The user ID number with alternative text', 'data_type': None, + 'meta': {}, }, model_data['columns']['id'] ) @@ -88,6 +92,7 @@ def test_postgres_alternative_docs_path(self): 'name': 'first_name', 'description': "The user's first name", 'data_type': None, + 'meta': {}, }, model_data['columns']['first_name'] ) @@ -97,6 +102,7 @@ def test_postgres_alternative_docs_path(self): 'name': 'last_name', 'description': "The user's last name in this other file", 'data_type': None, + 'meta': {}, }, model_data['columns']['last_name'] ) diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index 9e05585b65a..9231b0a182b 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -46,6 +46,7 @@ def test_basic_uncompiled(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, 'compiled': False, 'extra_ctes': [], 'extra_ctes_injected': False, @@ -69,6 +70,7 @@ def test_basic_uncompiled(self): alias='bar', tags=[], config=NodeConfig(), + meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, @@ -128,6 +130,7 @@ def test_basic_compiled(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, 'compiled': True, 'compiled_sql': 'select * from whatever', 'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}], @@ -154,6 +157,7 @@ def test_basic_compiled(self): alias='bar', tags=[], config=NodeConfig(), + meta={}, compiled=True, compiled_sql='select * from whatever', extra_ctes=[InjectedCTE('whatever', 'select * from other')], @@ -239,6 +243,7 @@ def test_basic_uncompiled(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, 'compiled': False, 'extra_ctes': [], 'extra_ctes_injected': False, @@ -262,6 +267,7 @@ def test_basic_uncompiled(self): alias='bar', tags=[], config=TestConfig(), + meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, @@ -322,6 +328,7 @@ def test_basic_compiled(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, 'compiled': True, 'compiled_sql': 'select * from whatever', 'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}], @@ -349,6 +356,7 @@ def test_basic_compiled(self): alias='bar', tags=[], config=TestConfig(severity='warn'), + meta={}, compiled=True, compiled_sql='select * from whatever', extra_ctes=[InjectedCTE('whatever', 'select * from other')], diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 350917a96fb..610bea7ffc0 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -91,6 +91,7 @@ def test_ok(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } node = self.ContractType( package_name='test', @@ -111,6 +112,7 @@ def test_ok(self): alias='bar', tags=[], config=NodeConfig(), + meta={}, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) @@ -154,6 +156,7 @@ def test_complex(self): 'schema': 'test_schema', 'alias': 'bar', 'tags': ['tag'], + 'meta': {}, 'config': { 'column_types': {'a': 'text'}, 'enabled': True, @@ -166,7 +169,7 @@ def test_complex(self): 'vars': {'foo': 100}, }, 'docrefs': [], - 'columns': {'a': {'name': 'a', 'description': 'a text field'}}, + 'columns': {'a': {'name': 'a', 'description': 'a text field', 'meta': {}}}, } node = self.ContractType( @@ -187,13 +190,14 @@ def test_complex(self): schema='test_schema', alias='bar', tags=['tag'], + meta={}, config=NodeConfig( column_types={'a': 'text'}, materialized='ephemeral', post_hook=[Hook(sql='insert into blah(a, b) select "1", 1')], vars={'foo': 100}, ), - columns={'a': ColumnInfo('a', 'a text field')}, + columns={'a': ColumnInfo('a', 'a text field', {})}, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) @@ -234,6 +238,7 @@ def test_invalid_bad_tags(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } self.assert_fails_validation(bad_tags) @@ -270,6 +275,7 @@ def test_invalid_bad_materialized(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } self.assert_fails_validation(bad_materialized) @@ -292,16 +298,18 @@ def test_patch_ok(self): schema='test_schema', alias='bar', tags=[], + meta={}, config=NodeConfig(), ) patch = ParsedNodePatch( name='foo', description='The foo model', original_file_path='/path/to/schema.yml', - columns={'a': ColumnInfo(name='a', description='a text field')}, + columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, docrefs=[ Docref(documentation_name='foo', documentation_package='test'), ], + meta={}, ) initial.patch(patch) @@ -324,6 +332,7 @@ def test_patch_ok(self): 'schema': 'test_schema', 'alias': 'bar', 'tags': [], + 'meta': {}, 'config': { 'column_types': {}, 'enabled': True, @@ -336,7 +345,7 @@ def test_patch_ok(self): 'vars': {}, }, 'patch_path': '/path/to/schema.yml', - 'columns': {'a': {'name': 'a', 'description': 'a text field'}}, + 'columns': {'a': {'name': 'a', 'description': 'a text field', 'meta':{}}}, 'docrefs': [ { 'documentation_name': 'foo', @@ -363,9 +372,10 @@ def test_patch_ok(self): schema='test_schema', alias='bar', tags=[], + meta={}, config=NodeConfig(), patch_path='/path/to/schema.yml', - columns={'a': ColumnInfo(name='a', description='a text field')}, + columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, docrefs=[ Docref(documentation_name='foo', documentation_package='test'), ], @@ -442,6 +452,7 @@ def test_ok(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } node = self.ContractType( package_name='test', @@ -504,6 +515,7 @@ def test_complex(self): 'schema': 'test_schema', 'alias': 'bar', 'tags': ['tag'], + 'meta': {}, 'config': { 'column_types': {'a': 'text'}, 'enabled': True, @@ -516,7 +528,7 @@ def test_complex(self): 'vars': {}, }, 'docrefs': [], - 'columns': {'a': {'name': 'a', 'description': 'a text field'}}, + 'columns': {'a': {'name': 'a', 'description': 'a text field', 'meta':{}}}, 'index': 13, } @@ -538,12 +550,13 @@ def test_complex(self): schema='test_schema', alias='bar', tags=['tag'], + meta={}, config=NodeConfig( column_types={'a': 'text'}, materialized='table', post_hook=[] ), - columns={'a': ColumnInfo('a', 'a text field')}, + columns={'a': ColumnInfo('a', 'a text field', {})}, index=13, ) self.assert_symmetric(node, node_dict) @@ -584,6 +597,7 @@ def test_invalid_index_type(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, 'index': 'a string!?', } self.assert_fails_validation(bad_index) @@ -611,6 +625,7 @@ def test_ok(self): 'schema': 'test_schema', 'alias': 'bar', 'tags': [], + 'meta': {}, 'config': { 'column_types': {}, 'enabled': True, @@ -625,6 +640,7 @@ def test_ok(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } node = self.ContractType( package_name='test', @@ -644,6 +660,7 @@ def test_ok(self): schema='test_schema', alias='bar', tags=[], + meta={}, config=TestConfig(), ) self.assert_symmetric(node, node_dict) @@ -665,6 +682,7 @@ def test_ok(self): 'database': 'test_db', 'schema': 'test_schema', 'alias': 'bar', + 'meta': {}, } self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node)) @@ -688,6 +706,7 @@ def test_complex(self): 'schema': 'test_schema', 'alias': 'bar', 'tags': ['tag'], + 'meta': {}, 'config': { 'column_types': {'a': 'text'}, 'enabled': True, @@ -702,7 +721,7 @@ def test_complex(self): 'extra_key': 'extra value' }, 'docrefs': [], - 'columns': {'a': {'name': 'a', 'description': 'a text field'}}, + 'columns': {'a': {'name': 'a', 'description': 'a text field', 'meta': {}}}, 'column_name': 'id', } @@ -731,8 +750,9 @@ def test_complex(self): schema='test_schema', alias='bar', tags=['tag'], + meta={}, config=cfg, - columns={'a': ColumnInfo('a', 'a text field')}, + columns={'a': ColumnInfo('a', 'a text field',{})}, column_name='id', ) self.assert_symmetric(node, node_dict) @@ -773,6 +793,7 @@ def test_invalid_column_name_type(self): 'docrefs': [], 'columns': {}, 'column_name': {}, + 'meta': {}, } self.assert_fails_validation(bad_column_name) @@ -810,6 +831,7 @@ def test_invalid_missing_severity(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } self.assert_fails_validation(missing_config_value) @@ -846,6 +868,7 @@ def test_invalid_severity(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } self.assert_fails_validation(invalid_config_value) @@ -1111,6 +1134,7 @@ def test_timestamp_ok(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } node = self.ContractType( @@ -1216,6 +1240,7 @@ def test_check_ok(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } node = self.ContractType( @@ -1319,6 +1344,7 @@ def test_invalid_bad_resource_type(self): }, 'docrefs': [], 'columns': {}, + 'meta': {}, } self.assert_fails_validation(bad_resource_type) @@ -1333,6 +1359,7 @@ def test_empty(self): 'original_file_path': '/path/to/schema.yml', 'columns': {}, 'docrefs': [], + 'meta': {}, } patch = ParsedNodePatch( name='foo', @@ -1340,6 +1367,7 @@ def test_empty(self): original_file_path='/path/to/schema.yml', columns={}, docrefs=[], + meta={}, ) self.assert_symmetric(patch, dct) @@ -1348,22 +1376,24 @@ def test_populated(self): 'name': 'foo', 'description': 'The foo model', 'original_file_path': '/path/to/schema.yml', - 'columns': {'a': {'name': 'a', 'description': 'a text field'}}, + 'columns': {'a': {'name': 'a', 'description': 'a text field', 'meta':{}}}, 'docrefs': [ { 'documentation_name': 'foo', 'documentation_package': 'test', } ], + 'meta': {'key': ['value']} } patch = ParsedNodePatch( name='foo', description='The foo model', original_file_path='/path/to/schema.yml', - columns={'a': ColumnInfo(name='a', description='a text field')}, + columns={'a': ColumnInfo(name='a', description='a text field', meta={})}, docrefs=[ Docref(documentation_name='foo', documentation_package='test'), ], + meta={'key': ['value']}, ) self.assert_symmetric(patch, dct) pickle.loads(pickle.dumps(patch)) @@ -1511,6 +1541,8 @@ def test_basic(self): 'columns': {}, 'quoting': {}, 'unique_id': 'test.source.my_source.my_source_table', + 'meta': {}, + 'source_meta': {}, } source_def = self.ContractType( columns={}, diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py index e2200d523b0..5f1e4f2916c 100644 --- a/test/unit/test_contracts_graph_unparsed.py +++ b/test/unit/test_contracts_graph_unparsed.py @@ -243,8 +243,18 @@ class TestUnparsedSourceDefinition(ContractTestCase): def test_defaults(self): minimum = self.ContractType(name='foo') - self.assert_from_dict(minimum, {'name': 'foo'}) - self.assert_to_dict(minimum, {'name': 'foo', 'description': '', 'freshness': {}, 'quoting': {}, 'tables': [], 'loader': ''}) + from_dict = {'name': 'foo'} + to_dict = { + 'name': 'foo', + 'description': '', + 'freshness': {}, + 'quoting': {}, + 'tables': [], + 'loader': '', + 'meta': {}, + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) def test_contents(self): empty = self.ContractType( @@ -254,6 +264,7 @@ def test_contents(self): loader='some_loader', freshness=FreshnessThreshold(), tables=[], + meta={}, ) dct = { 'name': 'foo', @@ -262,6 +273,7 @@ def test_contents(self): 'loader': 'some_loader', 'freshness': {}, 'tables': [], + 'meta': {}, } self.assert_symmetric(empty, dct) @@ -293,6 +305,7 @@ def test_table_defaults(self): 'loader': '', 'freshness': {}, 'quoting': {}, + 'meta': {}, 'tables': [ { 'name': 'table1', @@ -302,6 +315,7 @@ def test_table_defaults(self): 'quoting': {}, 'external': {}, 'freshness': {}, + 'meta': {}, }, { 'name': 'table2', @@ -311,6 +325,7 @@ def test_table_defaults(self): 'quoting': {'database': True}, 'external': {}, 'freshness': {}, + 'meta': {}, }, ], } @@ -361,7 +376,13 @@ class TestUnparsedNodeUpdate(ContractTestCase): def test_defaults(self): minimum = self.ContractType(name='foo') from_dict = {'name': 'foo'} - to_dict = {'name': 'foo', 'columns': [], 'description': '', 'tests': []} + to_dict = { + 'name': 'foo', + 'columns': [], + 'description': '', + 'tests': [], + 'meta': {}, + } self.assert_from_dict(minimum, from_dict) self.assert_to_dict(minimum, to_dict) @@ -370,15 +391,21 @@ def test_contents(self): name='foo', description='a description', tests=['table_test'], + meta={'key': ['value1', 'value2']}, columns=[ - NamedTested(name='x', description='x description'), + NamedTested( + name='x', + description='x description', + meta={'key2': 'value3'}, + ), NamedTested( name='y', description='y description', tests=[ 'unique', {'accepted_values': {'values': ['blue', 'green']}} - ] + ], + meta={}, ), ], ) @@ -386,8 +413,14 @@ def test_contents(self): 'name': 'foo', 'description': 'a description', 'tests': ['table_test'], + 'meta': {'key': ['value1', 'value2']}, 'columns': [ - {'name': 'x', 'description': 'x description', 'tests': []}, + { + 'name': 'x', + 'description': 'x description', + 'tests': [], + 'meta': {'key2': 'value3'}, + }, { 'name': 'y', 'description': 'y description', @@ -395,6 +428,7 @@ def test_contents(self): 'unique', {'accepted_values': {'values': ['blue', 'green']}} ], + 'meta': {}, }, ], } @@ -406,8 +440,14 @@ def test_bad_test_type(self): 'name': 'foo', 'description': 'a description', 'tests': ['table_test'], + 'meta': {'key': ['value1', 'value2']}, 'columns': [ - {'name': 'x', 'description': 'x description', 'tests': []}, + { + 'name': 'x', + 'description': 'x description', + 'tests': [], + 'meta': {'key2': 'value3'}, + }, { 'name': 'y', 'description': 'y description', @@ -415,6 +455,7 @@ def test_bad_test_type(self): 100, {'accepted_values': {'values': ['blue', 'green']}} ], + 'meta': {}, }, ], } @@ -424,9 +465,14 @@ def test_bad_test_type(self): 'name': 'foo', 'description': 'a description', 'tests': ['table_test'], + 'meta': {'key': ['value1', 'value2']}, 'columns': [ # column missing a name - {'description': 'x description', 'tests': []}, + { + 'description': 'x description', + 'tests': [], + 'meta': {'key2': 'value3'}, + }, { 'name': 'y', 'description': 'y description', @@ -434,6 +480,7 @@ def test_bad_test_type(self): 'unique', {'accepted_values': {'values': ['blue', 'green']}} ], + 'meta': {}, }, ], } @@ -443,8 +490,14 @@ def test_bad_test_type(self): dct = { 'description': 'a description', 'tests': ['table_test'], + 'meta': {'key': ['value1', 'value2']}, 'columns': [ - {'name': 'x', 'description': 'x description', 'tests': []}, + { + 'name': 'x', + 'description': 'x description', + 'tests': [], + 'meta': {'key2': 'value3'}, + }, { 'name': 'y', 'description': 'y description', @@ -452,6 +505,7 @@ def test_bad_test_type(self): 'unique', {'accepted_values': {'values': ['blue', 'green']}} ], + 'meta': {}, }, ], } diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 69c28c4dc22..47f3120aa96 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -16,7 +16,7 @@ REQUIRED_PARSED_NODE_KEYS = frozenset({ - 'alias', 'tags', 'config', 'unique_id', 'refs', 'sources', + 'alias', 'tags', 'config', 'unique_id', 'refs', 'sources', 'meta', 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'root_path', 'path', 'original_file_path', 'raw_sql', 'docrefs', 'description', 'columns', 'fqn', 'build_path', 'patch_path', @@ -64,6 +64,7 @@ def setUp(self): path='events.sql', original_file_path='events.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.events': ParsedModelNode( @@ -83,6 +84,7 @@ def setUp(self): path='events.sql', original_file_path='events.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.dep': ParsedModelNode( @@ -102,6 +104,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.nested': ParsedModelNode( @@ -121,6 +124,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.sibling': ParsedModelNode( @@ -140,6 +144,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.multi': ParsedModelNode( @@ -159,6 +164,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), } @@ -390,6 +396,7 @@ def setUp(self): original_file_path='events.sql', root_path='', raw_sql='does not matter', + meta={}, compiled=True, compiled_sql='also does not matter', extra_ctes_injected=True, @@ -414,6 +421,7 @@ def setUp(self): original_file_path='events.sql', root_path='', raw_sql='does not matter', + meta={}, compiled=True, compiled_sql='also does not matter', extra_ctes_injected=True, @@ -437,6 +445,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.nested': ParsedModelNode( @@ -456,6 +465,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.sibling': ParsedModelNode( @@ -475,6 +485,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), 'model.root.multi': ParsedModelNode( @@ -494,6 +505,7 @@ def setUp(self): path='multi.sql', original_file_path='multi.sql', root_path='', + meta={}, raw_sql='does not matter' ), } diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index eec07eef7f6..c9f0dc6fa9c 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -297,6 +297,7 @@ def test__parse_basic_model_tests(self): columns={'color': ColumnInfo(name='color', description='The color value')}, docrefs=[], original_file_path=normalize('models/test_one.yml'), + meta={}, ) self.assertEqual(patch, expected_patch)