From b7d9f53dbea90193c628a0e9ecb8c7d9a292cee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Fri, 8 Mar 2024 14:18:15 -0600 Subject: [PATCH 1/3] Add failing test --- tests/core/test_typing.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 59bfa41ab..37b74a888 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -263,6 +263,27 @@ def test_object_arrays_remove_types(caplog: pytest.LogCaptureFixture): ) +def test_conform_object_additional_properties(): + schema = PropertiesList( + Property( + "object", + PropertiesList(additional_properties=True), + ), + ).to_dict() + + record = {"object": {"extra": "value"}} + expected_output = {"object": {"extra": "value"}} + + actual_output = conform_record_data_types( + "test_stream", + record, + schema, + TypeConformanceLevel.RECURSIVE, + logger, + ) + assert actual_output == expected_output + + def test_conform_primitives(): assert ( _conform_primitive_property( From 42302e2cb9bfe345f267f058c5c0c1acce50b590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Fri, 8 Mar 2024 14:23:53 -0600 Subject: [PATCH 2/3] Implement fix --- singer_sdk/helpers/_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index fdaaecd6e..ccaa41412 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -421,6 +421,8 @@ def _conform_record_data_types( # noqa: PLR0912 for property_name, elem in input_object.items(): property_path = property_name if parent is None else f"{parent}.{property_name}" if property_name not in schema["properties"]: + if schema.get("additionalProperties"): + output_object[property_name] = elem unmapped_properties.append(property_path) continue From ba222ac83c8cb20301bc249d12e9910d6002712d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Fri, 8 Mar 2024 14:38:40 -0600 Subject: [PATCH 3/3] Some refactoring --- singer_sdk/helpers/_typing.py | 55 +++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index ccaa41412..74ed59cf5 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -393,7 +393,7 @@ def conform_record_data_types( # TODO: This is in dire need of refactoring. It's a mess. -def _conform_record_data_types( # noqa: PLR0912 +def _conform_record_data_types( input_object: dict[str, t.Any], schema: dict, level: TypeConformanceLevel, @@ -429,24 +429,14 @@ def _conform_record_data_types( # noqa: PLR0912 property_schema = schema["properties"][property_name] if isinstance(elem, list) and is_uniform_list(property_schema): if level == TypeConformanceLevel.RECURSIVE: - item_schema = property_schema["items"] - output = [] - for item in elem: - if is_object_type(item_schema) and isinstance(item, dict): - ( - output_item, - sub_unmapped_properties, - ) = _conform_record_data_types( - item, - item_schema, - level, - property_path, - ) - unmapped_properties.extend(sub_unmapped_properties) - output.append(output_item) - else: - output.append(_conform_primitive_property(item, item_schema)) + output, sub_unmapped_properties = _conform_uniform_list( + elem, + path=property_path, + schema=property_schema, + level=level, + ) output_object[property_name] = output + unmapped_properties.extend(sub_unmapped_properties) else: output_object[property_name] = elem elif ( @@ -475,6 +465,35 @@ def _conform_record_data_types( # noqa: PLR0912 return output_object, unmapped_properties +def _conform_uniform_list( + element: list, + *, + path: str, + schema: dict, + level: TypeConformanceLevel, +) -> tuple[list, list[str]]: + item_schema = schema["items"] + unmapped_properties = [] + output = [] + for item in element: + if is_object_type(item_schema) and isinstance(item, dict): + ( + output_item, + sub_unmapped_properties, + ) = _conform_record_data_types( + item, + item_schema, + level, + path, + ) + unmapped_properties.extend(sub_unmapped_properties) + output.append(output_item) + else: + output.append(_conform_primitive_property(item, item_schema)) + + return output, unmapped_properties + + def _conform_primitive_property( # noqa: PLR0911 elem: t.Any, # noqa: ANN401 property_schema: dict,