From b82b48a5587824c0b2a4dac47b408ba93c952e55 Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Thu, 23 Mar 2023 12:10:43 +0100 Subject: [PATCH] Add support for constants on empty lists, fixes #2102 --- .../hop/core/injection/bean/BeanInjector.java | 21 ++- .../mdi/0025-get-variables-parent.hpl | 49 +++--- ...0026-inject-using-constants-only-child.hpl | 123 ++++++++++++++ ...026-inject-using-constants-only-parent.hpl | 157 ++++++++++++++++++ .../datasets/golden-only-use-constants.csv | 2 + .../mdi/main-0025-inject-get-variables.hwf | 79 +++++++++ .../main-0026-inject-using-constants-only.hwf | 79 +++++++++ .../dataset/golden-only-use-constants.json | 24 +++ ...ject-using-constants-only-parent UNIT.json | 33 ++++ .../transforms/metainject/MetaInject.java | 2 - 10 files changed, 533 insertions(+), 36 deletions(-) create mode 100644 integration-tests/mdi/0026-inject-using-constants-only-child.hpl create mode 100644 integration-tests/mdi/0026-inject-using-constants-only-parent.hpl create mode 100644 integration-tests/mdi/datasets/golden-only-use-constants.csv create mode 100644 integration-tests/mdi/main-0025-inject-get-variables.hwf create mode 100644 integration-tests/mdi/main-0026-inject-using-constants-only.hwf create mode 100644 integration-tests/mdi/metadata/dataset/golden-only-use-constants.json create mode 100644 integration-tests/mdi/metadata/unit-test/0026-inject-using-constants-only-parent UNIT.json diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java index c4e9cee512c..aaa03a536e7 100644 --- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java +++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java @@ -17,12 +17,8 @@ package org.apache.hop.core.injection.bean; -import org.apache.hop.core.RowMetaAndData; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.injection.AfterInjection; -import org.apache.hop.metadata.api.IHopMetadata; -import org.apache.hop.metadata.api.IHopMetadataProvider; -import org.apache.hop.metadata.api.IHopMetadataSerializer; +import static com.google.common.collect.Lists.newLinkedList; +import static java.util.Objects.requireNonNull; import java.lang.reflect.Array; import java.lang.reflect.Constructor; @@ -34,9 +30,12 @@ import java.util.Optional; import java.util.Queue; import java.util.stream.Collectors; - -import static com.google.common.collect.Lists.newLinkedList; -import static java.util.Objects.requireNonNull; +import org.apache.hop.core.RowMetaAndData; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.injection.AfterInjection; +import org.apache.hop.metadata.api.IHopMetadata; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.api.IHopMetadataSerializer; /** Engine for get/set metadata injection properties from bean. */ public class BeanInjector { @@ -443,6 +442,10 @@ private List checkList(BeanLevelInfo s, Object obj, int index) throws Ex if (existList == null) { return null; } + // if constants are added without data we still want 1 row created + if (existList.size() == 0) { + return extendList(s, obj, 1); + } return index < existList.size() ? existList : null; } diff --git a/integration-tests/mdi/0025-get-variables-parent.hpl b/integration-tests/mdi/0025-get-variables-parent.hpl index 0e36aec6f0e..6fa8bb7d9f8 100644 --- a/integration-tests/mdi/0025-get-variables-parent.hpl +++ b/integration-tests/mdi/0025-get-variables-parent.hpl @@ -25,6 +25,7 @@ limitations under the License. Normal + 0 N @@ -34,8 +35,6 @@ limitations under the License. 2022/12/05 22:54:39.724 - 2022/12/05 22:54:39.724 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N @@ -147,46 +146,46 @@ limitations under the License. none + + + A + ${A} + String + + + B + ${B} + String + + + C + ${C} + String + + - N -1 - name -1 + N + name String - N -1 - variable -1 + N + variable String - N -1 - type -1 + N + type String - - - A - ${A} - 2 - - - B - ${B} - 2 - - - C - ${C} - 2 - - 128 diff --git a/integration-tests/mdi/0026-inject-using-constants-only-child.hpl b/integration-tests/mdi/0026-inject-using-constants-only-child.hpl new file mode 100644 index 00000000000..c47ce0b76ec --- /dev/null +++ b/integration-tests/mdi/0026-inject-using-constants-only-child.hpl @@ -0,0 +1,123 @@ + + + + + 0026-inject-using-constants-only-child + Y + + + + Normal + + + N + 1000 + 100 + - + 2022/03/23 12:29:35.111 + - + 2022/03/23 12:29:35.111 + + + + + + Data grid + Row denormaliser + Y + + + + Data grid + DataGrid + + Y + + 1 + + none + + + + + A + A + 100 + + + A + A + 20 + + + + + -1 + -1 + N + group + String + + + -1 + -1 + N + key + String + + + -1 + -1 + N + value + Number + + + + + 96 + 64 + + + + Row denormaliser + Denormaliser + + Y + + 1 + + none + + + + + + + + + 240 + 64 + + + + + + diff --git a/integration-tests/mdi/0026-inject-using-constants-only-parent.hpl b/integration-tests/mdi/0026-inject-using-constants-only-parent.hpl new file mode 100644 index 00000000000..6c209833ee2 --- /dev/null +++ b/integration-tests/mdi/0026-inject-using-constants-only-parent.hpl @@ -0,0 +1,157 @@ + + + + + 0026-inject-using-constants-only-parent + Y + + + + Normal + + + N + 1000 + 100 + - + 2023/03/23 11:08:16.063 + - + 2023/03/23 11:08:16.063 + + + + + + ETL metadata injection + Dummy (do nothing) + Y + + + + Dummy (do nothing) + Dummy + + Y + + 1 + + none + + + + + 368 + 112 + + + + ETL metadata injection + MetaInject + + Y + + 1 + + none + + + ${PROJECT_HOME}/0026-inject-using-constants-only-child.hpl + local + Row denormaliser + + + group + String + -1 + -1 + + + A + Number + -1 + -1 + + + + Y + N + + + + + Row denormaliser + TARGET_AGGREGATION + Y + + SUM + + + Row denormaliser + key_field + N + + key + + + Row denormaliser + TARGET_NAME + Y + + A + + + Row denormaliser + name + Y + + group + + + Row denormaliser + NAME + Y + + value + + + Row denormaliser + KEY_VALUE + Y + + A + + + Row denormaliser + TARGET_TYPE + Y + + Number + + + + + 144 + 112 + + + + + + diff --git a/integration-tests/mdi/datasets/golden-only-use-constants.csv b/integration-tests/mdi/datasets/golden-only-use-constants.csv new file mode 100644 index 00000000000..7a1e2e26f54 --- /dev/null +++ b/integration-tests/mdi/datasets/golden-only-use-constants.csv @@ -0,0 +1,2 @@ +group,A +A,120.0 diff --git a/integration-tests/mdi/main-0025-inject-get-variables.hwf b/integration-tests/mdi/main-0025-inject-get-variables.hwf new file mode 100644 index 00000000000..083acf1321a --- /dev/null +++ b/integration-tests/mdi/main-0025-inject-get-variables.hwf @@ -0,0 +1,79 @@ + + + + main-0025-inject-get-variables + Y + + + + - + 2022/12/05 15:56:29.618 + - + 2022/12/05 15:56:29.618 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 96 + 80 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0025-get-variables-parent UNIT + + + N + 304 + 80 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/mdi/main-0026-inject-using-constants-only.hwf b/integration-tests/mdi/main-0026-inject-using-constants-only.hwf new file mode 100644 index 00000000000..da9a1e5bb7f --- /dev/null +++ b/integration-tests/mdi/main-0026-inject-using-constants-only.hwf @@ -0,0 +1,79 @@ + + + + main-0026-inject-using-constants-only + Y + + + + - + 2022/12/05 15:56:29.618 + - + 2022/12/05 15:56:29.618 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 96 + 80 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0026-inject-using-constants-only-parent UNIT + + + N + 304 + 80 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/mdi/metadata/dataset/golden-only-use-constants.json b/integration-tests/mdi/metadata/dataset/golden-only-use-constants.json new file mode 100644 index 00000000000..19132d3a177 --- /dev/null +++ b/integration-tests/mdi/metadata/dataset/golden-only-use-constants.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-only-use-constants.csv", + "name": "golden-only-use-constants", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "group", + "field_format": "" + }, + { + "field_comment": "", + "field_length": 0, + "field_type": 1, + "field_precision": 0, + "field_name": "A", + "field_format": "####0.0#########;-####0.0#########" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/mdi/metadata/unit-test/0026-inject-using-constants-only-parent UNIT.json b/integration-tests/mdi/metadata/unit-test/0026-inject-using-constants-only-parent UNIT.json new file mode 100644 index 00000000000..3e4ff5e150c --- /dev/null +++ b/integration-tests/mdi/metadata/unit-test/0026-inject-using-constants-only-parent UNIT.json @@ -0,0 +1,33 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "group", + "data_set_field": "group" + }, + { + "transform_field": "A", + "data_set_field": "A" + } + ], + "field_order": [ + "group", + "A" + ], + "data_set_name": "golden-only-use-constants", + "transform_name": "Dummy (do nothing)" + } + ], + "input_data_sets": [], + "name": "0026-inject-using-constants-only-parent UNIT", + "description": "", + "persist_filename": "", + "trans_test_tweaks": [], + "pipeline_filename": "./0026-inject-using-constants-only-parent.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java index ef64d6e6321..f703c5a5051 100644 --- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java +++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java @@ -425,7 +425,6 @@ private void newInjection(String targetTransform, ITransformMeta targetTransform } } else { // target transform doesn't have specified key - just report but don't fail like in - // 6.0 (BACKLOG-6753) logError( BaseMessages.getString( PKG, @@ -474,7 +473,6 @@ private void newInjectionConstants(IVariables variables, String targetTransform, targetTransformMeta, target.getAttributeKey(), null, value); } else { // target transform doesn't have specified key - just report but don't fail like in 6.0 - // (BACKLOG-6753) logError( BaseMessages.getString( PKG,