diff --git a/integration-tests/sort_and_unique/0003-sort-order-ascending-case-sensitive-unique.hpl b/integration-tests/sort_and_unique/0003-sort-order-ascending-case-sensitive-unique.hpl
new file mode 100644
index 00000000000..cf539461641
--- /dev/null
+++ b/integration-tests/sort_and_unique/0003-sort-order-ascending-case-sensitive-unique.hpl
@@ -0,0 +1,224 @@
+
+
+
+
+ 0003-sort-order-ascending-case-sensitive-unique
+ Y
+
+
+
+ Normal
+
+
+ N
+ 1000
+ 100
+ -
+ 2020/12/11 14:08:48.348
+ -
+ 2020/12/11 14:08:48.348
+
+
+
+
+
+ Test strings
+ Sort rows
+ Y
+
+
+ Sort rows
+ concat values
+ Y
+
+
+ concat values
+ SORT_RESULT
+ Y
+
+
+
+ SORT_RESULT
+ SetVariable
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ value
+ SORT_RESULT
+ PARENT_WORKFLOW
+
+
+ N
+
+
+ 512
+ 80
+
+
+
+ Sort rows
+ SortRows
+
+ Y
+
+ 1
+
+ none
+
+
+ ${java.io.tmpdir}
+ out
+ 1000000
+
+ N
+
+ Y
+
+
+ value
+ Y
+ Y
+ N
+ 0
+ N
+
+
+
+
+ 256
+ 80
+
+
+
+ Test strings
+ DataGrid
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ - CCCC
+
+
+ - dddd
+
+
+ - BBBB
+
+
+ - EEEEE
+
+
+ - EEEEE
+
+
+ - bbbb
+
+
+ - ffff
+
+
+ - aaaa
+
+
+ - aaaa
+
+
+ - DDDD
+
+
+ - AAAA
+
+
+ - FFFFF
+
+
+ - FFFFF
+
+
+ - ccccc
+
+
+ - eeee
+
+
+
+
+ -1
+ -1
+ N
+ value
+ String
+
+
+
+
+ 128
+ 80
+
+
+
+ concat values
+ GroupBy
+
+ Y
+
+ 1
+
+ none
+
+
+ N
+ N
+ ${java.io.tmpdir}
+
+
+ value
+ value
+ CONCAT_COMMA
+
+
+ N
+
+
+ N
+ grp
+
+
+ 368
+ 80
+
+
+
+
+
+
diff --git a/integration-tests/sort_and_unique/0004-sort-order-ascending-case-insensitive-unique.hpl b/integration-tests/sort_and_unique/0004-sort-order-ascending-case-insensitive-unique.hpl
new file mode 100644
index 00000000000..74c1f55dd85
--- /dev/null
+++ b/integration-tests/sort_and_unique/0004-sort-order-ascending-case-insensitive-unique.hpl
@@ -0,0 +1,224 @@
+
+
+
+
+ 0004-sort-order-ascending-case-insensitive-unique
+ Y
+
+
+
+ Normal
+
+
+ N
+ 1000
+ 100
+ -
+ 2020/12/11 14:08:48.348
+ -
+ 2020/12/11 14:08:48.348
+
+
+
+
+
+ Test strings
+ Sort rows
+ Y
+
+
+ Sort rows
+ concat values
+ Y
+
+
+ concat values
+ SORT_RESULT
+ Y
+
+
+
+ SORT_RESULT
+ SetVariable
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ value
+ SORT_RESULT
+ PARENT_WORKFLOW
+
+
+ N
+
+
+ 512
+ 80
+
+
+
+ Sort rows
+ SortRows
+
+ Y
+
+ 1
+
+ none
+
+
+ ${java.io.tmpdir}
+ out
+ 1000000
+
+ N
+
+ Y
+
+
+ value
+ Y
+ N
+ N
+ 0
+ N
+
+
+
+
+ 256
+ 80
+
+
+
+ Test strings
+ DataGrid
+
+ Y
+
+ 1
+
+ none
+
+
+
+
+ - CCCC
+
+
+ - dddd
+
+
+ - BBBB
+
+
+ - EEEEE
+
+
+ - EEEEE
+
+
+ - bbbb
+
+
+ - ffff
+
+
+ - aaaa
+
+
+ - DDDD
+
+
+ - AAAA
+
+
+ - FFFFF
+
+
+ - FFFFF
+
+
+ - ccccc
+
+
+ - ccccc
+
+
+ - eeee
+
+
+
+
+ -1
+ -1
+ N
+ value
+ String
+
+
+
+
+ 128
+ 80
+
+
+
+ concat values
+ GroupBy
+
+ Y
+
+ 1
+
+ none
+
+
+ N
+ N
+ ${java.io.tmpdir}
+
+
+ value
+ value
+ CONCAT_COMMA
+
+
+ N
+
+
+ N
+ grp
+
+
+ 368
+ 80
+
+
+
+
+
+
diff --git a/integration-tests/sort_and_unique/main-0003-sort-order-ascending-case-sensitive-unique.hwf b/integration-tests/sort_and_unique/main-0003-sort-order-ascending-case-sensitive-unique.hwf
new file mode 100644
index 00000000000..d66692f0faf
--- /dev/null
+++ b/integration-tests/sort_and_unique/main-0003-sort-order-ascending-case-sensitive-unique.hwf
@@ -0,0 +1,117 @@
+
+
+
+ main-0003-sort-order-ascending-case-sensitive-unique
+ Y
+
+
+
+ -
+ 2020/12/11 14:07:57.181
+ -
+ 2020/12/11 14:07:57.181
+
+
+
+
+ Start
+
+ SPECIAL
+
+ 1
+ 12
+ 60
+ 0
+ 0
+ N
+ 0
+ 1
+ N
+ 112
+ 96
+
+
+
+ 0003-sort-order-ascending-case-sensitive-unique.hpl
+
+ PIPELINE
+
+ N
+ N
+ N
+ N
+ N
+ N
+ ${PROJECT_HOME}/0003-sort-order-ascending-case-sensitive-unique.hpl
+
+
+ Basic
+
+ Y
+
+ N
+ local
+ N
+ N
+ Y
+ N
+ 432
+ 96
+
+
+
+ check SORT_RESULT value
+
+ SIMPLE_EVAL
+
+ AAAA, BBBB, CCCC, DDDD, EEEEE, FFFFF, aaaa, bbbb, ccccc, dddd, eeee, ffff
+ string
+ false
+ equal
+ equal
+ N
+ variable
+ SORT_RESULT
+ N
+ 768
+ 96
+
+
+
+
+
+ Start
+ 0003-sort-order-ascending-case-sensitive-unique.hpl
+ Y
+ Y
+ Y
+
+
+ 0003-sort-order-ascending-case-sensitive-unique.hpl
+ check SORT_RESULT value
+ Y
+ Y
+ N
+
+
+
+
+
+
diff --git a/integration-tests/sort_and_unique/main-0004-sort-order-ascending-case-insensitive-unique.hwf b/integration-tests/sort_and_unique/main-0004-sort-order-ascending-case-insensitive-unique.hwf
new file mode 100644
index 00000000000..e7e6765b4e4
--- /dev/null
+++ b/integration-tests/sort_and_unique/main-0004-sort-order-ascending-case-insensitive-unique.hwf
@@ -0,0 +1,119 @@
+
+
+
+ main-0004-sort-order-ascending-case-insensitive-unique
+ Y
+
+
+
+ -
+ 2020/12/11 14:07:57.181
+ -
+ 2020/12/11 14:07:57.181
+
+
+
+
+ Start
+
+ SPECIAL
+
+ 1
+ 12
+ 60
+ 0
+ 0
+ N
+ 0
+ 1
+ N
+ 112
+ 96
+
+
+
+ 0004-sort-order-ascending-case-insensitive-unique.hpl
+
+ PIPELINE
+
+ N
+ N
+ N
+ N
+ N
+ N
+ ${PROJECT_HOME}/0004-sort-order-ascending-case-insensitive-unique.hpl
+ Basic
+
+ Y
+
+ N
+ local
+ N
+ N
+ Y
+ N
+ 432
+ 96
+
+
+
+ check SORT_RESULT value
+
+ SIMPLE_EVAL
+
+ aaaa, BBBB, CCCC, ccccc, dddd, eeee, EEEEE, ffff, FFFFF
+
+ string
+
+
+
+ false
+ equal
+ equal
+ N
+ variable
+ SORT_RESULT
+ N
+ 768
+ 96
+
+
+
+
+
+ Start
+ 0004-sort-order-ascending-case-insensitive-unique.hpl
+ Y
+ Y
+ Y
+
+
+ 0004-sort-order-ascending-case-insensitive-unique.hpl
+ check SORT_RESULT value
+ Y
+ Y
+ N
+
+
+
+
+
+
diff --git a/plugins/transforms/json/src/test/java/org/apache/hop/TestUtilities.java b/plugins/transforms/json/src/test/java/org/apache/hop/TestUtilities.java
index 94ba8722d2e..9e989901f16 100644
--- a/plugins/transforms/json/src/test/java/org/apache/hop/TestUtilities.java
+++ b/plugins/transforms/json/src/test/java/org/apache/hop/TestUtilities.java
@@ -43,6 +43,7 @@
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.pipeline.transforms.sort.SortRowsField;
import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
public class TestUtilities {
@@ -300,31 +301,23 @@ public static String writeTextFile(String folderName, String fileName, String de
* Create and return a SortRows transform.
*
* @param name
- * @param sortFields [] Fields to sort by
- * @param ascending [] Boolean indicating whether the corresponding field is to be sorted in
- * ascending or descending order.
- * @param caseSensitive [] Boolean indicating whether the corresponding field is to have case as a
- * factor in the sort.
* @param directory The directory in the file system where the sort is to take place if it can't
* fit into memory?
* @param sortSize ???
+ * @param sortRowsFields the fields to sort by
* @param pluginRegistry The environment's Hop plugin registry.
* @return
*/
public static synchronized TransformMeta createSortRowsTransform(
String name,
- String[] sortFields,
- boolean[] ascending,
- boolean[] caseSensitive,
String directory,
int sortSize,
+ List sortRowsFields,
PluginRegistry pluginRegistry) {
SortRowsMeta sortRowsMeta = new SortRowsMeta();
sortRowsMeta.setSortSize(Integer.toString(sortSize / 10));
- sortRowsMeta.setFieldName(sortFields);
- sortRowsMeta.setAscending(ascending);
- sortRowsMeta.setCaseSensitive(caseSensitive);
+ sortRowsMeta.setSortFields(sortRowsFields);
sortRowsMeta.setDirectory(directory);
String sortRowsTransformPid =
diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
index aaa9faa3fdf..e25eac6e06b 100644
--- a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
+++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
@@ -370,7 +370,7 @@ public boolean processRow() throws HopException {
// wait for first for is available
Object[] r = getRow();
- List groupFields = null;
+ List groupFields = null;
if (first) {
this.first = false;
@@ -392,8 +392,10 @@ public boolean processRow() throws HopException {
groupFields = meta.getGroupFields();
data.groupnrs = new int[groupFields.size()];
+ for (int i = 0; i < data.groupnrs.length; i++) {}
+
for (int i = 0; i < groupFields.size(); i++) {
- data.groupnrs[i] = inputRowMeta.indexOfValue(groupFields.get(i));
+ data.groupnrs[i] = inputRowMeta.indexOfValue(groupFields.get(i).getFieldName());
if (data.groupnrs[i] < 0) {
logError(
BaseMessages.getString(
@@ -405,7 +407,10 @@ public boolean processRow() throws HopException {
}
}
- String[] fieldNames = meta.getFieldName();
+ String[] fieldNames = new String[meta.getSortFields().size()];
+ for (int i = 0; i < fieldNames.length; i++) {
+ fieldNames[i] = meta.getSortFields().get(i).getFieldName();
+ }
data.fieldnrs = new int[fieldNames.length];
List toConvert = new ArrayList<>();
@@ -414,14 +419,14 @@ public boolean processRow() throws HopException {
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
data.comparator = new RowTemapFileComparator(data.outputRowMeta, data.fieldnrs);
- for (int i = 0; i < fieldNames.length; i++) {
- data.fieldnrs[i] = inputRowMeta.indexOfValue(fieldNames[i]);
+ for (int i = 0; i < meta.getSortFields().size(); i++) {
+ data.fieldnrs[i] = inputRowMeta.indexOfValue(meta.getSortFields().get(i).getFieldName());
if (data.fieldnrs[i] < 0) {
throw new HopException(
BaseMessages.getString(
PKG,
"SortRowsMeta.CheckResult.TransformFieldNotInInputStream",
- meta.getFieldName()[i],
+ meta.getSortFields().get(i).getFieldName(),
getTransformName()));
}
// do we need binary conversion for this type?
@@ -429,6 +434,7 @@ public boolean processRow() throws HopException {
toConvert.add(data.fieldnrs[i]);
}
}
+
data.convertKeysToNative = toConvert.isEmpty() ? null : new int[toConvert.size()];
int i = 0;
for (Integer in : toConvert) {
@@ -566,7 +572,7 @@ public boolean init() {
data.rowbuffer = new ArrayList<>(5000);
data.compressFiles =
- getVariableBoolean(meta.getCompressFilesVariable(), meta.getCompressFiles());
+ getVariableBoolean(meta.getCompressFilesVariable(), meta.isCompressFiles());
data.tempRows = new ArrayList<>();
@@ -645,6 +651,7 @@ public void batchComplete() throws HopException {
preSortBeforeFlush();
passBuffer();
setOutputDone();
+ setOutputDone();
}
}
diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
index 8d33878bfcf..30996939d0e 100644
--- a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
+++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java
@@ -275,7 +275,7 @@ public void widgetSelected(SelectionEvent e) {
fdlFields.top = new FormAttachment(wUniqueRows, margin);
wlFields.setLayoutData(fdlFields);
- final int FieldsRows = input.getFieldName().length;
+ final int FieldsRows = input.getSortFields().size();
colinf =
new ColumnInfo[] {
@@ -397,41 +397,42 @@ public void getData() {
}
wSortSize.setText(Const.NVL(input.getSortSize(), ""));
wFreeMemory.setText(Const.NVL(input.getFreeMemoryLimit(), ""));
- wCompress.setSelection(input.getCompressFiles());
+ wCompress.setSelection(input.isCompressFiles());
wCompress.setVariableName(input.getCompressFilesVariable());
wUniqueRows.setSelection(input.isOnlyPassingUniqueRows());
Table table = wFields.table;
- if (input.getFieldName().length > 0) {
+ if (input.getSortFields().size() > 0) {
table.removeAll();
}
- for (int i = 0; i < input.getFieldName().length; i++) {
+ for (int i = 0; i < input.getSortFields().size(); i++) {
+ SortRowsField field = input.getSortFields().get(i);
TableItem ti = new TableItem(table, SWT.NONE);
ti.setText(0, "" + (i + 1));
- ti.setText(1, input.getFieldName()[i]);
+ ti.setText(1, field.getFieldName());
ti.setText(
2,
- input.getAscending()[i]
+ field.isAscending()
? BaseMessages.getString(PKG, "System.Combo.Yes")
: BaseMessages.getString(PKG, "System.Combo.No"));
ti.setText(
3,
- input.getCaseSensitive()[i]
+ field.isCaseSensitive()
? BaseMessages.getString(PKG, "System.Combo.Yes")
: BaseMessages.getString(PKG, "System.Combo.No"));
ti.setText(
4,
- input.getCollatorEnabled()[i]
+ field.isCollatorEnabled()
? BaseMessages.getString(PKG, "System.Combo.Yes")
: BaseMessages.getString(PKG, "System.Combo.No"));
ti.setText(
5,
- input.getCollatorStrength()[i] == 0
+ field.getCollatorStrength() == 0
? BaseMessages.getString(PKG, "System.Combo.Primary")
- : Integer.toString(input.getCollatorStrength()[i]));
+ : Integer.toString(field.getCollatorStrength()));
ti.setText(
6,
- input.getPreSortedField()[i]
+ field.isPreSortedField()
? BaseMessages.getString(PKG, "System.Combo.Yes")
: BaseMessages.getString(PKG, "System.Combo.No"));
}
@@ -468,28 +469,30 @@ private void ok() {
int nrFields = wFields.nrNonEmpty();
- input.allocate(nrFields);
+ List fields = new ArrayList<>(nrFields);
for (int i = 0; i < nrFields; i++) {
TableItem ti = wFields.getNonEmpty(i);
- input.getFieldName()[i] = ti.getText(1);
- input.getAscending()[i] =
+ SortRowsField field = new SortRowsField();
+ field.setFieldName(ti.getText(1));
+ field.setAscending(
Utils.isEmpty(ti.getText(2))
- || BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(2));
- input.getCaseSensitive()[i] =
- BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(3));
- input.getCollatorEnabled()[i] =
- BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(4));
+ || BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(2)));
+ field.setCaseSensitive(
+ BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(3)));
+ field.setCollatorEnabled(
+ BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(4)));
if (ti.getText(5) == "") {
- input.getCollatorStrength()[i] =
- Integer.parseInt(BaseMessages.getString(PKG, "System.Combo.Primary"));
+ field.setCollatorStrength(
+ Integer.parseInt(BaseMessages.getString(PKG, "System.Combo.Primary")));
} else {
- input.getCollatorStrength()[i] = Integer.parseInt(ti.getText(5));
+ field.setCollatorStrength(Integer.parseInt(ti.getText(5)));
}
- input.getPreSortedField()[i] =
- BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(6));
+ field.setPreSortedField(
+ BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(6)));
+ fields.add(field);
}
-
+ input.setSortFields(fields);
dispose();
}
diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
new file mode 100644
index 00000000000..673517435a5
--- /dev/null
+++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.sort;
+
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+public class SortRowsField {
+
+ public SortRowsField() {}
+
+ public SortRowsField(
+ String fieldName,
+ boolean ascending,
+ boolean caseSensitive,
+ boolean collatorEnabled,
+ int collatorStrength,
+ boolean preSortedField) {
+ this.fieldName = fieldName;
+ this.ascending = ascending;
+ this.caseSensitive = caseSensitive;
+ this.collatorEnabled = collatorEnabled;
+ this.collatorStrength = collatorStrength;
+ this.preSortedField = preSortedField;
+ }
+
+ /** order by which fields? */
+ @HopMetadataProperty(key = "name", injectionKey = "NAME")
+ private String fieldName;
+
+ /** false : descending, true=ascending */
+ @HopMetadataProperty(key = "ascending", injectionKey = "SORT_ASCENDING")
+ private boolean ascending = true;
+
+ /** false : case insensitive, true=case sensitive */
+ @HopMetadataProperty(key = "case_sensitive", injectionKey = "IGNORE_CASE")
+ private boolean caseSensitive = true;
+
+ /** false : collator disabeld, true=collator enabled */
+ @HopMetadataProperty(key = "collator_enabled", injectionKey = "COLLATOR_ENABLED")
+ private boolean collatorEnabled = false;
+
+ // collator strength, 0,1,2,3
+ @HopMetadataProperty(key = "collator_strength", injectionKey = "COLLATOR_STRENGTH")
+ private int collatorStrength = 0;
+
+ /** false : not a presorted field, true=presorted field */
+ @HopMetadataProperty(key = "presorted", injectionKey = "PRESORTED")
+ private boolean preSortedField = false;
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ public void setFieldName(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public boolean isAscending() {
+ return ascending;
+ }
+
+ public void setAscending(boolean ascending) {
+ this.ascending = ascending;
+ }
+
+ public boolean isCaseSensitive() {
+ return caseSensitive;
+ }
+
+ public void setCaseSensitive(boolean caseSensitive) {
+ this.caseSensitive = caseSensitive;
+ }
+
+ public boolean isCollatorEnabled() {
+ return collatorEnabled;
+ }
+
+ public void setCollatorEnabled(boolean collatorEnabled) {
+ this.collatorEnabled = collatorEnabled;
+ }
+
+ public int getCollatorStrength() {
+ return collatorStrength;
+ }
+
+ public void setCollatorStrength(int collatorStrength) {
+ this.collatorStrength = collatorStrength;
+ }
+
+ public boolean isPreSortedField() {
+ return preSortedField;
+ }
+
+ public void setPreSortedField(boolean preSortedField) {
+ this.preSortedField = preSortedField;
+ }
+}
diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
index eb69daab314..251a9af14b9 100644
--- a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
+++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
@@ -28,25 +28,16 @@
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
-import org.apache.hop.core.injection.AfterInjection;
-import org.apache.hop.core.injection.Injection;
-import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
-import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.w3c.dom.Node;
-@InjectionSupported(
- localizationPrefix = "SortRows.Injection.",
- groups = {"FIELDS"})
@Transform(
id = "SortRows",
image = "sortrows.svg",
@@ -63,80 +54,49 @@ public class SortRowsMeta extends BaseTransformMeta
private static final String CONST_SPACE_LONG = " ";
private static final String CONST_FIELD = "field";
- /** order by which fields? */
- @Injection(name = "NAME", group = "FIELDS")
- private String[] fieldName;
-
- /** false : descending, true=ascending */
- @Injection(name = "SORT_ASCENDING", group = "FIELDS")
- private boolean[] ascending;
-
- /** false : case insensitive, true=case sensitive */
- @Injection(name = "IGNORE_CASE", group = "FIELDS")
- private boolean[] caseSensitive;
-
- /** false : collator disabeld, true=collator enabled */
- @Injection(name = "COLLATOR_ENABLED", group = "FIELDS")
- private boolean[] collatorEnabled;
-
- // collator strength, 0,1,2,3
- @Injection(name = "COLLATOR_STRENGTH", group = "FIELDS")
- private int[] collatorStrength;
-
- /** false : not a presorted field, true=presorted field */
- @Injection(name = "PRESORTED", group = "FIELDS")
- private boolean[] preSortedField;
-
- private List groupFields;
+ @HopMetadataProperty(groupKey = "fields", key = "field", injectionGroupKey = "FIELDS")
+ private List sortFields = new ArrayList<>();
/** Directory to store the temp files */
- @Injection(name = "SORT_DIRECTORY")
+ @HopMetadataProperty(key = "directory", injectionKey = "SORT_DIRECTORY")
private String directory;
/** Temp files prefix... */
- @Injection(name = "SORT_FILE_PREFIX")
+ @HopMetadataProperty(key = "sort_prefix", injectionKey = "SORT_FILE_PREFIX")
private String prefix;
/** The sort size: number of rows sorted and kept in memory */
- @Injection(name = "SORT_SIZE_ROWS")
+ @HopMetadataProperty(key = "sort_size", injectionKey = "SORT_SIZE_ROWS")
private String sortSize;
- /** The free memory limit in percentages in case we don't use the sort size */
- @Injection(name = "FREE_MEMORY_TRESHOLD")
+ /**
+ * The free memory limit in percentages in case we don't use the sort size We need the keep the
+ * missing 'H' in FREE_MEMORY_TRESHOLD for backwards compatibility.
+ */
+ @HopMetadataProperty(key = "free_memory", injectionKey = "FREE_MEMORY_TRESHOLD")
private String freeMemoryLimit;
/** only pass unique rows to the output stream(s) */
- @Injection(name = "ONLY_PASS_UNIQUE_ROWS")
+ @HopMetadataProperty(key = "unique_rows", injectionKey = "ONLY_PASS_UNIQUE_ROWS")
private boolean onlyPassingUniqueRows;
/**
* Compress files: if set to true, temporary files are compressed, thus reducing I/O at the cost
* of slightly higher CPU usage
*/
- @Injection(name = "COMPRESS_TEMP_FILES")
+ @HopMetadataProperty(key = "compress", injectionKey = "COMPRESS_TEMP_FILES")
private boolean compressFiles;
/** The variable to use to set the compressFiles option boolean */
+ @HopMetadataProperty(key = "compress_variables", injectionKey = "COMPRESS_VARIABLE")
private String compressFilesVariable;
+ private List groupFields;
+
public SortRowsMeta() {
super(); // allocate BaseTransformMeta
}
- /**
- * @return Returns the ascending.
- */
- public boolean[] getAscending() {
- return ascending;
- }
-
- /**
- * @param ascending The ascending to set.
- */
- public void setAscending(boolean[] ascending) {
- this.ascending = ascending;
- }
-
/**
* @return Returns the directory.
*/
@@ -151,20 +111,6 @@ public void setDirectory(String directory) {
this.directory = directory;
}
- /**
- * @return Returns the fieldName.
- */
- public String[] getFieldName() {
- return fieldName;
- }
-
- /**
- * @param fieldName The fieldName to set.
- */
- public void setFieldName(String[] fieldName) {
- this.fieldName = fieldName;
- }
-
/**
* @return Returns the prefix.
*/
@@ -179,77 +125,21 @@ public void setPrefix(String prefix) {
this.prefix = prefix;
}
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider)
- throws HopXmlException {
- readData(transformNode);
+ public List getSortFields() {
+ return sortFields;
}
- public void allocate(int nrFields) {
- fieldName = new String[nrFields]; // order by
- ascending = new boolean[nrFields];
- caseSensitive = new boolean[nrFields];
- collatorEnabled = new boolean[nrFields];
- collatorStrength = new int[nrFields];
- preSortedField = new boolean[nrFields];
- groupFields = null;
+ public void setSortFields(List sortFields) {
+ this.sortFields = sortFields;
}
@Override
public Object clone() {
SortRowsMeta retval = (SortRowsMeta) super.clone();
- int nrFields = fieldName.length;
-
- retval.allocate(nrFields);
- System.arraycopy(fieldName, 0, retval.fieldName, 0, nrFields);
- System.arraycopy(ascending, 0, retval.ascending, 0, nrFields);
- System.arraycopy(caseSensitive, 0, retval.caseSensitive, 0, nrFields);
- System.arraycopy(collatorEnabled, 0, retval.collatorEnabled, 0, nrFields);
- System.arraycopy(collatorStrength, 0, retval.collatorStrength, 0, nrFields);
- System.arraycopy(preSortedField, 0, retval.preSortedField, 0, nrFields);
-
return retval;
}
- private void readData(Node transformNode) throws HopXmlException {
- try {
- directory = XmlHandler.getTagValue(transformNode, "directory");
- prefix = XmlHandler.getTagValue(transformNode, "prefix");
- sortSize = XmlHandler.getTagValue(transformNode, "sort_size");
- freeMemoryLimit = XmlHandler.getTagValue(transformNode, "free_memory");
- compressFiles = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "compress"));
- compressFilesVariable = XmlHandler.getTagValue(transformNode, "compress_variable");
- onlyPassingUniqueRows =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "unique_rows"));
-
- Node fields = XmlHandler.getSubNode(transformNode, "fields");
- int nrFields = XmlHandler.countNodes(fields, CONST_FIELD);
-
- allocate(nrFields);
- String defaultStrength = Integer.toString(this.getDefaultCollationStrength());
-
- for (int i = 0; i < nrFields; i++) {
- Node fnode = XmlHandler.getSubNodeByNr(fields, CONST_FIELD, i);
-
- fieldName[i] = XmlHandler.getTagValue(fnode, "name");
- String asc = XmlHandler.getTagValue(fnode, "ascending");
- ascending[i] = "Y".equalsIgnoreCase(asc);
- String sens = XmlHandler.getTagValue(fnode, "case_sensitive");
- String coll = Const.NVL(XmlHandler.getTagValue(fnode, "collator_enabled"), "N");
- caseSensitive[i] = Utils.isEmpty(sens) || "Y".equalsIgnoreCase(sens);
- collatorEnabled[i] = "Y".equalsIgnoreCase(coll);
- collatorStrength[i] =
- Integer.parseInt(
- Const.NVL(XmlHandler.getTagValue(fnode, "collator_strength"), defaultStrength));
- String presorted = XmlHandler.getTagValue(fnode, "presorted");
- preSortedField[i] = "Y".equalsIgnoreCase(presorted);
- }
- } catch (Exception e) {
- throw new HopXmlException("Unable to load transform info from XML", e);
- }
- }
-
@Override
public void setDefault() {
directory = "${java.io.tmpdir}";
@@ -261,53 +151,6 @@ public void setDefault() {
onlyPassingUniqueRows = false;
int nrFields = 0;
-
- allocate(nrFields);
-
- for (int i = 0; i < nrFields; i++) {
- fieldName[i] = CONST_FIELD + i;
- caseSensitive[i] = true;
- collatorEnabled[i] = false;
- collatorStrength[i] = 0;
- preSortedField[i] = false;
- }
- }
-
- @Override
- public String getXml() {
- StringBuilder retval = new StringBuilder(256);
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("directory", directory));
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("prefix", prefix));
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("sort_size", sortSize));
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("free_memory", freeMemoryLimit));
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("compress", compressFiles));
- retval
- .append(CONST_SPACE)
- .append(XmlHandler.addTagValue("compress_variable", compressFilesVariable));
- retval.append(CONST_SPACE).append(XmlHandler.addTagValue("unique_rows", onlyPassingUniqueRows));
-
- retval.append(" ").append(Const.CR);
- for (int i = 0; i < fieldName.length; i++) {
- retval.append(" ").append(Const.CR);
- retval.append(CONST_SPACE_LONG).append(XmlHandler.addTagValue("name", fieldName[i]));
- retval.append(CONST_SPACE_LONG).append(XmlHandler.addTagValue("ascending", ascending[i]));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue("case_sensitive", caseSensitive[i]));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue("collator_enabled", collatorEnabled[i]));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue("collator_strength", collatorStrength[i]));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue("presorted", preSortedField[i]));
- retval.append(" ").append(Const.CR);
- }
- retval.append(" ").append(Const.CR);
-
- return retval.toString();
}
// Returns the default collation strength based on the users' default locale.
@@ -343,24 +186,15 @@ public void getFields(
}
public void assignSortingCriteria(IRowMeta inputRowMeta) {
- for (int i = 0; i < fieldName.length; i++) {
- int idx = inputRowMeta.indexOfValue(fieldName[i]);
+ for (int i = 0; i < sortFields.size(); i++) {
+ SortRowsField field = sortFields.get(i);
+ int idx = inputRowMeta.indexOfValue(field.getFieldName());
if (idx >= 0) {
IValueMeta valueMeta = inputRowMeta.getValueMeta(idx);
- // On all these valueMetas, check to see if the value actually exists before we try to
- // set them.
- if (ascending.length > i) {
- valueMeta.setSortedDescending(!ascending[i]);
- }
- if (caseSensitive.length > i) {
- valueMeta.setCaseInsensitive(!caseSensitive[i]);
- }
- if (collatorEnabled.length > i) {
- valueMeta.setCollatorDisabled(!collatorEnabled[i]);
- }
- if (collatorStrength.length > i) {
- valueMeta.setCollatorStrength(collatorStrength[i]);
- }
+ valueMeta.setSortedDescending(!field.isAscending());
+ valueMeta.setCaseInsensitive(!field.isCaseSensitive());
+ valueMeta.setCollatorDisabled(!field.isCollatorEnabled());
+ valueMeta.setCollatorStrength(field.getCollatorStrength());
// Also see if lazy conversion is active on these key fields.
// If so we want to automatically convert them to the normal storage type.
// This will improve performance
@@ -397,13 +231,15 @@ public void check(
boolean errorFound = false;
// Starting from selected fields in ...
- for (int i = 0; i < fieldName.length; i++) {
- int idx = prev.indexOfValue(fieldName[i]);
+ for (int i = 0; i < sortFields.size(); i++) {
+ SortRowsField field = sortFields.get(i);
+ int idx = prev.indexOfValue(field.getFieldName());
if (idx < 0) {
- errorMessage += "\t\t" + fieldName[i] + Const.CR;
+ errorMessage += "\t\t" + field.getFieldName() + Const.CR;
errorFound = true;
}
}
+
if (errorFound) {
errorMessage =
BaseMessages.getString(PKG, "SortRowsMeta.CheckResult.SortKeysNotFound", errorMessage);
@@ -411,7 +247,7 @@ public void check(
cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage, transformMeta);
remarks.add(cr);
} else {
- if (fieldName.length > 0) {
+ if (sortFields.size() > 0) {
cr =
new CheckResult(
ICheckResult.TYPE_RESULT_OK,
@@ -503,7 +339,7 @@ public void setSortSize(String sortSize) {
/**
* @return Returns whether temporary files should be compressed
*/
- public boolean getCompressFiles() {
+ public boolean isCompressFiles() {
return compressFiles;
}
@@ -542,48 +378,6 @@ public void setCompressFilesVariable(String compressFilesVariable) {
this.compressFilesVariable = compressFilesVariable;
}
- /**
- * @return the caseSensitive
- */
- public boolean[] getCaseSensitive() {
- return caseSensitive;
- }
-
- /**
- * @param caseSensitive the caseSensitive to set
- */
- public void setCaseSensitive(boolean[] caseSensitive) {
- this.caseSensitive = caseSensitive;
- }
-
- /**
- * @return the collatorEnabled
- */
- public boolean[] getCollatorEnabled() {
- return collatorEnabled;
- }
-
- /**
- * @param collatorEnabled the collatorEnabled to set
- */
- public void setCollatorEnabled(boolean[] collatorEnabled) {
- this.collatorEnabled = collatorEnabled;
- }
-
- /**
- * @return the collatorStrength
- */
- public int[] getCollatorStrength() {
- return collatorStrength;
- }
-
- /**
- * @param collatorStrength the collatorStrength to set
- */
- public void setCollatorStrength(int[] collatorStrength) {
- this.collatorStrength = collatorStrength;
- }
-
/**
* @return the freeMemoryLimit
*/
@@ -598,56 +392,26 @@ public void setFreeMemoryLimit(String freeMemoryLimit) {
this.freeMemoryLimit = freeMemoryLimit;
}
- /**
- * @return the preSortedField
- */
- public boolean[] getPreSortedField() {
- return preSortedField;
+ public boolean isGroupSortEnabled() {
+ return this.getSortFields() != null;
}
- /**
- * @param preSorted the preSorteField to set
- */
- public void setPreSortedField(boolean[] preSorted) {
- preSortedField = preSorted;
+ public void setGroupFields(List groupFields) {
+ this.groupFields = groupFields;
}
- public List getGroupFields() {
+ public List getGroupFields() {
if (this.groupFields == null) {
- for (int i = 0; i < preSortedField.length; i++) {
- if (preSortedField[i]) {
- if (groupFields == null) {
- groupFields = new ArrayList<>();
- }
- groupFields.add(this.fieldName[i]);
+ groupFields = new ArrayList<>();
+ for (int i = 0; i < getSortFields().size(); i++) {
+ if (getSortFields().get(i).isPreSortedField()) {
+ // if (groupFields == null) {
+ // groupFields = new ArrayList<>();
+ // }
+ groupFields.add(getSortFields().get(i));
}
}
}
return groupFields;
}
-
- public boolean isGroupSortEnabled() {
- return this.getGroupFields() != null;
- }
-
- /**
- * If we use injection we can have different arrays lengths. We need synchronize them for
- * consistency behavior with UI
- */
- @AfterInjection
- public void afterInjectionSynchronization() {
- int nrFields = (fieldName == null) ? -1 : fieldName.length;
- if (nrFields <= 0) {
- return;
- }
- boolean[][] rtnBooleanArrays =
- Utils.normalizeArrays(nrFields, ascending, caseSensitive, collatorEnabled, preSortedField);
- ascending = rtnBooleanArrays[0];
- caseSensitive = rtnBooleanArrays[1];
- collatorEnabled = rtnBooleanArrays[2];
- preSortedField = rtnBooleanArrays[3];
-
- int[][] rtnIntArrays = Utils.normalizeArrays(nrFields, collatorStrength);
- collatorStrength = rtnIntArrays[0];
- }
}
diff --git a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
index 4f2040d8dee..603742c11fd 100644
--- a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
+++ b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java
@@ -17,6 +17,8 @@
package org.apache.hop.pipeline.transforms.sort;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hop.core.injection.BaseMetadataInjectionTest;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironment;
import org.junit.Before;
@@ -28,7 +30,11 @@ public class SortRowsMetaInjectionTest extends BaseMetadataInjectionTest sortRowsFields = new ArrayList<>();
+ sortRowsFields.add(sortRowsField);
+ setup(sortRowsMeta);
}
@Test
@@ -38,12 +44,19 @@ public void test() throws Exception {
check("SORT_SIZE_ROWS", () -> meta.getSortSize());
check("FREE_MEMORY_TRESHOLD", () -> meta.getFreeMemoryLimit());
check("ONLY_PASS_UNIQUE_ROWS", () -> meta.isOnlyPassingUniqueRows());
- check("COMPRESS_TEMP_FILES", () -> meta.getCompressFiles());
- check("NAME", () -> meta.getFieldName()[0]);
- check("SORT_ASCENDING", () -> meta.getAscending()[0]);
- check("IGNORE_CASE", () -> meta.getCaseSensitive()[0]);
- check("PRESORTED", () -> meta.getPreSortedField()[0]);
- check("COLLATOR_STRENGTH", () -> meta.getCollatorStrength()[0]);
- check("COLLATOR_ENABLED", () -> meta.getCollatorEnabled()[0]);
+ check("COMPRESS_TEMP_FILES", () -> meta.isCompressFiles());
+ check("NAME", () -> meta.getSortFields().get(0).getFieldName());
+ check("SORT_ASCENDING", () -> meta.getSortFields().get(0).isAscending());
+ check("IGNORE_CASE", () -> meta.getSortFields().get(0).isCaseSensitive());
+ check("PRESORTED", () -> meta.getSortFields().get(0).isPreSortedField());
+ check("COLLATOR_ENABLED", () -> meta.getSortFields().get(0).isCollatorEnabled());
+ check("COLLATOR_STRENGTH", () -> meta.getSortFields().get(0).getCollatorStrength());
+ check("COMPRESS_VARIABLE", () -> meta.getCompressFilesVariable());
+ // check("NAME", () -> meta.getFieldName()[0]);
+ // check("SORT_ASCENDING", () -> meta.getAscending()[0]);
+ // check("IGNORE_CASE", () -> meta.getCaseSensitive()[0]);
+ // check("PRESORTED", () -> meta.getPreSortedField()[0]);
+ // check("COLLATOR_STRENGTH", () -> meta.getCollatorStrength()[0]);
+ // check("COLLATOR_ENABLED", () -> meta.getCollatorEnabled()[0]);
}
}
diff --git a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
index 04a7e21e9de..8b67bc5be1e 100644
--- a/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
+++ b/plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java
@@ -20,20 +20,19 @@
import static org.junit.Assert.assertEquals;
import java.text.Collator;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.UUID;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironment;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
import org.apache.hop.pipeline.transforms.loadsave.validator.BooleanLoadSaveValidator;
import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import org.apache.hop.pipeline.transforms.loadsave.validator.IntLoadSaveValidator;
-import org.apache.hop.pipeline.transforms.loadsave.validator.PrimitiveBooleanArrayLoadSaveValidator;
-import org.apache.hop.pipeline.transforms.loadsave.validator.PrimitiveIntArrayLoadSaveValidator;
+import org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator;
import org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
import org.junit.Assert;
import org.junit.ClassRule;
@@ -47,7 +46,7 @@ public class SortRowsMetaTest {
* @throws HopException
*/
@Test
- public void testRoundTrips() throws HopException {
+ public void testRoundTrips() throws Exception {
List attributes =
Arrays.asList(
"Directory",
@@ -57,30 +56,23 @@ public void testRoundTrips() throws HopException {
"CompressFiles",
"CompressFilesVariable",
"OnlyPassingUniqueRows",
- "FieldName",
- "Ascending",
- "CaseSensitive",
- "CollatorEnabled",
- "CollatorStrength",
- "PreSortedField");
+ "SortFields");
Map getterMap = new HashMap<>();
Map setterMap = new HashMap<>();
Map> fieldLoadSaveValidatorAttributeMap = new HashMap<>();
- IFieldLoadSaveValidator stringArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 25);
- IFieldLoadSaveValidator booleanArrayLoadSaveValidator =
- new PrimitiveBooleanArrayLoadSaveValidator(new BooleanLoadSaveValidator(), 25);
- IFieldLoadSaveValidator intArrayLoadSaveValidator =
- new PrimitiveIntArrayLoadSaveValidator(new IntLoadSaveValidator(), 25);
-
- fieldLoadSaveValidatorAttributeMap.put("FieldName", stringArrayLoadSaveValidator);
- fieldLoadSaveValidatorAttributeMap.put("Ascending", booleanArrayLoadSaveValidator);
- fieldLoadSaveValidatorAttributeMap.put("CaseSensitive", booleanArrayLoadSaveValidator);
- fieldLoadSaveValidatorAttributeMap.put("CollatorEnabled", booleanArrayLoadSaveValidator);
- fieldLoadSaveValidatorAttributeMap.put("CollatorStrength", intArrayLoadSaveValidator);
- fieldLoadSaveValidatorAttributeMap.put("PreSortedField", booleanArrayLoadSaveValidator);
+
+ IFieldLoadSaveValidator stringFieldLoadSaveValidator = new StringLoadSaveValidator();
+ IFieldLoadSaveValidator booleanFieldLoadSaveValidator = new BooleanLoadSaveValidator();
+
+ fieldLoadSaveValidatorAttributeMap.put("Directory", stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("Prefix", stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("SortSize", stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("FreeMemoryLimit", stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("CompressFiles", booleanFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("CompressFilesVariable", stringFieldLoadSaveValidator);
+ fieldLoadSaveValidatorAttributeMap.put("OnlyPassingUniqueRows", booleanFieldLoadSaveValidator);
LoadSaveTester loadSaveTester =
new LoadSaveTester<>(
@@ -91,9 +83,37 @@ public void testRoundTrips() throws HopException {
fieldLoadSaveValidatorAttributeMap,
new HashMap<>());
+ loadSaveTester
+ .getFieldLoadSaveValidatorFactory()
+ .registerValidator(
+ SortRowsMeta.class.getDeclaredField("sortFields").getGenericType().toString(),
+ new ListLoadSaveValidator<>(new SortRowsFieldLoadSaveValidator()));
loadSaveTester.testSerialization();
}
+ private final class SortRowsFieldLoadSaveValidator
+ implements IFieldLoadSaveValidator {
+
+ @Override
+ public SortRowsField getTestObject() {
+ return new SortRowsField(UUID.randomUUID().toString(), true, true, true, 0, true);
+ }
+
+ @Override
+ public boolean validateTestObject(SortRowsField testObject, Object actual) throws HopException {
+ if (!(actual instanceof SortRowsField)) {
+ return false;
+ }
+ SortRowsField actualObject = (SortRowsField) actual;
+ return testObject.getFieldName().equals(actualObject.getFieldName())
+ && testObject.isAscending() == actualObject.isAscending()
+ && testObject.isCaseSensitive() == actualObject.isCaseSensitive()
+ && testObject.isCollatorEnabled() == actualObject.isCollatorEnabled()
+ && testObject.getCollatorStrength() == actualObject.getCollatorStrength()
+ && testObject.isPreSortedField() == actualObject.isPreSortedField();
+ }
+ }
+
@Test
public void testGetDefaultStrength() {
SortRowsMeta srm = new SortRowsMeta();
@@ -107,30 +127,25 @@ public void testPDI16559() throws Exception {
SortRowsMeta sortRowsReal = new SortRowsMeta();
SortRowsMeta sortRows = Mockito.spy(sortRowsReal);
sortRows.setDirectory("/tmp");
- sortRows.setFieldName(new String[] {"field1", "field2", "field3", "field4", "field5"});
- sortRows.setAscending(new boolean[] {false, true, false});
- sortRows.setCaseSensitive(new boolean[] {true, false, true, false});
- sortRows.setCollatorEnabled(new boolean[] {false, false, true});
- sortRows.setCollatorStrength(new int[] {2, 1, 3});
- sortRows.setPreSortedField(new boolean[] {true, true, false});
-
- try {
- String badXml = sortRows.getXml();
- Assert.fail(
- "Before calling afterInjectionSynchronization, should have thrown an ArrayIndexOOB");
- } catch (Exception expected) {
- // Do Nothing
- }
- sortRows.afterInjectionSynchronization();
- // run without a exception
- String ktrXml = sortRows.getXml();
+ List sortRowsFields = new ArrayList<>();
+ sortRowsFields.add(new SortRowsField("field1", false, true, false, 2, true));
+ sortRowsFields.add(new SortRowsField("field2", true, false, false, 1, true));
+ sortRowsFields.add(new SortRowsField("field3", false, true, true, 3, false));
+ sortRowsFields.add(new SortRowsField("field4", true, false, false, 0, false));
+ sortRowsFields.add(new SortRowsField("field5", true, true, false, 0, false));
+ sortRows.setSortFields(sortRowsFields);
- int targetSz = sortRows.getFieldName().length;
+ // check the field names
+ for (int i = 0; i < sortRows.getSortFields().size(); i++) {
+ SortRowsField sortRowsField = sortRows.getSortFields().get(i);
+ Assert.assertEquals("field" + (i + 1), sortRowsField.getFieldName());
+ }
- Assert.assertEquals(targetSz, sortRows.getAscending().length);
- Assert.assertEquals(targetSz, sortRows.getCaseSensitive().length);
- Assert.assertEquals(targetSz, sortRows.getCollatorEnabled().length);
- Assert.assertEquals(targetSz, sortRows.getCollatorStrength().length);
- Assert.assertEquals(targetSz, sortRows.getPreSortedField().length);
+ // check the properties for SortRowField 3.
+ Assert.assertFalse(sortRowsFields.get(2).isAscending());
+ Assert.assertTrue(sortRowsFields.get(2).isCaseSensitive());
+ Assert.assertTrue(sortRowsFields.get(2).isCollatorEnabled());
+ Assert.assertEquals(3, sortRowsFields.get(2).getCollatorStrength());
+ Assert.assertFalse(sortRowsFields.get(2).isPreSortedField());
}
}
diff --git a/plugins/transforms/streamschemamerge/src/test/java/org/apache/hop/pipeline/transforms/streamschemamerge/TestUtilities.java b/plugins/transforms/streamschemamerge/src/test/java/org/apache/hop/pipeline/transforms/streamschemamerge/TestUtilities.java
index 42292e88037..b3a26da678a 100755
--- a/plugins/transforms/streamschemamerge/src/test/java/org/apache/hop/pipeline/transforms/streamschemamerge/TestUtilities.java
+++ b/plugins/transforms/streamschemamerge/src/test/java/org/apache/hop/pipeline/transforms/streamschemamerge/TestUtilities.java
@@ -40,6 +40,7 @@
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.pipeline.transforms.sort.SortRowsField;
import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
public class TestUtilities {
@@ -294,30 +295,24 @@ public static String writeTextFile(String folderName, String fileName, String de
*
* @param name
* @param sortFields [] Fields to sort by
- * @param ascending [] Boolean indicating whether the corresponding field is to be sorted in
- * ascending or descending order.
- * @param caseSensitive [] Boolean indicating whether the corresponding field is to have case as a
- * factor in the sort.
* @param directory The directory in the file system where the sort is to take place if it can't
* fit into memory?
* @param sortSize ???
+ * @param sortRowsFields list of SortRowsFields to configure the sort per field.
* @param pluginRegistry The environment's Hop plugin registry.
* @return
*/
public static synchronized TransformMeta createSortRowsTransform(
String name,
String[] sortFields,
- boolean[] ascending,
- boolean[] caseSensitive,
String directory,
int sortSize,
+ List sortRowsFields,
PluginRegistry pluginRegistry) {
SortRowsMeta sortRowsMeta = new SortRowsMeta();
sortRowsMeta.setSortSize(Integer.toString(sortSize / 10));
- sortRowsMeta.setFieldName(sortFields);
- sortRowsMeta.setAscending(ascending);
- sortRowsMeta.setCaseSensitive(caseSensitive);
+ sortRowsMeta.setSortFields(sortRowsFields);
sortRowsMeta.setDirectory(directory);
String sortRowsTransformPid =