additional updates and fixes for Sort Rows XML cleanup. #1933

plugins/transforms/json/src/test/java/org/apache/hop/TestUtilities.java

@@ -43,6 +43,7 @@
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
 import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.pipeline.transforms.sort.SortRowsField;
 import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
 
 public class TestUtilities {
@@ -300,31 +301,23 @@ public static String writeTextFile(String folderName, String fileName, String de
    * Create and return a SortRows transform.
    *
    * @param name
-   * @param sortFields [] Fields to sort by
-   * @param ascending [] Boolean indicating whether the corresponding field is to be sorted in
-   *     ascending or descending order.
-   * @param caseSensitive [] Boolean indicating whether the corresponding field is to have case as a
-   *     factor in the sort.
    * @param directory The directory in the file system where the sort is to take place if it can't
    *     fit into memory?
    * @param sortSize ???
+   * @param sortRowsFields the fields to sort by
    * @param pluginRegistry The environment's Hop plugin registry.
    * @return
    */
   public static synchronized TransformMeta createSortRowsTransform(
       String name,
-      String[] sortFields,
-      boolean[] ascending,
-      boolean[] caseSensitive,
       String directory,
       int sortSize,
+      List<SortRowsField> sortRowsFields,
       PluginRegistry pluginRegistry) {
 
     SortRowsMeta sortRowsMeta = new SortRowsMeta();
     sortRowsMeta.setSortSize(Integer.toString(sortSize / 10));
-    sortRowsMeta.setFieldName(sortFields);
-    sortRowsMeta.setAscending(ascending);
-    sortRowsMeta.setCaseSensitive(caseSensitive);
+    sortRowsMeta.setSortFields(sortRowsFields);
     sortRowsMeta.setDirectory(directory);
 
     String sortRowsTransformPid =

plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java

@@ -407,21 +407,26 @@ public boolean processRow() throws HopException {
         }
       }
 
+      String[] fieldNames = new String[meta.getSortFields().size()];
+      for (int i = 0; i < fieldNames.length; i++) {
+        fieldNames[i] = meta.getSortFields().get(i).getFieldName();
+      }
+      data.fieldnrs = new int[fieldNames.length];
       List<Integer> toConvert = new ArrayList<>();
 
       // Metadata
       data.outputRowMeta = inputRowMeta.clone();
       meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
       data.comparator = new RowTemapFileComparator(data.outputRowMeta, data.fieldnrs);
 
-      for (int i = 0; i < meta.getGroupFields().size(); i++) {
-        data.fieldnrs[i] = inputRowMeta.indexOfValue(meta.getGroupFields().get(i).getFieldName());
+      for (int i = 0; i < meta.getSortFields().size(); i++) {
+        data.fieldnrs[i] = inputRowMeta.indexOfValue(meta.getSortFields().get(i).getFieldName());
         if (data.fieldnrs[i] < 0) {
           throw new HopException(
               BaseMessages.getString(
                   PKG,
                   "SortRowsMeta.CheckResult.TransformFieldNotInInputStream",
-                  meta.getGroupFields().get(i).getFieldName(),
+                  meta.getSortFields().get(i).getFieldName(),
                   getTransformName()));
         }
         // do we need binary conversion for this type?
@@ -646,6 +651,7 @@ public void batchComplete() throws HopException {
       preSortBeforeFlush();
       passBuffer();
       setOutputDone();
+      setOutputDone();
     }
   }
 

plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java

@@ -275,7 +275,7 @@ public void widgetSelected(SelectionEvent e) {
     fdlFields.top = new FormAttachment(wUniqueRows, margin);
     wlFields.setLayoutData(fdlFields);
 
-    final int FieldsRows = input.getGroupFields().size();
+    final int FieldsRows = input.getSortFields().size();
 
     colinf =
         new ColumnInfo[] {
@@ -402,11 +402,11 @@ public void getData() {
     wUniqueRows.setSelection(input.isOnlyPassingUniqueRows());
 
     Table table = wFields.table;
-    if (input.getGroupFields().size() > 0) {
+    if (input.getSortFields().size() > 0) {
       table.removeAll();
     }
-    for (int i = 0; i < input.getGroupFields().size(); i++) {
-      SortRowsField field = input.getGroupFields().get(i);
+    for (int i = 0; i < input.getSortFields().size(); i++) {
+      SortRowsField field = input.getSortFields().get(i);
       TableItem ti = new TableItem(table, SWT.NONE);
       ti.setText(0, "" + (i + 1));
       ti.setText(1, field.getFieldName());
@@ -470,7 +470,6 @@ private void ok() {
     int nrFields = wFields.nrNonEmpty();
 
     List<SortRowsField> fields = new ArrayList<>(nrFields);
-    //    input.allocate(nrFields);
 
     for (int i = 0; i < nrFields; i++) {
       TableItem ti = wFields.getNonEmpty(i);
@@ -491,7 +490,9 @@ private void ok() {
       }
       field.setPreSortedField(
           BaseMessages.getString(PKG, "System.Combo.Yes").equalsIgnoreCase(ti.getText(6)));
+      fields.add(field);
     }
+    input.setSortFields(fields);
     dispose();
   }
 

plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java

@@ -52,7 +52,7 @@ public class SortRowsMeta extends BaseTransformMeta<SortRows, SortRowsData>
   private static final Class<?> PKG = SortRowsMeta.class; // For Translator
 
   @HopMetadataProperty(groupKey = "fields", key = "field", injectionGroupKey = "FIELDS")
-  private List<SortRowsField> groupFields = new ArrayList<>();
+  private List<SortRowsField> sortFields = new ArrayList<>();
 
   /** Directory to store the temp files */
   @HopMetadataProperty(key = "directory", injectionKey = "SORT_DIRECTORY")
@@ -88,6 +88,8 @@ public class SortRowsMeta extends BaseTransformMeta<SortRows, SortRowsData>
   @HopMetadataProperty(key = "compress_variables", injectionKey = "COMPRESS_VARIABLE")
   private String compressFilesVariable;
 
+  private List<SortRowsField> groupFields;
+
   public SortRowsMeta() {
     super(); // allocate BaseTransformMeta
   }
@@ -120,12 +122,12 @@ public void setPrefix(String prefix) {
     this.prefix = prefix;
   }
 
-  public List<SortRowsField> getGroupFields() {
-    return groupFields;
+  public List<SortRowsField> getSortFields() {
+    return sortFields;
   }
 
-  public void setGroupFields(List<SortRowsField> groupFields) {
-    this.groupFields = groupFields;
+  public void setSortFields(List<SortRowsField> sortFields) {
+    this.sortFields = sortFields;
   }
 
   @Override
@@ -181,8 +183,8 @@ public void getFields(
   }
 
   public void assignSortingCriteria(IRowMeta inputRowMeta) {
-    for (int i = 0; i < groupFields.size(); i++) {
-      SortRowsField field = groupFields.get(i);
+    for (int i = 0; i < sortFields.size(); i++) {
+      SortRowsField field = sortFields.get(i);
       int idx = inputRowMeta.indexOfValue(field.getFieldName());
       if (idx >= 0) {
         IValueMeta valueMeta = inputRowMeta.getValueMeta(idx);
@@ -226,8 +228,8 @@ public void check(
       boolean errorFound = false;
 
       // Starting from selected fields in ...
-      for (int i = 0; i < groupFields.size(); i++) {
-        SortRowsField field = groupFields.get(i);
+      for (int i = 0; i < sortFields.size(); i++) {
+        SortRowsField field = sortFields.get(i);
         int idx = prev.indexOfValue(field.getFieldName());
         if (idx < 0) {
           errorMessage += "\t\t" + field.getFieldName() + Const.CR;
@@ -242,7 +244,7 @@ public void check(
         cr = new CheckResult(ICheckResult.TYPE_RESULT_ERROR, errorMessage, transformMeta);
         remarks.add(cr);
       } else {
-        if (groupFields.size() > 0) {
+        if (sortFields.size() > 0) {
           cr =
               new CheckResult(
                   ICheckResult.TYPE_RESULT_OK,
@@ -388,6 +390,25 @@ public void setFreeMemoryLimit(String freeMemoryLimit) {
   }
 
   public boolean isGroupSortEnabled() {
-    return this.getGroupFields() != null;
+    return this.getSortFields() != null;
+  }
+
+  public void setGroupFields(List<SortRowsField> groupFields) {
+    this.groupFields = groupFields;
+  }
+
+  public List<SortRowsField> getGroupFields() {
+    if (this.groupFields == null) {
+      groupFields = new ArrayList<>();
+      for (int i = 0; i < getSortFields().size(); i++) {
+        if (getSortFields().get(i).isPreSortedField()) {
+          //          if (groupFields == null) {
+          //            groupFields = new ArrayList<>();
+          //          }
+          groupFields.add(getSortFields().get(i));
+        }
+      }
+    }
+    return groupFields;
   }
 }

plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaInjectionTest.java

@@ -45,12 +45,12 @@ public void test() throws Exception {
     check("FREE_MEMORY_TRESHOLD", () -> meta.getFreeMemoryLimit());
     check("ONLY_PASS_UNIQUE_ROWS", () -> meta.isOnlyPassingUniqueRows());
     check("COMPRESS_TEMP_FILES", () -> meta.isCompressFiles());
-    check("NAME", () -> meta.getGroupFields().get(0).getFieldName());
-    check("SORT_ASCENDING", () -> meta.getGroupFields().get(0).isAscending());
-    check("IGNORE_CASE", () -> meta.getGroupFields().get(0).isCaseSensitive());
-    check("PRESORTED", () -> meta.getGroupFields().get(0).isPreSortedField());
-    check("COLLATOR_ENABLED", () -> meta.getGroupFields().get(0).isCollatorEnabled());
-    check("COLLATOR_STRENGTH", () -> meta.getGroupFields().get(0).getCollatorStrength());
+    check("NAME", () -> meta.getSortFields().get(0).getFieldName());
+    check("SORT_ASCENDING", () -> meta.getSortFields().get(0).isAscending());
+    check("IGNORE_CASE", () -> meta.getSortFields().get(0).isCaseSensitive());
+    check("PRESORTED", () -> meta.getSortFields().get(0).isPreSortedField());
+    check("COLLATOR_ENABLED", () -> meta.getSortFields().get(0).isCollatorEnabled());
+    check("COLLATOR_STRENGTH", () -> meta.getSortFields().get(0).getCollatorStrength());
     check("COMPRESS_VARIABLE", () -> meta.getCompressFilesVariable());
     //    check("NAME", () -> meta.getFieldName()[0]);
     //    check("SORT_ASCENDING", () -> meta.getAscending()[0]);

plugins/transforms/sort/src/test/java/org/apache/hop/pipeline/transforms/sort/SortRowsMetaTest.java

@@ -56,7 +56,7 @@ public void testRoundTrips() throws Exception {
             "CompressFiles",
             "CompressFilesVariable",
             "OnlyPassingUniqueRows",
-            "GroupFields");
+            "SortFields");
 
     Map<String, String> getterMap = new HashMap<>();
     Map<String, String> setterMap = new HashMap<>();
@@ -86,7 +86,7 @@ public void testRoundTrips() throws Exception {
     loadSaveTester
         .getFieldLoadSaveValidatorFactory()
         .registerValidator(
-            SortRowsMeta.class.getDeclaredField("groupFields").getGenericType().toString(),
+            SortRowsMeta.class.getDeclaredField("sortFields").getGenericType().toString(),
             new ListLoadSaveValidator<>(new SortRowsFieldLoadSaveValidator()));
     loadSaveTester.testSerialization();
   }
@@ -133,11 +133,11 @@ public void testPDI16559() throws Exception {
     sortRowsFields.add(new SortRowsField("field3", false, true, true, 3, false));
     sortRowsFields.add(new SortRowsField("field4", true, false, false, 0, false));
     sortRowsFields.add(new SortRowsField("field5", true, true, false, 0, false));
-    sortRows.setGroupFields(sortRowsFields);
+    sortRows.setSortFields(sortRowsFields);
 
     // check the field names
-    for (int i = 0; i < sortRows.getGroupFields().size(); i++) {
-      SortRowsField sortRowsField = sortRows.getGroupFields().get(i);
+    for (int i = 0; i < sortRows.getSortFields().size(); i++) {
+      SortRowsField sortRowsField = sortRows.getSortFields().get(i);
       Assert.assertEquals("field" + (i + 1), sortRowsField.getFieldName());
     }
 

plugins/transforms/streamschemamerge/src/test/java/org/apache/hop/pipeline/transforms/streamschemamerge/TestUtilities.java

@@ -40,6 +40,7 @@
 import org.apache.hop.pipeline.transform.TransformMeta;
 import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
 import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.apache.hop.pipeline.transforms.sort.SortRowsField;
 import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
 
 public class TestUtilities {
@@ -294,30 +295,24 @@ public static String writeTextFile(String folderName, String fileName, String de
    *
    * @param name
    * @param sortFields [] Fields to sort by
-   * @param ascending [] Boolean indicating whether the corresponding field is to be sorted in
-   *     ascending or descending order.
-   * @param caseSensitive [] Boolean indicating whether the corresponding field is to have case as a
-   *     factor in the sort.
    * @param directory The directory in the file system where the sort is to take place if it can't
    *     fit into memory?
    * @param sortSize ???
+   * @param sortRowsFields list of SortRowsFields to configure the sort per field.
    * @param pluginRegistry The environment's Hop plugin registry.
    * @return
    */
   public static synchronized TransformMeta createSortRowsTransform(
       String name,
       String[] sortFields,
-      boolean[] ascending,
-      boolean[] caseSensitive,
       String directory,
       int sortSize,
+      List<SortRowsField> sortRowsFields,
       PluginRegistry pluginRegistry) {
 
     SortRowsMeta sortRowsMeta = new SortRowsMeta();
     sortRowsMeta.setSortSize(Integer.toString(sortSize / 10));
-    sortRowsMeta.setFieldName(sortFields);
-    sortRowsMeta.setAscending(ascending);
-    sortRowsMeta.setCaseSensitive(caseSensitive);
+    sortRowsMeta.setSortFields(sortRowsFields);
     sortRowsMeta.setDirectory(directory);
 
     String sortRowsTransformPid =