Merge pull request #41417 from poorna2152/ws_in_invalid_tokens

Fix removal of whitespace between invalid tokens in formatter

misc/formatter/modules/formatter-core/src/main/java/org/ballerinalang/formatter/core/FormattingTreeModifier.java

@@ -4355,7 +4355,9 @@ private MinutiaeList getTrailingMinutiae(Token token) {
 
         // Preserve the necessary trailing minutiae coming from the original token
         int consecutiveNewlines = 0;
-        for (Minutiae minutiae : token.trailingMinutiae()) {
+        int size = token.trailingMinutiae().size();
+        for (int i = 0; i < size; i++) {
+            Minutiae minutiae = token.trailingMinutiae().get(i);
             switch (minutiae.kind()) {
                 case END_OF_LINE_MINUTIAE:
                     preserveIndentation(true);
@@ -4369,16 +4371,26 @@ private MinutiaeList getTrailingMinutiae(Token token) {
                         continue;
                     }
 
-                    addWhitespace(env.trailingWS, trailingMinutiae);
+                    // We reach here when the prevMinutiae is an invalid node/token
+                    if (i == size - 1) {
+                        addWhitespace(env.trailingWS, trailingMinutiae);
+                    } else {
+                        addWhitespace(1, trailingMinutiae);
+                    }
                     break;
                 case COMMENT_MINUTIAE:
-                    addWhitespace(1, trailingMinutiae);
+                    if (!matchesMinutiaeKind(prevMinutiae, SyntaxKind.WHITESPACE_MINUTIAE)) {
+                        addWhitespace(1, trailingMinutiae);
+                    }
                     trailingMinutiae.add(minutiae);
                     consecutiveNewlines = 0;
                     break;
                 case INVALID_TOKEN_MINUTIAE_NODE:
                 case INVALID_NODE_MINUTIAE:
                 default:
+                    if (matchesMinutiaeKind(prevMinutiae, SyntaxKind.END_OF_LINE_MINUTIAE)) {
+                        addWhitespace(env.currentIndentation, trailingMinutiae);
+                    }
                     trailingMinutiae.add(minutiae);
                     consecutiveNewlines = 0;
                     break;
@@ -4687,6 +4699,10 @@ private boolean hasTrailingNL(Token token) {
         return false;
     }
 
+    private boolean matchesMinutiaeKind(Minutiae minutiae, SyntaxKind kind) {
+        return minutiae != null && minutiae.kind() == kind;
+    }
+
     private NodeList<ImportDeclarationNode> sortAndGroupImportDeclarationNodes(
             NodeList<ImportDeclarationNode> importDeclarationNodes) {
         // moduleImports would collect only module level imports if grouping is enabled,

misc/formatter/modules/formatter-core/src/test/java/org/ballerinalang/formatter/core/ParserTestFormatter.java

@@ -89,11 +89,6 @@ public List<String> skipList() {
                 "service_decl_source_02.bal", "service_decl_source_05.bal", "service_decl_source_17.bal",
                 "service_decl_source_20.bal",
 
-                // The following tests are disabled due to tokens merging together after formatting. issue #35240
-                "query_expr_source_121.bal", "query_expr_source_123.bal", "query_expr_source_124.bal",
-                "query_expr_source_126.bal", "match_stmt_source_21.bal",
-                "func_params_source_27.bal",
-
                 "separated_node_list_import_decl.bal", "node_location_test_03.bal",
 
                 // parser tests with syntax errors that cannot be handled by the formatter

misc/formatter/modules/formatter-core/src/test/resources/actions/query/assert/query_action_4.bal

@@ -0,0 +1,5 @@
+public function main() {
+    var c = from var i in 0 ... 5
+        select i
+        where i %2 != 0;
+}

misc/formatter/modules/formatter-core/src/test/resources/actions/query/source/query_action_4.bal

@@ -0,0 +1,5 @@
+public function main() {
+    var c = from var i in 0 ... 5
+        select i
+     where  i  %2  !=  0 ;
+}