Merge pull request #207 from malakaganga/fix_hang_list

Fix hanging when listing large file volume
wso2-extensions · Mar 26, 2024 · 9ed6aac · 9ed6aac
2 parents 58027ac + c627c84
commit 9ed6aac
Showing 1 changed file with 19 additions and 8 deletions.
diff --git a/src/main/java/org/wso2/carbon/connector/operations/ListFiles.java b/src/main/java/org/wso2/carbon/connector/operations/ListFiles.java
@@ -106,9 +106,7 @@ public void connect(MessageContext messageContext) throws ConnectException {
             if (StringUtils.isEmpty(responseFormat)) {
                 responseFormat = HIERARCHICAL_FORMAT;
             }
-            if (StringUtils.isEmpty(fileMatchingPattern)) {
-                fileMatchingPattern = Const.MATCH_ALL_REGEX;
-            }
+
             String recursiveStr = (String) ConnectorUtils.
                     lookupTemplateParamater(messageContext, RECURSIVE_PARAM);
             recursive = Boolean.parseBoolean(recursiveStr);
@@ -122,9 +120,10 @@ public void connect(MessageContext messageContext) throws ConnectException {
             if (folder.exists()) {
 
                 if (folder.isFolder()) {
-
+                    //Added debug logs to track the flow and identify bottlenecks
                     OMElement fileListEle = listFilesInFolder(folder, fileMatchingPattern,
                             recursive, responseFormat, sortingAttribute, sortingOrder);
+                    log.debug(Const.CONNECTOR_NAME + ": " + OPERATION_NAME + " operation completed.");
                     FileOperationResult result = new FileOperationResult(
                             OPERATION_NAME,
                             true,
@@ -286,13 +285,25 @@ private void listFilesInFlatFormat(FileObject folder, String pattern,
      */
     private FileObject[] getFilesAndFolders(FileObject folder, String pattern) throws FileSystemException {
 
-        FileFilter fileFilter = new SimpleFileFiler(pattern);
-        FileFilterSelector fileFilterSelector = new FileFilterSelector(fileFilter);
-        ArrayList<FileObject> matchingFilesAndFolders =
-                new ArrayList<>(Arrays.asList(folder.findFiles(fileFilterSelector)));
+        ArrayList<FileObject> matchingFilesAndFolders;
+        log.debug(Const.CONNECTOR_NAME + ": Start listing matching files and folders.");
+        if (StringUtils.isEmpty(pattern)) {
+            matchingFilesAndFolders = new ArrayList<>(Arrays.asList(folder.getChildren()));
+            log.debug(Const.CONNECTOR_NAME + ": Pattern is empty , Children count : "
+                    + matchingFilesAndFolders.size());
+        } else {
+            FileFilter fileFilter = new SimpleFileFiler(pattern);
+            FileFilterSelector fileFilterSelector = new FileFilterSelector(fileFilter);
+            matchingFilesAndFolders =
+                    new ArrayList<>(Arrays.asList(folder.findFiles(fileFilterSelector)));
+            log.debug(Const.CONNECTOR_NAME + ": Pattern is : " + pattern + " : children count : "
+                    + matchingFilesAndFolders.size());
+        }
         //when a pattern exists folder.findFiles does not return folders
         if (!StringUtils.isEmpty(pattern)) {
             FileObject[] children = folder.getChildren();
+            log.debug(Const.CONNECTOR_NAME + ": Start traversing to add folders, children count : "
+                    + children.length );
             for (FileObject child : children) {
                 if (child.isFolder() && !matchingFilesAndFolders.contains(child)) {
                     matchingFilesAndFolders.add(child);