diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java index cdf3b8e2e..22a53dd49 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java @@ -130,7 +130,7 @@ public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedO PluginPropertyUtils.pluginProp(expectedOutputFilePath), bucketName); } else { String outputFileSinkBucketName = PluginPropertyUtils.pluginProp(fileSinkBucketPath); - ValidationHelper.listBucketObjects(outputFileSinkBucketName.substring(5), + ValidationHelper.listBucketObjectsForXml(outputFileSinkBucketName.substring(5), PluginPropertyUtils.pluginProp(expectedOutputFilePath)); } } diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java index 09164bf47..b5f666eb0 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java @@ -185,4 +185,28 @@ private static void validateBothFilesDataForXml(List outputFileData, Str counter ); } + + public static void listBucketObjectsForXml(String bucketName, String expectedOutputFilePath) { + List bucketObjectNames = new ArrayList<>(); + List fileData = new ArrayList<>(); + String sourceBucketName = TestSetupHooks.fileSourceBucket; + Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); + Page blobs = storage.list(bucketName); + + // Adding all the Objects which have data in a list. + List bucketObjects = StreamSupport.stream(blobs.iterateAll().spliterator(), true) + .filter(blob -> blob.getSize() != 0) + .collect(Collectors.toList()); + + Stream objectNamesWithData = bucketObjects.stream().map(blob -> blob.getName()); + objectNamesWithData.forEach(objectName -> bucketObjectNames.add(objectName)); + + // Fetching the data for all the part-r Files + for (String objectName : bucketObjectNames) { + if (objectName.contains("part-r")) { + Collections.addAll(fileData, fetchObjectData(projectId, bucketName, objectName)); + } + } + validateBothFilesDataForXml(fileData, expectedOutputFilePath, sourceBucketName); + } }