Destination S3-V2: Maybe fix metadata issue

airbytehq · Jan 31, 2025 · 858a8ef · 858a8ef
1 parent da3198c
commit 858a8ef
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/...n/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt b/...n/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt
@@ -56,6 +56,7 @@ class ObjectStorageDestinationState(
         return client
             .list(pathFactory.getLongestStreamConstantPrefix(stream, isStaging = false))
             .filter { matcher.match(it.key) != null }
+            .toList() // Force the list call to complete before initiating metadata calls
             .mapNotNull { obj ->
                 val generationId =
                     client.getMetadata(obj.key)[METADATA_GENERATION_ID_KEY]?.toLongOrNull() ?: 0L
@@ -65,7 +66,6 @@ class ObjectStorageDestinationState(
                     null
                 }
             }
-            .toList()
     }
 
     /**

diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt
@@ -68,7 +68,7 @@ class S3Client(
                 lastKey = obj.key
                 emit(S3Object(obj.key!!, bucketConfig))
             } // null contents => empty list, not error
-            if (client.listObjects(request).isTruncated == false) {
+            if (response.isTruncated == false) {
                 break
             }
             request = request.copy { marker = lastKey }