From fb3491cb17fc653802155aef40968edd04bb8357 Mon Sep 17 00:00:00 2001 From: Johnny Schmidt Date: Fri, 31 Jan 2025 13:44:27 -0800 Subject: [PATCH] Destination S3-V2: Maybe fix metadata issue --- .../object_storage/ObjectStorageDestinationStateManager.kt | 2 +- .../src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt | 2 +- airbyte-integrations/connectors/destination-s3/metadata.yaml | 2 +- docs/integrations/destinations/s3.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt index 607b3976e790b..0397ba8cc7d6d 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/state/object_storage/ObjectStorageDestinationStateManager.kt @@ -56,6 +56,7 @@ class ObjectStorageDestinationState( return client .list(pathFactory.getLongestStreamConstantPrefix(stream, isStaging = false)) .filter { matcher.match(it.key) != null } + .toList() // Force the list call to complete before initiating metadata calls .mapNotNull { obj -> val generationId = client.getMetadata(obj.key)[METADATA_GENERATION_ID_KEY]?.toLongOrNull() ?: 0L @@ -65,7 +66,6 @@ class ObjectStorageDestinationState( null } } - .toList() } /** diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index 5666fbd55a605..bfdbc6d791f6f 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -68,7 +68,7 @@ class S3Client( lastKey = obj.key emit(S3Object(obj.key!!, bucketConfig)) } // null contents => empty list, not error - if (client.listObjects(request).isTruncated == false) { + if (response.isTruncated == false) { break } request = request.copy { marker = lastKey } diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 73b0a154acba8..cb7fc16a725e2 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 1.5.0-rc.17 + dockerImageTag: 1.5.0-rc.18 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index 164528a40458f..0a37873f8c7ca 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -544,6 +544,7 @@ To see connector limitations, or troubleshoot your S3 connector, see more [in ou | Version | Date | Pull Request | Subject | |:------------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.5.0-rc.18 | 2025-01-29 | [52703](https://github.com/airbytehq/airbyte/pull/52703) | Force list call evaluation before making head calls | | 1.5.0-rc.17 | 2025-01-29 | [52610](https://github.com/airbytehq/airbyte/pull/52642) | Pin CDK 0.296 | | 1.5.0-rc.16 | 2025-01-29 | [52610](https://github.com/airbytehq/airbyte/pull/52610) | Fix assume role behavior | | 1.5.0-rc.15 | 2025-01-23 | [52103](https://github.com/airbytehq/airbyte/pull/52103) | Make the connector use our non root base image. |