🐛 Source Datadog : Fix the Pagination in the logs stream (#54180)

Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Feb 20, 2025 · 7abac4a · 7abac4a
1 parent fc80e61
commit 7abac4a
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 22 deletions.
diff --git a/airbyte-integrations/connectors/source-datadog/manifest.yaml b/airbyte-integrations/connectors/source-datadog/manifest.yaml
@@ -1,4 +1,4 @@
-version: 5.16.0
+version: 6.33.4
 
 type: DeclarativeSource
 
@@ -34,8 +34,8 @@ definitions:
           type: DefaultPaginator
           page_token_option:
             type: RequestOption
-            inject_into: request_parameter
             field_name: page[cursor]
+            inject_into: request_parameter
           pagination_strategy:
             type: CursorPagination
             cursor_value: >-
@@ -144,8 +144,8 @@ definitions:
           type: DefaultPaginator
           page_token_option:
             type: RequestOption
-            inject_into: request_parameter
             field_name: page[offset]
+            inject_into: request_parameter
           pagination_strategy:
             type: CursorPagination
             cursor_value: >-
@@ -195,15 +195,12 @@ definitions:
         paginator:
           type: DefaultPaginator
           page_token_option:
-            type: RequestOption
-            inject_into: request_parameter
-            field_name: page[cursor]
+            type: RequestPath
           pagination_strategy:
             type: CursorPagination
-            cursor_value: >-
-              {{ last_record['meta']['page']['after'] if last_record else None
-              }}
-            stop_condition: "{{ 'page' not in last_record['meta'] }}"
+            page_size: 1000
+            cursor_value: "{{ response.get(\"links\", {}).get(\"next\", {}) }}"
+            stop_condition: "{{ not response.get(\"links\", {}).get(\"next\", {}) }}"
       incremental_sync:
         type: DatetimeBasedCursor
         cursor_field: sync_date
@@ -356,12 +353,12 @@ definitions:
           type: DefaultPaginator
           page_token_option:
             type: RequestOption
-            inject_into: request_parameter
             field_name: page[number]
+            inject_into: request_parameter
           page_size_option:
             type: RequestOption
-            inject_into: request_parameter
             field_name: page[size]
+            inject_into: request_parameter
           pagination_strategy:
             type: PageIncrement
             page_size: 100
@@ -429,10 +426,10 @@ spec:
           syncs.
         order: 3
         title: Start date
+        default: "2023-12-01T00:00:00Z"
         pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
         examples:
           - "2022-10-01T00:00:00Z"
-        default: "2023-12-01T00:00:00Z"
       site:
         type: string
         description: The site where Datadog data resides in.
@@ -454,10 +451,10 @@ spec:
           syncs.
         order: 5
         title: End date
+        default: "2024-01-01T00:00:00Z"
         pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
         examples:
           - "2022-10-01T00:00:00Z"
-        default: "2024-01-01T00:00:00Z"
       max_records_per_request:
         type: integer
         description: Maximum number of records to collect per request.
@@ -516,12 +513,12 @@ metadata:
     users: false
   testedStreams:
     audit_logs:
+      hasRecords: true
       streamHash: 4938d5b333c624c8f301eac284c5a174d944f466
       hasResponse: true
-      responsesAreSuccessful: true
-      hasRecords: true
-      primaryKeysArePresent: true
       primaryKeysAreUnique: true
+      primaryKeysArePresent: true
+      responsesAreSuccessful: true
     dashboards:
       hasRecords: true
       streamHash: 277a1ea39fde86d246bc648ae60ff3b8ebd8ec41
@@ -551,12 +548,12 @@ metadata:
       primaryKeysArePresent: true
       responsesAreSuccessful: true
     logs:
-      streamHash: 6c2d948af333d7285c98d4f6776d79ff50310acb
+      hasRecords: true
+      streamHash: ec198a7eb4754773f1778c2b338ab477cf666ea6
       hasResponse: true
-      responsesAreSuccessful: true
-      hasRecords: false
-      primaryKeysArePresent: true
       primaryKeysAreUnique: true
+      primaryKeysArePresent: true
+      responsesAreSuccessful: true
     metrics:
       hasRecords: true
       streamHash: f4ec990f551bab54c17838ed70867b1ec35f43e5

diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml
@@ -26,7 +26,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 1cfc30c7-82db-43f4-9fd7-ac1b42312cda
-  dockerImageTag: 2.0.11
+  dockerImageTag: 2.0.12
   dockerRepository: airbyte/source-datadog
   githubIssueLabel: source-datadog
   icon: datadog.svg

diff --git a/docs/integrations/sources/datadog.md b/docs/integrations/sources/datadog.md
@@ -76,6 +76,7 @@ The Datadog source connector supports the following [sync modes](https://docs.ai
 
 | Version | Date       | Pull Request                                             | Subject                                                                      |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------|
+| 2.0.12 | 2025-02-20 | [54180](https://github.com/airbytehq/airbyte/pull/54180) | 🐛 Source Datadog : Fix the Pagination in the logs stream |
 | 2.0.11 | 2025-02-15 | [53705](https://github.com/airbytehq/airbyte/pull/53705) | Update dependencies |
 | 2.0.10 | 2025-02-08 | [53382](https://github.com/airbytehq/airbyte/pull/53382) | Update dependencies |
 | 2.0.9 | 2025-02-01 | [52866](https://github.com/airbytehq/airbyte/pull/52866) | Update dependencies |