fix(es): add indexed retry object for http logs (#3543)

## Changes Contributes to https://linear.app/nango/issue/NAN-2736/new-retry-strategy-for-proxy Contributes to https://linear.app/nango/issue/NAN-2683/log-retries-in-a-way-that-it-can-be-use-to-graph - Prepare the **retry** field It will contain information related to each http log, whether they are part of a retry, the attempt number, and the previous wait time. It's all based on theoretical test so hopefully I nailed it. Note it's the first undefined field I create, I realized nullable field are not ideal in terms of storage cost, so I had to change also some types
NangoHQ · Feb 20, 2025 · 04abcd3 · 04abcd3
1 parent 2f8dec2
commit 04abcd3
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 5 deletions.
diff --git a/packages/logs/lib/es/__snapshots__/index.integration.test.ts.snap b/packages/logs/lib/es/__snapshots__/index.integration.test.ts.snap
@@ -115,6 +115,19 @@ exports[`mapping > should create one index automatically on log 1`] = `
           },
         },
       },
+      "retry": {
+        "properties": {
+          "attempt": {
+            "type": "integer",
+          },
+          "max": {
+            "type": "integer",
+          },
+          "waited": {
+            "type": "integer",
+          },
+        },
+      },
       "source": {
         "type": "keyword",
       },

diff --git a/packages/logs/lib/es/schema.ts b/packages/logs/lib/es/schema.ts
@@ -80,13 +80,19 @@ const props: Record<keyof MessageRow, estypes.MappingProperty> = {
             headers: { type: 'object', enabled: false }
         }
     },
-
     response: {
         properties: {
             code: { type: 'integer' },
             headers: { type: 'object', enabled: false }
         }
     },
+    retry: {
+        properties: {
+            max: { type: 'integer' },
+            attempt: { type: 'integer' },
+            waited: { type: 'integer' }
+        }
+    },
 
     createdAt: { type: 'date' },
     updatedAt: { type: 'date' },

diff --git a/packages/logs/lib/models/helpers.ts b/packages/logs/lib/models/helpers.ts
@@ -47,7 +47,9 @@ export function getFormattedOperation(
         meta: meta || data.meta || null,
 
         userId: user?.id || data.userId || null,
-        parentId: null
+        parentId: null,
+
+        expiresAt: data.expiresAt || defaultOperationExpiration.sync()
     };
 }
 export function getFormattedMessage(data: Partial<MessageRow>, { meta }: FormatMessageData = {}): MessageRow {
@@ -88,6 +90,7 @@ export function getFormattedMessage(data: Partial<MessageRow>, { meta }: FormatM
         error: data.error || null,
         request: data.request || null,
         response: data.response || null,
+        retry: data.retry || undefined,
         meta: meta || data.meta || null,
 
         createdAt: data.createdAt || now.toISOString(),

diff --git a/packages/persist/lib/routes/environment/environmentId/postLog.ts b/packages/persist/lib/routes/environment/environmentId/postLog.ts
@@ -77,7 +77,8 @@ export const logBodySchema = z.object({
             .optional()
             .default(null),
         meta: z.record(z.any()).nullable().optional().default(null),
-        createdAt: z.string()
+        createdAt: z.string(),
+        retry: z.object({ max: z.number(), waited: z.number(), attempt: z.number() }).optional()
     })
 });
 

diff --git a/packages/types/lib/logs/messages.ts b/packages/types/lib/logs/messages.ts
@@ -1,4 +1,4 @@
-import type { Merge } from 'type-fest';
+import type { Merge, SetNonNullable } from 'type-fest';
 
 /**
  * Level of the log and operation
@@ -138,6 +138,13 @@ export interface MessageRow {
         headers: Record<string, string>;
     } | null;
     meta: MessageMeta | null;
+    retry?:
+        | {
+              attempt: number;
+              max: number;
+              waited: number;
+          }
+        | undefined;
 
     // Dates
     createdAt: string;
@@ -151,7 +158,12 @@ export interface MessageRow {
  * What is required to insert a Message
  */
 export type OperationRowInsert = Omit<Merge<Partial<MessageRow>, { operation: OperationList }>, 'message'>;
-export type OperationRow = Merge<Required<OperationRowInsert>, { message: string; accountId: number; accountName: string }>;
+export type OperationRow = SetNonNullable<
+    Omit<MessageRow, 'operation'>,
+    'id' | 'message' | 'source' | 'createdAt' | 'accountId' | 'accountName' | 'type' | 'level' | 'expiresAt' | 'state'
+> & {
+    operation: OperationList;
+};
 
 /**
  * What is required to insert a Message