Merge branch 'master' into alan/icon-mistake

docs-v2/snippets/generated/google-drive/PreBuiltUseCases.mdx

@@ -4,13 +4,22 @@
 <AccordionGroup>
 
 
-<Accordion title="Others">
+<Accordion title="Documents">
 | Endpoint | Description | Readme |
 | - | - | - |
-| `GET /fetch-document` | Fetches the content of a file given its ID, processes the data using<br />a response stream, and encodes it into a base64 string. This base64-encoded<br />string can be used to recreate the file in its original format using an external tool. | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/actions/fetch-document.md) |
+| `GET /fetch-document` | Fetches the content of a file given its ID, processes the data using<br />a response stream, and encodes it into a base64 string. This base64-encoded<br />string can be used to recreate the file in its original format using an external tool.<br />If this is a native google file type then use the fetch-google-sheet or fetch-google-doc<br />actions. | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/actions/fetch-document.md) |
+| `GET /fetch-google-sheet` | Fetches the content of a native google spreadsheet given its ID. Outputs<br />a JSON representation of a google sheet. | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/actions/fetch-google-sheet.md) |
+| `GET /fetch-google-document` | Fetches the content of a native google document given its ID. Outputs <br />a JSON reprensentation of a google doc. | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/actions/fetch-google-doc.md) |
 | `GET /documents` | Sync the metadata of a specified file or folders from Google Drive,<br />handling both individual files and nested folders.<br />Metadata required to filter on a particular folder, or file(s). Metadata<br />fields should be `{"files": ["<some-id>"]}` OR<br />`{"folders": ["<some-id>"]}`. The ID should be able to be provided<br />by using the Google Picker API<br />(https://developers.google.com/drive/picker/guides/overview)<br />and using the ID field provided by the response<br />(https://developers.google.com/drive/picker/reference/results) | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/syncs/documents.md) |
 </Accordion>
 
+
+<Accordion title="Folders">
+| Endpoint | Description | Readme |
+| - | - | - |
+| `GET /root-folders` | Sync the folders at the root level of a google drive. | [🔗](https://github.com/NangoHQ/integration-templates/blob/main/integrations/google-drive/syncs/folders.md) |
+</Accordion>
+
 </AccordionGroup>
 
 <Tip>Not seeing the integration you need? [Build your own](https://docs.nango.dev/guides/custom-integrations/overview) independently.</Tip>

packages/cli/lib/services/__snapshots__/model.service.unit.test.ts.snap

@@ -429,6 +429,7 @@ export declare class NangoSync extends NangoAction {
     batchUpdate<T extends object>(results: T[], model: string): Promise<boolean | null>;
     getMetadata<T = Metadata>(): Promise<T>;
     setMergingStrategy(merging: { strategy: 'ignore_if_modified_after' | 'override' }, model: string): Promise<void>;
+    getRecordsByIds<K = string | number, T = any>(ids: K[], model: string): Promise<Map<K, T>>;
 }
 /**
  * @internal

packages/cli/lib/services/sdk.ts

@@ -1,7 +1,7 @@
 import { Nango } from '@nangohq/node';
 import type { ProxyConfiguration } from '@nangohq/runner-sdk';
 import { InvalidRecordSDKError, NangoActionBase, NangoSyncBase } from '@nangohq/runner-sdk';
-import type { AdminAxiosProps } from '@nangohq/node';
+import type { AdminAxiosProps, ListRecordsRequestConfig } from '@nangohq/node';
 import type { Metadata, NangoProps, UserLogParameters } from '@nangohq/types';
 import type { AxiosResponse } from 'axios';
 import type { DryRunService } from './dryrun.service';
@@ -213,4 +213,45 @@ export class NangoSyncCLI extends NangoSyncBase {
 
         return super.getMetadata<TMetadata>();
     }
+
+    public override async getRecordsByIds<K = string | number, T = any>(ids: K[], model: string): Promise<Map<K, T>> {
+        const objects = new Map<K, T>();
+
+        if (ids.length === 0) {
+            return objects;
+        }
+
+        const externalIds = ids.map((id) => String(id).replaceAll('\x00', ''));
+        const externalIdMap = new Map<string, K>(ids.map((id) => [String(id), id]));
+
+        let cursor: string | null = null;
+        for (let i = 0; i < ids.length; i += 100) {
+            const batchIds = externalIds.slice(i, i + 100);
+
+            const props: ListRecordsRequestConfig = {
+                providerConfigKey: this.providerConfigKey,
+                connectionId: this.connectionId,
+                model,
+                ids: batchIds
+            };
+            if (cursor) {
+                props.cursor = cursor;
+            }
+
+            const response = await this.nango.listRecords<any>(props);
+
+            const batchRecords = response.records;
+            cursor = response.next_cursor;
+
+            for (const record of batchRecords) {
+                const stringId = String(record.id);
+                const realId = externalIdMap.get(stringId);
+                if (realId !== undefined) {
+                    objects.set(realId, record as T);
+                }
+            }
+        }
+
+        return objects;
+    }
 }

packages/runner-sdk/lib/sync.ts

@@ -32,6 +32,8 @@ export abstract class NangoSyncBase extends NangoActionBase {
 
     public abstract batchUpdate<T extends object>(results: T[], model: string): MaybePromise<boolean>;
 
+    public abstract getRecordsByIds<K = string | number, T = any>(ids: K[], model: string): MaybePromise<Map<K, T>>;
+
     protected validateRecords(model: string, records: unknown[]): { data: any; validation: ValidateDataError[] }[] {
         // Validate records
         const hasErrors: { data: any; validation: ValidateDataError[] }[] = [];

packages/runner-sdk/models.d.ts

@@ -386,6 +386,7 @@ export declare class NangoSync extends NangoAction {
     batchUpdate<T extends object>(results: T[], model: string): Promise<boolean | null>;
     getMetadata<T = Metadata>(): Promise<T>;
     setMergingStrategy(merging: { strategy: 'ignore_if_modified_after' | 'override' }, model: string): Promise<void>;
+    getRecordsByIds<K = string | number, T = any>(ids: K[], model: string): Promise<Map<K, T>>;
 }
 /**
  * @internal

packages/runner/lib/sdk/persist.ts

@@ -6,7 +6,15 @@ import { getUserAgent } from '@nangohq/node';
 import { httpRetryStrategy, retryWithBackoff, Ok, Err } from '@nangohq/utils';
 import { logger } from '../logger.js';
 import type { Result } from '@nangohq/utils';
-import type { CursorOffset, DeleteRecordsSuccess, GetCursorSuccess, MergingStrategy, PostRecordsSuccess, PutRecordsSuccess } from '@nangohq/types';
+import type {
+    CursorOffset,
+    DeleteRecordsSuccess,
+    GetCursorSuccess,
+    GetRecordsSuccess,
+    MergingStrategy,
+    PostRecordsSuccess,
+    PutRecordsSuccess
+} from '@nangohq/types';
 
 export class PersistClient {
     private httpClient: AxiosInstance;
@@ -225,4 +233,32 @@ export class PersistClient {
         }
         return res;
     }
+
+    public async getRecords({
+        environmentId,
+        nangoConnectionId,
+        model,
+        cursor,
+        externalIds
+    }: {
+        environmentId: number;
+        nangoConnectionId: number;
+        model: string;
+        cursor?: string | undefined;
+        externalIds?: string[] | undefined;
+    }): Promise<Result<GetRecordsSuccess>> {
+        const res = await this.makeRequest<GetRecordsSuccess>({
+            method: 'GET',
+            url: `/environment/${environmentId}/connection/${nangoConnectionId}/records`,
+            params: {
+                model,
+                cursor,
+                externalIds
+            }
+        });
+        if (res.isErr()) {
+            return Err(new Error(`Failed to get records: ${res.error.message}`));
+        }
+        return res;
+    }
 }

packages/runner/lib/sdk/sdk.ts

@@ -194,6 +194,7 @@ export class NangoSyncRunner extends NangoSyncBase {
 
     protected persistClient: PersistClient;
     private batchSize = 1000;
+    private getRecordsBatchSize = 100;
     private mergingByModel = new Map<string, MergingStrategy>();
 
     constructor(props: NangoProps, runnerProps?: { persistClient?: PersistClient }) {
@@ -381,12 +382,54 @@ export class NangoSyncRunner extends NangoSyncBase {
         }
         return true;
     }
+
+    public async getRecordsByIds<K = string | number, T = any>(ids: K[], model: string): Promise<Map<K, T>> {
+        this.throwIfAborted();
+
+        const objects = new Map<K, T>();
+
+        if (ids.length === 0) {
+            return objects;
+        }
+
+        let cursor: string | undefined = undefined;
+        for (let i = 0; i < ids.length; i += this.getRecordsBatchSize) {
+            const externalIdMap = new Map<string, K>(ids.slice(i, i + this.getRecordsBatchSize).map((id) => [String(id), id]));
+
+            const res = await this.persistClient.getRecords({
+                model,
+                externalIds: Array.from(externalIdMap.keys()),
+                environmentId: this.environmentId,
+                nangoConnectionId: this.nangoConnectionId!,
+                cursor
+            });
+
+            if (res.isErr()) {
+                throw res.error;
+            }
+
+            const { nextCursor, records } = res.unwrap();
+            cursor = nextCursor;
+
+            for (const record of records) {
+                const stringId = String(record.id);
+                const realId = externalIdMap.get(stringId);
+                if (realId !== undefined) {
+                    objects.set(realId, record as T);
+                }
+            }
+        }
+
+        return objects;
+    }
 }
 
 const TELEMETRY_ALLOWED_METHODS: (keyof NangoSyncBase)[] = [
     'batchDelete',
     'batchSave',
+    'batchUpdate',
     'batchSend',
+    'getRecordsByIds',
     'getConnection',
     'getEnvironmentVariables',
     'getMetadata',

packages/runner/lib/sdk/sdk.unit.test.ts

@@ -485,3 +485,58 @@ describe('Aborted script', () => {
         expect(nango.log('hello')).rejects.toThrowError(new AbortedSDKError());
     });
 });
+
+describe('getRecordsById', () => {
+    it('show throw if aborted', () => {
+        const ac = new AbortController();
+        const nango = new NangoSyncRunner({ ...nangoProps, abortSignal: ac.signal });
+        ac.abort();
+        expect(nango.getRecordsByIds(['a', 'b', 'c'], 'hello')).rejects.toThrowError(new AbortedSDKError());
+    });
+
+    it('should return empty map if no ids', async () => {
+        const mockPersistClient = new PersistClient({ secretKey: '***' });
+        mockPersistClient.getRecords = vi.fn();
+
+        const nango = new NangoSyncRunner({ ...nangoProps }, { persistClient: mockPersistClient });
+        const result = await nango.getRecordsByIds([], 'Wello');
+        expect(result).toEqual(new Map());
+        expect(mockPersistClient.getRecords).not.toHaveBeenCalled();
+    });
+
+    it('should call getRecords once for less than the batch size', async () => {
+        const records = new Map<number, { id: string }>();
+        for (let i = 0; i < 10; i++) {
+            records.set(i, { id: i.toString() });
+        }
+
+        const mockPersistClient = new PersistClient({ secretKey: '***' });
+        mockPersistClient.getRecords = vi.fn().mockResolvedValueOnce(Ok({ records: Array.from(records.values()), nextCursor: undefined }));
+
+        const nango = new NangoSyncRunner({ ...nangoProps }, { persistClient: mockPersistClient });
+        const result = await nango.getRecordsByIds(Array.from(records.keys()), 'Whatever');
+
+        expect(result).toEqual(records);
+        expect(mockPersistClient.getRecords).toHaveBeenCalledOnce();
+    });
+
+    it("should call getRecords multiple times if there's more than the batch size", async () => {
+        const records = new Map<number, { id: string }>();
+        for (let i = 0; i < 200; i++) {
+            records.set(i, { id: i.toString() });
+        }
+
+        const mockPersistClient = new PersistClient({ secretKey: '***' });
+        const recordsArray = Array.from(records.values());
+        mockPersistClient.getRecords = vi
+            .fn()
+            .mockResolvedValueOnce(Ok({ records: recordsArray.slice(0, 100), nextCursor: 'next' }))
+            .mockResolvedValueOnce(Ok({ records: recordsArray.slice(100, 200), nextCursor: 'next' }));
+
+        const nango = new NangoSyncRunner({ ...nangoProps }, { persistClient: mockPersistClient });
+        const result = await nango.getRecordsByIds(Array.from(records.keys()), 'Whatever');
+
+        expect(result).toEqual(records);
+        expect(mockPersistClient.getRecords).toHaveBeenCalledTimes(2);
+    });
+});

packages/shared/flows.yaml

@@ -4792,16 +4792,28 @@ integrations:
                     (https://developers.google.com/drive/picker/reference/results)
                 input: DocumentMetadata
                 auto_start: false
-                version: 1.0.3
+                version: 1.0.4
                 output: Document
                 sync_type: full
                 endpoint:
                     method: GET
                     path: /documents
+                    group: Documents
+                scopes: https://www.googleapis.com/auth/drive.readonly
+            folders:
+                runs: every day
+                track_deletes: true
+                sync_type: full
+                description: Sync the folders at the root level of a google drive.
+                output: Folder
+                endpoint:
+                    method: GET
+                    path: /root-folders
+                    group: Folders
                 scopes: https://www.googleapis.com/auth/drive.readonly
         actions:
             fetch-document:
-                input: DocumentId
+                input: IdEntity
                 description: >
                     Fetches the content of a file given its ID, processes the data using
 
@@ -4810,14 +4822,44 @@ integrations:
 
                     string can be used to recreate the file in its original format using
                     an external tool.
+
+                    If this is a native google file type then use the fetch-google-sheet
+                    or fetch-google-doc
+
+                    actions.
                 output: string
-                version: 2.0.0
+                version: 2.0.1
                 endpoint:
                     method: GET
                     path: /fetch-document
+                    group: Documents
+                scopes: https://www.googleapis.com/auth/drive.readonly
+            fetch-google-sheet:
+                input: IdEntity
+                description: >
+                    Fetches the content of a native google spreadsheet given its ID.
+                    Outputs
+
+                    a JSON representation of a google sheet.
+                output: JSONSpreadsheet
+                endpoint:
+                    method: GET
+                    path: /fetch-google-sheet
+                    group: Documents
+                scopes: https://www.googleapis.com/auth/drive.readonly
+            fetch-google-doc:
+                input: IdEntity
+                description: |
+                    Fetches the content of a native google document given its ID. Outputs 
+                    a JSON reprensentation of a google doc.
+                output: JSONDocument
+                endpoint:
+                    method: GET
+                    path: /fetch-google-document
+                    group: Documents
                 scopes: https://www.googleapis.com/auth/drive.readonly
         models:
-            DocumentId:
+            IdEntity:
                 id: string
             DocumentMetadata:
                 files: string[] | undefined
@@ -4826,6 +4868,40 @@ integrations:
                 id: string
                 url: string
                 title: string
+                mimeType: string
+            Folder:
+                id: string
+                url: string
+                title: string
+                mimeType: string
+            JSONSpreadsheet:
+                spreadsheetId: string
+                properties: object
+                sheets: object[]
+                namedRanges: object[]
+                spreadsheetUrl: string
+                developerMetadata: object[]
+                dataSources: object[]
+                dataSourceSchedules: object[]
+            JSONDocument:
+                documentId: string
+                title: string
+                url: string
+                tabs: object[]
+                revisionId: string
+                suggestionsViewMode: "DEFAULT_FOR_CURRENT_ACCESS | SUGGESTIONS_INLINE | PREVIEW_SUGGESTIONS_ACCEPTED\t| PREVIEW_WITHOUT_SUGGESTIONS"
+                body: object
+                headers: object
+                footers: object
+                footnotes: object
+                documentStyle: object
+                suggestedDocumentStyleChanges: object
+                namedStyles: object
+                suggestedNamedStylesChanges: object
+                lists: object
+                namedRanges: object
+                inlineObjects: object
+                positionedObjects: object
     google-mail:
         syncs:
             emails: