From 6742213050a9f99e9435a6eed4c4e0c6330faa7c Mon Sep 17 00:00:00 2001 From: Almog Mesilaty Date: Thu, 6 Feb 2025 17:27:35 +0200 Subject: [PATCH 1/4] Added to skiplist --- packages/salesforce-adapter/src/fetch_profile/metadata_query.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts b/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts index 3ca3f98041a..2b1be7f6eb7 100644 --- a/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts +++ b/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts @@ -58,6 +58,8 @@ const PERMANENT_SKIP_LIST: MetadataQueryParams[] = [ { metadataType: 'EscalationRule' }, // May conflict with the MetadataType ForecastingCategoryMapping { metadataType: 'CustomObject', name: 'ForecastingCategoryMapping' }, + // Retrieve on GenAiPromptTemplate fails on INSUFFICIENT_ACCESS + { metadataType: 'GenAiPromptTemplate' }, ] // Instances of these types will match all namespaces From ab7045ad2447b6b8c94550fcb1a0f9e3ab8e0fde Mon Sep 17 00:00:00 2001 From: Almog Mesilaty Date: Mon, 10 Feb 2025 12:29:30 +0200 Subject: [PATCH 2/4] handle insufficient error in retrieve --- .../salesforce-adapter/src/client/client.ts | 2 +- packages/salesforce-adapter/src/fetch.ts | 43 +++++++++- .../test/adapter.discover.test.ts | 81 ++++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/packages/salesforce-adapter/src/client/client.ts b/packages/salesforce-adapter/src/client/client.ts index 19cb7fdade2..80a0c4fda0d 100644 --- a/packages/salesforce-adapter/src/client/client.ts +++ b/packages/salesforce-adapter/src/client/client.ts @@ -924,7 +924,7 @@ export default class SalesforceClient implements ISalesforceClient { @logDecorator() @requiresLogin() public async retrieve(retrieveRequest: RetrieveRequest): Promise { - return flatValues(await this.retryOnBadResponse(() => this.conn.metadata.retrieve(retrieveRequest).complete())) + return flatValues(await this.conn.metadata.retrieve(retrieveRequest).complete()) } private async reportDeployProgressUntilComplete( diff --git a/packages/salesforce-adapter/src/fetch.ts b/packages/salesforce-adapter/src/fetch.ts index d793e27f1e2..9ec2a74938d 100644 --- a/packages/salesforce-adapter/src/fetch.ts +++ b/packages/salesforce-adapter/src/fetch.ts @@ -8,7 +8,7 @@ import _ from 'lodash' import JSZip from 'jszip' import { inspectValue, safeJsonStringify } from '@salto-io/adapter-utils' -import { FileProperties, MetadataInfo, MetadataObject } from '@salto-io/jsforce-types' +import { FileProperties, MetadataInfo, MetadataObject, RetrieveRequest, RetrieveResult } from '@salto-io/jsforce-types' import { InstanceElement, ObjectType, TypeElement } from '@salto-io/adapter-api' import { collections, objects, values as lowerDashValues } from '@salto-io/lowerdash' import { logger } from '@salto-io/logging' @@ -458,7 +458,37 @@ export const retrieveMetadataInstances = async ({ const typesToRetrieve = [...new Set(filesToRetrieve.map(prop => prop.type))].join(',') log.debug('retrieving types %s', typesToRetrieve) const request = toRetrieveRequest(filesToRetrieve) - const result = await client.retrieve(request) + + const typesWithInsufficientAccess = new Set() + const retrieveWithRetry = async ( + currentRequest: RetrieveRequest, + CurrentFilesToRetrieve: FileProperties[], + ): Promise => { + try { + return await client.retrieve(currentRequest) + } catch (error) { + const errorPattern = /INSUFFICIENT_ACCESS: insufficient access rights on entity: (\w+)/g + const matches = [...error.message.matchAll(errorPattern)] + if (matches.length > 0) { + matches.forEach(match => { + const failedEntity = match[1] + log.debug(`Failed to retrieve ${failedEntity} due to insufficient access rights`) + typesWithInsufficientAccess.add(failedEntity) + }) + const updatedFilesToRetrieve = CurrentFilesToRetrieve.filter( + fileProp => ![...matches.map(match => match[1])].includes(fileProp.type), + ) + if (updatedFilesToRetrieve.length === 0) { + throw new Error('No files left to retrieve after filtering out failed entities') + } + const updatedRequest = toRetrieveRequest(updatedFilesToRetrieve) + return retrieveWithRetry(updatedRequest, updatedFilesToRetrieve) + } + throw error + } + } + + const result = await retrieveWithRetry(request, filesToRetrieve) log.debug('retrieve result for types %s: %o', typesToRetrieve, _.omit(result, ['zipFile', 'fileProperties'])) @@ -505,7 +535,14 @@ export const retrieveMetadataInstances = async ({ ) ).flat() } - + typesWithInsufficientAccess.forEach(entity => { + configChanges.push( + createSkippedListConfigChange({ + type: entity, + reason: `Insufficient access rights on entity: ${entity}`, + }), + ) + }) const newConfigChanges = createRetrieveConfigChange(result).filter(change => !configChangeAlreadyExists(change)) configChanges.push(...newConfigChanges) // if we get an error then result.zipFile will be a single 'nil' XML element, which will be parsed as an object by diff --git a/packages/salesforce-adapter/test/adapter.discover.test.ts b/packages/salesforce-adapter/test/adapter.discover.test.ts index 2dcfea9c05a..698c7a262bf 100644 --- a/packages/salesforce-adapter/test/adapter.discover.test.ts +++ b/packages/salesforce-adapter/test/adapter.discover.test.ts @@ -27,7 +27,7 @@ import { import { MetadataInfo } from '@salto-io/jsforce' import { collections, values } from '@salto-io/lowerdash' import { MockInterface } from '@salto-io/test-utils' -import { FileProperties } from '@salto-io/jsforce-types' +import { FileProperties, RetrieveRequest } from '@salto-io/jsforce-types' import { buildElementsSourceFromElements } from '@salto-io/adapter-utils' import SalesforceAdapter from '../src/adapter' import Connection from '../src/client/jsforce' @@ -47,7 +47,7 @@ import { mockRetrieveLocator, mockRetrieveResult, } from './connection' -import { ConfigChangeSuggestion, FetchElements, MAX_ITEMS_IN_RETRIEVE_REQUEST } from '../src/types' +import { ConfigChangeSuggestion, FetchElements, FetchProfile, MAX_ITEMS_IN_RETRIEVE_REQUEST } from '../src/types' import * as fetchModule from '../src/fetch' import { fetchMetadataInstances, retrieveMetadataInstances } from '../src/fetch' import * as xmlTransformerModule from '../src/transformers/xml_transformer' @@ -68,7 +68,7 @@ import { SOCKET_TIMEOUT, } from '../src/constants' import { apiNameSync, isInstanceOfType, isInstanceOfTypeSync } from '../src/filters/utils' -import { NON_TRANSIENT_SALESFORCE_ERRORS } from '../src/config_change' +import { createSkippedListConfigChange, NON_TRANSIENT_SALESFORCE_ERRORS } from '../src/config_change' import SalesforceClient from '../src/client/client' import createMockClient from './client' import { mockInstances, mockTypes } from './mock_elements' @@ -2751,6 +2751,81 @@ describe('Fetch via retrieve API', () => { }, ) + describe('when retrieve fails for INSUFFICIENT_ACCESS rights on entity', () => { + let fetchProfile: FetchProfile + let mockRetrieve: jest.Mock + + beforeEach(async () => { + const instances = [ + { + type: mockTypes.CustomObject, + instanceName: 'CustomObject1', + }, + { + type: mockTypes.CustomObject, + instanceName: 'CustomObject2', + }, + { + type: mockTypes.Flow, + instanceName: 'Flow1', + }, + { + type: mockTypes.Flow, + instanceName: 'Flow2', + }, + { + type: mockTypes.Opportunity, + instanceName: 'Opportunity1', + }, + { + type: mockTypes.Opportunity, + instanceName: 'Opportunity2', + }, + ] + await setupMocks(instances) + fetchProfile = buildFetchProfile({ + fetchParams: {}, + }) + mockRetrieve = jest.fn().mockImplementation((retrieveRequest: RetrieveRequest) => { + if ( + retrieveRequest.unpackaged?.types.some(t => t.name === 'CustomObject') && + retrieveRequest.unpackaged?.types.some(t => t.name === 'Flow') + ) { + throw new Error( + `Retrieve request for ${retrieveRequest.unpackaged.types} failed. messages: INSUFFICIENT_ACCESS: insufficient access rights on entity: CustomObject INSUFFICIENT_ACCESS: insufficient access rights on entity: Flow`, + ) + } + if (retrieveRequest.unpackaged?.types.some(t => t.name === 'CustomObject')) { + throw new Error( + `Retrieve request for ${retrieveRequest.unpackaged.types} failed. messages: INSUFFICIENT_ACCESS: insufficient access rights on entity: CustomObject`, + ) + } + return connection.metadata.retrieve(retrieveRequest).complete() + }) + }) + it('should try again without the failed entity and create a config suggestion', async () => { + const expectedConfigChanges = [ + createSkippedListConfigChange({ + type: 'CustomObject', + reason: 'Insufficient access rights on entity: CustomObject', + }), + createSkippedListConfigChange({ + type: 'Flow', + reason: 'Insufficient access rights on entity: Flow', + }), + ] + jest.spyOn(client, 'retrieve').mockImplementation(mockRetrieve) + const { configChanges } = await retrieveMetadataInstances({ + client, + types: [mockTypes.CustomObject, mockTypes.Flow, mockTypes.Opportunity], + fetchProfile, + }) + // expect(elements).toHaveLength(2) + expect(client.retrieve).toHaveBeenCalledTimes(2) + expect(configChanges).toIncludeSameMembers(expectedConfigChanges) + }) + }) + describe('Config changes', () => { let configChanges: ConfigChangeSuggestion[] From ba316a49651c853b17836a2af6235885488a0de2 Mon Sep 17 00:00:00 2001 From: Almog Mesilaty Date: Mon, 10 Feb 2025 12:46:43 +0200 Subject: [PATCH 3/4] Removed from permanent skip list --- packages/salesforce-adapter/src/fetch_profile/metadata_query.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts b/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts index 2b1be7f6eb7..3ca3f98041a 100644 --- a/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts +++ b/packages/salesforce-adapter/src/fetch_profile/metadata_query.ts @@ -58,8 +58,6 @@ const PERMANENT_SKIP_LIST: MetadataQueryParams[] = [ { metadataType: 'EscalationRule' }, // May conflict with the MetadataType ForecastingCategoryMapping { metadataType: 'CustomObject', name: 'ForecastingCategoryMapping' }, - // Retrieve on GenAiPromptTemplate fails on INSUFFICIENT_ACCESS - { metadataType: 'GenAiPromptTemplate' }, ] // Instances of these types will match all namespaces From 833addfeee426754c3b4c45b44c857f64c1a868b Mon Sep 17 00:00:00 2001 From: Almog Mesilaty Date: Mon, 10 Feb 2025 13:21:48 +0200 Subject: [PATCH 4/4] Call with retrayonbadresponse --- packages/salesforce-adapter/src/client/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/salesforce-adapter/src/client/client.ts b/packages/salesforce-adapter/src/client/client.ts index 80a0c4fda0d..19cb7fdade2 100644 --- a/packages/salesforce-adapter/src/client/client.ts +++ b/packages/salesforce-adapter/src/client/client.ts @@ -924,7 +924,7 @@ export default class SalesforceClient implements ISalesforceClient { @logDecorator() @requiresLogin() public async retrieve(retrieveRequest: RetrieveRequest): Promise { - return flatValues(await this.conn.metadata.retrieve(retrieveRequest).complete()) + return flatValues(await this.retryOnBadResponse(() => this.conn.metadata.retrieve(retrieveRequest).complete())) } private async reportDeployProgressUntilComplete(