From b1155a374ea307bafaa4efb6cc712cd85c58f70b Mon Sep 17 00:00:00 2001 From: Nilesh Sarupriya <20905988+nsarupr@users.noreply.github.com> Date: Mon, 17 Feb 2025 13:01:46 +0530 Subject: [PATCH 1/2] add postSaveHook --- .../external/models/DatasourceStorage.java | 6 ++ .../external/plugins/PluginExecutor.java | 7 ++ .../base/DatasourceServiceCEImpl.java | 78 ++++++++++++------- .../base/DatasourceServiceImpl.java | 10 ++- .../base/DatasourceStorageServiceCEImpl.java | 15 +++- 5 files changed, 85 insertions(+), 31 deletions(-) diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java index a038402c7cbe..43a330c4125c 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java @@ -89,6 +89,12 @@ public class DatasourceStorage extends GitSyncedDomain { @Transient Boolean isMock; + @Transient + String tenantId; + + @Transient + String instanceId; + public DatasourceStorage( String datasourceId, String environmentId, diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java index a95a7c2988ac..70acc28eeb9d 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/plugins/PluginExecutor.java @@ -185,6 +185,13 @@ default Mono preDeleteHook(DatasourceStorage datasourceStorag return Mono.just(datasourceStorage); } + /** + * This function is being called as a hook after saving a datasource. + */ + default Mono postSaveHook(DatasourceStorage datasourceStorage) { + return Mono.just(datasourceStorage); + } + /** * This function fetches the structure of the tables/collections in the datasource. It's used to make query creation * easier for the user. diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java index 9d03ab90ae00..34ba9031b688 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java @@ -28,10 +28,12 @@ import com.appsmith.server.repositories.DatasourceRepository; import com.appsmith.server.repositories.NewActionRepository; import com.appsmith.server.services.AnalyticsService; +import com.appsmith.server.services.ConfigService; import com.appsmith.server.services.DatasourceContextService; import com.appsmith.server.services.FeatureFlagService; import com.appsmith.server.services.SequenceService; import com.appsmith.server.services.SessionUserService; +import com.appsmith.server.services.TenantService; import com.appsmith.server.services.WorkspaceService; import com.appsmith.server.solutions.DatasourcePermission; import com.appsmith.server.solutions.EnvironmentPermission; @@ -93,6 +95,8 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE { private final RateLimitService rateLimitService; private final FeatureFlagService featureFlagService; private final ObservationRegistry observationRegistry; + private final TenantService tenantService; + private final ConfigService configService; // Defines blocking duration for test as well as connection created for query execution // This will block the creation of datasource connection for 5 minutes, in case of more than 3 failed connection @@ -119,7 +123,9 @@ public DatasourceServiceCEImpl( EnvironmentPermission environmentPermission, RateLimitService rateLimitService, FeatureFlagService featureFlagService, - ObservationRegistry observationRegistry) { + ObservationRegistry observationRegistry, + TenantService tenantService, + ConfigService configService) { this.workspaceService = workspaceService; this.sessionUserService = sessionUserService; @@ -138,6 +144,8 @@ public DatasourceServiceCEImpl( this.rateLimitService = rateLimitService; this.featureFlagService = featureFlagService; this.observationRegistry = observationRegistry; + this.tenantService = tenantService; + this.configService = configService; } @Override @@ -182,6 +190,12 @@ private Mono createEx( datasource.nullifyStorageReplicaFields(); Mono datasourceMono = Mono.just(datasource); + // + Mono> tenantIdAndInstanceIdMonoCached = tenantService + .getDefaultTenantId() + .zipWith(configService.getInstanceId()) + .cache(); + // First check if this is an existing datasource or whether we need to create one if (!hasText(datasource.getId())) { // We need to create the datasource as well @@ -222,34 +236,42 @@ private Mono createEx( new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE)))); } - return datasourceMono.flatMap(savedDatasource -> this.organiseDatasourceStorages(savedDatasource) - .flatMap(datasourceStorage -> { - // Make sure that we are creating entries only if the id is not already populated - if (hasText(datasourceStorage.getId())) { - return Mono.just(datasourceStorage); - } - - return datasourceStorageService - .create(datasourceStorage, isDryOps) - .map(datasourceStorage1 -> { - if (datasourceStorageDryRunQueries != null && isDryOps) { - List datasourceStorages = - datasourceStorageDryRunQueries.get(SAVE); - if (datasourceStorages == null) { - datasourceStorages = new ArrayList<>(); + return datasourceMono.zipWith(tenantIdAndInstanceIdMonoCached).flatMap(tuple2 -> { + Datasource savedDatasource = tuple2.getT1(); + String tenantId = tuple2.getT2().getT1(); + String instanceId = tuple2.getT2().getT2(); + return this.organiseDatasourceStorages(savedDatasource) + .flatMap(datasourceStorage -> { + // Set the tenantId and instanceId + datasourceStorage.setTenantId(tenantId); + datasourceStorage.setInstanceId(instanceId); + // Make sure that we are creating entries only if the id is not already populated + if (hasText(datasourceStorage.getId())) { + return Mono.just(datasourceStorage); + } + + return datasourceStorageService + .create(datasourceStorage, isDryOps) + .map(datasourceStorage1 -> { + if (datasourceStorageDryRunQueries != null && isDryOps) { + List datasourceStorages = + datasourceStorageDryRunQueries.get(SAVE); + if (datasourceStorages == null) { + datasourceStorages = new ArrayList<>(); + } + datasourceStorages.add(datasourceStorage1); + datasourceStorageDryRunQueries.put(SAVE, datasourceStorages); } - datasourceStorages.add(datasourceStorage1); - datasourceStorageDryRunQueries.put(SAVE, datasourceStorages); - } - return datasourceStorage1; - }); - }) - .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) - .collectMap(DatasourceStorageDTO::getEnvironmentId) - .map(savedStorages -> { - savedDatasource.setDatasourceStorages(savedStorages); - return savedDatasource; - })); + return datasourceStorage1; + }); + }) + .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) + .collectMap(DatasourceStorageDTO::getEnvironmentId) + .map(savedStorages -> { + savedDatasource.setDatasourceStorages(savedStorages); + return savedDatasource; + }); + }); } // this requires an EE override multiple environments diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceImpl.java index 1fc7bf66b43d..67a8e8f3d94f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceImpl.java @@ -8,10 +8,12 @@ import com.appsmith.server.repositories.DatasourceRepository; import com.appsmith.server.repositories.NewActionRepository; import com.appsmith.server.services.AnalyticsService; +import com.appsmith.server.services.ConfigService; import com.appsmith.server.services.DatasourceContextService; import com.appsmith.server.services.FeatureFlagService; import com.appsmith.server.services.SequenceService; import com.appsmith.server.services.SessionUserService; +import com.appsmith.server.services.TenantService; import com.appsmith.server.services.WorkspaceService; import com.appsmith.server.solutions.DatasourcePermission; import com.appsmith.server.solutions.EnvironmentPermission; @@ -41,7 +43,9 @@ public DatasourceServiceImpl( EnvironmentPermission environmentPermission, RateLimitService rateLimitService, FeatureFlagService featureFlagService, - ObservationRegistry observationRegistry) { + ObservationRegistry observationRegistry, + TenantService tenantService, + ConfigService configService) { super( repository, @@ -60,6 +64,8 @@ public DatasourceServiceImpl( environmentPermission, rateLimitService, featureFlagService, - observationRegistry); + observationRegistry, + tenantService, + configService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java index 556880330cc5..5b2a16b0f8b9 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java @@ -183,6 +183,16 @@ public Mono executePreSaveActions(DatasourceStorage datasourc return pluginExecutorMono.flatMap(pluginExecutor -> pluginExecutor.preSaveHook(datasourceStorage)); } + public Mono executePostSaveActions(DatasourceStorage datasourceStorage) { + Mono pluginMono = pluginService.findById(datasourceStorage.getPluginId()); + Mono pluginExecutorMono = pluginExecutorHelper + .getPluginExecutor(pluginMono) + .switchIfEmpty(Mono.error(new AppsmithException( + AppsmithError.NO_RESOURCE_FOUND, FieldName.PLUGIN, datasourceStorage.getPluginId()))); + + return pluginExecutorMono.flatMap(pluginExecutor -> pluginExecutor.postSaveHook(datasourceStorage)); + } + @Override public Mono validateDatasourceStorage(DatasourceStorage datasourceStorage) { @@ -242,7 +252,10 @@ private Mono validateAndSaveDatasourceStorageToRepository( unsavedDatasourceStorage.updateForBulkWriteOperation(); return Mono.just(unsavedDatasourceStorage); } - return repository.save(unsavedDatasourceStorage).thenReturn(unsavedDatasourceStorage); + return repository + .save(unsavedDatasourceStorage) + .then(this.executePostSaveActions(unsavedDatasourceStorage)) + .thenReturn(unsavedDatasourceStorage); }); } From 260ad934fc1295d2ad5ed035ff593c9950158c74 Mon Sep 17 00:00:00 2001 From: Nilesh Sarupriya <20905988+nsarupr@users.noreply.github.com> Date: Tue, 18 Feb 2025 14:33:04 +0530 Subject: [PATCH 2/2] add metadata instead of individual fields for extensibility --- .../external/models/DatasourceStorage.java | 5 +- .../server/constants/ce/FieldNameCE.java | 1 + .../base/DatasourceServiceCEImpl.java | 83 ++++++++++--------- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java index 43a330c4125c..3a0e7898f685 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/DatasourceStorage.java @@ -90,10 +90,7 @@ public class DatasourceStorage extends GitSyncedDomain { Boolean isMock; @Transient - String tenantId; - - @Transient - String instanceId; + Map metadata; public DatasourceStorage( String datasourceId, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java index 2ffdc1ba5c7f..aeab3822f0c4 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java @@ -197,6 +197,7 @@ public class FieldNameCE { public static final String REMOTE_PLUGINS = "remotePlugins"; public static final String INSTANCE_ID = "instanceId"; + public static final String TENANT_ID = "tenantId"; public static final String IP_ADDRESS = "ipAddress"; public static final String VERSION = "version"; public static final String PUBLISHED = "published"; diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java index 34ba9031b688..3bd7a58a0cd1 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java @@ -66,6 +66,8 @@ import static com.appsmith.external.constants.spans.DatasourceSpan.FETCH_ALL_DATASOURCES_WITH_STORAGES; import static com.appsmith.external.constants.spans.DatasourceSpan.FETCH_ALL_PLUGINS_IN_WORKSPACE; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; +import static com.appsmith.server.constants.ce.FieldNameCE.INSTANCE_ID; +import static com.appsmith.server.constants.ce.FieldNameCE.TENANT_ID; import static com.appsmith.server.dtos.DBOpsType.SAVE; import static com.appsmith.server.helpers.CollectionUtils.isNullOrEmpty; import static com.appsmith.server.helpers.DatasourceAnalyticsUtils.getAnalyticsProperties; @@ -190,12 +192,6 @@ private Mono createEx( datasource.nullifyStorageReplicaFields(); Mono datasourceMono = Mono.just(datasource); - // - Mono> tenantIdAndInstanceIdMonoCached = tenantService - .getDefaultTenantId() - .zipWith(configService.getInstanceId()) - .cache(); - // First check if this is an existing datasource or whether we need to create one if (!hasText(datasource.getId())) { // We need to create the datasource as well @@ -236,41 +232,48 @@ private Mono createEx( new AppsmithException(AppsmithError.NO_RESOURCE_FOUND, FieldName.DATASOURCE)))); } - return datasourceMono.zipWith(tenantIdAndInstanceIdMonoCached).flatMap(tuple2 -> { - Datasource savedDatasource = tuple2.getT1(); - String tenantId = tuple2.getT2().getT1(); - String instanceId = tuple2.getT2().getT2(); - return this.organiseDatasourceStorages(savedDatasource) - .flatMap(datasourceStorage -> { - // Set the tenantId and instanceId - datasourceStorage.setTenantId(tenantId); - datasourceStorage.setInstanceId(instanceId); - // Make sure that we are creating entries only if the id is not already populated - if (hasText(datasourceStorage.getId())) { - return Mono.just(datasourceStorage); - } - - return datasourceStorageService - .create(datasourceStorage, isDryOps) - .map(datasourceStorage1 -> { - if (datasourceStorageDryRunQueries != null && isDryOps) { - List datasourceStorages = - datasourceStorageDryRunQueries.get(SAVE); - if (datasourceStorages == null) { - datasourceStorages = new ArrayList<>(); + return datasourceMono.flatMap(savedDatasource -> this.organiseDatasourceStorages(savedDatasource) + .flatMap(datasourceStorageX -> setAdditionalMetadataInDatasourceStorage(datasourceStorageX) + .flatMap(datasourceStorage -> { + // Make sure that we are creating entries only if the id is not already populated + if (hasText(datasourceStorage.getId())) { + return Mono.just(datasourceStorage); + } + + return datasourceStorageService + .create(datasourceStorage, isDryOps) + .map(datasourceStorage1 -> { + if (datasourceStorageDryRunQueries != null && isDryOps) { + List datasourceStorages = + datasourceStorageDryRunQueries.get(SAVE); + if (datasourceStorages == null) { + datasourceStorages = new ArrayList<>(); + } + datasourceStorages.add(datasourceStorage1); + datasourceStorageDryRunQueries.put(SAVE, datasourceStorages); } - datasourceStorages.add(datasourceStorage1); - datasourceStorageDryRunQueries.put(SAVE, datasourceStorages); - } - return datasourceStorage1; - }); - }) - .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) - .collectMap(DatasourceStorageDTO::getEnvironmentId) - .map(savedStorages -> { - savedDatasource.setDatasourceStorages(savedStorages); - return savedDatasource; - }); + return datasourceStorage1; + }); + })) + .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) + .collectMap(DatasourceStorageDTO::getEnvironmentId) + .map(savedStorages -> { + savedDatasource.setDatasourceStorages(savedStorages); + return savedDatasource; + })); + } + + private Mono setAdditionalMetadataInDatasourceStorage(DatasourceStorage datasourceStorage) { + Mono tenantIdMono = tenantService.getDefaultTenantId(); + Mono instanceIdMono = configService.getInstanceId(); + + Map metadata = new HashMap<>(); + + return tenantIdMono.zipWith(instanceIdMono).map(tuple -> { + metadata.put(TENANT_ID, tuple.getT1()); + metadata.put(INSTANCE_ID, tuple.getT2()); + datasourceStorage.setMetadata(metadata); + return datasourceStorage; }); }