chore: json to map conversion with test template

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceType.java

@@ -6,7 +6,7 @@ public enum GitResourceType {
     ROOT_CONFIG,
     DATASOURCE_CONFIG,
     JSLIB_CONFIG,
-    PAGE_CONFIG,
+    CONTEXT_CONFIG,
     JSOBJECT_CONFIG,
     JSOBJECT_DATA,
     QUERY_CONFIG,

app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java

@@ -1,13 +1,17 @@
 package com.appsmith.server.applications.git;
 
 import com.appsmith.external.git.FileInterface;
+import com.appsmith.external.git.models.GitResourceIdentity;
+import com.appsmith.external.git.models.GitResourceMap;
+import com.appsmith.external.git.models.GitResourceType;
 import com.appsmith.external.helpers.AppsmithBeanUtils;
 import com.appsmith.external.models.ActionDTO;
 import com.appsmith.external.models.ApplicationGitReference;
 import com.appsmith.external.models.ArtifactGitReference;
 import com.appsmith.external.models.DatasourceStorage;
 import com.appsmith.external.models.PluginType;
 import com.appsmith.git.files.FileUtilsImpl;
+import com.appsmith.git.helpers.DSLTransformerHelper;
 import com.appsmith.server.actioncollections.base.ActionCollectionService;
 import com.appsmith.server.constants.FieldName;
 import com.appsmith.server.domains.ActionCollection;
@@ -124,6 +128,62 @@ public void addArtifactReferenceFromExportedJson(
         setCustomJSLibsInApplicationReference(applicationJson, applicationReference);
     }
 
+    @Override
+    public void setArtifactDependentResources(
+            ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap) {
+
+        ApplicationJson applicationJson = (ApplicationJson) artifactExchangeJson;
+        Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
+
+        // application
+        Application application = applicationJson.getExportedApplication();
+        removeUnwantedFieldsFromApplication(application);
+        GitResourceIdentity applicationIdentity =
+                new GitResourceIdentity(GitResourceType.ROOT_CONFIG, "application.json");
+        resourceMap.put(applicationIdentity, application);
+
+        // metadata
+        Iterable<String> keys = AppsmithBeanUtils.getAllFields(applicationJson.getClass())
+                .map(Field::getName)
+                .filter(name -> !getBlockedMetadataFields().contains(name))
+                .collect(Collectors.toList());
+
+        ApplicationJson applicationMetadata = new ApplicationJson();
+        applicationJson.setModifiedResources(null);
+        copyProperties(applicationJson, applicationMetadata, keys);
+        GitResourceIdentity metadataIdentity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, "metadata.json");
+        resourceMap.put(metadataIdentity, applicationMetadata);
+
+        // pages and widgets
+        applicationJson.getPageList().stream()
+                // As we are expecting the commit will happen only after the application is published, so we can safely
+                // assume if the unpublished version is deleted entity should not be committed to git
+                .filter(newPage -> newPage.getUnpublishedPage() != null
+                        && newPage.getUnpublishedPage().getDeletedAt() == null)
+                .forEach(newPage -> {
+                    removeUnwantedFieldsFromPage(newPage);
+                    JSONObject dsl =
+                            newPage.getUnpublishedPage().getLayouts().get(0).getDsl();
+                    // Get MainContainer widget data, remove the children and club with Canvas.json file
+                    JSONObject mainContainer = new JSONObject(dsl);
+                    mainContainer.remove("children");
+                    newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer);
+                    // pageName will be used for naming the json file
+                    GitResourceIdentity pageIdentity =
+                            new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId());
+                    resourceMap.put(pageIdentity, newPage);
+
+                    Map<String, org.json.JSONObject> result =
+                            DSLTransformerHelper.flatten(new org.json.JSONObject(dsl.toString()));
+                    result.forEach((key, jsonObject) -> {
+                        String widgetId = newPage.getGitSyncId() + "-" + jsonObject.getString("widgetId");
+                        GitResourceIdentity widgetIdentity =
+                                new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId);
+                        resourceMap.put(widgetIdentity, jsonObject);
+                    });
+                });
+    }
+
     private void setApplicationInApplicationReference(
             ApplicationJson applicationJson, ApplicationGitReference applicationReference) {
         Application application = applicationJson.getExportedApplication();

app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java

@@ -133,4 +133,9 @@ public void setThemes(Theme unpublishedTheme, Theme publishedTheme) {
     public Theme getUnpublishedTheme() {
         return this.getEditModeTheme();
     }
+
+    @Override
+    public List getContextList() {
+        return this.pageList;
+    }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java

@@ -3,12 +3,14 @@
 import com.appsmith.external.dtos.ModifiedResources;
 import com.appsmith.external.models.DatasourceStorage;
 import com.appsmith.external.models.DecryptedSensitiveFields;
+import com.appsmith.external.views.Views;
 import com.appsmith.server.constants.ArtifactType;
 import com.appsmith.server.domains.ActionCollection;
 import com.appsmith.server.domains.Artifact;
 import com.appsmith.server.domains.CustomJSLib;
 import com.appsmith.server.domains.NewAction;
 import com.appsmith.server.domains.Theme;
+import com.fasterxml.jackson.annotation.JsonView;
 
 import java.util.List;
 import java.util.Map;
@@ -62,4 +64,7 @@ default Theme getUnpublishedTheme() {
     default Theme getPublishedTheme() {
         return null;
     }
+
+    @JsonView(Views.Internal.class)
+    List getContextList();
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/CommonGitFileUtils.java

@@ -4,11 +4,12 @@
 import com.appsmith.external.git.operations.FileOperations;
 import com.appsmith.external.models.ApplicationGitReference;
 import com.appsmith.git.files.FileUtilsImpl;
+import com.appsmith.server.actioncollections.base.ActionCollectionService;
 import com.appsmith.server.helpers.ce.CommonGitFileUtilsCE;
 import com.appsmith.server.migrations.JsonSchemaVersions;
+import com.appsmith.server.newactions.base.NewActionService;
 import com.appsmith.server.services.AnalyticsService;
 import com.appsmith.server.services.SessionUserService;
-import com.google.gson.Gson;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Import;
 import org.springframework.stereotype.Component;
@@ -24,14 +25,17 @@ public CommonGitFileUtils(
             FileOperations fileOperations,
             AnalyticsService analyticsService,
             SessionUserService sessionUserService,
-            Gson gson,
+            NewActionService newActionService,
+            ActionCollectionService actionCollectionService,
             JsonSchemaVersions jsonSchemaVersions) {
         super(
                 applicationGitFileUtils,
                 fileUtils,
                 fileOperations,
                 analyticsService,
                 sessionUserService,
+                newActionService,
+                actionCollectionService,
                 jsonSchemaVersions);
     }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java

@@ -1,5 +1,6 @@
 package com.appsmith.server.helpers.ce;
 
+import com.appsmith.external.git.models.GitResourceMap;
 import com.appsmith.external.models.ArtifactGitReference;
 import com.appsmith.server.dtos.ArtifactExchangeJson;
 import lombok.NonNull;
@@ -12,6 +13,8 @@ public interface ArtifactGitFileUtilsCE<T extends ArtifactGitReference> {
 
     T createArtifactReferenceObject();
 
+    void setArtifactDependentResources(ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap);
+
     Mono<ArtifactExchangeJson> reconstructArtifactExchangeJsonFromFilesInRepository(
             String workspaceId, String baseArtifactId, String repoName, String branchName);
 

app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java

@@ -2,24 +2,33 @@
 
 import com.appsmith.external.constants.AnalyticsEvents;
 import com.appsmith.external.git.FileInterface;
+import com.appsmith.external.git.models.GitResourceIdentity;
+import com.appsmith.external.git.models.GitResourceMap;
+import com.appsmith.external.git.models.GitResourceType;
 import com.appsmith.external.git.operations.FileOperations;
 import com.appsmith.external.helpers.Stopwatch;
 import com.appsmith.external.models.ApplicationGitReference;
 import com.appsmith.external.models.ArtifactGitReference;
 import com.appsmith.external.models.BaseDomain;
 import com.appsmith.external.models.DatasourceStorage;
+import com.appsmith.external.models.PluginType;
 import com.appsmith.git.constants.CommonConstants;
 import com.appsmith.git.files.FileUtilsImpl;
+import com.appsmith.server.actioncollections.base.ActionCollectionService;
 import com.appsmith.server.constants.ArtifactType;
 import com.appsmith.server.constants.FieldName;
+import com.appsmith.server.domains.ActionCollection;
 import com.appsmith.server.domains.GitArtifactMetadata;
+import com.appsmith.server.domains.NewAction;
+import com.appsmith.server.domains.Theme;
 import com.appsmith.server.dtos.ApplicationJson;
 import com.appsmith.server.dtos.ArtifactExchangeJson;
 import com.appsmith.server.dtos.PageDTO;
 import com.appsmith.server.exceptions.AppsmithError;
 import com.appsmith.server.exceptions.AppsmithException;
 import com.appsmith.server.helpers.ArtifactGitFileUtils;
 import com.appsmith.server.migrations.JsonSchemaVersions;
+import com.appsmith.server.newactions.base.NewActionService;
 import com.appsmith.server.services.AnalyticsService;
 import com.appsmith.server.services.SessionUserService;
 import com.google.gson.Gson;
@@ -47,6 +56,7 @@
 import static com.appsmith.git.constants.CommonConstants.CLIENT_SCHEMA_VERSION;
 import static com.appsmith.git.constants.CommonConstants.FILE_FORMAT_VERSION;
 import static com.appsmith.git.constants.CommonConstants.SERVER_SCHEMA_VERSION;
+import static com.appsmith.git.files.FileUtilsCEImpl.getJsLibFileName;
 import static org.springframework.util.StringUtils.hasText;
 
 @Slf4j
@@ -61,6 +71,9 @@ public class CommonGitFileUtilsCE {
     private final AnalyticsService analyticsService;
     private final SessionUserService sessionUserService;
 
+    private final NewActionService newActionService;
+    private final ActionCollectionService actionCollectionService;
+
     // Number of seconds after lock file is stale
     @Value("${appsmith.index.lock.file.time}")
     public final int INDEX_LOCK_FILE_STALE_TIME = 300;
@@ -177,6 +190,172 @@ public ArtifactGitReference createArtifactReference(ArtifactExchangeJson artifac
         return artifactGitReference;
     }
 
+    public GitResourceMap createGitResourceMap(ArtifactExchangeJson artifactExchangeJson) {
+        ArtifactGitFileUtils<?> artifactGitFileUtils =
+                getArtifactBasedFileHelper(artifactExchangeJson.getArtifactJsonType());
+        GitResourceMap gitResourceMap = new GitResourceMap();
+        gitResourceMap.setModifiedResources(artifactExchangeJson.getModifiedResources());
+
+        setArtifactIndependentResources(artifactExchangeJson, gitResourceMap);
+
+        artifactGitFileUtils.setArtifactDependentResources(artifactExchangeJson, gitResourceMap);
+
+        return gitResourceMap;
+    }
+
+    protected void setArtifactIndependentResources(
+            ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap) {
+        Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();
+
+        // datasources
+        artifactExchangeJson.getDatasourceList().forEach(datasource -> {
+            removeUnwantedFieldsFromDatasource(datasource);
+            GitResourceIdentity identity =
+                    new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId());
+            resourceMap.put(identity, datasource);
+        });
+
+        // themes
+        Theme theme = artifactExchangeJson.getUnpublishedTheme();
+        // Only proceed if the current artifact supports themes
+        if (theme != null) {
+            // Reset published mode theme since it is not required
+            artifactExchangeJson.setThemes(theme, null);
+            // Remove internal fields from the themes
+            removeUnwantedFieldsFromBaseDomain(theme);
+            GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, "theme.json");
+            resourceMap.put(identity, theme);
+        }
+
+        // custom js libs
+        artifactExchangeJson.getCustomJSLibList().forEach(jsLib -> {
+            removeUnwantedFieldsFromBaseDomain(jsLib);
+            String jsLibFileName = getJsLibFileName(jsLib.getUidString());
+            GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName);
+            resourceMap.put(identity, jsLib);
+        });
+
+        // actions
+        setNewActionsInResourceMap(artifactExchangeJson, resourceMap);
+
+        // action collections
+        setActionCollectionsInResourceMap(artifactExchangeJson, resourceMap);
+    }
+
+    protected void setNewActionsInResourceMap(
+            ArtifactExchangeJson artifactExchangeJson, Map<GitResourceIdentity, Object> resourceMap) {
+        if (artifactExchangeJson.getActionList() == null) {
+            return;
+        }
+        artifactExchangeJson.getActionList().stream()
+                // As we are expecting the commit will happen only after the application is published, so we can safely
+                // assume if the unpublished version is deleted entity should not be committed to git
+                .filter(newAction -> newAction.getUnpublishedAction() != null
+                        && newAction.getUnpublishedAction().getDeletedAt() == null)
+                .peek(newAction -> newActionService.generateActionByViewMode(newAction, false))
+                .forEach(newAction -> {
+                    removeUnwantedFieldFromAction(newAction);
+                    String body = newAction.getUnpublishedAction().getActionConfiguration() != null
+                                    && newAction
+                                                    .getUnpublishedAction()
+                                                    .getActionConfiguration()
+                                                    .getBody()
+                                            != null
+                            ? newAction
+                                    .getUnpublishedAction()
+                                    .getActionConfiguration()
+                                    .getBody()
+                            : "";
+
+                    // This is a special case where we are handling REMOTE type plugins based actions such as Twilio
+                    // The user configured values are stored in an attribute called formData which is a map unlike the
+                    // body
+                    if (PluginType.REMOTE.equals(newAction.getPluginType())
+                            && newAction.getUnpublishedAction().getActionConfiguration() != null
+                            && newAction
+                                            .getUnpublishedAction()
+                                            .getActionConfiguration()
+                                            .getFormData()
+                                    != null) {
+                        body = new Gson()
+                                .toJson(
+                                        newAction
+                                                .getUnpublishedAction()
+                                                .getActionConfiguration()
+                                                .getFormData(),
+                                        Map.class);
+                        newAction
+                                .getUnpublishedAction()
+                                .getActionConfiguration()
+                                .setFormData(null);
+                    }
+                    // This is a special case where we are handling JS actions as we don't want to commit the body of JS
+                    // actions
+                    if (PluginType.JS.equals(newAction.getPluginType())) {
+                        if (newAction.getUnpublishedAction().getActionConfiguration() != null) {
+                            newAction
+                                    .getUnpublishedAction()
+                                    .getActionConfiguration()
+                                    .setBody(null);
+                            newAction.getUnpublishedAction().setJsonPathKeys(null);
+                        }
+                    } else {
+                        // For the regular actions we save the body field to git repo
+                        GitResourceIdentity actionDataIdentity =
+                                new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId());
+                        resourceMap.put(actionDataIdentity, body);
+                    }
+                    GitResourceIdentity actionConfigIdentity =
+                            new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId());
+                    resourceMap.put(actionConfigIdentity, newAction);
+                });
+    }
+
+    protected void setActionCollectionsInResourceMap(
+            ArtifactExchangeJson artifactExchangeJson, Map<GitResourceIdentity, Object> resourceMap) {
+        if (artifactExchangeJson.getActionCollectionList() == null) {
+            return;
+        }
+        artifactExchangeJson.getActionCollectionList().stream()
+                // As we are expecting the commit will happen only after the application is published, so we can safely
+                // assume if the unpublished version is deleted entity should not be committed to git
+                .filter(collection -> collection.getUnpublishedCollection() != null
+                        && collection.getUnpublishedCollection().getDeletedAt() == null)
+                .peek(actionCollection ->
+                        actionCollectionService.generateActionCollectionByViewMode(actionCollection, false))
+                .forEach(actionCollection -> {
+                    removeUnwantedFieldFromActionCollection(actionCollection);
+                    String body = actionCollection.getUnpublishedCollection().getBody() != null
+                            ? actionCollection.getUnpublishedCollection().getBody()
+                            : "";
+                    actionCollection.getUnpublishedCollection().setBody(null);
+
+                    GitResourceIdentity collectionConfigIdentity =
+                            new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId());
+                    resourceMap.put(collectionConfigIdentity, actionCollection);
+
+                    GitResourceIdentity collectionDataIdentity =
+                            new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId());
+                    resourceMap.put(collectionDataIdentity, body);
+                });
+    }
+
+    private void removeUnwantedFieldFromAction(NewAction action) {
+        // As we are publishing the app and then committing to git we expect the published and unpublished ActionDTO
+        // will be same, so we only commit unpublished ActionDTO.
+        action.setPublishedAction(null);
+        action.getUnpublishedAction().sanitiseToExportDBObject();
+        removeUnwantedFieldsFromBaseDomain(action);
+    }
+
+    private void removeUnwantedFieldFromActionCollection(ActionCollection actionCollection) {
+        // As we are publishing the app and then committing to git we expect the published and unpublished
+        // ActionCollectionDTO will be same, so we only commit unpublished ActionCollectionDTO.
+        actionCollection.setPublishedCollection(null);
+        actionCollection.getUnpublishedCollection().sanitiseForExport();
+        removeUnwantedFieldsFromBaseDomain(actionCollection);
+    }
+
     private void setDatasourcesInArtifactReference(
             ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) {
         Map<String, Object> resourceMap = new HashMap<>();