From 28c791191f6665228758119cc81149592bb88b87 Mon Sep 17 00:00:00 2001 From: Nidhi Nair Date: Tue, 3 Dec 2024 20:14:45 +0530 Subject: [PATCH] chore: Git resource map conversions --- .../appsmith/git/files/FileUtilsCEImpl.java | 219 +++++++++++++- .../com/appsmith/git/files/FileUtilsImpl.java | 6 +- .../git/helpers/DSLTransformerHelper.java | 34 ++- .../git/helpers/FileUtilsImplTest.java | 4 +- .../external/dtos/ModifiedResources.java | 55 +--- .../external/dtos/ce/ModifiedResourcesCE.java | 86 ++++++ .../appsmith/external/git/FileInterface.java | 4 + .../git/models/GitResourceIdentity.java | 4 +- ...tionCollectionExportableServiceCEImpl.java | 18 +- .../git/ApplicationGitFileUtilsCEImpl.java | 142 ++++++++- .../git/ApplicationGitFileUtilsImpl.java | 4 +- .../server/dtos/ce/ApplicationJsonCE.java | 11 + .../dtos/ce/ArtifactExchangeJsonCE.java | 4 + .../ce/MappedExportableResourcesCE_DTO.java | 1 + .../AutoCommitEventHandlerCEImpl.java | 24 +- .../helpers/ce/ArtifactGitFileUtilsCE.java | 4 + .../helpers/ce/CommonGitFileUtilsCE.java | 278 ++++++++++++++---- .../CustomJSLibExportableServiceCEImpl.java | 6 + .../NewActionExportableServiceCEImpl.java | 17 +- .../NewPageExportableServiceCEImpl.java | 13 +- .../ExchangeJsonConversionTests.java | 64 +++- .../ExchangeJsonTestTemplateProviderCE.java | 66 ++++- 22 files changed, 901 insertions(+), 163 deletions(-) create mode 100644 app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java index d919338cef95..b7f395022124 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java @@ -6,6 +6,9 @@ import com.appsmith.external.git.FileInterface; import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.constants.GitSpan; +import com.appsmith.external.git.models.GitResourceIdentity; +import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.external.helpers.Stopwatch; @@ -14,10 +17,12 @@ import com.appsmith.git.configurations.GitServiceConfig; import com.appsmith.git.constants.CommonConstants; import com.appsmith.git.helpers.DSLTransformerHelper; +import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.tracing.Span; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.json.JSONObject; @@ -28,6 +33,8 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; import java.io.BufferedWriter; import java.io.File; @@ -48,7 +55,10 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST; import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST; @@ -74,6 +84,7 @@ public class FileUtilsCEImpl implements FileInterface { private final GitExecutor gitExecutor; protected final FileOperations fileOperations; private final ObservationHelper observationHelper; + protected final ObjectMapper objectMapper; private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}"; @@ -90,11 +101,23 @@ public FileUtilsCEImpl( GitServiceConfig gitServiceConfig, GitExecutor gitExecutor, FileOperations fileOperations, - ObservationHelper observationHelper) { + ObservationHelper observationHelper, + ObjectMapper objectMapper) { this.gitServiceConfig = gitServiceConfig; this.gitExecutor = gitExecutor; this.fileOperations = fileOperations; this.observationHelper = observationHelper; + this.objectMapper = objectMapper; + } + + protected Map getModifiedResourcesTypes() { + return Map.of( + GitResourceType.JSLIB_CONFIG, GitResourceType.JSLIB_CONFIG, + GitResourceType.CONTEXT_CONFIG, GitResourceType.CONTEXT_CONFIG, + GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_CONFIG, + GitResourceType.QUERY_DATA, GitResourceType.QUERY_CONFIG, + GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_CONFIG, + GitResourceType.JSOBJECT_DATA, GitResourceType.JSOBJECT_CONFIG); } /** @@ -215,6 +238,101 @@ public Mono saveApplicationToGitRepo( .subscribeOn(scheduler); } + @Override + public Mono saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName) + throws GitAPIException, IOException { + + // Repo path will be: + // baseRepo : root/orgId/defaultAppId/repoName/{applicationData} + // Checkout to mentioned branch if not already checked-out + return gitExecutor + .resetToLastCommit(baseRepoSuffix, branchName) + .flatMap(isSwitched -> { + Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix); + + try { + updateEntitiesInRepo(gitResourceMap, baseRepo); + } catch (IOException e) { + return Mono.error(e); + } + + return Mono.just(baseRepo); + }) + .subscribeOn(scheduler); + } + + protected Set getExistingFilesInRepo(Path baseRepo) throws IOException { + try (Stream stream = Files.walk(baseRepo).parallel()) { + return stream.filter(path -> { + try { + return Files.isRegularFile(path) || FileUtils.isEmptyDirectory(path.toFile()); + } catch (IOException e) { + log.error("Unable to find file details. Please check the file at file path: {}", path); + log.error("Assuming that it does not exist for now ..."); + return false; + } + }) + .map(baseRepo::relativize) + .map(Path::toString) + .collect(Collectors.toSet()); + } + } + + protected Set updateEntitiesInRepo(GitResourceMap gitResourceMap, Path baseRepo) throws IOException { + ModifiedResources modifiedResources = gitResourceMap.getModifiedResources(); + Map resourceMap = gitResourceMap.getGitResourceMap(); + + Set filesInRepo = getExistingFilesInRepo(baseRepo); + + Set updatedFilesToBeSerialized = resourceMap.keySet().parallelStream() + .map(gitResourceIdentity -> gitResourceIdentity.getFilePath()) + .collect(Collectors.toSet()); + + // Remove all files that need to be serialized from the existing files list, as well as the README file + // What we are left with are all the files to be deleted + filesInRepo.removeAll(updatedFilesToBeSerialized); + filesInRepo.remove("README.md"); + + // Delete all the files because they are no longer needed + // This covers both older structures of storing files and, + // legitimate changes in the artifact that might cause deletions + filesInRepo.stream().parallel().forEach(filePath -> { + try { + Files.deleteIfExists(baseRepo.resolve(filePath)); + } catch (IOException e) { + // We ignore files that could not be deleted and expect to come back to this at a later point + // Just log the path for now + log.error("Unable to delete file at path: {}", filePath); + } + }); + + // Now go through the resource map and based on resource type, check if the resource is modified before + // serialization + // Or simply choose the mechanism for serialization + Map modifiedResourcesTypes = getModifiedResourcesTypes(); + return resourceMap.entrySet().parallelStream() + .map(entry -> { + GitResourceIdentity key = entry.getKey(); + boolean resourceUpdated = true; + if (modifiedResourcesTypes.containsKey(key.getResourceType()) && modifiedResources != null) { + GitResourceType comparisonType = modifiedResourcesTypes.get(key.getResourceType()); + + resourceUpdated = + modifiedResources.isResourceUpdatedNew(comparisonType, key.getResourceIdentifier()); + } + + if (resourceUpdated) { + String filePath = key.getFilePath(); + saveResourceCommon(entry.getValue(), baseRepo.resolve(filePath)); + + return filePath; + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + } + protected Set updateEntitiesInRepo(ApplicationGitReference applicationGitReference, Path baseRepo) { Set validDatasourceFileNames = new HashSet<>(); @@ -434,6 +552,23 @@ protected boolean saveResource(Object sourceEntity, Path path) { return false; } + protected void saveResourceCommon(Object sourceEntity, Path path) { + try { + Files.createDirectories(path.getParent()); + if (sourceEntity instanceof String s) { + writeStringToFile(s, path); + return; + } + if (sourceEntity instanceof JSONObject) { + sourceEntity = objectMapper.readTree(sourceEntity.toString()); + } + fileOperations.writeToFile(sourceEntity, path); + } catch (IOException e) { + log.error("Error while writing resource to file {} with {}", path, e.getMessage()); + log.debug(e.getMessage()); + } + } + /** * This method is used to write actionCollection specific resource to file system. We write the data in two steps * 1. Actual js code @@ -514,9 +649,9 @@ private void writeStringToFile(String sourceEntity, Path path) throws IOExceptio /** * This will reconstruct the application from the repo * - * @param organisationId To which organisation application needs to be rehydrated + * @param organisationId To which organisation application needs to be rehydrated * @param baseApplicationId To which organisation application needs to be rehydrated - * @param branchName for which the application needs to be rehydrate + * @param branchName for which the application needs to be rehydrate * @return application reference from which entire application can be rehydrated */ public Mono reconstructApplicationReferenceFromGitRepo( @@ -672,6 +807,84 @@ private Object readPageMetadata(Path directoryPath) { directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION)); } + protected GitResourceMap fetchGitResourceMap(Path baseRepoPath) throws IOException { + // Extract application metadata from the json + Object metadata = fileOperations.readFile( + baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION)); + Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata); + // Check if fileFormat of the saved files in repo is compatible + if (!isFileFormatCompatible(fileFormatVersion)) { + throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT); + } + + GitResourceMap gitResourceMap = new GitResourceMap(); + Map resourceMap = gitResourceMap.getGitResourceMap(); + + Set filesInRepo = getExistingFilesInRepo(baseRepoPath); + + filesInRepo.parallelStream() + .filter(path -> !Files.isDirectory(baseRepoPath.resolve(path))) + .forEach(filePath -> { + Tuple2 identity = getGitResourceIdentity(baseRepoPath, filePath); + + resourceMap.put(identity.getT1(), identity.getT2()); + }); + + return gitResourceMap; + } + + protected Tuple2 getGitResourceIdentity(Path baseRepoPath, String filePath) { + Path path = baseRepoPath.resolve(filePath); + GitResourceIdentity identity; + Object contents = fileOperations.readFile(path); + if (!filePath.contains("/")) { + identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath); + } else if (filePath.matches(DATASOURCE_DIRECTORY + "/.*")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(JS_LIB_DIRECTORY + "/.*")) { + String fileName = FilenameUtils.getBaseName(filePath); + identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, fileName, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/[^/]*.json]")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*/metadata.json")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.QUERY_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*\\.txt")) { + Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json")); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.QUERY_DATA, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*/metadata.json")) { + String gitSyncId = + objectMapper.valueToTree(contents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*\\.js")) { + Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json")); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId, filePath); + } else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/widgets/.*\\.json")) { + Pattern pageDirPattern = Pattern.compile("(" + PAGE_DIRECTORY + "/([^/]*))/widgets/.*\\.json"); + Matcher matcher = pageDirPattern.matcher(filePath); + matcher.find(); + String pageDirectory = matcher.group(1); + String pageName = matcher.group(2) + ".json"; + Object configContents = + fileOperations.readFile(baseRepoPath.resolve(pageDirectory).resolve(pageName)); + String gitSyncId = + objectMapper.valueToTree(configContents).get("gitSyncId").asText(); + String widgetId = objectMapper.valueToTree(contents).get("widgetId").asText(); + identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId + "-" + widgetId, filePath); + } else return null; + + return Tuples.of(identity, contents); + } + private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) { ApplicationGitReference applicationGitReference = new ApplicationGitReference(); // Extract application metadata from the json diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java index b45aba4a7236..34256a08096a 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java @@ -5,6 +5,7 @@ import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.git.configurations.GitServiceConfig; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Import; @@ -22,7 +23,8 @@ public FileUtilsImpl( GitServiceConfig gitServiceConfig, GitExecutor gitExecutor, FileOperations fileOperations, - ObservationHelper observationHelper) { - super(gitServiceConfig, gitExecutor, fileOperations, observationHelper); + ObservationHelper observationHelper, + ObjectMapper objectMapper) { + super(gitServiceConfig, gitExecutor, fileOperations, observationHelper, objectMapper); } } diff --git a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java index 9d444b857dfb..67b7aafc0026 100644 --- a/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java +++ b/app/server/appsmith-git/src/main/java/com/appsmith/git/helpers/DSLTransformerHelper.java @@ -16,6 +16,10 @@ import java.util.stream.Collectors; import static com.appsmith.git.constants.CommonConstants.CANVAS_WIDGET; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_POINT; +import static com.appsmith.git.constants.CommonConstants.EMPTY_STRING; +import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER; @Component @RequiredArgsConstructor @@ -24,7 +28,7 @@ public class DSLTransformerHelper { public static Map flatten(JSONObject jsonObject) { Map flattenedMap = new HashMap<>(); - flattenObject(jsonObject, CommonConstants.EMPTY_STRING, flattenedMap); + flattenObject(jsonObject, EMPTY_STRING, flattenedMap); return new TreeMap<>(flattenedMap); } @@ -44,8 +48,7 @@ private static void flattenObject(JSONObject jsonObject, String prefix, Map> calculateParentDirectories(List Map> parentDirectories = new HashMap<>(); paths = paths.stream() - .map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, CommonConstants.EMPTY_STRING)) + .map(currentPath -> currentPath.replace(CommonConstants.JSON_EXTENSION, EMPTY_STRING)) .collect(Collectors.toList()); for (String path : paths) { - String[] directories = path.split(CommonConstants.DELIMITER_PATH); + String[] directories = path.split(DELIMITER_PATH); int lastDirectoryIndex = directories.length - 1; - if (lastDirectoryIndex > 0 && directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) { + if (lastDirectoryIndex <= 0) { + // This is not a valid path anymore, ignore + continue; + } + if (directories[lastDirectoryIndex].equals(directories[lastDirectoryIndex - 1])) { if (lastDirectoryIndex - 2 >= 0) { String parentDirectory = directories[lastDirectoryIndex - 2]; List pathsList = parentDirectories.getOrDefault(parentDirectory, new ArrayList<>()); @@ -143,10 +150,10 @@ public static JSONObject getNestedDSL( Map jsonMap, Map> pathMapping, JSONObject mainContainer) { // start from the root // Empty page with no widgets - if (!pathMapping.containsKey(CommonConstants.MAIN_CONTAINER)) { + if (!pathMapping.containsKey(MAIN_CONTAINER)) { return mainContainer; } - for (String path : pathMapping.get(CommonConstants.MAIN_CONTAINER)) { + for (String path : pathMapping.get(MAIN_CONTAINER)) { JSONObject child = getChildren(path, jsonMap, pathMapping); JSONArray children = mainContainer.optJSONArray(CommonConstants.CHILDREN); if (children == null) { @@ -179,7 +186,7 @@ public static JSONObject getChildren( } public static String getWidgetName(String path) { - String[] directories = path.split(CommonConstants.DELIMITER_PATH); + String[] directories = path.split(DELIMITER_PATH); return directories[directories.length - 1]; } @@ -229,15 +236,16 @@ private static Map getWidgetIdWidgetNameMapping(JSONArray ex public static String getPathToWidgetFile(String key, JSONObject jsonObject, String widgetName) { // get path with splitting the name via key - String childPath = key.replace(CommonConstants.MAIN_CONTAINER, CommonConstants.EMPTY_STRING) - .replace(CommonConstants.DELIMITER_POINT, CommonConstants.DELIMITER_PATH); + String childPath = key.replace(MAIN_CONTAINER, EMPTY_STRING).replace(DELIMITER_POINT, DELIMITER_PATH); // Replace the canvas Widget as a child and add it to the same level as parent - childPath = childPath.replaceAll(CANVAS_WIDGET, CommonConstants.EMPTY_STRING); + childPath = childPath.replaceAll(CANVAS_WIDGET, EMPTY_STRING); if (!DSLTransformerHelper.hasChildren(jsonObject) && !DSLTransformerHelper.isTabsWidget(jsonObject)) { // Save the widget as a directory or Save the widget as a file // Only consider widgetName at the end of the childPath to reset // For example, "foobar/bar" should convert into "foobar/" - childPath = childPath.replaceAll(widgetName + "$", CommonConstants.EMPTY_STRING); + childPath = childPath.replaceAll(widgetName + "$", EMPTY_STRING); + } else { + childPath += DELIMITER_PATH; } return childPath; diff --git a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java index 58b4c3bfc775..e222feff2a92 100644 --- a/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java +++ b/app/server/appsmith-git/src/test/java/com/appsmith/git/helpers/FileUtilsImplTest.java @@ -7,6 +7,7 @@ import com.appsmith.git.files.FileUtilsImpl; import com.appsmith.git.files.operations.FileOperationsImpl; import com.appsmith.git.service.GitExecutorImpl; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.AfterEach; @@ -42,7 +43,8 @@ public void setUp() { GitServiceConfig gitServiceConfig = new GitServiceConfig(); gitServiceConfig.setGitRootPath(localTestDirectoryPath.toString()); FileOperations fileOperations = new FileOperationsImpl(null, ObservationHelper.NOOP); - fileUtils = new FileUtilsImpl(gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP); + fileUtils = new FileUtilsImpl( + gitServiceConfig, gitExecutor, fileOperations, ObservationHelper.NOOP, new ObjectMapper()); } @AfterEach diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java index 8157d6c27633..166b379c4265 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ModifiedResources.java @@ -1,13 +1,7 @@ package com.appsmith.external.dtos; +import com.appsmith.external.dtos.ce.ModifiedResourcesCE; import lombok.Data; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * This DTO class is used to store which resources have been updated after the last commit. @@ -17,49 +11,4 @@ * the pages to file system for difference git processes e.g. check git status, commit etc */ @Data -public class ModifiedResources { - // boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default - private boolean isAllModified; - - // a map to store the type of the resources and related entries - Map> modifiedResourceMap = new ConcurrentHashMap<>(); - - /** - * Checks whether the provided resource name should be considered as modified or not. - * It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap - * @param resourceType String, type of the resource e.g. PAGE_LIST - * @param resourceName String, name of the resource e.g. "Home Page" - * @return true if modified, false otherwise - */ - public boolean isResourceUpdated(String resourceType, String resourceName) { - return StringUtils.isNotEmpty(resourceType) - && (isAllModified - || (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType)) - && modifiedResourceMap.get(resourceType).contains(resourceName))); - } - - /** - * Adds a new resource to the map. Will create a new set if no set found for the provided resource type. - * @param resourceType String, type of the resource e.g. PAGE_LST - * @param resourceName String, name of the resource e.g. Home Page - */ - public void putResource(String resourceType, String resourceName) { - if (!this.modifiedResourceMap.containsKey(resourceType)) { - this.modifiedResourceMap.put(resourceType, new HashSet<>()); - } - this.modifiedResourceMap.get(resourceType).add(resourceName); - } - - /** - * Adds a set of resources to the map. Will create a new set if no set found for the provided resource type. - * It'll append the resources to the set. - * @param resourceType String, type of the resource e.g. PAGE_LST - * @param resourceNames Set of String, names of the resource e.g. Home Page, About page - */ - public void putResource(String resourceType, Set resourceNames) { - if (!this.modifiedResourceMap.containsKey(resourceType)) { - this.modifiedResourceMap.put(resourceType, new HashSet<>()); - } - this.modifiedResourceMap.get(resourceType).addAll(resourceNames); - } -} +public class ModifiedResources extends ModifiedResourcesCE {} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java new file mode 100644 index 000000000000..9e3e5a3aa8d0 --- /dev/null +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/dtos/ce/ModifiedResourcesCE.java @@ -0,0 +1,86 @@ +package com.appsmith.external.dtos.ce; + +import com.appsmith.external.git.models.GitResourceType; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * This DTO class is used to store which resources have been updated after the last commit. + * Primarily the export process sets this information and git import process uses this information to identify + * which resources need to be written in file system. For example, if a page has not been updated after the last commit, + * the name of the page should not be part of the modifiedResourceMap so that git will skip this page when it writes + * the pages to file system for difference git processes e.g. check git status, commit etc + */ +@Data +public class ModifiedResourcesCE { + // boolean flag to set whether all the resources should be considered as updated or not, it'll be false by default + private boolean isAllModified; + + // a map to store the type of the resources and related entries + Map> modifiedResourceMap = new ConcurrentHashMap<>(); + + Map> modifiedResourceIdentifiers = new ConcurrentHashMap<>(); + + public Map> getModifiedResourceIdentifiers() { + if (this.modifiedResourceIdentifiers.isEmpty()) { + this.modifiedResourceIdentifiers.putAll(Map.of( + GitResourceType.CONTEXT_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.JSLIB_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.QUERY_CONFIG, ConcurrentHashMap.newKeySet(), + GitResourceType.JSOBJECT_CONFIG, ConcurrentHashMap.newKeySet())); + } + return modifiedResourceIdentifiers; + } + + /** + * Checks whether the provided resource name should be considered as modified or not. + * It'll return true if the isAllModified flag is set or the resource is present in the modifiedResourceMap + * @param resourceType String, type of the resource e.g. PAGE_LIST + * @param resourceName String, name of the resource e.g. "Home Page" + * @return true if modified, false otherwise + */ + public boolean isResourceUpdated(String resourceType, String resourceName) { + return StringUtils.isNotEmpty(resourceType) + && (isAllModified + || (!CollectionUtils.isEmpty(modifiedResourceMap.get(resourceType)) + && modifiedResourceMap.get(resourceType).contains(resourceName))); + } + + public boolean isResourceUpdatedNew(GitResourceType resourceType, String resourceIdentifier) { + return StringUtils.isNotEmpty(resourceIdentifier) + && (isAllModified + || (!CollectionUtils.isEmpty(modifiedResourceIdentifiers.get(resourceType)) + && modifiedResourceIdentifiers.get(resourceType).contains(resourceIdentifier))); + } + + /** + * Adds a new resource to the map. Will create a new set if no set found for the provided resource type. + * @param resourceType String, type of the resource e.g. PAGE_LST + * @param resourceName String, name of the resource e.g. Home Page + */ + public void putResource(String resourceType, String resourceName) { + if (!this.modifiedResourceMap.containsKey(resourceType)) { + this.modifiedResourceMap.put(resourceType, new HashSet<>()); + } + this.modifiedResourceMap.get(resourceType).add(resourceName); + } + + /** + * Adds a set of resources to the map. Will create a new set if no set found for the provided resource type. + * It'll append the resources to the set. + * @param resourceType String, type of the resource e.g. PAGE_LST + * @param resourceNames Set of String, names of the resource e.g. Home Page, About page + */ + public void putResource(String resourceType, Set resourceNames) { + if (!this.modifiedResourceMap.containsKey(resourceType)) { + this.modifiedResourceMap.put(resourceType, new HashSet<>()); + } + this.modifiedResourceMap.get(resourceType).addAll(resourceNames); + } +} diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java index fd17dd1e51cc..d5422a24ef48 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/FileInterface.java @@ -1,5 +1,6 @@ package com.appsmith.external.git; +import com.appsmith.external.git.models.GitResourceMap; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import org.eclipse.jgit.api.errors.GitAPIException; @@ -34,6 +35,9 @@ Mono saveApplicationToGitRepo( Path baseRepoSuffix, ArtifactGitReference artifactGitReference, String branchName) throws IOException, GitAPIException; + Mono saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName) + throws GitAPIException, IOException; + /** * This method will reconstruct the application from the repo * diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java index e14282166566..2bf12f779f4d 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/git/models/GitResourceIdentity.java @@ -8,8 +8,6 @@ @Data @RequiredArgsConstructor public class GitResourceIdentity { - // TODO @Nidhi should we persist the info from parsing this filePath ? - String filePath; // TODO @Nidhi should we persist this sha against the Appsmith domain to integrate with the isModified logic? String sha; @@ -25,4 +23,6 @@ public class GitResourceIdentity { // root dir files -> fileName @NonNull @EqualsAndHashCode.Include String resourceIdentifier; + + @NonNull String filePath; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java index 34f46f26e3c4..dd4d34a12c6c 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/actioncollections/exportable/ActionCollectionExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.actioncollections.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.ActionCollection; @@ -11,7 +12,6 @@ import com.appsmith.server.dtos.MappedExportableResourcesDTO; import com.appsmith.server.exports.exportable.ExportableServiceCE; import com.appsmith.server.exports.exportable.artifactbased.ArtifactBasedExportableService; -import com.appsmith.server.helpers.ImportExportUtils; import com.appsmith.server.solutions.ActionPermission; import lombok.RequiredArgsConstructor; import reactor.core.publisher.Flux; @@ -67,6 +67,7 @@ public Mono getExportableEntities( // Because the actions will have a reference to the collection Set updatedActionCollectionSet = new HashSet<>(); + Set updatedIdentifiers = new HashSet<>(); actionCollections.forEach(actionCollection -> { ActionCollectionDTO publishedActionCollectionDTO = actionCollection.getPublishedCollection(); ActionCollectionDTO unpublishedActionCollectionDTO = @@ -78,9 +79,12 @@ public Mono getExportableEntities( // we've replaced page id with page name in previous step String contextNameAtIdReference = artifactBasedExportableService.getContextNameAtIdReference(actionCollectionDTO); - String contextListPath = artifactBasedExportableService.getContextListPath(); - boolean isContextUpdated = ImportExportUtils.isContextNameInUpdatedList( - artifactExchangeJson, contextNameAtIdReference, contextListPath); + String contextGitSyncId = mappedExportableResourcesDTO + .getContextNameToGitSyncIdMap() + .get(contextNameAtIdReference); + boolean isContextUpdated = artifactExchangeJson + .getModifiedResources() + .isResourceUpdatedNew(GitResourceType.CONTEXT_CONFIG, contextGitSyncId); String actionCollectionName = actionCollectionDTO.getUserExecutableName() + NAME_SEPARATOR + contextNameAtIdReference; Instant actionCollectionUpdatedAt = actionCollection.getUpdatedAt(); @@ -92,6 +96,7 @@ public Mono getExportableEntities( || exportingMetaDTO.getArtifactLastCommittedAt().isBefore(actionCollectionUpdatedAt); if (isActionCollectionUpdated) { updatedActionCollectionSet.add(actionCollectionName); + updatedIdentifiers.add(actionCollection.getGitSyncId()); } actionCollection.sanitiseToExportDBObject(); }); @@ -100,6 +105,11 @@ public Mono getExportableEntities( artifactExchangeJson .getModifiedResources() .putResource(FieldName.ACTION_COLLECTION_LIST, updatedActionCollectionSet); + artifactExchangeJson + .getModifiedResources() + .getModifiedResourceIdentifiers() + .get(GitResourceType.JSOBJECT_CONFIG) + .addAll(updatedIdentifiers); return actionCollections; }) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java index dbe2f5dc2a67..a5628db2df12 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java @@ -19,6 +19,7 @@ import com.appsmith.server.domains.Application; import com.appsmith.server.domains.ApplicationPage; import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; import com.appsmith.server.domains.Theme; @@ -32,9 +33,12 @@ import com.appsmith.server.helpers.ce.ArtifactGitFileUtilsCE; import com.appsmith.server.migrations.JsonSchemaMigration; import com.appsmith.server.newactions.base.NewActionService; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; @@ -60,6 +64,11 @@ import static com.appsmith.external.git.constants.GitConstants.NAME_SEPARATOR; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyProperties; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; +import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.MAIN_CONTAINER; +import static com.appsmith.git.constants.CommonConstants.WIDGETS; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY; import static com.appsmith.server.constants.FieldName.ACTION_COLLECTION_LIST; import static com.appsmith.server.constants.FieldName.ACTION_LIST; import static com.appsmith.server.constants.FieldName.CHILDREN; @@ -70,20 +79,36 @@ import static com.appsmith.server.constants.FieldName.EXPORTED_APPLICATION; import static com.appsmith.server.constants.FieldName.PAGE_LIST; import static com.appsmith.server.constants.FieldName.WIDGET_ID; +import static com.appsmith.server.constants.ce.FieldNameCE.WIDGET_NAME; import static com.appsmith.server.helpers.ce.CommonGitFileUtilsCE.removeUnwantedFieldsFromBaseDomain; @Slf4j @Component -@RequiredArgsConstructor @Import({FileUtilsImpl.class}) public class ApplicationGitFileUtilsCEImpl implements ArtifactGitFileUtilsCE { private final Gson gson; + private final ObjectMapper objectMapper; private final NewActionService newActionService; private final FileInterface fileUtils; private final JsonSchemaMigration jsonSchemaMigration; private final ActionCollectionService actionCollectionService; + public ApplicationGitFileUtilsCEImpl( + Gson gson, + ObjectMapper objectMapper, + NewActionService newActionService, + FileInterface fileUtils, + JsonSchemaMigration jsonSchemaMigration, + ActionCollectionService actionCollectionService) { + this.gson = gson; + this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS); + this.newActionService = newActionService; + this.fileUtils = fileUtils; + this.jsonSchemaMigration = jsonSchemaMigration; + this.actionCollectionService = actionCollectionService; + } + // Only include the application helper fields in metadata object protected Set getBlockedMetadataFields() { return Set.of( @@ -109,6 +134,11 @@ public ApplicationGitReference createArtifactReferenceObject() { return new ApplicationGitReference(); } + @Override + public ArtifactExchangeJson createArtifactExchangeJsonObject() { + return new ApplicationJson(); + } + @Override public void addArtifactReferenceFromExportedJson( ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) { @@ -141,8 +171,9 @@ public void setArtifactDependentResources( // application Application application = applicationJson.getExportedApplication(); removeUnwantedFieldsFromApplication(application); - GitResourceIdentity applicationIdentity = new GitResourceIdentity( - GitResourceType.ROOT_CONFIG, CommonConstants.APPLICATION + CommonConstants.JSON_EXTENSION); + final String applicationFilePath = CommonConstants.APPLICATION + JSON_EXTENSION; + GitResourceIdentity applicationIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, applicationFilePath, applicationFilePath); resourceMap.put(applicationIdentity, application); // metadata @@ -154,9 +185,11 @@ public void setArtifactDependentResources( ApplicationJson applicationMetadata = new ApplicationJson(); applicationJson.setModifiedResources(null); copyProperties(applicationJson, applicationMetadata, keys); - GitResourceIdentity metadataIdentity = new GitResourceIdentity( - GitResourceType.ROOT_CONFIG, CommonConstants.METADATA + CommonConstants.JSON_EXTENSION); - resourceMap.put(metadataIdentity, applicationMetadata); + final String metadataFilePath = CommonConstants.METADATA + JSON_EXTENSION; + ObjectNode metadata = objectMapper.valueToTree(applicationMetadata); + GitResourceIdentity metadataIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, metadataFilePath, metadataFilePath); + resourceMap.put(metadataIdentity, metadata); // pages and widgets applicationJson.getPageList().stream() @@ -166,23 +199,30 @@ public void setArtifactDependentResources( && newPage.getUnpublishedPage().getDeletedAt() == null) .forEach(newPage -> { removeUnwantedFieldsFromPage(newPage); - JSONObject dsl = - newPage.getUnpublishedPage().getLayouts().get(0).getDsl(); + PageDTO pageDTO = newPage.getUnpublishedPage(); + JSONObject dsl = pageDTO.getLayouts().get(0).getDsl(); // Get MainContainer widget data, remove the children and club with Canvas.json file JSONObject mainContainer = new JSONObject(dsl); mainContainer.remove(CHILDREN); - newPage.getUnpublishedPage().getLayouts().get(0).setDsl(mainContainer); + pageDTO.getLayouts().get(0).setDsl(mainContainer); // pageName will be used for naming the json file - GitResourceIdentity pageIdentity = - new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId()); + final String pagePathPrefix = PAGE_DIRECTORY + DELIMITER_PATH + pageDTO.getName() + DELIMITER_PATH; + final String pageFilePath = pagePathPrefix + pageDTO.getName() + JSON_EXTENSION; + GitResourceIdentity pageIdentity = new GitResourceIdentity( + GitResourceType.CONTEXT_CONFIG, newPage.getGitSyncId(), pageFilePath); resourceMap.put(pageIdentity, newPage); Map result = DSLTransformerHelper.flatten(new org.json.JSONObject(dsl.toString())); result.forEach((key, jsonObject) -> { String widgetId = newPage.getGitSyncId() + "-" + jsonObject.getString(WIDGET_ID); + String widgetsPath = pagePathPrefix + WIDGETS + DELIMITER_PATH; + String widgetName = jsonObject.getString(WIDGET_NAME); + String subPath = DSLTransformerHelper.getPathToWidgetFile(key, jsonObject, widgetName); + + String widgetPath = widgetsPath + subPath + widgetName + JSON_EXTENSION; GitResourceIdentity widgetIdentity = - new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId); + new GitResourceIdentity(GitResourceType.WIDGET_CONFIG, widgetId, widgetPath); resourceMap.put(widgetIdentity, jsonObject); }); }); @@ -629,4 +669,80 @@ public Path getRepoSuffixPath(String workspaceId, String artifactId, String repo varargs.addAll(List.of(args)); return Paths.get(workspaceId, varargs.toArray(new String[0])); } + + @Override + public void setArtifactDependentPropertiesInJson( + GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson) { + Map resourceMap = gitResourceMap.getGitResourceMap(); + + // exported application + final String applicationFilePath = CommonConstants.APPLICATION + JSON_EXTENSION; + GitResourceIdentity applicationJsonIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, applicationFilePath, applicationFilePath); + + Object applicationObject = resourceMap.get(applicationJsonIdentity); + Application application = objectMapper.convertValue(applicationObject, Application.class); + artifactExchangeJson.setArtifact(application); + + // metadata + final String metadataFilePath = CommonConstants.METADATA + JSON_EXTENSION; + GitResourceIdentity metadataIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, metadataFilePath, metadataFilePath); + + Object metadataObject = resourceMap.get(metadataIdentity); + ApplicationJson metadata = objectMapper.convertValue(metadataObject, ApplicationJson.class); + copyNestedNonNullProperties(metadata, artifactExchangeJson); + + // pages + List pageList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.CONTEXT_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(pageObject -> objectMapper.convertValue(pageObject, NewPage.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setContextList(pageList); + + // widgets + + pageList.parallelStream().forEach(newPage -> { + Map widgetsData = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.WIDGET_CONFIG.equals(key.getResourceType()) + && key.getResourceIdentifier().startsWith(newPage.getGitSyncId() + "-"); + }) + .collect(Collectors.toMap( + entry -> entry.getKey() + .getFilePath() + .replaceFirst( + PAGE_DIRECTORY + + newPage.getUnpublishedPage() + .getName() + + DELIMITER_PATH + + WIDGETS + + DELIMITER_PATH, + MAIN_CONTAINER + DELIMITER_PATH), + entry -> (org.json.JSONObject) entry.getValue())); + + Layout layout = newPage.getUnpublishedPage().getLayouts().get(0); + org.json.JSONObject mainContainer; + try { + mainContainer = new org.json.JSONObject(objectMapper.writeValueAsString(layout.getDsl())); + + Map> parentDirectories = DSLTransformerHelper.calculateParentDirectories( + widgetsData.keySet().stream().toList()); + org.json.JSONObject nestedDSL = + DSLTransformerHelper.getNestedDSL(widgetsData, parentDirectories, mainContainer); + + JSONParser jsonParser = new JSONParser(); + JSONObject parsedDSL = jsonParser.parse(nestedDSL.toString(), JSONObject.class); + + layout.setDsl(parsedDSL); + } catch (ParseException | JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java index a5535c5f055d..fcf0b2682ada 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsImpl.java @@ -6,6 +6,7 @@ import com.appsmith.server.helpers.ArtifactGitFileUtils; import com.appsmith.server.migrations.JsonSchemaMigration; import com.appsmith.server.newactions.base.NewActionService; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import org.springframework.stereotype.Component; @@ -15,10 +16,11 @@ public class ApplicationGitFileUtilsImpl extends ApplicationGitFileUtilsCEImpl public ApplicationGitFileUtilsImpl( Gson gson, + ObjectMapper objectMapper, NewActionService newActionService, FileInterface fileUtils, JsonSchemaMigration jsonSchemaMigration, ActionCollectionService actionCollectionService) { - super(gson, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService); + super(gson, objectMapper, newActionService, fileUtils, jsonSchemaMigration, actionCollectionService); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java index 7f4ec76ee8a0..348ca06e01a1 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ApplicationJsonCE.java @@ -10,6 +10,7 @@ import com.appsmith.server.domains.ActionCollection; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.Artifact; +import com.appsmith.server.domains.Context; import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.NewPage; @@ -123,6 +124,11 @@ public Artifact getArtifact() { return this.getExportedApplication(); } + @Override + public void setArtifact(T application) { + this.exportedApplication = (Application) application; + } + @Override public void setThemes(Theme unpublishedTheme, Theme publishedTheme) { this.setEditModeTheme(unpublishedTheme); @@ -138,4 +144,9 @@ public Theme getUnpublishedTheme() { public List getContextList() { return this.pageList; } + + @Override + public void setContextList(List contextList) { + this.pageList = (List) contextList; + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java index 47bf907e3d0b..f22d68d36999 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/ArtifactExchangeJsonCE.java @@ -30,6 +30,8 @@ public interface ArtifactExchangeJsonCE { Artifact getArtifact(); + void setArtifact(T artifact); + default void setThemes(Theme unpublishedTheme, Theme publishedTheme) {} default List getCustomJSLibList() { @@ -68,4 +70,6 @@ default Theme getPublishedTheme() { @JsonView(Views.Internal.class) List getContextList(); + + void setContextList(List contextList); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java index 0490dbc2d59e..d69a88660264 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedExportableResourcesCE_DTO.java @@ -15,6 +15,7 @@ public class MappedExportableResourcesCE_DTO { Map datasourceIdToNameMap = new HashMap<>(); Map datasourceNameToUpdatedAtMap = new HashMap<>(); Map contextIdToNameMap = new HashMap<>(); + Map contextNameToGitSyncIdMap = new HashMap<>(); Map actionIdToNameMap = new HashMap<>(); Map collectionIdToNameMap = new HashMap<>(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java index 8e9313b90cb4..bfb09ac1e5c0 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerCEImpl.java @@ -4,12 +4,14 @@ import com.appsmith.external.dtos.ModifiedResources; import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.constants.GitConstants.GitCommandConstants; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.configurations.ProjectProperties; import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Layout; import com.appsmith.server.domains.NewPage; import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.dtos.PageDTO; import com.appsmith.server.events.AutoCommitEvent; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; @@ -28,14 +30,16 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static com.appsmith.external.git.constants.GitConstants.PAGE_LIST; import static java.lang.Boolean.TRUE; @@ -212,9 +216,17 @@ private Mono migrateUnpublishedPageDSLs( ApplicationJson to file system conversion will use this field to decide which pages need to be written back to file system. */ - Set pageNamesSet = new HashSet<>(updatedPageNamesList); + + Set pageNamesSet = + updatedPageNamesList.stream().map(Tuple2::getT1).collect(Collectors.toSet()); + Set pageIdentifiersSet = + updatedPageNamesList.stream().map(Tuple2::getT2).collect(Collectors.toSet()); ModifiedResources modifiedResources = new ModifiedResources(); modifiedResources.putResource(PAGE_LIST, pageNamesSet); + modifiedResources + .getModifiedResourceIdentifiers() + .get(GitResourceType.CONTEXT_CONFIG) + .addAll(pageIdentifiersSet); modifiedResources.setAllModified(true); applicationJson.setModifiedResources(modifiedResources); return applicationJson; @@ -236,7 +248,7 @@ private Mono migrateUnpublishedPageDSLs( * @param latestSchemaVersion latest dsl schema version obtained from RTS * @return list of names of the pages that have been migrated. */ - private Mono> migratePageDsl(List newPageList, Integer latestSchemaVersion) { + private Mono>> migratePageDsl(List newPageList, Integer latestSchemaVersion) { return Flux.fromIterable(newPageList) .filter(newPage -> { // filter the pages which have unpublished page with layouts and where dsl version is not latest @@ -249,8 +261,8 @@ private Mono> migratePageDsl(List newPageList, Integer lat } return false; }) - .map(NewPage::getUnpublishedPage) - .flatMap(pageDTO -> { + .flatMap(newPage -> { + PageDTO pageDTO = newPage.getUnpublishedPage(); Layout layout = pageDTO.getLayouts().get(0); return dslMigrationUtils .migratePageDsl(layout.getDsl()) @@ -258,7 +270,7 @@ private Mono> migratePageDsl(List newPageList, Integer lat layout.setDsl(migratedDsl); return migratedDsl; }) - .thenReturn(pageDTO.getName()); + .thenReturn(Tuples.of(pageDTO.getName(), newPage.getGitSyncId())); }) .collectList(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java index 0af8ce932354..f92820c047ac 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/ArtifactGitFileUtilsCE.java @@ -13,6 +13,8 @@ public interface ArtifactGitFileUtilsCE { T createArtifactReferenceObject(); + ArtifactExchangeJson createArtifactExchangeJsonObject(); + void setArtifactDependentResources(ArtifactExchangeJson artifactExchangeJson, GitResourceMap gitResourceMap); Mono reconstructArtifactExchangeJsonFromFilesInRepository( @@ -24,4 +26,6 @@ void addArtifactReferenceFromExportedJson( Map getConstantsMap(); Path getRepoSuffixPath(String workspaceId, String artifactId, String repoName, @NonNull String... args); + + void setArtifactDependentPropertiesInJson(GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java index e0955455d6d2..486406d6422b 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/helpers/ce/CommonGitFileUtilsCE.java @@ -7,6 +7,8 @@ import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.git.operations.FileOperations; import com.appsmith.external.helpers.Stopwatch; +import com.appsmith.external.models.ActionConfiguration; +import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.ApplicationGitReference; import com.appsmith.external.models.ArtifactGitReference; import com.appsmith.external.models.BaseDomain; @@ -22,6 +24,7 @@ import com.appsmith.server.domains.GitArtifactMetadata; import com.appsmith.server.domains.NewAction; import com.appsmith.server.domains.Theme; +import com.appsmith.server.dtos.ActionCollectionDTO; import com.appsmith.server.dtos.ApplicationJson; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.dtos.PageDTO; @@ -32,10 +35,12 @@ import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.eclipse.jgit.api.errors.GitAPIException; import org.json.JSONObject; @@ -52,19 +57,30 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collector; +import java.util.stream.Collectors; import static com.appsmith.external.git.constants.ce.GitConstantsCE.GitCommandConstantsCE.CHECKOUT_BRANCH; import static com.appsmith.external.git.constants.ce.GitConstantsCE.RECONSTRUCT_PAGE; import static com.appsmith.git.constants.CommonConstants.CLIENT_SCHEMA_VERSION; +import static com.appsmith.git.constants.CommonConstants.DELIMITER_PATH; import static com.appsmith.git.constants.CommonConstants.FILE_FORMAT_VERSION; import static com.appsmith.git.constants.CommonConstants.JSON_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.JS_EXTENSION; +import static com.appsmith.git.constants.CommonConstants.METADATA; import static com.appsmith.git.constants.CommonConstants.SERVER_SCHEMA_VERSION; +import static com.appsmith.git.constants.CommonConstants.TEXT_FILE_EXTENSION; import static com.appsmith.git.constants.CommonConstants.THEME; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_COLLECTION_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.ACTION_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.DATASOURCE_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.JS_LIB_DIRECTORY; +import static com.appsmith.git.constants.ce.GitDirectoriesCE.PAGE_DIRECTORY; import static com.appsmith.git.files.FileUtilsCEImpl.getJsLibFileName; import static org.springframework.util.StringUtils.hasText; @Slf4j -@RequiredArgsConstructor @Component @Import({FileUtilsImpl.class}) public class CommonGitFileUtilsCE { @@ -83,6 +99,28 @@ public class CommonGitFileUtilsCE { public final int INDEX_LOCK_FILE_STALE_TIME = 300; private final JsonSchemaVersions jsonSchemaVersions; + protected final ObjectMapper objectMapper; + + public CommonGitFileUtilsCE( + ArtifactGitFileUtils applicationGitFileUtils, + FileInterface fileUtils, + FileOperations fileOperations, + AnalyticsService analyticsService, + SessionUserService sessionUserService, + NewActionService newActionService, + ActionCollectionService actionCollectionService, + JsonSchemaVersions jsonSchemaVersions, + ObjectMapper objectMapper) { + this.applicationGitFileUtils = applicationGitFileUtils; + this.fileUtils = fileUtils; + this.fileOperations = fileOperations; + this.analyticsService = analyticsService; + this.sessionUserService = sessionUserService; + this.newActionService = newActionService; + this.actionCollectionService = actionCollectionService; + this.jsonSchemaVersions = jsonSchemaVersions; + this.objectMapper = objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS); + } private ArtifactGitFileUtils getArtifactBasedFileHelper(ArtifactType artifactType) { if (ArtifactType.APPLICATION.equals(artifactType)) { @@ -120,6 +158,19 @@ public Mono saveArtifactToLocalRepo( } } + public Mono saveArtifactToLocalRepoNew( + Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) + throws IOException, GitAPIException { + + // this should come from the specific files + GitResourceMap gitResourceMap = createGitResourceMap(artifactExchangeJson); + + // Save application to git repo + return fileUtils + .saveArtifactToGitRepo(baseRepoSuffix, gitResourceMap, branchName) + .subscribeOn(Schedulers.boundedElastic()); + } + public Mono saveArtifactToLocalRepoWithAnalytics( Path baseRepoSuffix, ArtifactExchangeJson artifactExchangeJson, String branchName) { @@ -216,8 +267,9 @@ protected void setArtifactIndependentResources( if (datasourceList != null) { datasourceList.forEach(datasource -> { removeUnwantedFieldsFromDatasource(datasource); + final String filePath = DATASOURCE_DIRECTORY + DELIMITER_PATH + datasource.getName() + JSON_EXTENSION; GitResourceIdentity identity = - new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId()); + new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, datasource.getGitSyncId(), filePath); resourceMap.put(identity, datasource); }); } @@ -230,7 +282,8 @@ protected void setArtifactIndependentResources( artifactExchangeJson.setThemes(theme, null); // Remove internal fields from the themes removeUnwantedFieldsFromBaseDomain(theme); - GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, THEME + JSON_EXTENSION); + final String filePath = THEME + JSON_EXTENSION; + GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath); resourceMap.put(identity, theme); } @@ -240,7 +293,9 @@ protected void setArtifactIndependentResources( customJSLibList.forEach(jsLib -> { removeUnwantedFieldsFromBaseDomain(jsLib); String jsLibFileName = getJsLibFileName(jsLib.getUidString()); - GitResourceIdentity identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName); + final String filePath = JS_LIB_DIRECTORY + DELIMITER_PATH + jsLibFileName + JSON_EXTENSION; + GitResourceIdentity identity = + new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, jsLibFileName, filePath); resourceMap.put(identity, jsLib); }); } @@ -265,58 +320,47 @@ protected void setNewActionsInResourceMap( .peek(newAction -> newActionService.generateActionByViewMode(newAction, false)) .forEach(newAction -> { removeUnwantedFieldFromAction(newAction); - String body = newAction.getUnpublishedAction().getActionConfiguration() != null - && newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - != null - ? newAction - .getUnpublishedAction() - .getActionConfiguration() - .getBody() - : ""; + ActionDTO action = newAction.getUnpublishedAction(); + final String actionFileName = action.getValidName().replace(".", "-"); + final String filePathPrefix = PAGE_DIRECTORY + + DELIMITER_PATH + + action.calculateContextId() + + DELIMITER_PATH + + ACTION_DIRECTORY + + DELIMITER_PATH + + actionFileName + + DELIMITER_PATH; + String body = action.getActionConfiguration() != null + && action.getActionConfiguration().getBody() != null + ? action.getActionConfiguration().getBody() + : null; // This is a special case where we are handling REMOTE type plugins based actions such as Twilio // The user configured values are stored in an attribute called formData which is a map unlike the // body if (PluginType.REMOTE.equals(newAction.getPluginType()) - && newAction.getUnpublishedAction().getActionConfiguration() != null - && newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData() - != null) { - body = new Gson() - .toJson( - newAction - .getUnpublishedAction() - .getActionConfiguration() - .getFormData(), - Map.class); - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setFormData(null); + && action.getActionConfiguration() != null + && action.getActionConfiguration().getFormData() != null) { + body = new Gson().toJson(action.getActionConfiguration().getFormData(), Map.class); + action.getActionConfiguration().setFormData(null); } // This is a special case where we are handling JS actions as we don't want to commit the body of JS // actions if (PluginType.JS.equals(newAction.getPluginType())) { - if (newAction.getUnpublishedAction().getActionConfiguration() != null) { - newAction - .getUnpublishedAction() - .getActionConfiguration() - .setBody(null); - newAction.getUnpublishedAction().setJsonPathKeys(null); + if (action.getActionConfiguration() != null) { + action.getActionConfiguration().setBody(null); + action.setJsonPathKeys(null); } - } else { + } else if (body != null) { // For the regular actions we save the body field to git repo + final String filePath = filePathPrefix + actionFileName + TEXT_FILE_EXTENSION; GitResourceIdentity actionDataIdentity = - new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId()); + new GitResourceIdentity(GitResourceType.QUERY_DATA, newAction.getGitSyncId(), filePath); resourceMap.put(actionDataIdentity, body); } + final String filePath = filePathPrefix + METADATA + JSON_EXTENSION; GitResourceIdentity actionConfigIdentity = - new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId()); + new GitResourceIdentity(GitResourceType.QUERY_CONFIG, newAction.getGitSyncId(), filePath); resourceMap.put(actionConfigIdentity, newAction); }); } @@ -335,18 +379,29 @@ protected void setActionCollectionsInResourceMap( actionCollectionService.generateActionCollectionByViewMode(actionCollection, false)) .forEach(actionCollection -> { removeUnwantedFieldFromActionCollection(actionCollection); - String body = actionCollection.getUnpublishedCollection().getBody() != null - ? actionCollection.getUnpublishedCollection().getBody() - : ""; - actionCollection.getUnpublishedCollection().setBody(null); - - GitResourceIdentity collectionConfigIdentity = - new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId()); + ActionCollectionDTO collection = actionCollection.getUnpublishedCollection(); + final String filePathPrefix = PAGE_DIRECTORY + + DELIMITER_PATH + + collection.calculateContextId() + + DELIMITER_PATH + + ACTION_COLLECTION_DIRECTORY + + DELIMITER_PATH + + collection.getName() + + DELIMITER_PATH; + String body = collection.getBody(); + collection.setBody(null); + + String configFilePath = filePathPrefix + METADATA + JSON_EXTENSION; + GitResourceIdentity collectionConfigIdentity = new GitResourceIdentity( + GitResourceType.JSOBJECT_CONFIG, actionCollection.getGitSyncId(), configFilePath); resourceMap.put(collectionConfigIdentity, actionCollection); - GitResourceIdentity collectionDataIdentity = - new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId()); - resourceMap.put(collectionDataIdentity, body); + if (body != null) { + String dataFilePath = filePathPrefix + collection.getName() + JS_EXTENSION; + GitResourceIdentity collectionDataIdentity = new GitResourceIdentity( + GitResourceType.JSOBJECT_DATA, actionCollection.getGitSyncId(), dataFilePath); + resourceMap.put(collectionDataIdentity, body); + } }); } @@ -366,6 +421,125 @@ private void removeUnwantedFieldFromActionCollection(ActionCollection actionColl removeUnwantedFieldsFromBaseDomain(actionCollection); } + public ArtifactExchangeJson createArtifactExchangeJson(GitResourceMap gitResourceMap, ArtifactType artifactType) { + ArtifactGitFileUtils artifactGitFileUtils = getArtifactBasedFileHelper(artifactType); + + ArtifactExchangeJson artifactExchangeJson = artifactGitFileUtils.createArtifactExchangeJsonObject(); + + artifactGitFileUtils.setArtifactDependentPropertiesInJson(gitResourceMap, artifactExchangeJson); + + setArtifactIndependentPropertiesInJson(gitResourceMap, artifactExchangeJson); + + return artifactExchangeJson; + } + + protected void setArtifactIndependentPropertiesInJson( + GitResourceMap gitResourceMap, ArtifactExchangeJson artifactExchangeJson) { + Map resourceMap = gitResourceMap.getGitResourceMap(); + + // datasources + List datasourceList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.DATASOURCE_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(value -> objectMapper.convertValue(value, DatasourceStorage.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setDatasourceList(datasourceList); + + // themes + final String themeFilePath = THEME + JSON_EXTENSION; + GitResourceIdentity themeIdentity = + new GitResourceIdentity(GitResourceType.ROOT_CONFIG, themeFilePath, themeFilePath); + Object themeObject = resourceMap.get(themeIdentity); + Theme theme = objectMapper.convertValue(themeObject, Theme.class); + artifactExchangeJson.setThemes(theme, null); + + // custom js libs + List jsLibList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return GitResourceType.JSLIB_CONFIG.equals(key.getResourceType()); + }) + .map(Map.Entry::getValue) + .map(value -> objectMapper.convertValue(value, CustomJSLib.class)) + .collect(Collectors.toList()); + artifactExchangeJson.setCustomJSLibList(jsLibList); + + // actions + final Set queryTypes = Set.of(GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_DATA); + List actionList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return queryTypes.contains(key.getResourceType()); + }) + .collect(collectByGitSyncId()) + .entrySet() + .parallelStream() + .map(entry -> { + Object config = entry.getValue().get(GitResourceType.QUERY_CONFIG); + NewAction newAction = objectMapper.convertValue(config, NewAction.class); + ActionDTO actionDTO = newAction.getUnpublishedAction(); + Object data = entry.getValue().get(GitResourceType.QUERY_DATA); + ActionConfiguration actionConfiguration = actionDTO.getActionConfiguration(); + if (actionConfiguration == null) { + // This shouldn't happen but safe-guarding just in case + actionConfiguration = new ActionConfiguration(); + } + + if (PluginType.REMOTE.equals(newAction.getPluginType())) { + Map formData = objectMapper.convertValue(data, new TypeReference<>() {}); + actionConfiguration.setFormData(formData); + } else if (data != null) { + String body = String.valueOf(data); + actionConfiguration.setBody(body); + } + + return newAction; + }) + .collect(Collectors.toList()); + artifactExchangeJson.setActionList(actionList); + + // action collections + final Set jsObjectTypes = + Set.of(GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_DATA); + List collectionList = resourceMap.entrySet().stream() + .filter(entry -> { + GitResourceIdentity key = entry.getKey(); + return jsObjectTypes.contains(key.getResourceType()); + }) + .collect(collectByGitSyncId()) + .entrySet() + .parallelStream() + .map(entry -> { + Object config = entry.getValue().get(GitResourceType.JSOBJECT_CONFIG); + ActionCollection actionCollection = objectMapper.convertValue(config, ActionCollection.class); + Object data = entry.getValue().get(GitResourceType.JSOBJECT_DATA); + String body = String.valueOf(data); + actionCollection.getUnpublishedCollection().setBody(body); + + return actionCollection; + }) + .collect(Collectors.toList()); + artifactExchangeJson.setActionCollectionList(collectionList); + } + + private Collector, ?, Map>> + collectByGitSyncId() { + return Collectors.toMap( + entry -> entry.getKey().getResourceIdentifier(), + entry -> { + HashMap map = new HashMap<>(); + map.put(entry.getKey().getResourceType(), entry.getValue()); + return map; + }, + (x, y) -> { + x.putAll(y); + return x; + }); + } + private void setDatasourcesInArtifactReference( ArtifactExchangeJson artifactExchangeJson, ArtifactGitReference artifactGitReference) { Map resourceMap = new HashMap<>(); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java index dde27edae161..8df63d51380b 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/exportable/CustomJSLibExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.jslibs.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.constants.FieldName; import com.appsmith.server.domains.Application; import com.appsmith.server.domains.Artifact; @@ -77,6 +78,11 @@ public Mono getExportableEntities( artifactExchangeJson .getModifiedResources() .putResource(FieldName.CUSTOM_JS_LIB_LIST, updatedCustomJSLibSet); + artifactExchangeJson + .getModifiedResources() + .getModifiedResourceIdentifiers() + .get(GitResourceType.JSLIB_CONFIG) + .addAll(updatedCustomJSLibSet); /** * Previously it was a Set and as Set is an unordered collection of elements that diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/exportable/NewActionExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/exportable/NewActionExportableServiceCEImpl.java index 0a315b05afa8..395e9c808af2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/exportable/NewActionExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/exportable/NewActionExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.newactions.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.external.models.ActionDTO; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.constants.FieldName; @@ -70,6 +71,7 @@ public Mono getExportableEntities( List actionList = tuple.getT1(); Set dbNamesUsedInActions = tuple.getT2(); Set updatedActionSet = new HashSet<>(); + Set updatedIdentities = new HashSet<>(); actionList.forEach(newAction -> { ActionDTO unpublishedActionDTO = newAction.getUnpublishedAction(); ActionDTO publishedActionDTO = newAction.getPublishedAction(); @@ -85,9 +87,12 @@ public Mono getExportableEntities( actionDTO, exportingMetaDTO.getArtifactLastCommittedAt()); - String contextListPath = artifactBasedExportableService.getContextListPath(); - boolean isContextUpdated = ImportExportUtils.isContextNameInUpdatedList( - artifactExchangeJson, contextNameAtIdReference, contextListPath); + String contextGitSyncId = mappedExportableResourcesDTO + .getContextNameToGitSyncIdMap() + .get(contextNameAtIdReference); + boolean isContextUpdated = artifactExchangeJson + .getModifiedResources() + .isResourceUpdatedNew(GitResourceType.CONTEXT_CONFIG, contextGitSyncId); Instant newActionUpdatedAt = newAction.getUpdatedAt(); boolean isNewActionUpdated = exportingMetaDTO.isClientSchemaMigrated() || exportingMetaDTO.isServerSchemaMigrated() @@ -98,10 +103,16 @@ public Mono getExportableEntities( || exportingMetaDTO.getArtifactLastCommittedAt().isBefore(newActionUpdatedAt); if (isNewActionUpdated && newActionName != null) { updatedActionSet.add(newActionName); + updatedIdentities.add(newAction.getGitSyncId()); } newAction.sanitiseToExportDBObject(); }); artifactExchangeJson.getModifiedResources().putResource(FieldName.ACTION_LIST, updatedActionSet); + artifactExchangeJson + .getModifiedResources() + .getModifiedResourceIdentifiers() + .get(GitResourceType.QUERY_CONFIG) + .addAll(updatedIdentities); artifactExchangeJson.setActionList(actionList); // This is where we're removing global datasources that are unused in this application diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/exportable/NewPageExportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/exportable/NewPageExportableServiceCEImpl.java index da7ecf8a6d45..0066a6be65cf 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/exportable/NewPageExportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newpages/exportable/NewPageExportableServiceCEImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.newpages.exportable; +import com.appsmith.external.git.models.GitResourceType; import com.appsmith.server.acl.AclPermission; import com.appsmith.server.constants.FieldName; import com.appsmith.server.constants.SerialiseArtifactObjective; @@ -66,7 +67,8 @@ public Mono getExportableEntities( // Extract mongoEscapedWidgets from pages and save it to applicationJson object as this // field is JsonIgnored. Also remove any ids those are present in the page objects - Set updatedPageSet = new HashSet(); + Set updatedPageSet = new HashSet<>(); + Set updatedIdentities = new HashSet<>(); // check the application object for the page reference in the page list // Exclude the deleted pages that are present in view mode because the app is not @@ -79,6 +81,9 @@ public Mono getExportableEntities( .put( newPage.getId() + EDIT, newPage.getUnpublishedPage().getName()); + mappedExportableResourcesDTO + .getContextNameToGitSyncIdMap() + .put(newPage.getUnpublishedPage().getName(), newPage.getGitSyncId()); PageDTO unpublishedPageDTO = newPage.getUnpublishedPage(); if (!CollectionUtils.isEmpty(unpublishedPageDTO.getLayouts())) { unpublishedPageDTO.getLayouts().forEach(layout -> { @@ -114,11 +119,17 @@ public Mono getExportableEntities( : null; if (isNewPageUpdated && newPageName != null) { updatedPageSet.add(newPageName); + updatedIdentities.add(newPage.getGitSyncId()); } newPage.sanitiseToExportDBObject(); }); applicationJson.setPageList(newPageList); applicationJson.getModifiedResources().putResource(FieldName.PAGE_LIST, updatedPageSet); + applicationJson + .getModifiedResources() + .getModifiedResourceIdentifiers() + .get(GitResourceType.CONTEXT_CONFIG) + .addAll(updatedIdentities); return newPageList; }) diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java index c1df11a4be14..56de587dcb81 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/ExchangeJsonConversionTests.java @@ -1,18 +1,26 @@ package com.appsmith.server.git.resourcemap; +import com.appsmith.external.git.GitExecutor; import com.appsmith.external.git.models.GitResourceIdentity; import com.appsmith.external.git.models.GitResourceMap; +import com.appsmith.server.constants.ArtifactType; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext; import com.appsmith.server.git.resourcemap.templates.providers.ExchangeJsonTestTemplateProvider; import com.appsmith.server.helpers.CommonGitFileUtils; import com.appsmith.server.migrations.JsonSchemaMigration; -import com.google.gson.Gson; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.io.FileUtils; +import org.assertj.core.api.Assertions; +import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.core.io.ClassPathResource; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; @@ -20,6 +28,8 @@ import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -33,7 +43,7 @@ public class ExchangeJsonConversionTests { public ExchangeJsonTestTemplateProvider templateProvider; @Autowired - Gson gson; + ObjectMapper objectMapper; @Autowired JsonSchemaMigration jsonSchemaMigration; @@ -41,16 +51,20 @@ public class ExchangeJsonConversionTests { @Autowired CommonGitFileUtils commonGitFileUtils; + @SpyBean + GitExecutor gitExecutor; + @TestTemplate public void testConvertArtifactJsonToGitResourceMap_whenArtifactIsFullyPopulated_returnsCorrespondingResourceMap( ExchangeJsonContext context) throws IOException { - Mono artifactJsonMono = - createArtifactJson(context).cache(); + Mono artifactJsonMono = createArtifactJson(context); + + Mono artifactJsonCloneMono = createArtifactJson(context); Mono> gitResourceMapAndArtifactJsonMono = artifactJsonMono .map(artifactJson -> commonGitFileUtils.createGitResourceMap(artifactJson)) - .zipWith(artifactJsonMono); + .zipWith(artifactJsonCloneMono); StepVerifier.create(gitResourceMapAndArtifactJsonMono) .assertNext(tuple2 -> { @@ -87,8 +101,46 @@ private Mono createArtifactJson(ExchangeJsonCont Class exchangeJsonType = context.getArtifactExchangeJsonType(); - ArtifactExchangeJson artifactExchangeJson = gson.fromJson(artifactJson, exchangeJsonType); + ArtifactExchangeJson artifactExchangeJson = + objectMapper.copy().disable(MapperFeature.USE_ANNOTATIONS).readValue(artifactJson, exchangeJsonType); return jsonSchemaMigration.migrateArtifactExchangeJsonToLatestSchema(artifactExchangeJson, null, null); } + + @TestTemplate + public void testConvertGitResourceMapToArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingJson( + ExchangeJsonContext context) throws IOException { + ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block(); + + GitResourceMap gitResourceMap = commonGitFileUtils.createGitResourceMap(originalArtifactJson); + + ArtifactExchangeJson artifactExchangeJson = + commonGitFileUtils.createArtifactExchangeJson(gitResourceMap, ArtifactType.APPLICATION); + + assertThat(artifactExchangeJson).isNotNull(); + + templateProvider.assertResourceComparisons(originalArtifactJson, artifactExchangeJson); + } + + @TestTemplate + public void testSerializeArtifactExchangeJson_whenArtifactIsFullyPopulated_returnsCorrespondingBaseRepoPath( + ExchangeJsonContext context) throws IOException, GitAPIException { + ArtifactExchangeJson originalArtifactJson = createArtifactJson(context).block(); + + Mockito.doReturn(Mono.just(true)).when(gitExecutor).resetToLastCommit(Mockito.any(), Mockito.anyString()); + + Files.createDirectories(Path.of("./container-volumes/git-storage/test123")); + + Mono responseMono = + commonGitFileUtils.saveArtifactToLocalRepoNew(Path.of("test123"), originalArtifactJson, "irrelevant"); + + StepVerifier.create(responseMono) + .assertNext(response -> { + Assertions.assertThat(response).isNotNull(); + }) + .verifyComplete(); + + FileUtils.deleteDirectory( + Path.of("./container-volumes/git-storage/test123").toFile()); + } } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java index 0724d60170d5..bbf07f5f62d3 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/resourcemap/templates/providers/ce/ExchangeJsonTestTemplateProviderCE.java @@ -2,7 +2,12 @@ import com.appsmith.external.git.models.GitResourceIdentity; import com.appsmith.external.git.models.GitResourceType; +import com.appsmith.external.models.DatasourceStorage; import com.appsmith.external.models.PluginType; +import com.appsmith.server.domains.ActionCollection; +import com.appsmith.server.domains.Context; +import com.appsmith.server.domains.CustomJSLib; +import com.appsmith.server.domains.NewAction; import com.appsmith.server.dtos.ApplicationJson; import com.appsmith.server.dtos.ArtifactExchangeJson; import com.appsmith.server.git.resourcemap.templates.contexts.ExchangeJsonContext; @@ -27,7 +32,7 @@ public boolean supportsTestTemplate(ExtensionContext extensionContext) { @Override public Stream provideTestTemplateInvocationContexts( ExtensionContext extensionContext) { - ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 23); + ExchangeJsonContext context = new ExchangeJsonContext("valid-application.json", ApplicationJson.class, 22); return Stream.of(context); } @@ -68,7 +73,14 @@ public long assertResourceComparisons( List jsObjectDataResources = getResourceListByType(resourceMap, GitResourceType.JSOBJECT_DATA); long resourceMapJsObjectDataCount = jsObjectDataResources.size(); - assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectCount); + int jsonJsObjectDataCount = exchangeJson.getActionCollectionList() != null + ? exchangeJson.getActionCollectionList().parallelStream() + .filter(collection -> + collection.getUnpublishedCollection().getBody() != null) + .collect(Collectors.toList()) + .size() + : 0; + assertThat(resourceMapJsObjectDataCount).isEqualTo(jsonJsObjectDataCount); List actionConfigResources = getResourceListByType(resourceMap, GitResourceType.QUERY_CONFIG); long resourceMapActionConfigCount = actionConfigResources.size(); @@ -82,7 +94,17 @@ public long assertResourceComparisons( long jsonActionDataCount = 0; if (exchangeJson.getActionList() != null) { jsonActionDataCount = exchangeJson.getActionList().stream() - .filter(action -> !PluginType.JS.equals(action.getPluginType())) + .filter(action -> !PluginType.JS.equals(action.getPluginType()) + && action.getUnpublishedAction().getActionConfiguration() != null + && !(action.getUnpublishedAction() + .getActionConfiguration() + .getBody() + == null + || (action.getPluginType().equals(PluginType.REMOTE) + && action.getUnpublishedAction() + .getActionConfiguration() + .getFormData() + == null))) .count(); } assertThat(resourceMapActionDataCount).isEqualTo(jsonActionDataCount); @@ -111,4 +133,42 @@ protected List getResourceListByType( .map(Map.Entry::getValue) .collect(Collectors.toList()); } + + public void assertResourceComparisons( + ArtifactExchangeJson originalExchangeJson, ArtifactExchangeJson convertedExchangeJson) { + List datasourceResources = convertedExchangeJson.getDatasourceList(); + long convertedDatasourceCount = datasourceResources.size(); + int jsonDatasourceCount = originalExchangeJson.getDatasourceList() != null + ? originalExchangeJson.getDatasourceList().size() + : 0; + assertThat(convertedDatasourceCount).isEqualTo(jsonDatasourceCount); + + List jsLibResources = convertedExchangeJson.getCustomJSLibList(); + long convertedJsLibCount = jsLibResources.size(); + int jsonJsLibCount = originalExchangeJson.getCustomJSLibList() != null + ? originalExchangeJson.getCustomJSLibList().size() + : 0; + assertThat(convertedJsLibCount).isEqualTo(jsonJsLibCount); + + List contextResources = convertedExchangeJson.getContextList(); + long convertedContextCount = contextResources.size(); + int jsonContextCount = originalExchangeJson.getContextList() != null + ? originalExchangeJson.getContextList().size() + : 0; + assertThat(convertedContextCount).isEqualTo(jsonContextCount); + + List jsObjectResources = convertedExchangeJson.getActionCollectionList(); + long convertedJsObjectCount = jsObjectResources.size(); + int jsonJsObjectCount = originalExchangeJson.getActionCollectionList() != null + ? originalExchangeJson.getActionCollectionList().size() + : 0; + assertThat(convertedJsObjectCount).isEqualTo(jsonJsObjectCount); + + List actionResources = convertedExchangeJson.getActionList(); + long convertedActionCount = actionResources.size(); + int jsonActionCount = originalExchangeJson.getActionList() != null + ? originalExchangeJson.getActionList().size() + : 0; + assertThat(convertedActionCount).isEqualTo(jsonActionCount); + } }