Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Git resource map conversions #37920

Merged
merged 3 commits into from
Dec 3, 2024
Merged

chore: Git resource map conversions #37920

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
28c7911
chore: Git resource map conversions
nidhi-nair Dec 3, 2024
ca6935e
chore: Git resource map conversions
nidhi-nair Dec 3, 2024
c78857b
chore: Git resource map conversions
nidhi-nair Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 216 additions & 3 deletions app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.helpers.Stopwatch;
Expand All @@ -14,10 +17,12 @@
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.helpers.DSLTransformerHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.json.JSONObject;
Expand All @@ -28,6 +33,8 @@
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;

import java.io.BufferedWriter;
import java.io.File;
Expand All @@ -48,7 +55,10 @@
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST;
import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST;
Expand All @@ -74,6 +84,7 @@ public class FileUtilsCEImpl implements FileInterface {
private final GitExecutor gitExecutor;
protected final FileOperations fileOperations;
private final ObservationHelper observationHelper;
protected final ObjectMapper objectMapper;

private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}";

Expand All @@ -90,11 +101,23 @@ public FileUtilsCEImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
this.gitServiceConfig = gitServiceConfig;
this.gitExecutor = gitExecutor;
this.fileOperations = fileOperations;
this.observationHelper = observationHelper;
this.objectMapper = objectMapper;
}

protected Map<GitResourceType, GitResourceType> getModifiedResourcesTypes() {
return Map.of(
GitResourceType.JSLIB_CONFIG, GitResourceType.JSLIB_CONFIG,
GitResourceType.CONTEXT_CONFIG, GitResourceType.CONTEXT_CONFIG,
GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_CONFIG,
GitResourceType.QUERY_DATA, GitResourceType.QUERY_CONFIG,
GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_CONFIG,
GitResourceType.JSOBJECT_DATA, GitResourceType.JSOBJECT_CONFIG);
}

/**
Expand Down Expand Up @@ -215,6 +238,101 @@ public Mono<Path> saveApplicationToGitRepo(
.subscribeOn(scheduler);
}

@Override
public Mono<Path> saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName)
throws GitAPIException, IOException {

// Repo path will be:
// baseRepo : root/orgId/defaultAppId/repoName/{applicationData}
// Checkout to mentioned branch if not already checked-out
return gitExecutor
.resetToLastCommit(baseRepoSuffix, branchName)
.flatMap(isSwitched -> {
Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix);

try {
updateEntitiesInRepo(gitResourceMap, baseRepo);
} catch (IOException e) {
return Mono.error(e);
}

return Mono.just(baseRepo);
})
.subscribeOn(scheduler);
}

protected Set<String> getExistingFilesInRepo(Path baseRepo) throws IOException {
try (Stream<Path> stream = Files.walk(baseRepo).parallel()) {
return stream.filter(path -> {
try {
return Files.isRegularFile(path) || FileUtils.isEmptyDirectory(path.toFile());
} catch (IOException e) {
log.error("Unable to find file details. Please check the file at file path: {}", path);
log.error("Assuming that it does not exist for now ...");
return false;
}
})
.map(baseRepo::relativize)
.map(Path::toString)
.collect(Collectors.toSet());
}
}

protected Set<String> updateEntitiesInRepo(GitResourceMap gitResourceMap, Path baseRepo) throws IOException {
ModifiedResources modifiedResources = gitResourceMap.getModifiedResources();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();

Set<String> filesInRepo = getExistingFilesInRepo(baseRepo);

Set<String> updatedFilesToBeSerialized = resourceMap.keySet().parallelStream()
.map(gitResourceIdentity -> gitResourceIdentity.getFilePath())
.collect(Collectors.toSet());

// Remove all files that need to be serialized from the existing files list, as well as the README file
// What we are left with are all the files to be deleted
filesInRepo.removeAll(updatedFilesToBeSerialized);
filesInRepo.remove("README.md");

// Delete all the files because they are no longer needed
// This covers both older structures of storing files and,
// legitimate changes in the artifact that might cause deletions
filesInRepo.stream().parallel().forEach(filePath -> {
try {
Files.deleteIfExists(baseRepo.resolve(filePath));
} catch (IOException e) {
// We ignore files that could not be deleted and expect to come back to this at a later point
// Just log the path for now
log.error("Unable to delete file at path: {}", filePath);
}
});

// Now go through the resource map and based on resource type, check if the resource is modified before
// serialization
// Or simply choose the mechanism for serialization
Map<GitResourceType, GitResourceType> modifiedResourcesTypes = getModifiedResourcesTypes();
return resourceMap.entrySet().parallelStream()
.map(entry -> {
GitResourceIdentity key = entry.getKey();
boolean resourceUpdated = true;
if (modifiedResourcesTypes.containsKey(key.getResourceType()) && modifiedResources != null) {
GitResourceType comparisonType = modifiedResourcesTypes.get(key.getResourceType());

resourceUpdated =
modifiedResources.isResourceUpdatedNew(comparisonType, key.getResourceIdentifier());
}

if (resourceUpdated) {
String filePath = key.getFilePath();
saveResourceCommon(entry.getValue(), baseRepo.resolve(filePath));

return filePath;
}
return null;
})
.filter(Objects::nonNull)
.collect(Collectors.toSet());
}

protected Set<String> updateEntitiesInRepo(ApplicationGitReference applicationGitReference, Path baseRepo) {

Set<String> validDatasourceFileNames = new HashSet<>();
Expand Down Expand Up @@ -434,6 +552,23 @@ protected boolean saveResource(Object sourceEntity, Path path) {
return false;
}

protected void saveResourceCommon(Object sourceEntity, Path path) {
try {
Files.createDirectories(path.getParent());
if (sourceEntity instanceof String s) {
writeStringToFile(s, path);
return;
}
if (sourceEntity instanceof JSONObject) {
sourceEntity = objectMapper.readTree(sourceEntity.toString());
}
fileOperations.writeToFile(sourceEntity, path);
} catch (IOException e) {
log.error("Error while writing resource to file {} with {}", path, e.getMessage());
log.debug(e.getMessage());
}
}

/**
* This method is used to write actionCollection specific resource to file system. We write the data in two steps
* 1. Actual js code
Expand Down Expand Up @@ -514,9 +649,9 @@ private void writeStringToFile(String sourceEntity, Path path) throws IOExceptio
/**
* This will reconstruct the application from the repo
*
* @param organisationId To which organisation application needs to be rehydrated
* @param organisationId To which organisation application needs to be rehydrated
* @param baseApplicationId To which organisation application needs to be rehydrated
* @param branchName for which the application needs to be rehydrate
* @param branchName for which the application needs to be rehydrate
* @return application reference from which entire application can be rehydrated
*/
public Mono<ApplicationGitReference> reconstructApplicationReferenceFromGitRepo(
Expand Down Expand Up @@ -672,6 +807,84 @@ private Object readPageMetadata(Path directoryPath) {
directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION));
}

protected GitResourceMap fetchGitResourceMap(Path baseRepoPath) throws IOException {
// Extract application metadata from the json
Object metadata = fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata);
// Check if fileFormat of the saved files in repo is compatible
if (!isFileFormatCompatible(fileFormatVersion)) {
throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT);
}

GitResourceMap gitResourceMap = new GitResourceMap();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();

Set<String> filesInRepo = getExistingFilesInRepo(baseRepoPath);

filesInRepo.parallelStream()
.filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
.forEach(filePath -> {
Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);

resourceMap.put(identity.getT1(), identity.getT2());
});
Comment on lines +825 to +831
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Handle Potential Null Values

getGitResourceIdentity may return null. Add a null check before using the returned value to prevent a NullPointerException.

Apply this diff:

.stream()
    .filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
    .forEach(filePath -> {
        Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);
+       if (identity != null) {
            resourceMap.put(identity.getT1(), identity.getT2());
+       }
    });
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
filesInRepo.parallelStream()
.filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
.forEach(filePath -> {
Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);
resourceMap.put(identity.getT1(), identity.getT2());
});
filesInRepo.stream()
.filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
.forEach(filePath -> {
Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);
if (identity != null) {
resourceMap.put(identity.getT1(), identity.getT2());
}
});


return gitResourceMap;
}

protected Tuple2<GitResourceIdentity, Object> getGitResourceIdentity(Path baseRepoPath, String filePath) {
Path path = baseRepoPath.resolve(filePath);
GitResourceIdentity identity;
Object contents = fileOperations.readFile(path);
if (!filePath.contains("/")) {
identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath);
} else if (filePath.matches(DATASOURCE_DIRECTORY + "/.*")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(JS_LIB_DIRECTORY + "/.*")) {
String fileName = FilenameUtils.getBaseName(filePath);
identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, fileName, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/[^/]*.json]")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*\\.txt")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*\\.js")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/widgets/.*\\.json")) {
Pattern pageDirPattern = Pattern.compile("(" + PAGE_DIRECTORY + "/([^/]*))/widgets/.*\\.json");
Matcher matcher = pageDirPattern.matcher(filePath);
matcher.find();
String pageDirectory = matcher.group(1);
String pageName = matcher.group(2) + ".json";
Object configContents =
fileOperations.readFile(baseRepoPath.resolve(pageDirectory).resolve(pageName));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
String widgetId = objectMapper.valueToTree(contents).get("widgetId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId + "-" + widgetId, filePath);
} else return null;

return Tuples.of(identity, contents);
}

private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) {
ApplicationGitReference applicationGitReference = new ApplicationGitReference();
// Extract application metadata from the json
Expand Down
6 changes: 4 additions & 2 deletions app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.git.configurations.GitServiceConfig;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Import;
Expand All @@ -22,7 +23,8 @@ public FileUtilsImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper);
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper, objectMapper);
}
}
Loading
Loading