From 9dbbe6b9d9cad4021815ae0fa3f0b7cffc05cc82 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:40:05 +0200 Subject: [PATCH 1/5] Fix detecting of obsolete files --- .../functionality/ObsoleteSourcesUtils.java | 7 +--- .../ObsoleteSourcesUtilsTest.java | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java diff --git a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java index 9356cb271..769abbafc 100644 --- a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java +++ b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java @@ -22,16 +22,11 @@ public static Map findObsoleteProjectFiles( Predicate patternCheck = ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(pattern, ignorePattern, preserveHierarchy); return projectFiles.entrySet().stream() - .filter(entry -> patternCheck.test(entry.getKey()) && checkExportPattern(exportPattern, entry.getValue())) + .filter(entry -> patternCheck.test(entry.getKey())) .filter(entry -> isFileDontHaveUpdate(filesToUpload, entry.getKey(), preserveHierarchy)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private static boolean checkExportPattern(String exportPattern, File file) { - String fileExportPattern = ProjectFilesUtils.getExportPattern(file.getExportOptions()); - return exportPattern.equals(fileExportPattern != null ? Utils.normalizePath(fileExportPattern) : null); - } - public static SortedMap findObsoleteProjectDirectories( @NonNull Map projectFiles, @NonNull Map directoryIds, @NonNull List filesToUpload, @NonNull Map obsoleteDeletedProjectFiles diff --git a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java new file mode 100644 index 000000000..bf843b373 --- /dev/null +++ b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java @@ -0,0 +1,37 @@ +package com.crowdin.cli.commands.functionality; + +import com.crowdin.cli.utils.Utils; +import com.crowdin.client.sourcefiles.model.File; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ObsoleteSourcesUtilsTest { + + @Test + public void testCreatePath_PathExists() { + Map projectFiles = new HashMap() { + { + put(Utils.normalizePath("test/en/test.md"), new File()); + put(Utils.normalizePath("test/en/support.md"), new File()); + put(Utils.normalizePath("test/en/help.md"), new File()); + } + }; + boolean preserveHierarchy = true; + List filesToUpload = List.of("test/en/test.md", "test/en/help.md"); + String pattern = "/test/en/*.md"; + String exportPattern = "/test/%two_letters_code%/%original_path%/%original_file_name%"; + List ignorePattern = List.of("**/.*"); + + Map obsoleteFiles = ObsoleteSourcesUtils.findObsoleteProjectFiles(projectFiles, preserveHierarchy, + filesToUpload, pattern, exportPattern, ignorePattern); + + assertEquals(1, obsoleteFiles.size()); + } + +} From 301bcf05f230786c8a1aee613a7227816f1bcfac Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:37:49 +0200 Subject: [PATCH 2/5] fix list --- .../cli/commands/functionality/ObsoleteSourcesUtilsTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java index bf843b373..bcd4799e7 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,10 +24,10 @@ public void testCreatePath_PathExists() { } }; boolean preserveHierarchy = true; - List filesToUpload = List.of("test/en/test.md", "test/en/help.md"); + List filesToUpload = Arrays.asList("test/en/test.md", "test/en/help.md"); String pattern = "/test/en/*.md"; String exportPattern = "/test/%two_letters_code%/%original_path%/%original_file_name%"; - List ignorePattern = List.of("**/.*"); + List ignorePattern = Arrays.asList("**/.*"); Map obsoleteFiles = ObsoleteSourcesUtils.findObsoleteProjectFiles(projectFiles, preserveHierarchy, filesToUpload, pattern, exportPattern, ignorePattern); From f2e081264470bb80c0aef68b791c2c6f80223288 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Tue, 7 May 2024 11:43:46 +0200 Subject: [PATCH 3/5] adjust code and test --- .../commands/functionality/ObsoleteSourcesUtils.java | 10 +++++++++- .../functionality/ObsoleteSourcesUtilsTest.java | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java index 769abbafc..b90da8e30 100644 --- a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java +++ b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java @@ -22,11 +22,19 @@ public static Map findObsoleteProjectFiles( Predicate patternCheck = ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(pattern, ignorePattern, preserveHierarchy); return projectFiles.entrySet().stream() - .filter(entry -> patternCheck.test(entry.getKey())) + .filter(entry -> patternCheck.test(entry.getKey()) && checkExportPattern(exportPattern, entry.getValue())) .filter(entry -> isFileDontHaveUpdate(filesToUpload, entry.getKey(), preserveHierarchy)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + private static boolean checkExportPattern(String exportPattern, File file) { + String fileExportPattern = ProjectFilesUtils.getExportPattern(file.getExportOptions()); + if (fileExportPattern == null) { + return true; + } + return exportPattern.equals(Utils.normalizePath(fileExportPattern)); + } + public static SortedMap findObsoleteProjectDirectories( @NonNull Map projectFiles, @NonNull Map directoryIds, @NonNull List filesToUpload, @NonNull Map obsoleteDeletedProjectFiles diff --git a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java index bcd4799e7..cb54ab06b 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java @@ -15,7 +15,7 @@ public class ObsoleteSourcesUtilsTest { @Test - public void testCreatePath_PathExists() { + public void testFindObsoleteProjectFiles() { Map projectFiles = new HashMap() { { put(Utils.normalizePath("test/en/test.md"), new File()); @@ -33,6 +33,7 @@ public void testCreatePath_PathExists() { filesToUpload, pattern, exportPattern, ignorePattern); assertEquals(1, obsoleteFiles.size()); + assertEquals(true, obsoleteFiles.containsKey(Utils.normalizePath("test/en/support.md"))); } } From 5068328db70c8f3c426f1e496a53339b1c02190e Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Tue, 7 May 2024 11:55:01 +0200 Subject: [PATCH 4/5] normalize paths --- .../cli/commands/functionality/ObsoleteSourcesUtilsTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java index cb54ab06b..9164dad0d 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java @@ -24,7 +24,8 @@ public void testFindObsoleteProjectFiles() { } }; boolean preserveHierarchy = true; - List filesToUpload = Arrays.asList("test/en/test.md", "test/en/help.md"); + List filesToUpload = Arrays.asList(Utils.normalizePath("test/en/test.md"), + Utils.normalizePath("test/en/help.md")); String pattern = "/test/en/*.md"; String exportPattern = "/test/%two_letters_code%/%original_path%/%original_file_name%"; List ignorePattern = Arrays.asList("**/.*"); From 4578ccb33a9744d788a0ecb4f3dcf2fedbd83243 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 8 May 2024 13:09:32 +0200 Subject: [PATCH 5/5] cleanup var namings --- .../functionality/ObsoleteSourcesUtils.java | 11 ++++++----- .../functionality/ObsoleteSourcesUtilsTest.java | 13 +++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java index b90da8e30..9b56a01b3 100644 --- a/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java +++ b/src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java @@ -17,13 +17,14 @@ public class ObsoleteSourcesUtils { public static Map findObsoleteProjectFiles( @NonNull Map projectFiles, boolean preserveHierarchy, - @NonNull List filesToUpload, @NonNull String pattern, @NonNull String exportPattern, List ignorePattern + @NonNull List filesToUpload, @NonNull String sourcePattern, @NonNull String exportPattern, List ignorePatterns ) { Predicate patternCheck = - ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(pattern, ignorePattern, preserveHierarchy); + ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(sourcePattern, ignorePatterns, preserveHierarchy); return projectFiles.entrySet().stream() - .filter(entry -> patternCheck.test(entry.getKey()) && checkExportPattern(exportPattern, entry.getValue())) - .filter(entry -> isFileDontHaveUpdate(filesToUpload, entry.getKey(), preserveHierarchy)) + .filter(entry -> patternCheck.test(entry.getKey())) + .filter(entry -> checkExportPattern(exportPattern, entry.getValue())) + .filter(entry -> isFileNotInList(filesToUpload, entry.getKey(), preserveHierarchy)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } @@ -74,7 +75,7 @@ public static SortedMap findObsoleteProjectDirectories( return obsoleteDirs; } - private static boolean isFileDontHaveUpdate(List filesToUpload, String filePath, boolean preserveHierarchy) { + private static boolean isFileNotInList(List filesToUpload, String filePath, boolean preserveHierarchy) { String filePathRegex = "^" + (preserveHierarchy ? "" : Utils.PRESERVE_HIERARCHY_REGEX_PART) + Utils.regexPath(filePath) + "$"; return filesToUpload.stream() .noneMatch(Pattern.compile(filePathRegex).asPredicate()); diff --git a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java index 9164dad0d..82076e86f 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java @@ -16,7 +16,7 @@ public class ObsoleteSourcesUtilsTest { @Test public void testFindObsoleteProjectFiles() { - Map projectFiles = new HashMap() { + Map projectFilesFromApi = new HashMap() { { put(Utils.normalizePath("test/en/test.md"), new File()); put(Utils.normalizePath("test/en/support.md"), new File()); @@ -26,12 +26,13 @@ public void testFindObsoleteProjectFiles() { boolean preserveHierarchy = true; List filesToUpload = Arrays.asList(Utils.normalizePath("test/en/test.md"), Utils.normalizePath("test/en/help.md")); - String pattern = "/test/en/*.md"; - String exportPattern = "/test/%two_letters_code%/%original_path%/%original_file_name%"; - List ignorePattern = Arrays.asList("**/.*"); + String sourcePattern = Utils.normalizePath("/test/en/*.md"); + String exportPattern = Utils.normalizePath("/test/%two_letters_code%/%original_path%/%original_file_name%"); + List ignorePatterns = Arrays.asList(Utils.normalizePath("**/.*")); - Map obsoleteFiles = ObsoleteSourcesUtils.findObsoleteProjectFiles(projectFiles, preserveHierarchy, - filesToUpload, pattern, exportPattern, ignorePattern); + Map obsoleteFiles = ObsoleteSourcesUtils.findObsoleteProjectFiles(projectFilesFromApi, + preserveHierarchy, + filesToUpload, sourcePattern, exportPattern, ignorePatterns); assertEquals(1, obsoleteFiles.size()); assertEquals(true, obsoleteFiles.containsKey(Utils.normalizePath("test/en/support.md")));