diff --git a/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/api/traversal/TreeNode.java b/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/api/traversal/TreeNode.java index fe409bca7d33..7197222caa28 100644 --- a/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/api/traversal/TreeNode.java +++ b/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/api/traversal/TreeNode.java @@ -9,6 +9,7 @@ import com.dotcms.model.asset.FolderSync; import com.dotcms.model.asset.FolderView; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.function.Predicate; @@ -85,6 +86,16 @@ public List children() { return this.children; } + /** + * Sorts the children of the TreeNode based on their folder names in ascending order. If the + * TreeNode has no children or is null, this method does nothing. + */ + public void sortChildren() { + if (this.children != null && !this.children.isEmpty()) { + this.children.sort(Comparator.comparing(a -> a.folder().name())); + } + } + /** * Returns a list of child nodes of this TreeNode * Given that this is a recursive structure, this method returns a flattened list of all the @@ -105,6 +116,17 @@ public List assets() { return this.assets; } + /** + * Sorts the assets within the current TreeNode based on their names. The sorting is done in + * ascending order. If the TreeNode does not contain any assets or is null, this method does + * nothing. + */ + public void sortAssets() { + if (this.assets != null && !this.assets.isEmpty()) { + this.assets.sort(Comparator.comparing(AssetView::name)); + } + } + /** * Adds a child node to this {@code TreeNode}. * diff --git a/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/contenttype/model/type/ContentType.java b/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/contenttype/model/type/ContentType.java index 8c58f888d3ef..aae8257f40d4 100644 --- a/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/contenttype/model/type/ContentType.java +++ b/tools/dotcms-cli/api-data-model/src/main/java/com/dotcms/contenttype/model/type/ContentType.java @@ -167,7 +167,7 @@ public Boolean system() { @Nullable @Value.Default public Map metadata() { - return null; + return Collections.emptyMap(); } @Value.Default diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorService.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorService.java new file mode 100644 index 000000000000..c9f85649c51e --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorService.java @@ -0,0 +1,20 @@ +package com.dotcms.api.client; + +import java.nio.file.Path; + +/** + * This is a service interface for calculating file hashes. It is responsible for providing a method + * to calculate the SHA-256 hash of a given file. + */ +public interface FileHashCalculatorService { + + /** + * Calculates the SHA-256 hash of the content of the supplied file represented by the provided + * {@link Path}. + * + * @param path the path to the file whose content hash needs to be calculated + * @return the SHA-256 hash as a UNIX-formatted string + */ + String sha256toUnixHash(Path path); + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorServiceImpl.java new file mode 100644 index 000000000000..a85cb14784db --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/FileHashCalculatorServiceImpl.java @@ -0,0 +1,63 @@ +package com.dotcms.api.client; + +import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; +import com.dotcms.security.Encryptor; +import com.dotcms.security.HashBuilder; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import org.jboss.logging.Logger; + +/** + * This is a service class for calculating file hashes. It is responsible for providing a method to + * calculate the SHA-256 hash of a given file. This class is marked as + * {@link javax.enterprise.context.ApplicationScoped}, meaning that a single instance is shared + * across the entire application. + */ +@ApplicationScoped +public class FileHashCalculatorServiceImpl implements FileHashCalculatorService { + + @Inject + Logger logger; + + /** + * Calculates the SHA-256 hash of the content of the supplied file represented by the provided + * {@link Path}. The function will throw a {@link TraversalTaskException} if the SHA-256 + * algorithm is not found or there is an error reading the file. + * + * @param path the path to the file whose content hash needs to be calculated + * @return the SHA-256 hash as a UNIX-formatted string + * @throws TraversalTaskException if there is an error calculating the hash + */ + public String sha256toUnixHash(final Path path) { + + try { + + final HashBuilder sha256Builder = Encryptor.Hashing.sha256(); + final byte[] buffer = new byte[4096]; + int countBytes; + + try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) { + + countBytes = inputStream.read(buffer); + while (countBytes > 0) { + + sha256Builder.append(buffer, countBytes); + countBytes = inputStream.read(buffer); + } + } + + return sha256Builder.buildUnixHash(); + } catch (NoSuchAlgorithmException | IOException e) { + var errorMessage = String.format("Error calculating sha256 for file [%s]", path); + logger.error(errorMessage, e); + throw new TraversalTaskException(errorMessage, e); + } + } + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/PushServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/PushServiceImpl.java index 5db617801ea3..4b628ef616b4 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/PushServiceImpl.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/PushServiceImpl.java @@ -26,6 +26,7 @@ import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; @DefaultBean @@ -41,6 +42,9 @@ public class PushServiceImpl implements PushService { @Inject RemoteTraversalService remoteTraversalService; + @Inject + ManagedExecutor executor; + /** * Traverses the local folders and retrieves the hierarchical tree representation of their * contents with the push related information for each file and folder. Each folder is @@ -251,17 +255,16 @@ public void processTreeNodes(OutputOptionMixin output, ); var isRetry = retryAttempts > 0; - CompletableFuture> pushTreeFuture = CompletableFuture.supplyAsync( + CompletableFuture> pushTreeFuture = executor.supplyAsync( () -> remoteTraversalService.pushTreeNode( PushTraverseParams.builder().from(traverseParams) .progressBar(progressBar) - .logger(logger) .isRetry(isRetry).build() ) ); progressBar.setFuture(pushTreeFuture); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( progressBar ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/AbstractPushTraverseParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/AbstractPushTraverseParams.java index 63c5db564fa8..800a5d613524 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/AbstractPushTraverseParams.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/AbstractPushTraverseParams.java @@ -1,6 +1,5 @@ package com.dotcms.api.client.files.traversal; -import com.dotcms.api.client.files.traversal.data.Pusher; import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.command.PushContext; import com.dotcms.cli.common.ConsoleProgressBar; @@ -10,7 +9,6 @@ import javax.annotation.Nullable; import org.immutables.value.Value; import org.immutables.value.Value.Default; -import org.jboss.logging.Logger; /** * Just a class to compile all the params shared by various Traverse APIs @@ -29,12 +27,6 @@ public interface AbstractPushTraverseParams extends Serializable { @Default default int maxRetryAttempts(){return 0;} - @Nullable - Logger logger(); - - @Nullable - Pusher pusher(); - @Nullable ConsoleProgressBar progressBar(); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/LocalTraversalServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/LocalTraversalServiceImpl.java index e595fc475955..ff01d14cf445 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/LocalTraversalServiceImpl.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/LocalTraversalServiceImpl.java @@ -2,10 +2,13 @@ import static com.dotcms.common.AssetsUtils.parseLocalPath; +import com.dotcms.api.client.FileHashCalculatorService; import com.dotcms.api.client.files.traversal.data.Downloader; import com.dotcms.api.client.files.traversal.data.Retriever; import com.dotcms.api.client.files.traversal.task.LocalFolderTraversalTask; +import com.dotcms.api.client.files.traversal.task.LocalFolderTraversalTaskParams; import com.dotcms.api.client.files.traversal.task.PullTreeNodeTask; +import com.dotcms.api.client.files.traversal.task.PullTreeNodeTaskParams; import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.common.ConsoleProgressBar; import com.dotcms.common.AssetsUtils; @@ -14,11 +17,11 @@ import java.io.File; import java.nio.file.Paths; import java.util.List; -import java.util.concurrent.ForkJoinPool; import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import javax.ws.rs.NotFoundException; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -39,6 +42,12 @@ public class LocalTraversalServiceImpl implements LocalTraversalService { @Inject protected Downloader downloader; + @Inject + ManagedExecutor executor; + + @Inject + FileHashCalculatorService fileHashCalculatorService; + /** * Traverses the file system directory at the specified path and builds a hierarchical tree * representation of its contents. The folders and contents are compared to the remote server in @@ -84,16 +93,25 @@ public TraverseResult traverseLocalFolder(final LocalTraverseParams params) { logger.debug(String.format("Language [%s] doesn't exist on remote server.", localPath.language())); } - var forkJoinPool = ForkJoinPool.commonPool(); + var task = new LocalFolderTraversalTask( + logger, + executor, + retriever, + fileHashCalculatorService + ); - var task = new LocalFolderTraversalTask(LocalTraverseParams.builder() - .from(params) - .logger(logger) - .retriever(retriever) + task.setTraversalParams(LocalFolderTraversalTaskParams.builder() .siteExists(siteExists) + .sourcePath(params.sourcePath()) + .workspace(params.workspace()) + .removeAssets(params.removeAssets()) + .removeFolders(params.removeFolders()) + .ignoreEmptyFolders(params.ignoreEmptyFolders()) + .failFast(params.failFast()) .build() ); - var result = forkJoinPool.invoke(task); + + var result = task.compute(); return TraverseResult.builder() .exceptions(result.getLeft()) .localPaths(localPath) @@ -125,18 +143,25 @@ public List pullTreeNode(final TreeNode rootNode, final String destin rootNode.folder().host()); // --- - var forkJoinPool = ForkJoinPool.commonPool(); var task = new PullTreeNodeTask( logger, + executor, downloader, - filteredRoot, - rootPath.toString(), - overwrite, - generateEmptyFolders, - failFast, - language, - progressBar); - return forkJoinPool.invoke(task); + fileHashCalculatorService + ); + + task.setTraversalParams(PullTreeNodeTaskParams.builder() + .rootNode(filteredRoot) + .destination(rootPath.toString()) + .overwrite(overwrite) + .generateEmptyFolders(generateEmptyFolders) + .failFast(failFast) + .language(language) + .progressBar(progressBar) + .build() + ); + + return task.compute(); } } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceImpl.java index d9866521a0f8..071b773a16fa 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceImpl.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceImpl.java @@ -3,21 +3,23 @@ import com.dotcms.api.client.files.traversal.data.Pusher; import com.dotcms.api.client.files.traversal.data.Retriever; import com.dotcms.api.client.files.traversal.task.PushTreeNodeTask; +import com.dotcms.api.client.files.traversal.task.PushTreeNodeTaskParams; import com.dotcms.api.client.files.traversal.task.RemoteFolderTraversalTask; +import com.dotcms.api.client.files.traversal.task.RemoteFolderTraversalTaskParams; import com.dotcms.api.traversal.Filter; import com.dotcms.api.traversal.TreeNode; import com.dotcms.common.AssetsUtils; import com.dotcms.model.asset.FolderView; import io.quarkus.arc.DefaultBean; -import org.apache.commons.lang3.tuple.Pair; -import org.jboss.logging.Logger; -import javax.enterprise.context.Dependent; -import javax.enterprise.context.control.ActivateRequestContext; -import javax.inject.Inject; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ForkJoinPool; +import javax.enterprise.context.Dependent; +import javax.enterprise.context.control.ActivateRequestContext; +import javax.inject.Inject; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; +import org.jboss.logging.Logger; /** * Service for traversing a dotCMS remote location and building a hierarchical tree representation of @@ -32,10 +34,13 @@ public class RemoteTraversalServiceImpl implements RemoteTraversalService { Logger logger; @Inject - protected Retriever retriever; + Retriever retriever; @Inject - protected Pusher pusher; + Pusher pusher; + + @Inject + ManagedExecutor executor; /** * Traverses the dotCMS remote location at the specified remote path and builds a hierarchical tree @@ -82,25 +87,28 @@ public Pair, TreeNode> traverseRemoteFolder( excludeAssetPatterns); // --- - var forkJoinPool = ForkJoinPool.commonPool(); - var task = new RemoteFolderTraversalTask( logger, - retriever, - filter, - dotCMSPath.site(), - FolderView.builder() + executor, + retriever + ); + + task.setTraversalParams(RemoteFolderTraversalTaskParams.builder() + .filter(filter) + .siteName(dotCMSPath.site()) + .folder(FolderView.builder() .host(dotCMSPath.site()) .path(dotCMSPath.folderPath().toString()) .name(dotCMSPath.folderName()) .level(0) - .build(), - true, - depthToUse, - failFast + .build()) + .isRoot(true) + .depth(depthToUse) + .failFast(failFast) + .build() ); - return forkJoinPool.invoke(task); + return task.compute(); } /** @@ -128,12 +136,25 @@ public List pushTreeNode(PushTraverseParams traverseParams) { } // --- - var forkJoinPool = ForkJoinPool.commonPool(); - var task = new PushTreeNodeTask(PushTraverseParams.builder() - .from(traverseParams) - .pusher(pusher) - .build()); - return forkJoinPool.invoke(task); + var task = new PushTreeNodeTask( + logger, + executor, + traverseParams.pushContext(), + pusher + ); + + task.setTraversalParams(PushTreeNodeTaskParams.builder() + .workspacePath(traverseParams.workspacePath()) + .localPaths(traverseParams.localPaths()) + .rootNode(traverseParams.rootNode()) + .failFast(traverseParams.failFast()) + .isRetry(traverseParams.isRetry()) + .maxRetryAttempts(traverseParams.maxRetryAttempts()) + .progressBar(traverseParams.progressBar()) + .build() + ); + + return task.compute(); } /** diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/data/Retriever.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/data/Retriever.java index 9fa8ccbd9ed1..8e806bed9162 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/data/Retriever.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/data/Retriever.java @@ -1,5 +1,8 @@ package com.dotcms.api.client.files.traversal.data; +import static com.dotcms.common.AssetsUtils.buildRemoteAssetURL; +import static com.dotcms.common.AssetsUtils.buildRemoteURL; + import com.dotcms.api.AssetAPI; import com.dotcms.api.LanguageAPI; import com.dotcms.api.client.model.RestClientFactory; @@ -8,17 +11,11 @@ import com.dotcms.model.asset.FolderView; import com.dotcms.model.language.Language; import com.google.common.collect.ImmutableList; - +import java.util.ArrayList; +import java.util.List; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -import static com.dotcms.common.AssetsUtils.buildRemoteAssetURL; -import static com.dotcms.common.AssetsUtils.buildRemoteURL; /** * Utility class for retrieving folder and asset information from the remote server using REST calls. @@ -133,12 +130,7 @@ public FolderView retrieveFolderInformation(String siteName, String folderPath, subFolders.add(child.withLevel(level + 1)); } - // Ordering foundFolder by name - List sortedFolders = subFolders.stream() - .sorted(Comparator.comparing(FolderView::name)) - .collect(Collectors.toList()); - - foundFolder = foundFolder.withSubFolders(ImmutableList.copyOf(sortedFolders)); + foundFolder = foundFolder.withSubFolders(ImmutableList.copyOf(subFolders)); } return foundFolder; diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractLocalFolderTraversalTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractLocalFolderTraversalTaskParams.java new file mode 100644 index 000000000000..3bedb1a7f8d0 --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractLocalFolderTraversalTaskParams.java @@ -0,0 +1,72 @@ +package com.dotcms.api.client.files.traversal.task; + +import com.dotcms.model.annotation.ValueType; +import java.io.File; +import java.io.Serializable; +import org.immutables.value.Value; +import org.immutables.value.Value.Default; + +/** + * This interface represents the parameters for a {@link LocalFolderTraversalTask}. + */ +@ValueType +@Value.Immutable +public interface AbstractLocalFolderTraversalTaskParams extends Serializable { + + // START-NOSCAN + + /** + * Returns a boolean indicating whether the site exists. + * + * @return true if the site exists, false otherwise. Default is false. + */ + @Default + default boolean siteExists() { + return false; + } + + /** + * Returns the source path for the traversal process. + * + * @return String representing the source path. + */ + String sourcePath(); + + /** + * Returns the workspace file. + * + * @return File representing the workspace. + */ + File workspace(); + + /** + * Returns a boolean flag indicating whether to remove assets. + * + * @return true if the assets should be removed, false otherwise. + */ + boolean removeAssets(); + + /** + * Returns a boolean flag indicating whether to remove folders. + * + * @return true if the folders should be removed, false otherwise. + */ + boolean removeFolders(); + + /** + * Returns a boolean flag indicating whether to ignore empty folders. + * + * @return true if the empty folders should be ignored, false otherwise. + */ + boolean ignoreEmptyFolders(); + + /** + * Returns a boolean flag indicating whether to fail fast. + * + * @return true if the operation should fail fast, false otherwise. + */ + boolean failFast(); + + // END-NOSCAN + +} \ No newline at end of file diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPullTreeNodeTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPullTreeNodeTaskParams.java new file mode 100644 index 000000000000..26e0f6309e7b --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPullTreeNodeTaskParams.java @@ -0,0 +1,69 @@ +package com.dotcms.api.client.files.traversal.task; + +import com.dotcms.api.traversal.TreeNode; +import com.dotcms.cli.common.ConsoleProgressBar; +import com.dotcms.model.annotation.ValueType; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * This interface represents the parameters for a {@link PullTreeNodeTask}. + */ +@ValueType +@Value.Immutable +public interface AbstractPullTreeNodeTaskParams extends Serializable { + + // START-NOSCAN + + /** + * Returns the root node of the file system tree as a {@link TreeNode}. + * + * @return the root {@link TreeNode} + */ + TreeNode rootNode(); + + /** + * Returns the destination path to save the pulled files. + * + * @return the destination path as a String + */ + String destination(); + + /** + * Determines if the operation should overwrite existing files. + * + * @return true if the operation should overwrite, false otherwise + */ + boolean overwrite(); + + /** + * Determines if empty folders should be generated. + * + * @return true if empty folders should be generated, false otherwise + */ + boolean generateEmptyFolders(); + + /** + * Determines if the operation should fail fast or continue on error. + * + * @return true if the operation should fail quickly on error, false if it should continue + */ + boolean failFast(); + + /** + * Returns the language of the assets. + * + * @return the language of the assets as a String + */ + String language(); + + /** + * Returns the {@link ConsoleProgressBar} for tracking the pull progress. + * + * @return the {@link ConsoleProgressBar} + */ + ConsoleProgressBar progressBar(); + + // END-NOSCAN + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPushTreeNodeTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPushTreeNodeTaskParams.java new file mode 100644 index 000000000000..6448350c0c45 --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractPushTreeNodeTaskParams.java @@ -0,0 +1,82 @@ +package com.dotcms.api.client.files.traversal.task; + +import com.dotcms.api.traversal.TreeNode; +import com.dotcms.cli.common.ConsoleProgressBar; +import com.dotcms.common.LocalPathStructure; +import com.dotcms.model.annotation.ValueType; +import java.io.Serializable; +import javax.annotation.Nullable; +import org.immutables.value.Value; +import org.immutables.value.Value.Default; + +/** + * This interface represents the parameters for a {@link PushTreeNodeTask}. + */ +@ValueType +@Value.Immutable +public interface AbstractPushTreeNodeTaskParams extends Serializable { + + // START-NOSCAN + + /** + * Returns the path to the workspace directory. + * + * @return the path to the workspace directory + */ + String workspacePath(); + + /** + * Returns the local path structure. + * + * @return the local path structure + */ + LocalPathStructure localPaths(); + + /** + * Returns the root node of the file system tree as a {@link TreeNode}. + * + * @return the root {@link TreeNode} + */ + TreeNode rootNode(); + + /** + * Determines if the operation should fail fast or continue on error. + * + * @return true if the operation should fail quickly on error, false if it should continue + */ + @Default + default boolean failFast() { + return true; + } + + /** + * Determine whether it is a retry or not. + * + * @return false by default + */ + @Default + default boolean isRetry() { + return false; + } + + /** + * Returns the maximum number of retry attempts for the push operation. + * + * @return the maximum number of retry attempts + */ + @Default + default int maxRetryAttempts() { + return 0; + } + + /** + * Returns the {@link ConsoleProgressBar} for tracking the push progress. + * + * @return the {@link ConsoleProgressBar} + */ + @Nullable + ConsoleProgressBar progressBar(); + + // END-NOSCAN + +} \ No newline at end of file diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractRemoteFolderTraversalTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractRemoteFolderTraversalTaskParams.java new file mode 100644 index 000000000000..fdc993a3fa92 --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/AbstractRemoteFolderTraversalTaskParams.java @@ -0,0 +1,63 @@ +package com.dotcms.api.client.files.traversal.task; + +import com.dotcms.api.traversal.Filter; +import com.dotcms.model.annotation.ValueType; +import com.dotcms.model.asset.FolderView; +import java.io.Serializable; +import org.immutables.value.Value; + +/** + * This interface represents the parameters for a {@link RemoteFolderTraversalTask}. + */ +@ValueType +@Value.Immutable +public interface AbstractRemoteFolderTraversalTaskParams extends Serializable { + + // START-NOSCAN + + /** + * Retrieves the {@link Filter} instance used to include or exclude folders and assets. + * + * @return The filter instance. + */ + Filter filter(); + + /** + * Retrieves the name of the site containing the folder to traverse. + * + * @return The site name. + */ + String siteName(); + + /** + * Retrieves the folder to traverse. + * + * @return The {@link FolderView} representing the folder to traverse. + */ + FolderView folder(); + + /** + * Indicates whether this task is for the root folder. + * + * @return true if this task is for the root folder, false otherwise. + */ + boolean isRoot(); + + /** + * Retrieves the maximum depth to traverse the directory tree. + * + * @return The maximum depth. + */ + int depth(); + + /** + * This method is used to determine if the traversal task should fail fast or continue on + * error. + * + * @return true if the traversal task should fail fast, false otherwise. + */ + boolean failFast(); + + // END-NOSCAN + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/LocalFolderTraversalTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/LocalFolderTraversalTask.java index d571b2170a98..a5ef48f40783 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/LocalFolderTraversalTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/LocalFolderTraversalTask.java @@ -5,8 +5,10 @@ import static com.dotcms.model.asset.BasicMetadataFields.PATH_META_KEY; import static com.dotcms.model.asset.BasicMetadataFields.SHA256_META_KEY; -import com.dotcms.api.client.files.traversal.LocalTraverseParams; +import com.dotcms.api.client.FileHashCalculatorService; +import com.dotcms.api.client.files.traversal.data.Retriever; import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.common.HiddenFileFilter; import com.dotcms.common.LocalPathStructure; @@ -17,7 +19,6 @@ import com.dotcms.model.asset.FolderSync; import com.dotcms.model.asset.FolderSync.Builder; import com.dotcms.model.asset.FolderView; -import com.dotcms.security.Utils; import com.google.common.base.Strings; import java.io.File; import java.util.ArrayList; @@ -25,90 +26,137 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import javax.enterprise.context.Dependent; import javax.ws.rs.NotFoundException; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** * Recursive task for traversing a file system directory and building a hierarchical tree - * representation of its contents. The folders and contents are compared to the remote server in order to determine - * if there are any differences between the local and remote file system. - * This task is used to split the traversal into smaller sub-tasks - * that can be executed in parallel, allowing for faster traversal of large directory structures. + * representation of its contents. The folders and contents are compared to the remote server in + * order to determine if there are any differences between the local and remote file system. This + * task is used to split the traversal into smaller sub-tasks that can be executed in parallel, + * allowing for faster traversal of large directory structures. */ -public class LocalFolderTraversalTask extends RecursiveTask, TreeNode>> { +@Dependent +public class LocalFolderTraversalTask extends TaskProcessor { - private final LocalTraverseParams params; + private final ManagedExecutor executor; private final Logger logger; + private final Retriever retriever; + + private final FileHashCalculatorService fileHashService; + + private LocalFolderTraversalTaskParams traversalTaskParams; + /** * Constructs a new LocalFolderTraversalTask instance. - * @param params the traverse parameters + * + * @param executor the executor for parallel execution of traversal tasks + * @param logger the logger for logging debug information + * @param retriever The retriever used for REST calls and other operations. + * @param fileHashService The file hash calculator service + */ + public LocalFolderTraversalTask(final Logger logger, final ManagedExecutor executor, + final Retriever retriever, final FileHashCalculatorService fileHashService) { + this.executor = executor; + this.logger = logger; + this.retriever = retriever; + this.fileHashService = fileHashService; + } + + /** + * Sets the traversal parameters for the LocalFolderTraversalTask. This method provides a way to + * inject necessary configuration after the instance of LocalFolderTraversalTask has been + * created by the container, which is a common pattern when working with frameworks like Quarkus + * that manage object creation and dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The traversal parameters */ - public LocalFolderTraversalTask(final LocalTraverseParams params) { - this.params = params; - this.logger = params.logger(); + public void setTraversalParams(final LocalFolderTraversalTaskParams params) { + this.traversalTaskParams = params; } /** * Executes the folder traversal task and returns a TreeNode representing the directory tree * rooted at the folder specified in the constructor. * - * @return A TreeNode representing the directory tree rooted at the folder specified in the - * constructor. + * @return A Pair object containing a list of exceptions encountered during traversal and the + * resulting TreeNode representing the directory tree at the specified folder. */ - @Override - protected Pair, TreeNode> compute() { + public Pair, TreeNode> compute() { + + CompletionService, TreeNode>> completionService = + new ExecutorCompletionService<>(executor); var errors = new ArrayList(); - File folderOrFile = new File(params.sourcePath()); + File folderOrFile = new File(traversalTaskParams.sourcePath()); - TreeNode currentNode = null; + AtomicReference currentNode = new AtomicReference<>(); try { - final var localPathStructure = parseLocalPath(params.workspace(), folderOrFile); - currentNode = gatherSyncInformation(params.workspace(), folderOrFile, localPathStructure); + final var localPathStructure = parseLocalPath(traversalTaskParams.workspace(), + folderOrFile); + currentNode.set(gatherSyncInformation(traversalTaskParams.workspace(), folderOrFile, + localPathStructure)); } catch (Exception e) { - if (params.failFast()) { + if (traversalTaskParams.failFast()) { throw e; } else { errors.add(e); } } - if ( null != currentNode && folderOrFile.isDirectory() ) { + if (null != currentNode.get() && folderOrFile.isDirectory()) { File[] files = folderOrFile.listFiles(new HiddenFileFilter()); if (files != null) { - List forks = new ArrayList<>(); + var toProcessCount = 0; for (File file : files) { if (file.isDirectory()) { - LocalFolderTraversalTask subTask = new LocalFolderTraversalTask( - LocalTraverseParams.builder() - .from(params) - .sourcePath(file.getAbsolutePath()) - .build() + + var subTask = new LocalFolderTraversalTask( + logger, executor, retriever, fileHashService ); - forks.add(subTask); - subTask.fork(); + + subTask.setTraversalParams(LocalFolderTraversalTaskParams.builder() + .from(traversalTaskParams) + .sourcePath(file.getAbsolutePath()) + .build() + ); + + completionService.submit(subTask::compute); + toProcessCount++; } } - for (LocalFolderTraversalTask task : forks) { - var taskResult = task.join(); + // Wait for all tasks to complete and gather the results + Function, TreeNode>, Void> processFunction = taskResult -> { errors.addAll(taskResult.getLeft()); - currentNode.addChild(taskResult.getRight()); - } + currentNode.get().addChild(taskResult.getRight()); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); } } - return Pair.of(errors, currentNode); + return Pair.of(errors, currentNode.get()); } /** @@ -120,7 +168,7 @@ protected Pair, TreeNode> compute() { * @return The TreeNode containing the synchronization information for the folder or file */ private TreeNode gatherSyncInformation(File workspaceFile, File folderOrFile, - LocalPathStructure localPathStructure) { + LocalPathStructure localPathStructure) { var live = statusToBoolean(localPathStructure.status()); var lang = localPathStructure.language(); @@ -153,7 +201,8 @@ private TreeNode gatherSyncInformation(File workspaceFile, File folderOrFile, // Checking if we need to push files checkAssetsToPush(workspaceFile, live, lang, assetVersions, files, remoteFolder); } catch (Exception e) { - var message = String.format("Error processing folder [%s]", folderOrFile.getAbsolutePath()); + var message = String.format("Error processing folder [%s]", + folderOrFile.getAbsolutePath()); logger.error(message, e); throw new TraversalTaskException(message, e); } @@ -172,7 +221,8 @@ private TreeNode gatherSyncInformation(File workspaceFile, File folderOrFile, } else { - final var parentLocalPathStructure = parseLocalPath(workspaceFile, folderOrFile.getParentFile()); + final var parentLocalPathStructure = parseLocalPath(workspaceFile, + folderOrFile.getParentFile()); var folder = folderViewFromFile(parentLocalPathStructure); var assetVersions = AssetVersionsView.builder(); @@ -203,11 +253,13 @@ private TreeNode gatherSyncInformation(File workspaceFile, File folderOrFile, asset.build() ); } else { - logger.debug(String.format("File [%s] - live [%b] - lang [%s] - already exist in the server.", + logger.debug(String.format( + "File [%s] - live [%b] - lang [%s] - already exist in the server.", localPathStructure.filePath(), live, lang)); } } catch (Exception e) { - var message = String.format("Error processing file [%s]", folderOrFile.getAbsolutePath()); + var message = String.format("Error processing file [%s]", + folderOrFile.getAbsolutePath()); logger.error(message, e); throw new TraversalTaskException(message, e); } @@ -221,16 +273,16 @@ private TreeNode gatherSyncInformation(File workspaceFile, File folderOrFile, /** * Checks if files need to be pushed to the remote server. * - * @param workspaceFile the workspace file - * @param live the live status - * @param lang the language + * @param workspaceFile the workspace file + * @param live the live status + * @param lang the language * @param assetVersionsBuilder the parent folder asset versions view builder - * @param folderChildren the files to check - * @param remoteFolder the remote folder + * @param folderChildren the files to check + * @param remoteFolder the remote folder */ private void checkAssetsToPush(File workspaceFile, boolean live, String lang, - AssetVersionsView.Builder assetVersionsBuilder, - File[] folderChildren, FolderView remoteFolder) { + AssetVersionsView.Builder assetVersionsBuilder, + File[] folderChildren, FolderView remoteFolder) { if (folderChildren != null) { for (File file : folderChildren) { @@ -248,9 +300,11 @@ private void checkAssetsToPush(File workspaceFile, boolean live, String lang, if (pushInfo.push()) { - logger.debug(String.format("Marking file [%s] - live [%b] - lang [%s] for push " + + logger.debug(String.format( + "Marking file [%s] - live [%b] - lang [%s] for push " + "- New [%b] - Modified [%b].", - file.toPath(), live, lang, pushInfo.isNew(), pushInfo.isModified())); + file.toPath(), live, lang, pushInfo.isNew(), + pushInfo.isModified())); final AssetSync syncData = AssetSync.builder() .markedForPush(true) @@ -263,7 +317,8 @@ private void checkAssetsToPush(File workspaceFile, boolean live, String lang, build() ); } else { - logger.debug(String.format("File [%s] - live [%b] - lang [%s] - already exist in the server.", + logger.debug(String.format( + "File [%s] - live [%b] - lang [%s] - already exist in the server.", file.toPath(), live, lang)); } } @@ -274,9 +329,9 @@ private void checkAssetsToPush(File workspaceFile, boolean live, String lang, /** * Checks if folders need to be pushed to the remote server. * - * @param folder the parent folder view builder - * @param remoteFolder the remote folder - * @param folderFiles the internal files of the folder + * @param folder the parent folder view builder + * @param remoteFolder the remote folder + * @param folderFiles the internal files of the folder */ private void checkFolderToPush(FolderView.Builder folder, FolderView remoteFolder, @@ -284,7 +339,7 @@ private void checkFolderToPush(FolderView.Builder folder, if (remoteFolder == null) { boolean markForPush = false; - if (params.ignoreEmptyFolders()) { + if (traversalTaskParams.ignoreEmptyFolders()) { if (folderFiles != null && folderFiles.length > 0) { // Does not exist on remote server, so we need to push it markForPush = true; @@ -300,18 +355,18 @@ private void checkFolderToPush(FolderView.Builder folder, /** * Checks if files need to be removed from the remote server. * - * @param live the live status - * @param lang the language - * @param assetVersions the parent folder asset versions view builder - * @param folderChildren the files to check - * @param remoteFolder the remote folder + * @param live the live status + * @param lang the language + * @param assetVersions the parent folder asset versions view builder + * @param folderChildren the files to check + * @param remoteFolder the remote folder */ private void checkAssetsToRemove(boolean live, String lang, AssetVersionsView.Builder assetVersions, File[] folderChildren, FolderView remoteFolder) { // The option to remove assets is disabled - if (!params.removeAssets()) { + if (!traversalTaskParams.removeAssets()) { return; } @@ -347,18 +402,18 @@ private void checkAssetsToRemove(boolean live, String lang, /** * Checks if folders need to be removed from the remote server. * - * @param live the live status - * @param lang the language - * @param folder the parent folder view builder - * @param folderChildren the files to check - * @param remoteFolder the remote folder + * @param live the live status + * @param lang the language + * @param folder the parent folder view builder + * @param folderChildren the files to check + * @param remoteFolder the remote folder */ private void checkFoldersToRemove(boolean live, String lang, FolderView.Builder folder, File[] folderChildren, FolderView remoteFolder) { // The option to remove folders is disabled - if (!params.removeFolders() && !params.removeAssets()) { + if (!traversalTaskParams.removeFolders() && !traversalTaskParams.removeAssets()) { return; } @@ -379,7 +434,7 @@ private void checkFoldersToRemove(boolean live, String lang, boolean ignore = false; - if (params.ignoreEmptyFolders()) { + if (traversalTaskParams.ignoreEmptyFolders()) { //This is basically a check for delete ignore = ignoreFolder(live, lang, remoteSubFolder); } @@ -394,12 +449,14 @@ private void checkFoldersToRemove(boolean live, String lang, subFolder = subFolder.withAssets(assetVersions.build()); // Folder exist on remote server, but not locally, so we need to remove it - logger.debug(String.format("Marking folder [%s] for delete.", subFolder.path())); - if (params.removeFolders()) { + logger.debug(String.format("Marking folder [%s] for delete.", + subFolder.path())); + if (traversalTaskParams.removeFolders()) { final Optional existingSyncData = subFolder.sync(); final Builder builder = FolderSync.builder(); existingSyncData.ifPresent(builder::from); - subFolder = subFolder.withSync(builder.markedForDelete(true).build()); + subFolder = subFolder.withSync( + builder.markedForDelete(true).build()); } folder.addSubFolders(subFolder); } @@ -410,8 +467,10 @@ private void checkFoldersToRemove(boolean live, String lang, } /** - * explore assets in the remote folder and figure out if they match status and language - * if so we can not ignore the remote folder cuz it has assets matching the status and language were in + * explore assets in the remote folder and figure out if they match status and language if so + * we can not ignore the remote folder cuz it has assets matching the status and language were + * in + * * @param live * @param lang * @param remoteSubFolder @@ -434,6 +493,7 @@ private boolean ignoreFolder(boolean live, String lang, FolderView remoteSubFold /** * Take the remote folder representation and find its equivalent locally + * * @param folderChildren * @param remote * @return @@ -453,11 +513,12 @@ private boolean findLocalFolderMatch(File[] folderChildren, FolderView remote) { * Retrieves an asset information from the remote server. * * @param localPathStructure the local path structure - * @return The AssetVersionsView representing the retrieved asset data, or null if it doesn't exist + * @return The AssetVersionsView representing the retrieved asset data, or null if it doesn't + * exist */ private AssetVersionsView retrieveAsset(LocalPathStructure localPathStructure) { - if (!params.siteExists()) { + if (!traversalTaskParams.siteExists()) { // Site doesn't exist on remote server // No need to pass a siteExists flag we could NullRetriever when the site doesn't exist return null; @@ -466,15 +527,16 @@ private AssetVersionsView retrieveAsset(LocalPathStructure localPathStructure) { AssetVersionsView remoteAsset = null; try { - remoteAsset = params.retriever().retrieveAssetInformation( + remoteAsset = retriever.retrieveAssetInformation( localPathStructure.site(), localPathStructure.folderPath(), localPathStructure.fileName() ); } catch (NotFoundException e) { // File doesn't exist on remote server - logger.debug(String.format("Local file [%s] in folder [%s] doesn't exist on remote server.", - localPathStructure.fileName(), localPathStructure.folderPath())); + logger.debug( + String.format("Local file [%s] in folder [%s] doesn't exist on remote server.", + localPathStructure.fileName(), localPathStructure.folderPath())); } return remoteAsset; @@ -499,7 +561,7 @@ private FolderView retrieveFolder(LocalPathStructure localPathStructure) { */ private FolderView retrieveFolder(final String site, final String folderPath) { - if (!params.siteExists()) { + if (!traversalTaskParams.siteExists()) { // Site doesn't exist on remote server return null; } @@ -507,10 +569,11 @@ private FolderView retrieveFolder(final String site, final String folderPath) { FolderView remoteFolder = null; try { - remoteFolder = params.retriever().retrieveFolderInformation(site, folderPath); + remoteFolder = retriever.retrieveFolderInformation(site, folderPath); } catch (NotFoundException e) { // Folder doesn't exist on remote server - logger.debug(String.format("Local folder [%s] doesn't exist on remote server.", folderPath)); + logger.debug( + String.format("Local folder [%s] doesn't exist on remote server.", folderPath)); } return remoteFolder; @@ -526,7 +589,7 @@ private FolderView retrieveFolder(final String site, final String folderPath) { * @return true if the file needs to be removed, false otherwise */ private boolean shouldRemoveAsset(final boolean live, final String lang, - AssetView version, File[] localFolderFiles) { + AssetView version, File[] localFolderFiles) { // Make sure we are handling the proper status and language boolean match = matchByStatusAndLang(live, lang, version); @@ -562,7 +625,7 @@ private boolean shouldRemoveAsset(final boolean live, final String lang, * @return The PushInfo object representing the push information for the file */ private PushInfo shouldPushFile(final boolean live, final String lang, - File file, final AssetVersionsView remoteAsset) { + File file, final AssetVersionsView remoteAsset) { if (remoteAsset == null) { return pushInfoForNoRemote(file); @@ -587,7 +650,7 @@ private PushInfo shouldPushFile(final boolean live, final String lang, } // Local SHA-256 - final String localFileHash = Utils.Sha256toUnixHash(file.toPath()); + final String localFileHash = fileHashService.sha256toUnixHash(file.toPath()); var push = true; var isNew = true; @@ -616,7 +679,7 @@ private PushInfo shouldPushFile(final boolean live, final String lang, private PushInfo pushInfoForNoRemote(File file) { // Local SHA-256 - final String localFileHash = Utils.Sha256toUnixHash(file.toPath()); + final String localFileHash = fileHashService.sha256toUnixHash(file.toPath()); return new PushInfo(true, true, false, localFileHash); } @@ -774,7 +837,7 @@ String fileHash() { */ PushType pushType() { PushType pushType = isNew() ? PushType.NEW : PushType.UNKNOWN; - if(pushType == PushType.UNKNOWN){ + if (pushType == PushType.UNKNOWN) { pushType = isModified() ? PushType.MODIFIED : PushType.UNKNOWN; } return pushType; diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PullTreeNodeTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PullTreeNodeTask.java index c17c5dfa2f1b..b24e743671ba 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PullTreeNodeTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PullTreeNodeTask.java @@ -1,88 +1,94 @@ package com.dotcms.api.client.files.traversal.task; +import static com.dotcms.common.AssetsUtils.buildRemoteAssetURL; +import static com.dotcms.common.AssetsUtils.buildRemoteURL; +import static com.dotcms.model.asset.BasicMetadataFields.SHA256_META_KEY; + +import com.dotcms.api.client.FileHashCalculatorService; import com.dotcms.api.client.files.traversal.data.Downloader; import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.api.traversal.TreeNode; -import com.dotcms.cli.common.ConsoleProgressBar; import com.dotcms.model.asset.AssetRequest; import com.dotcms.model.asset.AssetView; import com.dotcms.model.asset.FolderView; -import com.dotcms.security.Utils; -import org.jboss.logging.Logger; - import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.RecursiveTask; - -import static com.dotcms.common.AssetsUtils.buildRemoteAssetURL; -import static com.dotcms.common.AssetsUtils.buildRemoteURL; -import static com.dotcms.model.asset.BasicMetadataFields.SHA256_META_KEY; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.context.ManagedExecutor; +import org.jboss.logging.Logger; /** * Recursive task for pulling the contents of a tree node from a remote server. */ -public class PullTreeNodeTask extends RecursiveTask> { +@Dependent +public class PullTreeNodeTask extends TaskProcessor { - private final TreeNode rootNode; - private final String destination; - private final boolean overwrite; - private final boolean generateEmptyFolders; - private final boolean failFast; - private final String language; + private final ManagedExecutor executor; private final Downloader downloader; - private final ConsoleProgressBar progressBar; + private final Logger logger; - private Logger logger; + private final FileHashCalculatorService fileHashService; + + private PullTreeNodeTaskParams traversalTaskParams; /** * Constructs a new PullTreeNodeTask. * - * @param logger logger for logging messages - * @param downloader class responsible for handling the downloading of assets - * from a given path through the AssetAPI - * @param rootNode the root node of the file system tree - * @param destination the destination path to save the pulled files - * @param overwrite true to overwrite existing files, false otherwise - * @param generateEmptyFolders true to generate empty folders, false otherwise - * @param failFast true to fail fast, false to continue on error - * @param language the language of the assets - * @param progressBar the progress bar for tracking the pull progress + * @param logger logger for logging messages + * @param executor the executor for parallel execution of pulling tasks + * @param downloader class responsible for handling the downloading of assets from a given + * path through the AssetAPI + * @param fileHashService The file hash calculator service */ - public PullTreeNodeTask(final Logger logger, - final Downloader downloader, - final TreeNode rootNode, - final String destination, - final boolean overwrite, - final boolean generateEmptyFolders, - final boolean failFast, - final String language, - final ConsoleProgressBar progressBar) { + public PullTreeNodeTask(final Logger logger, final ManagedExecutor executor, + final Downloader downloader, final FileHashCalculatorService fileHashService) { this.logger = logger; + this.executor = executor; this.downloader = downloader; - this.rootNode = rootNode; - this.overwrite = overwrite; - this.destination = destination; - this.generateEmptyFolders = generateEmptyFolders; - this.failFast = failFast; - this.language = language; - this.progressBar = progressBar; + this.fileHashService = fileHashService; + } + + /** + * Sets the traversal parameters for the PullTreeNodeTask. This method provides a way to inject + * necessary configuration after the instance of PullTreeNodeTask has been created by the + * container, which is a common pattern when working with frameworks like Quarkus that manage + * object creation and dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The traversal parameters + */ + public void setTraversalParams(final PullTreeNodeTaskParams params) { + this.traversalTaskParams = params; } - @Override - protected List compute() { + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); var errors = new ArrayList(); // Create the folder for the current node try { - createFolderInFileSystem(destination, rootNode.folder()); + createFolderInFileSystem( + traversalTaskParams.destination(), + traversalTaskParams.rootNode().folder() + ); } catch (Exception e) { - if (failFast) { + if (traversalTaskParams.failFast()) { throw e; } else { errors.add(e); @@ -90,11 +96,15 @@ protected List compute() { } // Create files for the assets in the current node - for (AssetView asset : rootNode.assets()) { + for (AssetView asset : traversalTaskParams.rootNode().assets()) { try { - createFileInFileSystem(destination, rootNode.folder(), asset); + createFileInFileSystem( + traversalTaskParams.destination(), + traversalTaskParams.rootNode().folder(), + asset + ); } catch (Exception e) { - if (failFast) { + if (traversalTaskParams.failFast()) { throw e; } else { errors.add(e); @@ -103,29 +113,33 @@ protected List compute() { } // Recursively build the file system tree for the children nodes - if (rootNode.children() != null && !rootNode.children().isEmpty()) { + if (traversalTaskParams.rootNode().children() != null && + !traversalTaskParams.rootNode().children().isEmpty()) { + + var toProcessCount = 0; - List tasks = new ArrayList<>(); - for (TreeNode child : rootNode.children()) { + for (TreeNode child : traversalTaskParams.rootNode().children()) { var task = new PullTreeNodeTask( logger, + executor, downloader, - child, - destination, - overwrite, - generateEmptyFolders, - failFast, - language, - progressBar + fileHashService ); - task.fork(); - tasks.add(task); + task.setTraversalParams(PullTreeNodeTaskParams.builder() + .from(traversalTaskParams) + .rootNode(child) + .build() + ); + completionService.submit(task::compute); + toProcessCount++; } - for (PullTreeNodeTask task : tasks) { - var taskErrors = task.join(); - errors.addAll(taskErrors); - } + // Wait for all tasks to complete and gather the results + Function, Void> processFunction = taskResult -> { + errors.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); } return errors; @@ -180,8 +194,8 @@ private void createFileInFileSystem(final String destination, final FolderView f String localFileHash = null; if (Files.exists(assetFilePath)) { - if (overwrite) { - localFileHash = Utils.Sha256toUnixHash(assetFilePath); + if (traversalTaskParams.overwrite()) { + localFileHash = fileHashService.sha256toUnixHash(assetFilePath); } else { // If the file already exist, and we are not overwriting files, there is no point in downloading it localFileHash = remoteFileHash; // Fixing hashes so the download is skipped @@ -196,7 +210,7 @@ private void createFileInFileSystem(final String destination, final FolderView f // Download the file try (var inputStream = this.downloader.download(AssetRequest.builder(). assetPath(remoteAssetURL). - language(language). + language(traversalTaskParams.language()). live(asset.live()). build())) { @@ -206,7 +220,7 @@ private void createFileInFileSystem(final String destination, final FolderView f } } else { logger.debug(String.format("Skipping file [%s], it already exists in the file system - Override flag [%b]", - remoteAssetURL, overwrite)); + remoteAssetURL, traversalTaskParams.overwrite())); } } catch (Exception e) { var message = String.format("Error pulling file [%s] to [%s]", remoteAssetURL, assetFilePath); @@ -214,7 +228,7 @@ private void createFileInFileSystem(final String destination, final FolderView f throw new TraversalTaskException(message, e); } finally { // File processed, updating the progress bar - progressBar.incrementStep(); + traversalTaskParams.progressBar().incrementStep(); } } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PushTreeNodeTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PushTreeNodeTask.java index ceed30e073f4..06a8f814863b 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PushTreeNodeTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/PushTreeNodeTask.java @@ -4,13 +4,12 @@ import static com.dotcms.common.AssetsUtils.isMarkedForPush; import static com.dotcms.model.asset.BasicMetadataFields.SHA256_META_KEY; -import com.dotcms.api.client.files.traversal.PushTraverseParams; import com.dotcms.api.client.files.traversal.data.Pusher; import com.dotcms.api.client.files.traversal.exception.SiteCreationException; import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.command.PushContext; -import com.dotcms.cli.common.ConsoleProgressBar; import com.dotcms.model.asset.AssetView; import com.dotcms.model.asset.FolderView; import com.dotcms.model.site.SiteView; @@ -19,43 +18,80 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; import javax.ws.rs.NotFoundException; import javax.ws.rs.WebApplicationException; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** * Represents a task that pushes the contents of a tree node to a remote server. */ -public class PushTreeNodeTask extends RecursiveTask> { +@Dependent +public class PushTreeNodeTask extends TaskProcessor { + + private final ManagedExecutor executor; + + private final PushContext pushContext; - private final PushTraverseParams params; private final Pusher pusher; + private final Logger logger; - private final transient ConsoleProgressBar progressBar; + + private PushTreeNodeTaskParams traversalTaskParams; /** * Constructs a new PushTreeNodeTask with the specified parameters. - * @param traverseParams the parameters for the task + * + * @param logger logger for logging messages + * @param executor the executor for parallel execution of pushing tasks + * @param pushContext the push shared Context + * @param pusher the pusher to use for pushing the contents of the tree node */ - public PushTreeNodeTask(final PushTraverseParams traverseParams) { - this.params = traverseParams; - this.pusher = traverseParams.pusher(); - this.logger = traverseParams.logger(); - this.progressBar = traverseParams.progressBar(); + public PushTreeNodeTask(final Logger logger, + final ManagedExecutor executor, + final PushContext pushContext, + final Pusher pusher) { + this.executor = executor; + this.pushContext = pushContext; + this.pusher = pusher; + this.logger = logger; } - @Override - protected List compute() { - var pushContext = params.pushContext(); + /** + * Sets the traversal parameters for the PushTreeNodeTask. This method provides a way to inject + * necessary configuration after the instance of PushTreeNodeTask has been created by the + * container, which is a common pattern when working with frameworks like Quarkus that manage + * object creation and dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The traversal parameters + */ + public void setTraversalParams(final PushTreeNodeTaskParams params) { + this.traversalTaskParams = params; + } + + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); + var errors = new ArrayList(); - final TreeNode rootNode = params.rootNode(); + final TreeNode rootNode = traversalTaskParams.rootNode(); + // Handle the folder for the current node try { processFolder(rootNode.folder(), pushContext); } catch (Exception e) { - if (params.failFast()) { + if (traversalTaskParams.failFast()) { throw e; } else { errors.add(e); @@ -72,9 +108,9 @@ protected List compute() { try { processAsset(rootNode.folder(), asset, pushContext); } catch (Exception e) { - if (params.failFast()) { + if (traversalTaskParams.failFast()) { //This adds a line so when the exception gets written to the console it looks consistent - progressBar.done(); + traversalTaskParams.progressBar().done(); throw e; } else { errors.add(e); @@ -82,37 +118,46 @@ protected List compute() { } } - handleChildren(errors, rootNode); + handleChildren(errors, rootNode, completionService); return errors; } /** * Recursively build the file system tree for the children nodes - * @param errors the list of errors to add to + * + * @param errors the list of errors to add to * @param rootNode the root node to process */ - private void handleChildren(ArrayList errors, TreeNode rootNode) { + private void handleChildren(ArrayList errors, TreeNode rootNode, + CompletionService> completionService) { if (rootNode.children() != null && !rootNode.children().isEmpty()) { - List tasks = new ArrayList<>(); + var toProcessCount = 0; for (TreeNode child : rootNode.children()) { var task = new PushTreeNodeTask( - PushTraverseParams.builder() - .from(params).rootNode(child).build() + logger, + executor, + pushContext, + pusher + ); + task.setTraversalParams(PushTreeNodeTaskParams.builder() + .from(traversalTaskParams).rootNode(child).build() ); - task.fork(); - tasks.add(task); + completionService.submit(task::compute); + toProcessCount++; } - for (var task : tasks) { - var taskErrors = task.join(); - errors.addAll(taskErrors); - } + // Wait for all tasks to complete and gather the results + Function, Void> processFunction = taskResult -> { + errors.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); } } @@ -123,33 +168,34 @@ private void handleChildren(ArrayList errors, TreeNode rootNode) { */ private void processFolder(final FolderView folder, PushContext pushContext) { - if (isMarkedForDelete(folder)) {// Delete - doDeleteFolder(folder, pushContext); - } else if (isMarkedForPush(folder)) {// Push - doPushFolder(folder, pushContext); - } + if (isMarkedForDelete(folder)) {// Delete + doDeleteFolder(folder, pushContext); + } else if (isMarkedForPush(folder)) {// Push + doPushFolder(folder, pushContext); + } } /** * Processes the asset associated with the specified AssetView. - * @param folder the folder associated with the asset + * + * @param folder the folder associated with the asset * @param pushContext the push context */ - private void doPushFolder(FolderView folder, PushContext pushContext) { + private void doPushFolder(FolderView folder, PushContext pushContext) { var isSite = Objects.equals(folder.path(), "/") && Objects.equals(folder.name(), "/"); try { if (isSite) { - final String status = params.localPaths().status(); + final String status = traversalTaskParams.localPaths().status(); // And we need to create the non-existing site final Optional optional = pushContext.execPush(folder.host(), () -> Optional.of( pusher.pushSite(folder.host(), status)) ); - if (optional.isPresent()){ - logger.debug(String.format("Site [%s] created", folder.host())); + if (optional.isPresent()) { + logger.debug(String.format("Site [%s] created", folder.host())); } else { - logger.debug(String.format("Site [%s] already pushed", folder.host())); + logger.debug(String.format("Site [%s] already pushed", folder.host())); } } else { @@ -164,11 +210,11 @@ private void doPushFolder(FolderView folder, PushContext pushContext) { } return Optional.empty(); }); - if (optional.isPresent()) { - logger.debug(String.format("Folder [%s] created", folder.path())); - } else { - logger.debug(String.format("Folder [%s] already exist", folder.path())); - } + if (optional.isPresent()) { + logger.debug(String.format("Folder [%s] created", folder.path())); + } else { + logger.debug(String.format("Folder [%s] already exist", folder.path())); + } } } catch (Exception e) { var message = String.format("Error creating %s [%s]", @@ -181,7 +227,7 @@ private void doPushFolder(FolderView folder, PushContext pushContext) { var alreadyExist = checkIfSiteAlreadyExist(e); // If we are trying to create a site that already exist we could ignore the error on retries - if (!params.isRetry() || !alreadyExist) { + if (!traversalTaskParams.isRetry() || !alreadyExist) { logger.error(message, e); throw new SiteCreationException(message, e); } @@ -190,25 +236,27 @@ private void doPushFolder(FolderView folder, PushContext pushContext) { var alreadyExist = checkIfAssetOrFolderAlreadyExist(e); // If we are trying to create a folder that already exist we could ignore the error on retries - if (!params.isRetry() || !alreadyExist) { + if (!traversalTaskParams.isRetry() || !alreadyExist) { logger.error(message, e); throw new TraversalTaskException(message, e); } } } finally { // Folder processed, updating the progress bar - progressBar.incrementStep(); + traversalTaskParams.progressBar().incrementStep(); } } /** * Performs the deletion of the specified folder. - * @param folder the folder to check + * + * @param folder the folder to check * @param pushContext the push context */ private void doDeleteFolder(FolderView folder, PushContext pushContext) { try { - final Optional delete = pushContext.execDelete(String.format("%s/%s",folder.host(),folder.path()), + final Optional delete = pushContext.execDelete( + String.format("%s/%s", folder.host(), folder.path()), () -> Optional.of(pusher.deleteFolder(folder.host(), folder.path()))); if (delete.isPresent()) { logger.debug(String.format("Folder [%s] deleted", folder.path())); @@ -219,7 +267,7 @@ private void doDeleteFolder(FolderView folder, PushContext pushContext) { } catch (Exception e) { // If we are trying to delete a folder that does not exist anymore we could ignore the error on retries - if (!params.isRetry() || !(e instanceof NotFoundException)) { + if (!traversalTaskParams.isRetry() || !(e instanceof NotFoundException)) { var message = String.format("Error deleting folder [%s]", folder.path()); logger.error(message, e); @@ -228,7 +276,7 @@ private void doDeleteFolder(FolderView folder, PushContext pushContext) { } finally { // Folder processed, updating the progress bar - this.progressBar.incrementStep(); + traversalTaskParams.progressBar().incrementStep(); } } @@ -238,7 +286,8 @@ private void doDeleteFolder(FolderView folder, PushContext pushContext) { * @param folder the folder containing the asset * @param asset the asset to process */ - private void processAsset(final FolderView folder, final AssetView asset, final PushContext pushContext) { + private void processAsset(final FolderView folder, final AssetView asset, + final PushContext pushContext) { if (isMarkedForDelete(asset)) { doDeleteAsset(folder, asset, pushContext); } else if (isMarkedForPush(asset)) { @@ -248,8 +297,9 @@ private void processAsset(final FolderView folder, final AssetView asset, final /** * Performs a push operation on the specified asset. - * @param folder the folder containing the asset - * @param asset the asset to push + * + * @param folder the folder containing the asset + * @param asset the asset to push * @param pushContext the push context */ private void doPushAsset(FolderView folder, AssetView asset, PushContext pushContext) { @@ -260,19 +310,20 @@ private void doPushAsset(FolderView folder, AssetView asset, PushContext pushCon pushAssetKey, () -> { // Pushing the asset (and creating the folder if needed - final AssetView assetView = pusher.push(params.workspacePath(), - params.localPaths().status(), - params.localPaths().language(), - params.localPaths().site(), + final AssetView assetView = pusher.push(traversalTaskParams.workspacePath(), + traversalTaskParams.localPaths().status(), + traversalTaskParams.localPaths().language(), + traversalTaskParams.localPaths().site(), folder.path(), asset.name() ); return Optional.of(assetView); }); - if(optional.isPresent()){ - logger.debug(String.format("Asset [%s%s] pushed", folder.path(), asset.name())); + if (optional.isPresent()) { + logger.debug(String.format("Asset [%s%s] pushed", folder.path(), asset.name())); } else { - logger.debug(String.format("Asset [%s%s] already pushed", folder.path(), asset.name())); + logger.debug( + String.format("Asset [%s%s] already pushed", folder.path(), asset.name())); } } catch (Exception e) { @@ -280,30 +331,34 @@ private void doPushAsset(FolderView folder, AssetView asset, PushContext pushCon var alreadyExist = checkIfAssetOrFolderAlreadyExist(e); // If we are trying to push an asset that already exist we could ignore the error on retries - if (!params.isRetry() || !alreadyExist) { - var message = String.format("Error pushing asset [%s%s]", folder.path(), asset.name()); + if (!traversalTaskParams.isRetry() || !alreadyExist) { + var message = String.format("Error pushing asset [%s%s]", folder.path(), + asset.name()); logger.error(message, e); throw new TraversalTaskException(message, e); } } finally { // Asset processed, updating the progress bar - this.progressBar.incrementStep(); + traversalTaskParams.progressBar().incrementStep(); } } /** * Performs a delete operation on the specified asset. - * @param folder the folder containing the asset - * @param asset the asset to delete + * + * @param folder the folder containing the asset + * @param asset the asset to delete * @param pushContext the push context */ - private void doDeleteAsset(final FolderView folder, final AssetView asset, final PushContext pushContext) { + private void doDeleteAsset(final FolderView folder, final AssetView asset, + final PushContext pushContext) { try { // Check if we already deleted the folder if (isMarkedForDelete(folder)) { // Folder already deleted, we don't need to delete the asset - logger.debug(String.format("Folder [%s] already deleted, ignoring deletion of [%s] asset", + logger.debug(String.format( + "Folder [%s] already deleted, ignoring deletion of [%s] asset", folder.path(), asset.name())); } else { final Optional optional = pushContext.execArchive( @@ -317,24 +372,27 @@ private void doDeleteAsset(final FolderView folder, final AssetView asset, final return Optional.empty(); }); if (optional.isPresent()) { - logger.debug(String.format("Asset [%s%s] archived", folder.path(), asset.name())); + logger.debug( + String.format("Asset [%s%s] archived", folder.path(), asset.name())); } else { - logger.debug(String.format("Asset [%s%s] already archived", folder.path(), asset.name())); + logger.debug(String.format("Asset [%s%s] already archived", folder.path(), + asset.name())); } } } catch (Exception e) { // If we are trying to delete an asset that does not exist anymore we could ignore the error on retries - if (!params.isRetry() || !(e instanceof NotFoundException)) { + if (!traversalTaskParams.isRetry() || !(e instanceof NotFoundException)) { - var message = String.format("Error deleting asset [%s%s]", folder.path(), asset.name()); + var message = String.format("Error deleting asset [%s%s]", folder.path(), + asset.name()); logger.error(message, e); throw new TraversalTaskException(message, e); } } finally { // Asset processed, updating the progress bar - this.progressBar.incrementStep(); + traversalTaskParams.progressBar().incrementStep(); } } @@ -353,7 +411,7 @@ private String generatePushAssetKey(FolderView folder, AssetView asset) { // order, live first, working second. // If already pushed and the order is respected, we don't need to push the same file again. return String.format("%s/%s/%s/%s/%s", - params.localPaths().language(), + traversalTaskParams.localPaths().language(), folder.host(), folder.path(), asset.name(), diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/RemoteFolderTraversalTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/RemoteFolderTraversalTask.java index 4bcf7b67361d..6de91162b1fa 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/RemoteFolderTraversalTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/files/traversal/task/RemoteFolderTraversalTask.java @@ -2,101 +2,140 @@ import com.dotcms.api.client.files.traversal.data.Retriever; import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; -import com.dotcms.api.traversal.Filter; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.api.traversal.TreeNode; import com.dotcms.model.asset.FolderView; -import org.apache.commons.lang3.tuple.Pair; -import org.jboss.logging.Logger; - import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; +import org.jboss.logging.Logger; /** * Recursive task for traversing a dotCMS remote location and building a hierarchical tree * representation of its contents. This task is used to split the traversal into smaller sub-tasks * that can be executed in parallel, allowing for faster traversal of large directory structures. */ -public class RemoteFolderTraversalTask extends RecursiveTask, TreeNode>> { +@Dependent +public class RemoteFolderTraversalTask extends TaskProcessor { + private final ManagedExecutor executor; private final Logger logger; private final Retriever retriever; - private final Filter filter; - private final String siteName; - private final FolderView folder; - private final boolean root; - private final int depth; - private final boolean failFast; + + private RemoteFolderTraversalTaskParams traversalTaskParams; /** - * Constructs a new RemoteFolderTraversalTask instance with the specified site name, folder, root - * flag, and depth. + * Constructs a new RemoteFolderTraversalTask instance. * + * @param executor the executor for parallel execution of traversal tasks * @param logger the logger for logging debug information * @param retriever The retriever used for REST calls and other operations. - * @param filter The filter used to include or exclude folders and assets. - * @param siteName The name of the site containing the folder to traverse. - * @param folder The folder to traverse. - * @param root Whether this task is for the root folder. - * @param depth The maximum depth to traverse the directory tree. - * @param failFast true to fail fast, false to continue on error */ public RemoteFolderTraversalTask( final Logger logger, - Retriever retriever, - Filter filter, - final String siteName, - final FolderView folder, - final Boolean root, - final int depth, - final boolean failFast) { + final ManagedExecutor executor, + final Retriever retriever) { + this.executor = executor; this.logger = logger; this.retriever = retriever; - this.filter = filter; - this.siteName = siteName; - this.folder = folder; - this.root = root; - this.depth = depth; - this.failFast = failFast; + } + + /** + * Sets the traversal parameters for the RemoteFolderTraversalTask. This method provides a way + * to inject necessary configuration after the instance of RemoteFolderTraversalTask has been + * created by the container, which is a common pattern when working with frameworks like Quarkus + * that manage object creation and dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The traversal parameters + */ + public void setTraversalParams(final RemoteFolderTraversalTaskParams params) { + this.traversalTaskParams = params; } /** * Executes the folder traversal task and returns a TreeNode representing the directory tree * rooted at the folder specified in the constructor. * - * @return A TreeNode representing the directory tree rooted at the folder specified in the - * constructor. + * @return A Pair object containing a list of exceptions encountered during traversal and the + * resulting TreeNode representing the directory tree at the specified folder. */ - @Override - protected Pair, TreeNode> compute() { + public Pair, TreeNode> compute() { + + CompletionService, TreeNode>> completionService = + new ExecutorCompletionService<>(executor); var errors = new ArrayList(); - var currentNode = new TreeNode(folder); - var currentFolder = folder; + // Processing the current folder + var processResult = processCurrentFolder( + errors + ); + var currentNode = processResult.getLeft(); + var currentFolder = processResult.getRight(); - List forks = new LinkedList<>(); + // And now its subfolders + int toProcessCount = processSubFolders( + currentNode, + currentFolder, + errors, + completionService + ); + + // Wait for all tasks to complete and gather the results + Function, TreeNode>, Void> processFunction = taskResult -> { + errors.addAll(taskResult.getLeft()); + currentNode.addChild(taskResult.getRight()); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); + + return Pair.of(errors, currentNode); + } + + /** + * Processes the current folder and returns a TreeNode representing it. + * + * @param errors The list of exceptions to which any error should be added. + * @return A Pair object containing the TreeNode representing the processed folder and the + * FolderView object representing the current folder. + */ + private Pair processCurrentFolder(List errors) { + + var currentNode = new TreeNode(traversalTaskParams.folder()); + var currentFolder = traversalTaskParams.folder(); // Processing the very first level - if (root) { + if (traversalTaskParams.isRoot()) { try { // Make a REST call to fetch the root folder currentFolder = this.retrieveFolderInformation( - this.siteName, - folder.path(), - folder.level(), - folder.implicitGlobInclude(), - folder.explicitGlobInclude(), - folder.explicitGlobExclude() + traversalTaskParams.siteName(), + traversalTaskParams.folder().path(), + traversalTaskParams.folder().level(), + traversalTaskParams.folder().implicitGlobInclude(), + traversalTaskParams.folder().explicitGlobInclude(), + traversalTaskParams.folder().explicitGlobExclude() ); // Using the values set by the filter in the root folder - var detailedFolder = folder.withImplicitGlobInclude(currentFolder.implicitGlobInclude()); - detailedFolder = detailedFolder.withExplicitGlobInclude(currentFolder.explicitGlobInclude()); - detailedFolder = detailedFolder.withExplicitGlobExclude(currentFolder.explicitGlobExclude()); + var detailedFolder = traversalTaskParams.folder().withImplicitGlobInclude( + currentFolder.implicitGlobInclude()); + detailedFolder = detailedFolder.withExplicitGlobInclude( + currentFolder.explicitGlobInclude()); + detailedFolder = detailedFolder.withExplicitGlobExclude( + currentFolder.explicitGlobExclude()); currentNode = new TreeNode(detailedFolder); // Add the fetched files to the root folder @@ -104,38 +143,50 @@ protected Pair, TreeNode> compute() { currentNode.assets(currentFolder.assets().versions()); } } catch (Exception e) { - if (failFast) { - throw e; - } else { - errors.add(e); - } + handleException(errors, e); } } + return Pair.of(currentNode, currentFolder); + } + + /** + * Processes the sub-folders of the given folder. + * + * @param currentNode The current TreeNode representing the current folder. + * @param currentFolder The FolderView object representing the current folder to process. + * @param errors The list of exceptions to which any error should be added. + * @param completionService The CompletionService for parallel execution of traversal tasks. + * @return The number of sub-folders to be processed. + */ + private int processSubFolders(TreeNode currentNode, FolderView currentFolder, + List errors, + CompletionService, TreeNode>> completionService) { + + var toProcessCount = 0; + // Traverse all sub-folders of the current folder if (currentFolder.subFolders() != null) { for (FolderView subFolder : currentFolder.subFolders()) { - if (this.depth == -1 || subFolder.level() <= this.depth) { + if (traversalTaskParams.depth() == -1 + || subFolder.level() <= traversalTaskParams.depth()) { try { // Create a new task to traverse the sub-folder and add it to the list of sub-tasks var task = searchForFolder( - this.siteName, + traversalTaskParams.siteName(), subFolder.path(), subFolder.level(), subFolder.implicitGlobInclude(), subFolder.explicitGlobInclude(), subFolder.explicitGlobExclude() ); - forks.add(task); - task.fork(); + + completionService.submit(task::compute); + toProcessCount++; } catch (Exception e) { - if (failFast) { - throw e; - } else { - errors.add(e); - } + handleException(errors, e); } } else { @@ -145,14 +196,7 @@ protected Pair, TreeNode> compute() { } } - // Join all sub-tasks and add their results to the current node - for (RemoteFolderTraversalTask task : forks) { - var taskResult = task.join(); - errors.addAll(taskResult.getLeft()); - currentNode.addChild(taskResult.getRight()); - } - - return Pair.of(errors, currentNode); + return toProcessCount; } /** @@ -162,23 +206,27 @@ protected Pair, TreeNode> compute() { * @param siteName The name of the site containing the folder to search for. * @param folderPath The path of the folder to search for. * @param level The level of the folder to search for. - * @param implicitGlobInclude This property represents whether a folder should be implicitly included based on the - * absence of any include patterns. When implicitGlobInclude is set to true, it means - * that there are no include patterns specified, so all folders should be included by - * default. In other words, if there are no specific include patterns defined, the - * filter assumes that all folders should be included unless explicitly excluded. - * @param explicitGlobInclude This property represents whether a folder should be explicitly included based on the - * configured includes patterns for folders. When explicitGlobInclude is set to true, - * it means that the folder has matched at least one of the include patterns and should - * be included in the filtered result. The explicit inclusion takes precedence over other - * rules. If a folder is explicitly included, it will be included regardless of any other - * rules or patterns. - * @param explicitGlobExclude This property represents whether a folder should be explicitly excluded based on the - * configured excludes patterns for folders. When explicitGlobExclude is set to true, it - * means that the folder has matched at least one of the exclude patterns and should be - * excluded from the filtered result. The explicit exclusion takes precedence over other - * rules. If a folder is explicitly excluded, it will be excluded regardless of any other - * rules or patterns. + * @param implicitGlobInclude This property represents whether a folder should be implicitly + * included based on the absence of any include patterns. When + * implicitGlobInclude is set to true, it means that there are no + * include patterns specified, so all folders should be included by + * default. In other words, if there are no specific include patterns + * defined, the filter assumes that all folders should be included + * unless explicitly excluded. + * @param explicitGlobInclude This property represents whether a folder should be explicitly + * included based on the configured includes patterns for folders. + * When explicitGlobInclude is set to true, it means that the folder + * has matched at least one of the include patterns and should be + * included in the filtered result. The explicit inclusion takes + * precedence over other rules. If a folder is explicitly included, + * it will be included regardless of any other rules or patterns. + * @param explicitGlobExclude This property represents whether a folder should be explicitly + * excluded based on the configured excludes patterns for folders. + * When explicitGlobExclude is set to true, it means that the folder + * has matched at least one of the exclude patterns and should be + * excluded from the filtered result. The explicit exclusion takes + * precedence over other rules. If a folder is explicitly excluded, + * it will be excluded regardless of any other rules or patterns. * @return A new FolderTraversalTask instance to search for the specified folder. */ private RemoteFolderTraversalTask searchForFolder( @@ -193,15 +241,23 @@ private RemoteFolderTraversalTask searchForFolder( final var folder = this.retrieveFolderInformation(siteName, folderPath, level, implicitGlobInclude, explicitGlobInclude, explicitGlobExclude); - return new RemoteFolderTraversalTask( + var task = new RemoteFolderTraversalTask( this.logger, - this.retriever, - this.filter, - siteName, - folder, - false, - this.depth, - this.failFast); + this.executor, + this.retriever + ); + + task.setTraversalParams(RemoteFolderTraversalTaskParams.builder() + .filter(traversalTaskParams.filter()) + .siteName(siteName) + .folder(folder) + .isRoot(false) + .depth(traversalTaskParams.depth()) + .failFast(traversalTaskParams.failFast()) + .build() + ); + + return task; } /** @@ -210,29 +266,32 @@ private RemoteFolderTraversalTask searchForFolder( * @param siteName The name of the site containing the folder * @param folderPath The path of the folder to search for. * @param level The hierarchical level of the folder - * @param implicitGlobInclude This property represents whether a folder should be implicitly included based on the - * absence of any include patterns. When implicitGlobInclude is set to true, it means - * that there are no include patterns specified, so all folders should be included by - * default. In other words, if there are no specific include patterns defined, the - * filter assumes that all folders should be included unless explicitly excluded. - * @param explicitGlobInclude This property represents whether a folder should be explicitly included based on the - * configured includes patterns for folders. When explicitGlobInclude is set to true, - * it means that the folder has matched at least one of the include patterns and should - * be included in the filtered result. The explicit inclusion takes precedence over other - * rules. If a folder is explicitly included, it will be included regardless of any other - * rules or patterns. - * @param explicitGlobExclude This property represents whether a folder should be explicitly excluded based on the - * configured excludes patterns for folders. When explicitGlobExclude is set to true, it - * means that the folder has matched at least one of the exclude patterns and should be - * excluded from the filtered result. The explicit exclusion takes precedence over other - * rules. If a folder is explicitly excluded, it will be excluded regardless of any other - * rules or patterns. + * @param implicitGlobInclude This property represents whether a folder should be implicitly + * included based on the absence of any include patterns. When + * implicitGlobInclude is set to true, it means that there are no + * include patterns specified, so all folders should be included by + * default. In other words, if there are no specific include patterns + * defined, the filter assumes that all folders should be included + * unless explicitly excluded. + * @param explicitGlobInclude This property represents whether a folder should be explicitly + * included based on the configured includes patterns for folders. + * When explicitGlobInclude is set to true, it means that the folder + * has matched at least one of the include patterns and should be + * included in the filtered result. The explicit inclusion takes + * precedence over other rules. If a folder is explicitly included, + * it will be included regardless of any other rules or patterns. + * @param explicitGlobExclude This property represents whether a folder should be explicitly + * excluded based on the configured excludes patterns for folders. + * When explicitGlobExclude is set to true, it means that the folder + * has matched at least one of the exclude patterns and should be + * excluded from the filtered result. The explicit exclusion takes + * precedence over other rules. If a folder is explicitly excluded, + * it will be excluded regardless of any other rules or patterns. * @return an {@code FolderView} object containing the metadata for the requested folder */ - private FolderView retrieveFolderInformation(final String siteName, final String folderPath, final int level, - final boolean implicitGlobInclude, - final Boolean explicitGlobInclude, - final Boolean explicitGlobExclude) { + private FolderView retrieveFolderInformation(final String siteName, final String folderPath, + final int level, final boolean implicitGlobInclude, final Boolean explicitGlobInclude, + final Boolean explicitGlobExclude) { try { var foundFolder = this.retriever.retrieveFolderInformation( @@ -244,7 +303,7 @@ private FolderView retrieveFolderInformation(final String siteName, final String explicitGlobExclude ); - return this.filter.apply(foundFolder); + return traversalTaskParams.filter().apply(foundFolder); } catch (Exception e) { var message = String.format("Error retrieving folder information [%s]", folderPath); logger.error(message, e); @@ -252,4 +311,24 @@ private FolderView retrieveFolderInformation(final String siteName, final String } } -} \ No newline at end of file + /** + * Handles an exception that occurred during the execution of a traversal task. + * + * @param errors The list of exceptions to which the error should be added. + * @param e The exception that occurred. + */ + private void handleException(List errors, Exception e) { + + if (traversalTaskParams.failFast()) { + + if (e instanceof TraversalTaskException) { + throw (TraversalTaskException) e; + } + + throw new TraversalTaskException(e.getMessage(), e); + } else { + errors.add(e); + } + } + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/GeneralPullHandler.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/GeneralPullHandler.java index 8a1baed3141d..c35aa95f8714 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/GeneralPullHandler.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/GeneralPullHandler.java @@ -10,8 +10,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; import javax.inject.Inject; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -31,6 +31,9 @@ public abstract class GeneralPullHandler extends PullHandler { @Inject Logger logger; + @Inject + ManagedExecutor executor; + /** * Returns a display name of a given T element. Used for logging purposes. */ @@ -58,7 +61,7 @@ public boolean pull(List contents, contents.size() ); - CompletableFuture> pullFuture = CompletableFuture.supplyAsync( + CompletableFuture> pullFuture = executor.supplyAsync( () -> { final var format = InputOutputFormat.valueOf( @@ -66,23 +69,27 @@ public boolean pull(List contents, .orElse(InputOutputFormat.defaultFormat().toString()) ); - var forkJoinPool = ForkJoinPool.commonPool(); - var task = new PullTask<>(PullTaskParams.builder(). + PullTask task = new PullTask<>( + logger, + mapperService, + executor + ); + + task.setTaskParams(PullTaskParams.builder(). destination(pullOptions.destination()). contents(contents). format(format). failFast(pullOptions.failFast()). pullHandler(this). - mapperService(mapperService). output(output). - logger(logger). progressBar(progressBar).build() ); - return forkJoinPool.invoke(task); + + return task.compute(); }); progressBar.setFuture(pullFuture); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( progressBar ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/PullServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/PullServiceImpl.java index 4b75989d5f56..6376fbc70376 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/PullServiceImpl.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/PullServiceImpl.java @@ -11,6 +11,7 @@ import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -24,6 +25,9 @@ public class PullServiceImpl implements PullService { @Inject Logger logger; + @Inject + ManagedExecutor executor; + /** * {@inheritDoc} */ @@ -83,7 +87,7 @@ private List fetch( final PullHandler pullHandler) { CompletableFuture> - fetcherServiceFuture = CompletableFuture.supplyAsync( + fetcherServiceFuture = executor.supplyAsync( () -> { // Looking for a specific content @@ -113,7 +117,7 @@ private List fetch( fetcherServiceFuture ); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( consoleLoadingAnimation ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/ContentTypeFetcher.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/ContentTypeFetcher.java index 37d247fa0538..fa9522481529 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/ContentTypeFetcher.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/ContentTypeFetcher.java @@ -8,11 +8,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ForkJoinPool; import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import javax.ws.rs.NotFoundException; +import org.eclipse.microprofile.context.ManagedExecutor; @Dependent public class ContentTypeFetcher implements ContentFetcher, Serializable { @@ -22,6 +22,9 @@ public class ContentTypeFetcher implements ContentFetcher, Serializ @Inject protected RestClientFactory clientFactory; + @Inject + ManagedExecutor executor; + @ActivateRequestContext @Override public List fetch(Map customOptions) { @@ -37,13 +40,14 @@ public List fetch(Map customOptions) { allContentTypes.addAll(contentTypes); } - // Create a ForkJoinPool to process the content types in parallel + // Create an HttpRequestTask to process the content types in parallel // We need this extra logic because the content type API returns when calling all content // types an object that is not equal to the one returned when calling by id or by var name, // it is a reduced, so we need to call the API for each content type to get the full object. - var forkJoinPool = ForkJoinPool.commonPool(); - var task = new HttpRequestTask(allContentTypes, this); - return forkJoinPool.invoke(task); + var task = new HttpRequestTask(this, executor); + task.setTaskParams(allContentTypes); + + return task.compute(); } @ActivateRequestContext diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/HttpRequestTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/HttpRequestTask.java index 79903bbafe85..1c9f4d2b84fb 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/HttpRequestTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/contenttype/HttpRequestTask.java @@ -1,30 +1,65 @@ package com.dotcms.api.client.pull.contenttype; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.contenttype.model.type.ContentType; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.context.ManagedExecutor; /** - * Represents a task that performs HTTP requests concurrently using the Fork/Join framework. It - * extends the RecursiveTask class and returns a list of ContentType objects. + * Represents a task that performs HTTP requests concurrently. */ -public class HttpRequestTask extends RecursiveTask> { +@Dependent +public class HttpRequestTask extends TaskProcessor { private final ContentTypeFetcher contentTypeFetcher; - private final transient List contentTypes; + private List contentTypes; - private static final int THRESHOLD = 10; + private final ManagedExecutor executor; - public HttpRequestTask(final List contentTypes, - final ContentTypeFetcher contentTypeFetcher) { - this.contentTypes = contentTypes; + public HttpRequestTask(final ContentTypeFetcher contentTypeFetcher, + final ManagedExecutor executor) { this.contentTypeFetcher = contentTypeFetcher; + this.executor = executor; } - @Override - protected List compute() { + /** + * Sets the parameters for the HttpRequestTask. This method provides a way to inject necessary + * configuration after the instance of HttpRequestTask has been created by the container, which + * is a common pattern when working with frameworks like Quarkus that manage object creation and + * dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param contentTypes List of ContentType objects to process. + */ + public void setTaskParams(final List contentTypes) { + this.contentTypes = contentTypes; + } + + /** + * Processes a list of ContentType objects, either sequantially or in parallel, depending on the + * list size. If the size of the list is under a predefined threshold, items are processed + * individually in order. For larger lists, the work is divided into separate concurrent tasks, + * which are processed in parallel. + *

+ * Each ContentType object in the list is processed by making a request to fetch its full + * version. + * + * @return A List of fully fetched ContentType objects. + */ + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); if (contentTypes.size() <= THRESHOLD) { @@ -40,29 +75,44 @@ protected List compute() { } else { - // If the list is large, split it into two smaller tasks - int mid = contentTypes.size() / 2; - HttpRequestTask task1 = new HttpRequestTask( - contentTypes.subList(0, mid), - contentTypeFetcher - ); - HttpRequestTask task2 = new HttpRequestTask( - contentTypes.subList(mid, contentTypes.size()), - contentTypeFetcher - ); + // If the list is large, split it into smaller tasks + int toProcessCount = splitTasks(contentTypes, completionService); + + // Wait for all tasks to complete and gather the results + final var foundContentTypes = new ArrayList(); + Function, Void> processFunction = taskResult -> { + foundContentTypes.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); + return foundContentTypes; + } + } - // Start the first subtask in a new thread - task1.fork(); + /** + * Splits a list of ContentType objects into separate tasks. + * + * @param contentTypes List of ContentType objects to process. + * @param completionService The CompletionService to submit tasks to. + * @return The number of tasks to process. + */ + private int splitTasks(final List contentTypes, + final CompletionService> completionService) { - // Start the second subtask and wait for it to finish - List task2Result = task2.compute(); + int mid = contentTypes.size() / 2; + var subList1 = contentTypes.subList(0, mid); + var subList2 = contentTypes.subList(mid, contentTypes.size()); - // Wait for the first subtask to finish and combine the results - List task1Result = task1.join(); - task1Result.addAll(task2Result); + var task1 = new HttpRequestTask(contentTypeFetcher, executor); + task1.setTaskParams(subList1); - return task1Result; - } + var task2 = new HttpRequestTask(contentTypeFetcher, executor); + task2.setTaskParams(subList2); + + completionService.submit(task1::compute); + completionService.submit(task2::compute); + + return 2; } } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/file/FilePullHandler.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/file/FilePullHandler.java index 740845eae50a..4b5f40746e45 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/file/FilePullHandler.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/file/FilePullHandler.java @@ -23,6 +23,7 @@ import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -42,6 +43,9 @@ public class FilePullHandler extends PullHandler { @Inject Puller puller; + @Inject + ManagedExecutor executor; + @Override public String title() { return "Files"; @@ -158,7 +162,7 @@ private List pullTree( // ConsoleProgressBar instance to handle the download progress bar ConsoleProgressBar progressBar = new ConsoleProgressBar(output); - CompletableFuture> treeBuilderFuture = CompletableFuture.supplyAsync( + CompletableFuture> treeBuilderFuture = executor.supplyAsync( () -> puller.pull( tree, treeNodeInfo, @@ -171,7 +175,7 @@ private List pullTree( progressBar.setFuture(treeBuilderFuture); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( progressBar ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/HttpRequestTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/HttpRequestTask.java index 9f421837207b..81e43cbd5c50 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/HttpRequestTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/HttpRequestTask.java @@ -1,30 +1,65 @@ package com.dotcms.api.client.pull.site; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.model.site.Site; import com.dotcms.model.site.SiteView; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.context.ManagedExecutor; /** - * Represents a task that performs HTTP requests concurrently using the Fork/Join framework. It - * extends the RecursiveTask class and returns a list of SiteView objects. + * Represents a task that performs HTTP requests concurrently. */ -public class HttpRequestTask extends RecursiveTask> { +@Dependent +public class HttpRequestTask extends TaskProcessor { private final SiteFetcher siteFetcher; - private final List sites; + private List sites; - private static final int THRESHOLD = 10; + private final ManagedExecutor executor; + + public HttpRequestTask(final SiteFetcher siteFetcher, + final ManagedExecutor executor) { + this.siteFetcher = siteFetcher; + this.executor = executor; + } - public HttpRequestTask(final List sites, final SiteFetcher siteFetcher) { + /** + * Sets the parameters for the HttpRequestTask. This method provides a way to inject necessary + * configuration after the instance of HttpRequestTask has been created by the container, which + * is a common pattern when working with frameworks like Quarkus that manage object creation and + * dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param sites List of Site objects to process. + */ + public void setTaskParams(final List sites) { this.sites = sites; - this.siteFetcher = siteFetcher; } - @Override - protected List compute() { + /** + * Processes a list of Site objects, either sequantially or in parallel, depending on the list + * size. If the size of the list is under a predefined threshold, items are processed + * individually in order. For larger lists, the work is divided into separate concurrent tasks, + * which are processed in parallel. + *

+ * Each Site object in the list is processed by making a request to fetch its full version. + * + * @return A List of fully fetched SiteView objects. + */ + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); if (sites.size() <= THRESHOLD) { @@ -38,24 +73,44 @@ protected List compute() { } else { - // If the list is large, split it into two smaller tasks - int mid = sites.size() / 2; - HttpRequestTask task1 = new HttpRequestTask(sites.subList(0, mid), siteFetcher); - HttpRequestTask task2 = new HttpRequestTask(sites.subList(mid, sites.size()), - siteFetcher); + // If the list is large, split it into smaller tasks + int toProcessCount = splitTasks(sites, completionService); + + // Wait for all tasks to complete and gather the results + final var foundSites = new ArrayList(); + Function, Void> processFunction = taskResult -> { + foundSites.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); + return foundSites; + } + } + + /** + * Splits a list of Site objects into separate tasks. + * + * @param sites List of Site objects to process. + * @param completionService The CompletionService to submit tasks to. + * @return The number of tasks to process. + */ + private int splitTasks(List sites, + CompletionService> completionService) { - // Start the first subtask in a new thread - task1.fork(); + int mid = sites.size() / 2; + var subList1 = sites.subList(0, mid); + var subList2 = sites.subList(mid, sites.size()); - // Start the second subtask and wait for it to finish - List task2Result = task2.compute(); + var task1 = new HttpRequestTask(siteFetcher, executor); + task1.setTaskParams(subList1); - // Wait for the first subtask to finish and combine the results - List task1Result = task1.join(); - task1Result.addAll(task2Result); + var task2 = new HttpRequestTask(siteFetcher, executor); + task2.setTaskParams(subList2); - return task1Result; - } + completionService.submit(task1::compute); + completionService.submit(task2::compute); + + return 2; } } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/SiteFetcher.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/SiteFetcher.java index 4c36b7cdd084..c60c75e49aec 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/SiteFetcher.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/site/SiteFetcher.java @@ -12,11 +12,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ForkJoinPool; import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import javax.ws.rs.NotFoundException; +import org.eclipse.microprofile.context.ManagedExecutor; @Dependent public class SiteFetcher implements ContentFetcher, Serializable { @@ -26,6 +26,9 @@ public class SiteFetcher implements ContentFetcher, Serializable { @Inject protected RestClientFactory clientFactory; + @Inject + ManagedExecutor executor; + @ActivateRequestContext @Override public List fetch(Map customOptions) { @@ -33,13 +36,14 @@ public List fetch(Map customOptions) { // Fetching the all the existing sites final List allSites = allSites(); - // Create a ForkJoinPool to process the sites in parallel + // Create a HttpRequestTask to process the sites in parallel // We need this extra logic because the site API returns when calling all sites an object // that is not equal to the one returned when calling by id or by name, it is a reduced and // different version of a site, so we need to call the API for each site to get the full object. - var forkJoinPool = ForkJoinPool.commonPool(); - var task = new HttpRequestTask(allSites, this); - return forkJoinPool.invoke(task); + var task = new HttpRequestTask(this, executor); + task.setTaskParams(allSites); + + return task.compute(); } /** diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/AbstractPullTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/AbstractPullTaskParams.java index 26fe6808e2f2..db2fb8ca7b28 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/AbstractPullTaskParams.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/AbstractPullTaskParams.java @@ -1,6 +1,5 @@ package com.dotcms.api.client.pull.task; -import com.dotcms.api.client.MapperService; import com.dotcms.api.client.pull.GeneralPullHandler; import com.dotcms.cli.common.ConsoleProgressBar; import com.dotcms.cli.common.InputOutputFormat; @@ -10,7 +9,6 @@ import java.io.Serializable; import java.util.List; import org.immutables.value.Value; -import org.jboss.logging.Logger; /** * The AbstractPullTaskParams class is used to compile all the parameters shared by various Pull @@ -50,13 +48,6 @@ public interface AbstractPullTaskParams extends Serializable { */ InputOutputFormat format(); - /** - * Retrieves the mapper service used to map the content to the output format. - * - * @return the mapper service. - */ - MapperService mapperService(); - /** * Retrieves the output options for the pull operation. * @@ -64,13 +55,6 @@ public interface AbstractPullTaskParams extends Serializable { */ OutputOptionMixin output(); - /** - * Retrieves the logger for the pull operation. - * - * @return the logger. - */ - Logger logger(); - /** * Retrieves the progress bar for the pull operation. * diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/PullTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/PullTask.java index 8f9d026d6b39..c6c195dd55bb 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/PullTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/pull/task/PullTask.java @@ -1,12 +1,18 @@ package com.dotcms.api.client.pull.task; +import com.dotcms.api.client.MapperService; import com.dotcms.api.client.pull.exception.PullException; +import com.dotcms.api.client.task.TaskProcessor; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -14,24 +20,48 @@ * * @param the type of content being pulled and processed */ -public class PullTask extends RecursiveTask> { +@Dependent +public class PullTask extends TaskProcessor { - private final PullTaskParams params; + private PullTaskParams params; private final Logger logger; - private static final int THRESHOLD = 10; + private final MapperService mapperService; - public PullTask(final PullTaskParams params) { + private final ManagedExecutor executor; + + public PullTask(final Logger logger, final MapperService mapperService, + final ManagedExecutor executor) { + this.logger = logger; + this.mapperService = mapperService; + this.executor = executor; + } + + /** + * Sets the parameters for the PullTask. This method provides a way to inject necessary + * configuration after the instance of PullTask has been created by the container, which is a + * common pattern when working with frameworks like Quarkus that manage object creation and + * dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The parameters for the PullTask + */ + public void setTaskParams(final PullTaskParams params) { this.params = params; - this.logger = params.logger(); } /** * Computes the contents to pull */ - @Override - protected List compute() { + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); var errors = new ArrayList(); @@ -54,29 +84,46 @@ protected List compute() { } else { - // If the list is large, split it into two smaller tasks - int mid = this.params.contents().size() / 2; - var paramsTask1 = this.params.withContents( - this.params.contents().subList(0, mid) - ); - var paramsTask2 = this.params.withContents( - this.params.contents().subList(mid, this.params.contents().size()) - ); + // If the list is large, split it into smaller tasks + int toProcessCount = splitTasks(completionService); + + // Wait for all tasks to complete and gather the results + Function, Void> processFunction = taskResult -> { + errors.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); + } + + return errors; + } + + /** + * Splits a list of T objects into separate tasks. + * + * @param completionService The CompletionService to submit tasks to. + * @return The number of tasks to process. + */ + private int splitTasks(final CompletionService> completionService) { - var task1 = new PullTask<>(paramsTask1); - var task2 = new PullTask<>(paramsTask2); + int mid = this.params.contents().size() / 2; + var paramsTask1 = this.params.withContents( + this.params.contents().subList(0, mid) + ); + var paramsTask2 = this.params.withContents( + this.params.contents().subList(mid, this.params.contents().size()) + ); - // Start the first subtask in a new thread - task1.fork(); + PullTask task1 = new PullTask<>(logger, mapperService, executor); + task1.setTaskParams(paramsTask1); - // Start and wait for the second subtask to finish - errors.addAll(task2.compute()); + PullTask task2 = new PullTask<>(logger, mapperService, executor); + task2.setTaskParams(paramsTask2); - // Wait for the first subtask to finish - errors.addAll(task1.join()); - } + completionService.submit(task1::compute); + completionService.submit(task2::compute); - return errors; + return 2; } /** @@ -96,8 +143,7 @@ private void toDiskContent(final T content) { // Save the content to a file - ObjectMapper objectMapper = this.params.mapperService() - .objectMapper(this.params.format()); + ObjectMapper objectMapper = mapperService.objectMapper(this.params.format()); final String asString = objectMapper.writeValueAsString(content); if (this.params.output().isVerbose()) { this.params.output().info( String.format("%n%s", asString) ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/PushServiceImpl.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/PushServiceImpl.java index 919c2f92ea92..4c6e39354fa7 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/PushServiceImpl.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/PushServiceImpl.java @@ -21,11 +21,11 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; import javax.enterprise.context.Dependent; import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** @@ -47,6 +47,9 @@ public class PushServiceImpl implements PushService { @Inject Logger logger; + @Inject + ManagedExecutor executor; + /** * Analyzes and pushes the changes to a remote repository. * @@ -158,7 +161,7 @@ private Pair>, PushAnalysisSummary> analyze( final ContentComparator comparator) { CompletableFuture>> - pushAnalysisServiceFuture = CompletableFuture.supplyAsync( + pushAnalysisServiceFuture = executor.supplyAsync( () -> // Analyzing what push operations need to be performed pushAnalysisService.analyze( @@ -175,7 +178,7 @@ private Pair>, PushAnalysisSummary> analyze( pushAnalysisServiceFuture ); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( consoleLoadingAnimation ); @@ -237,28 +240,29 @@ private void processPush(List> analysisResults, summary.total ); - CompletableFuture> pushFuture = CompletableFuture.supplyAsync( + CompletableFuture> pushFuture = executor.supplyAsync( () -> { - var forkJoinPool = ForkJoinPool.commonPool(); - - var task = new PushTask<>( - PushTaskParams.builder(). - results(analysisResults). - allowRemove(options.allowRemove()). - disableAutoUpdate(options.disableAutoUpdate()). - failFast(options.failFast()). - customOptions(customOptions). - pushHandler(pushHandler). - mapperService(mapperService). - logger(logger). - progressBar(progressBar). - build() + + PushTask task = new PushTask<>( + logger, mapperService, executor ); - return forkJoinPool.invoke(task); + + task.setTaskParams(PushTaskParams.builder(). + results(analysisResults). + allowRemove(options.allowRemove()). + disableAutoUpdate(options.disableAutoUpdate()). + failFast(options.failFast()). + customOptions(customOptions). + pushHandler(pushHandler). + progressBar(progressBar). + build() + ); + + return task.compute(); }); progressBar.setFuture(pushFuture); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( progressBar ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/AbstractPushTaskParams.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/AbstractPushTaskParams.java index ac71e38dbf34..b13cc387969f 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/AbstractPushTaskParams.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/AbstractPushTaskParams.java @@ -1,6 +1,5 @@ package com.dotcms.api.client.push.task; -import com.dotcms.api.client.MapperService; import com.dotcms.api.client.push.PushHandler; import com.dotcms.cli.common.ConsoleProgressBar; import com.dotcms.model.annotation.ValueType; @@ -9,7 +8,6 @@ import java.util.List; import java.util.Map; import org.immutables.value.Value; -import org.jboss.logging.Logger; /** * Interface representing the parameters for the PushTask. @@ -34,13 +32,6 @@ public interface AbstractPushTaskParams extends Serializable { */ PushHandler pushHandler(); - /** - * Retrieves the mapper service used to map the content to the output format. - * - * @return the mapper service. - */ - MapperService mapperService(); - /** * Retrieves the custom push options. * @@ -74,11 +65,4 @@ public interface AbstractPushTaskParams extends Serializable { */ ConsoleProgressBar progressBar(); - /** - * Retrieves the logger for the pull operation. - * - * @return the logger. - */ - Logger logger(); - } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/PushTask.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/PushTask.java index 352ea85afb01..0b2edd25682d 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/PushTask.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/push/task/PushTask.java @@ -1,6 +1,8 @@ package com.dotcms.api.client.push.task; +import com.dotcms.api.client.MapperService; import com.dotcms.api.client.push.exception.PushException; +import com.dotcms.api.client.task.TaskProcessor; import com.dotcms.model.push.PushAnalysisResult; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -8,26 +10,51 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.RecursiveTask; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.function.Function; +import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.context.ManagedExecutor; import org.jboss.logging.Logger; /** - * Represents a task for pushing analysis results using a specified push handler. This class extends - * the `RecursiveTask` class from the `java.util.concurrent` package. + * Represents a task for pushing analysis results using a specified push handler. * * @param the type of analysis result */ -public class PushTask extends RecursiveTask> { +@Dependent +public class PushTask extends TaskProcessor { - private final PushTaskParams params; + private PushTaskParams params; private final Logger logger; - private static final int THRESHOLD = 10; + private final ManagedExecutor executor; - public PushTask(final PushTaskParams params) { + private final MapperService mapperService; + + public PushTask(final Logger logger, + final MapperService mapperService, final ManagedExecutor executor) { + this.logger = logger; + this.executor = executor; + this.mapperService = mapperService; + } + + /** + * Sets the parameters for the PushTask. This method provides a way to inject necessary + * configuration after the instance of PushTask has been created by the container, which is a + * common pattern when working with frameworks like Quarkus that manage object creation and + * dependency injection in a specific manner. + *

+ * This method is used as an alternative to constructor injection, which is not feasible due to + * the limitations or constraints of the framework's dependency injection mechanism. It allows + * for the explicit setting of traversal parameters after the object's instantiation, ensuring + * that the executor is properly configured before use. + * + * @param params The parameters for the PullTask + */ + public void setTaskParams(final PushTaskParams params) { this.params = params; - this.logger = params.logger(); } /** @@ -35,8 +62,10 @@ public PushTask(final PushTaskParams params) { * * @return a list of exceptions encountered during the computation */ - @Override - protected List compute() { + public List compute() { + + CompletionService> completionService = + new ExecutorCompletionService<>(executor); var errors = new ArrayList(); @@ -60,29 +89,46 @@ protected List compute() { } else { - // If the list is large, split it into two smaller tasks - int mid = this.params.results().size() / 2; - var paramsTask1 = this.params.withResults( - this.params.results().subList(0, mid) - ); - var paramsTask2 = this.params.withResults( - this.params.results().subList(mid, this.params.results().size()) - ); + // If the list is large, split it into smaller tasks + int toProcessCount = splitTasks(completionService); + + // Wait for all tasks to complete and gather the results + Function, Void> processFunction = taskResult -> { + errors.addAll(taskResult); + return null; + }; + processTasks(toProcessCount, completionService, processFunction); + } + + return errors; + } + + /** + * Splits a list of T objects into separate tasks. + * + * @param completionService The CompletionService to submit tasks to. + * @return The number of tasks to process. + */ + private int splitTasks(final CompletionService> completionService) { - var task1 = new PushTask<>(paramsTask1); - var task2 = new PushTask<>(paramsTask2); + int mid = this.params.results().size() / 2; + var paramsTask1 = this.params.withResults( + this.params.results().subList(0, mid) + ); + var paramsTask2 = this.params.withResults( + this.params.results().subList(mid, this.params.results().size()) + ); - // Start the first subtask in a new thread - task1.fork(); + PushTask task1 = new PushTask<>(logger, mapperService, executor); + task1.setTaskParams(paramsTask1); - // Start and wait for the second subtask to finish - errors.addAll(task2.compute()); + PushTask task2 = new PushTask<>(logger, mapperService, executor); + task2.setTaskParams(paramsTask2); - // Wait for the first subtask to finish - errors.addAll(task1.join()); - } + completionService.submit(task1::compute); + completionService.submit(task2::compute); - return errors; + return 2; } /** @@ -265,7 +311,7 @@ private void updateFile(final T content, final File localFile) { // Updating the file content - ObjectMapper objectMapper = this.params.mapperService().objectMapper(localFile); + ObjectMapper objectMapper = mapperService.objectMapper(localFile); final String asString = objectMapper.writeValueAsString(content); final Path path = Path.of(localFile.getAbsolutePath()); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/task/TaskProcessor.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/task/TaskProcessor.java new file mode 100644 index 000000000000..918c8d408da2 --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/api/client/task/TaskProcessor.java @@ -0,0 +1,93 @@ +package com.dotcms.api.client.task; + +import com.dotcms.api.client.files.traversal.exception.TraversalTaskException; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +public class TaskProcessor { + + private static final int MAX_RETRIES = 3;// Maximum number of retries for a single task + private static final int TASK_TIMEOUT = 15;// In seconds + protected static final int THRESHOLD = 10; + + /** + * Processes and waits for the results of the tasks submitted to the completion service. + * + * @param toProcessCount The number of tasks to process. + * @param completionService The CompletionService for parallel execution of tasks. + * @param processFunction The function to apply on each task result. + * @param The type of the task result. + */ + protected void processTasks(int toProcessCount, CompletionService completionService, + Function processFunction) { + + boolean interrupted = false; + int retryCount = 0; + int taskCount = 0; + + // Wait for all tasks to complete and gather the results + while (taskCount < toProcessCount) { + + try { + + Future future = completionService.poll( + TASK_TIMEOUT, TimeUnit.SECONDS + ); + if (future != null) { + // Task completed, process the result + T taskResult = future.get(); + processFunction.apply(taskResult); + + taskCount++; + retryCount = 0; // Reset retry count after a successful operation + } else { + // No task was completed in the given timeframe + if (retryCount < MAX_RETRIES) { + retryCount++; + } else { + throw new TraversalTaskException( + "Maximum retries reached for fetching task result" + ); + } + } + } catch (InterruptedException e) { + // Thread was interrupted, handle it outside the loop + interrupted = true; + Thread.currentThread().interrupt(); // Preserve interrupt status + } catch (ExecutionException e) { + handleExecutionException(e); + } + } + + if (interrupted) { + handleInterrupt(); + } + } + + /** + * Handles the case when a task execution throws an ExecutionException. + * + * @param e The ExecutionException encountered. + * @throws TraversalTaskException with the cause of the execution failure. + */ + private void handleExecutionException(ExecutionException e) throws TraversalTaskException { + if (e.getCause() instanceof TraversalTaskException) { + throw (TraversalTaskException) e.getCause(); + } else { + throw new TraversalTaskException("Error executing task", e); + } + } + + /** + * Handles the situation when the task processing is interrupted. + * + * @throws TraversalTaskException indicating that the task was interrupted. + */ + private void handleInterrupt() throws TraversalTaskException { + throw new TraversalTaskException("Task was interrupted and may not have completed fully"); + } + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/AbstractFilesListingCommand.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/AbstractFilesListingCommand.java new file mode 100644 index 000000000000..42a16c169a7a --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/AbstractFilesListingCommand.java @@ -0,0 +1,110 @@ +package com.dotcms.cli.command.files; + +import com.dotcms.api.LanguageAPI; +import com.dotcms.api.client.files.traversal.RemoteTraversalService; +import com.dotcms.api.traversal.TreeNode; +import com.dotcms.cli.common.ConsoleLoadingAnimation; +import com.dotcms.model.language.Language; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import javax.inject.Inject; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; +import picocli.CommandLine; + +/** + * This abstract class is used for implementing files listing commands. It provides common + * functionality for listing the contents of a remote directory. + */ +public abstract class AbstractFilesListingCommand extends AbstractFilesCommand { + + @CommandLine.Mixin + FilesListingMixin filesMixin; + + @Inject + RemoteTraversalService remoteTraversalService; + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @Inject + ManagedExecutor executor; + + /** + * Executes the listing of a remote folder at the specified depth. + * + * @param depth the depth of the folder traversal + * @return an exit code indicating the success of the operation + * @throws ExecutionException if an exception occurs during execution + * @throws InterruptedException if the execution is interrupted + */ + protected Integer listing(final Integer depth) throws ExecutionException, InterruptedException { + + // Checking for unmatched arguments + output.throwIfUnmatchedArguments(spec.commandLine()); + + var includeFolderPatterns = parsePatternOption( + filesMixin.globMixin.includeFolderPatternsOption + ); + var includeAssetPatterns = parsePatternOption( + filesMixin.globMixin.includeAssetPatternsOption + ); + var excludeFolderPatterns = parsePatternOption( + filesMixin.globMixin.excludeFolderPatternsOption + ); + var excludeAssetPatterns = parsePatternOption( + filesMixin.globMixin.excludeAssetPatternsOption + ); + + CompletableFuture, TreeNode>> folderTraversalFuture = executor.supplyAsync( + () -> + // Service to handle the traversal of the folder + remoteTraversalService.traverseRemoteFolder( + filesMixin.folderPath, + depth, + true, + includeFolderPatterns, + includeAssetPatterns, + excludeFolderPatterns, + excludeAssetPatterns + ) + ); + + // ConsoleLoadingAnimation instance to handle the waiting "animation" + ConsoleLoadingAnimation consoleLoadingAnimation = new ConsoleLoadingAnimation( + output, + folderTraversalFuture + ); + + CompletableFuture animationFuture = executor.runAsync( + consoleLoadingAnimation + ); + + // Waits for the completion of both the folder traversal and console loading animation tasks. + // This line blocks the current thread until both CompletableFuture instances + // (folderTraversalFuture and animationFuture) have completed. + CompletableFuture.allOf(folderTraversalFuture, animationFuture).join(); + final var result = folderTraversalFuture.get(); + + if (result == null) { + output.error(String.format( + "Error occurred while pulling folder info: [%s].", filesMixin.folderPath)); + return CommandLine.ExitCode.SOFTWARE; + } + + // We need to retrieve the languages + final LanguageAPI languageAPI = clientFactory.getClient(LanguageAPI.class); + final List languages = languageAPI.list().entity(); + + // Display the result + StringBuilder sb = new StringBuilder(); + TreePrinter.getInstance() + .filteredFormat(sb, result.getRight(), !filesMixin.excludeEmptyFolders, languages); + + output.info(sb.toString()); + + return CommandLine.ExitCode.OK; + } + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesGlobMixin.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesGlobMixin.java new file mode 100644 index 000000000000..96943228ac4b --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesGlobMixin.java @@ -0,0 +1,35 @@ +package com.dotcms.cli.command.files; + +import picocli.CommandLine; + +/** + * Mixin class that provides options for specifying glob patterns to include or exclude files and + * folders. + */ +public class FilesGlobMixin { + + @CommandLine.Option(names = {"-ef", "--excludeFolder"}, + paramLabel = "patterns", + description = "Exclude directories matching the given glob patterns. Multiple " + + "patterns can be specified, separated by commas.") + String excludeFolderPatternsOption; + + @CommandLine.Option(names = {"-ea", "--excludeAsset"}, + paramLabel = "patterns", + description = "Exclude assets matching the given glob patterns. Multiple " + + "patterns can be specified, separated by commas.") + String excludeAssetPatternsOption; + + @CommandLine.Option(names = {"-if", "--includeFolder"}, + paramLabel = "patterns", + description = "Include directories matching the given glob patterns. Multiple " + + "patterns can be specified, separated by commas.") + String includeFolderPatternsOption; + + @CommandLine.Option(names = {"-ia", "--includeAsset"}, + paramLabel = "patterns", + description = "Include assets matching the given glob patterns. Multiple " + + "patterns can be specified, separated by commas.") + String includeAssetPatternsOption; + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesListingMixin.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesListingMixin.java new file mode 100644 index 000000000000..60cafcc2711b --- /dev/null +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesListingMixin.java @@ -0,0 +1,29 @@ +package com.dotcms.cli.command.files; + +import picocli.CommandLine; +import picocli.CommandLine.Parameters; + +/** + * Mixin class that provides options for listing the contents of a remote dotCMS directory + */ +public class FilesListingMixin { + + @Parameters(index = "0", arity = "1", paramLabel = "path", + description = "dotCMS path to the directory to list the contents of " + + "- Format: //{site}/{folder}") + String folderPath; + + @CommandLine.Option(names = {"-ee", "--excludeEmptyFolders"}, defaultValue = "false", + description = + "When this option is enabled, the tree display will exclude folders that do " + + "not contain any assets, as well as folders that have no children with assets. " + + "This can be useful for users who want to focus on the folder structure that " + + "contains assets, making the output more concise and easier to navigate. By " + + "default, this option is disabled, and all folders, including empty ones, " + + "will be displayed in the tree.") + boolean excludeEmptyFolders; + + @CommandLine.Mixin + FilesGlobMixin globMixin; + +} diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesLs.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesLs.java index aaeceaf145e6..af86107dd6a8 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesLs.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesLs.java @@ -1,21 +1,10 @@ package com.dotcms.cli.command.files; -import com.dotcms.api.LanguageAPI; -import com.dotcms.api.client.files.traversal.RemoteTraversalService; -import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.command.DotCommand; -import com.dotcms.cli.common.ConsoleLoadingAnimation; import com.dotcms.cli.common.OutputOptionMixin; -import com.dotcms.model.language.Language; -import org.apache.commons.lang3.tuple.Pair; -import picocli.CommandLine; -import picocli.CommandLine.Parameters; - -import javax.enterprise.context.control.ActivateRequestContext; -import javax.inject.Inject; -import java.util.List; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; +import javax.enterprise.context.control.ActivateRequestContext; +import picocli.CommandLine; /** * Command to lists the files and directories in the specified directory. @@ -29,115 +18,13 @@ "" // empty string here so we can have a new line } ) -public class FilesLs extends AbstractFilesCommand implements Callable, DotCommand { +public class FilesLs extends AbstractFilesListingCommand implements Callable, DotCommand { static final String NAME = "ls"; - @Parameters(index = "0", arity = "1", paramLabel = "path", - description = "dotCMS path to the directory to list the contents of " - + "- Format: //{site}/{folder}") - String folderPath; - - @CommandLine.Option(names = {"-ee", "--excludeEmptyFolders"}, defaultValue = "false", - description = - "When this option is enabled, the tree display will exclude folders that do " - + "not contain any assets, as well as folders that have no children with assets. " - + "This can be useful for users who want to focus on the folder structure that " - + "contains assets, making the output more concise and easier to navigate. By " - + "default, this option is disabled, and all folders, including empty ones, " - + "will be displayed in the tree.") - boolean excludeEmptyFolders; - - @CommandLine.Option(names = {"-ef", "--excludeFolder"}, - paramLabel = "patterns", - description = "Exclude directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeFolderPatternsOption; - - @CommandLine.Option(names = {"-ea", "--excludeAsset"}, - paramLabel = "patterns", - description = "Exclude assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeAssetPatternsOption; - - @CommandLine.Option(names = {"-if", "--includeFolder"}, - paramLabel = "patterns", - description = "Include directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeFolderPatternsOption; - - @CommandLine.Option(names = {"-ia", "--includeAsset"}, - paramLabel = "patterns", - description = "Include assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeAssetPatternsOption; - - @Inject - RemoteTraversalService remoteTraversalService; - - @CommandLine.Spec - CommandLine.Model.CommandSpec spec; - @Override public Integer call() throws Exception { - - // Checking for unmatched arguments - output.throwIfUnmatchedArguments(spec.commandLine()); - - var includeFolderPatterns = parsePatternOption(includeFolderPatternsOption); - var includeAssetPatterns = parsePatternOption(includeAssetPatternsOption); - var excludeFolderPatterns = parsePatternOption(excludeFolderPatternsOption); - var excludeAssetPatterns = parsePatternOption(excludeAssetPatternsOption); - - - CompletableFuture, TreeNode>> folderTraversalFuture = CompletableFuture.supplyAsync( - () -> { - // Service to handle the traversal of the folder - return remoteTraversalService.traverseRemoteFolder( - folderPath, - 0, - true, - includeFolderPatterns, - includeAssetPatterns, - excludeFolderPatterns, - excludeAssetPatterns - ); - }); - - // ConsoleLoadingAnimation instance to handle the waiting "animation" - ConsoleLoadingAnimation consoleLoadingAnimation = new ConsoleLoadingAnimation( - output, - folderTraversalFuture - ); - - CompletableFuture animationFuture = CompletableFuture.runAsync( - consoleLoadingAnimation - ); - - // Waits for the completion of both the folder traversal and console loading animation tasks. - // This line blocks the current thread until both CompletableFuture instances - // (folderTraversalFuture and animationFuture) have completed. - CompletableFuture.allOf(folderTraversalFuture, animationFuture).join(); - final var result = folderTraversalFuture.get(); - - if (result == null) { - output.error(String.format( - "Error occurred while pulling folder info: [%s].", folderPath)); - return CommandLine.ExitCode.SOFTWARE; - } - - // We need to retrieve the languages - final LanguageAPI languageAPI = clientFactory.getClient(LanguageAPI.class); - final List languages = languageAPI.list().entity(); - - // Display the result - StringBuilder sb = new StringBuilder(); - TreePrinter.getInstance().filteredFormat(sb, result.getRight(), !excludeEmptyFolders, languages); - - output.info(sb.toString()); - - - return CommandLine.ExitCode.OK; + return listing(0); } @Override @@ -150,5 +37,4 @@ public OutputOptionMixin getOutput() { return output; } - } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPull.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPull.java index 202741f69e4c..d6a10feb8543 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPull.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPull.java @@ -17,7 +17,6 @@ import com.dotcms.cli.common.OutputOptionMixin; import com.dotcms.cli.common.PullMixin; import com.dotcms.cli.common.WorkspaceParams; -import com.dotcms.common.WorkspaceManager; import com.dotcms.model.config.Workspace; import com.dotcms.model.pull.PullOptions; import java.io.File; @@ -64,10 +63,7 @@ public class FilesPull extends AbstractFilesCommand implements Callable PullMixin pullMixin; @CommandLine.Mixin(name = FILE_PULL_MIXIN) - FilePullMixin filePullMixin; - - @Inject - WorkspaceManager workspaceManager; + FilesPullMixin filesPullMixin; @Inject PullService pullService; @@ -103,26 +99,34 @@ public Integer call() throws Exception { ); } - var includeFolderPatterns = parsePatternOption(filePullMixin.includeFolderPatternsOption); - var includeAssetPatterns = parsePatternOption(filePullMixin.includeAssetPatternsOption); - var excludeFolderPatterns = parsePatternOption(filePullMixin.excludeFolderPatternsOption); - var excludeAssetPatterns = parsePatternOption(filePullMixin.excludeAssetPatternsOption); + var includeFolderPatterns = parsePatternOption( + filesPullMixin.globMixin.includeFolderPatternsOption + ); + var includeAssetPatterns = parsePatternOption( + filesPullMixin.globMixin.includeAssetPatternsOption + ); + var excludeFolderPatterns = parsePatternOption( + filesPullMixin.globMixin.excludeFolderPatternsOption + ); + var excludeAssetPatterns = parsePatternOption( + filesPullMixin.globMixin.excludeAssetPatternsOption + ); var customOptions = Map.of( INCLUDE_FOLDER_PATTERNS, includeFolderPatterns, INCLUDE_ASSET_PATTERNS, includeAssetPatterns, EXCLUDE_FOLDER_PATTERNS, excludeFolderPatterns, EXCLUDE_ASSET_PATTERNS, excludeAssetPatterns, - NON_RECURSIVE, filePullMixin.nonRecursive, - PRESERVE, filePullMixin.preserve, - INCLUDE_EMPTY_FOLDERS, filePullMixin.includeEmptyFolders + NON_RECURSIVE, filesPullMixin.nonRecursive, + PRESERVE, filesPullMixin.preserve, + INCLUDE_EMPTY_FOLDERS, filesPullMixin.includeEmptyFolders ); // Execute the pull pullService.pull( PullOptions.builder(). destination(filesFolder). - contentKey(Optional.ofNullable(filePullMixin.path)). + contentKey(Optional.ofNullable(filesPullMixin.path)). isShortOutput(pullMixin.shortOutputOption().isShortOutput()). failFast(pullMixin.failFast). maxRetryAttempts(pullMixin.retryAttempts). @@ -151,11 +155,6 @@ public PullMixin getPullMixin() { return pullMixin; } - @Override - public Optional getCustomMixinName() { - return Optional.empty(); - } - @Override public int getOrder() { return ApplyCommandOrder.FILES.getOrder(); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilePullMixin.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPullMixin.java similarity index 51% rename from tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilePullMixin.java rename to tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPullMixin.java index 734d407c2d22..e5c755483cea 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilePullMixin.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPullMixin.java @@ -2,7 +2,7 @@ import picocli.CommandLine; -public class FilePullMixin { +public class FilesPullMixin { @CommandLine.Parameters(index = "0", arity = "0..1", paramLabel = "path", description = "dotCMS path to a specific site, directory or file to pull. " @@ -25,28 +25,7 @@ public class FilePullMixin { + "By default, this option is disabled, and empty folders will not be created.") boolean includeEmptyFolders; - @CommandLine.Option(names = {"-ef", "--excludeFolder"}, - paramLabel = "patterns", - description = "Exclude directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeFolderPatternsOption; - - @CommandLine.Option(names = {"-ea", "--excludeAsset"}, - paramLabel = "patterns", - description = "Exclude assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeAssetPatternsOption; - - @CommandLine.Option(names = {"-if", "--includeFolder"}, - paramLabel = "patterns", - description = "Include directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeFolderPatternsOption; - - @CommandLine.Option(names = {"-ia", "--includeAsset"}, - paramLabel = "patterns", - description = "Include assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeAssetPatternsOption; + @CommandLine.Mixin + FilesGlobMixin globMixin; } diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPush.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPush.java index b77b2a7db14a..df759bb8ff5e 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPush.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesPush.java @@ -29,6 +29,7 @@ import javax.enterprise.context.control.ActivateRequestContext; import javax.inject.Inject; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.context.ManagedExecutor; import picocli.CommandLine; @ActivateRequestContext @@ -61,6 +62,9 @@ public class FilesPush extends AbstractFilesCommand implements Callable @Inject PushContext pushContext; + @Inject + ManagedExecutor executor; + @Override public Integer call() throws Exception { @@ -76,7 +80,7 @@ public Integer call() throws Exception { File finalInputFile = resolvedWorkspaceAndPath.getRight(); CompletableFuture> - folderTraversalFuture = CompletableFuture.supplyAsync( + folderTraversalFuture = executor.supplyAsync( () -> // Service to handle the traversal of the folder pushService.traverseLocalFolders( @@ -91,7 +95,7 @@ public Integer call() throws Exception { folderTraversalFuture ); - CompletableFuture animationFuture = CompletableFuture.runAsync( + CompletableFuture animationFuture = executor.runAsync( consoleLoadingAnimation ); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesTree.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesTree.java index e26ee641ff24..589b4118aa1f 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesTree.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/FilesTree.java @@ -1,21 +1,10 @@ package com.dotcms.cli.command.files; -import com.dotcms.api.LanguageAPI; -import com.dotcms.api.client.files.traversal.RemoteTraversalService; -import com.dotcms.api.traversal.TreeNode; import com.dotcms.cli.command.DotCommand; -import com.dotcms.cli.common.ConsoleLoadingAnimation; import com.dotcms.cli.common.OutputOptionMixin; -import com.dotcms.model.language.Language; -import org.apache.commons.lang3.tuple.Pair; -import picocli.CommandLine; -import picocli.CommandLine.Parameters; - -import javax.enterprise.context.control.ActivateRequestContext; -import javax.inject.Inject; -import java.util.List; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; +import javax.enterprise.context.control.ActivateRequestContext; +import picocli.CommandLine; /** * Command to display a hierarchical tree view of the files and subdirectories within the specified @@ -31,15 +20,11 @@ "" // empty string here so we can have a new line } ) -public class FilesTree extends AbstractFilesCommand implements Callable, DotCommand { +public class FilesTree extends AbstractFilesListingCommand implements Callable, + DotCommand { static final String NAME = "tree"; - @Parameters(index = "0", arity = "1", paramLabel = "path", - description = "dotCMS path to the directory to list the contents of " - + "- Format: //{site}/{folder}") - String folderPath; - @CommandLine.Option(names = {"-d", "--depth"}, description = "Limits the depth of the directory tree to levels. " + "The default value is 0, which means that only the files and directories in " @@ -47,105 +32,9 @@ public class FilesTree extends AbstractFilesCommand implements Callable + "there is no limit on the depth of the directory tree.") Integer depth; - @CommandLine.Option(names = {"-ee", "--excludeEmptyFolders"}, defaultValue = "false", - description = - "When this option is enabled, the tree display will exclude folders that do " - + "not contain any assets, as well as folders that have no children with assets. " - + "This can be useful for users who want to focus on the folder structure that " - + "contains assets, making the output more concise and easier to navigate. By " - + "default, this option is disabled, and all folders, including empty ones, " - + "will be displayed in the tree.") - boolean excludeEmptyFolders; - - @CommandLine.Option(names = {"-ef", "--excludeFolder"}, - paramLabel = "patterns", - description = "Exclude directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeFolderPatternsOption; - - @CommandLine.Option(names = {"-ea", "--excludeAsset"}, - paramLabel = "patterns", - description = "Exclude assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String excludeAssetPatternsOption; - - @CommandLine.Option(names = {"-if", "--includeFolder"}, - paramLabel = "patterns", - description = "Include directories matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeFolderPatternsOption; - - @CommandLine.Option(names = {"-ia", "--includeAsset"}, - paramLabel = "patterns", - description = "Include assets matching the given glob patterns. Multiple " - + "patterns can be specified, separated by commas.") - String includeAssetPatternsOption; - - @Inject - RemoteTraversalService remoteTraversalService; - - @CommandLine.Spec - CommandLine.Model.CommandSpec spec; - @Override public Integer call() throws Exception { - - // Checking for unmatched arguments - output.throwIfUnmatchedArguments(spec.commandLine()); - - var includeFolderPatterns = parsePatternOption(includeFolderPatternsOption); - var includeAssetPatterns = parsePatternOption(includeAssetPatternsOption); - var excludeFolderPatterns = parsePatternOption(excludeFolderPatternsOption); - var excludeAssetPatterns = parsePatternOption(excludeAssetPatternsOption); - - CompletableFuture, TreeNode>> folderTraversalFuture = CompletableFuture.supplyAsync( - () -> { - // Service to handle the traversal of the folder - return remoteTraversalService.traverseRemoteFolder( - folderPath, - depth, - true, - includeFolderPatterns, - includeAssetPatterns, - excludeFolderPatterns, - excludeAssetPatterns - ); - }); - - // ConsoleLoadingAnimation instance to handle the waiting "animation" - ConsoleLoadingAnimation consoleLoadingAnimation = new ConsoleLoadingAnimation( - output, - folderTraversalFuture - ); - - CompletableFuture animationFuture = CompletableFuture.runAsync( - consoleLoadingAnimation - ); - - // Waits for the completion of both the folder traversal and console loading animation tasks. - // This line blocks the current thread until both CompletableFuture instances - // (folderTraversalFuture and animationFuture) have completed. - CompletableFuture.allOf(folderTraversalFuture, animationFuture).join(); - final var result = folderTraversalFuture.get(); - - if (result == null) { - output.error(String.format( - "Error occurred while pulling folder info: [%s].", folderPath)); - return CommandLine.ExitCode.SOFTWARE; - } - - // We need to retrieve the languages - final LanguageAPI languageAPI = clientFactory.getClient(LanguageAPI.class); - final List languages = languageAPI.list().entity(); - - // Display the result - StringBuilder sb = new StringBuilder(); - TreePrinter.getInstance() - .filteredFormat(sb, result.getRight(), !excludeEmptyFolders, languages); - - output.info(sb.toString()); - - return CommandLine.ExitCode.OK; + return listing(depth); } @Override diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/TreePrinter.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/TreePrinter.java index 6e0aa1c7c254..d45a5cc5c16b 100644 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/TreePrinter.java +++ b/tools/dotcms-cli/cli/src/main/java/com/dotcms/cli/command/files/TreePrinter.java @@ -10,7 +10,6 @@ import com.dotcms.model.asset.AssetView; import com.dotcms.model.asset.FolderView; import com.dotcms.model.language.Language; - import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; @@ -244,6 +243,7 @@ private void format(StringBuilder sb, String prefix, final TreeNode node, if (includeAssets) { // Adds the names of the node's files to the string representation. + node.sortAssets(); int assetCount = node.assets().size(); for (int i = 0; i < assetCount; i++) { @@ -275,6 +275,7 @@ private void format(StringBuilder sb, String prefix, final TreeNode node, } // Recursively creates string representations for the node's children. + node.sortChildren(); int childCount = node.children().size(); for (int i = 0; i < childCount; i++) { TreeNode child = node.children().get(i); diff --git a/tools/dotcms-cli/cli/src/main/java/com/dotcms/security/Utils.java b/tools/dotcms-cli/cli/src/main/java/com/dotcms/security/Utils.java deleted file mode 100644 index 28aebba13abb..000000000000 --- a/tools/dotcms-cli/cli/src/main/java/com/dotcms/security/Utils.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.dotcms.security; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.NoSuchAlgorithmException; - -public class Utils { - - private Utils() { - //Hide public constructor - } - - /** - * Figure out the sha256 of the file content, assumes that the file exists and can be read - * - * @param path {@link Path} - * @return String just as unix sha returns - * @throws NoSuchAlgorithmException - * @throws IOException - */ - public static String Sha256toUnixHash(final Path path) { - - try { - final HashBuilder sha256Builder = Encryptor.Hashing.sha256(); - final byte[] buffer = new byte[4096]; - int countBytes; - - try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) { - - countBytes = inputStream.read(buffer); - while (countBytes > 0) { - - sha256Builder.append(buffer, countBytes); - countBytes = inputStream.read(buffer); - } - } - - return sha256Builder.buildUnixHash(); - } catch (NoSuchAlgorithmException | IOException e) { - var errorMessage = String.format("Error calculating sha256 for file [%s]", path); - throw new RuntimeException(errorMessage, e); - } - } -} diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/security/UtilsTest.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/FileHashCalculatorServiceTest.java similarity index 74% rename from tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/security/UtilsTest.java rename to tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/FileHashCalculatorServiceTest.java index 5a854920778e..ce3ac7158ca1 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/security/UtilsTest.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/FileHashCalculatorServiceTest.java @@ -1,17 +1,19 @@ -package com.dotcms.cli.security; +package com.dotcms.api.client; -import com.dotcms.security.Utils; import io.quarkus.test.junit.QuarkusTest; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import javax.inject.Inject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; @QuarkusTest -class UtilsTest { +class FileHashCalculatorServiceTest { + + @Inject + FileHashCalculatorService fileHashCalculatorService; @Test void testSha256toUnixHash_withValidFile_returnsCorrectHash() throws Exception { @@ -23,7 +25,7 @@ void testSha256toUnixHash_withValidFile_returnsCorrectHash() throws Exception { Files.write(path, "Hello, world!".getBytes()); String expectedHash = "315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3"; - String actualHash = Utils.Sha256toUnixHash(path); + String actualHash = fileHashCalculatorService.sha256toUnixHash(path); Assertions.assertEquals(expectedHash, actualHash); } finally { @@ -43,13 +45,13 @@ void testSha256toUnixHash_withValidFile_afterRename_returnsSameHash() throws Exc originalPath = Files.createTempFile("test", ".txt"); Files.write(originalPath, "Hello, world!".getBytes()); - var originalHash = Utils.Sha256toUnixHash(originalPath); + var originalHash = fileHashCalculatorService.sha256toUnixHash(originalPath); Assertions.assertNotNull(originalHash); renamePath = Files.createTempFile("new-test", ".txt"); Files.move(originalPath, renamePath, StandardCopyOption.REPLACE_EXISTING); - var newHash = Utils.Sha256toUnixHash(renamePath); + var newHash = fileHashCalculatorService.sha256toUnixHash(renamePath); Assertions.assertNotNull(newHash); Assertions.assertEquals(originalHash, newHash); @@ -69,16 +71,17 @@ void testSha256toUnixHash_withNonExistentFile_throwsIOException() { Path path = Path.of("nonexistent.txt"); try { - Utils.Sha256toUnixHash(path); + fileHashCalculatorService.sha256toUnixHash(path); Assertions.fail("Expected NoSuchFileException to be thrown"); } catch (Exception e) { - Assertions.assertTrue(e.getCause() instanceof NoSuchFileException); + Assertions.assertInstanceOf(NoSuchFileException.class, e.getCause()); } } @Test void testSha256toUnixHash_withNullPath_throwsNullPointerException() { - Assertions.assertThrows(NullPointerException.class, () -> Utils.Sha256toUnixHash(null)); + Assertions.assertThrows(NullPointerException.class, + () -> fileHashCalculatorService.sha256toUnixHash(null)); } } diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/PushServiceIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/PushServiceIT.java index 840fdceadbfc..2a06cc78a053 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/PushServiceIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/PushServiceIT.java @@ -37,7 +37,6 @@ import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** @@ -185,7 +184,6 @@ void Test_Nothing_To_Push() throws IOException { * * @throws IOException if an I/O error occurs */ - @Disabled("Test is intermittently failing.") @Test void Test_Push_New_Site() throws IOException { @@ -298,6 +296,14 @@ void Test_Push_New_Site() throws IOException { ); var newSiteTreeNode = newSiteResults.getRight(); + Assertions.assertEquals(4, newSiteTreeNode.children().size()); + + // Sorting the children to make the test deterministic + newSiteTreeNode.sortChildren(); + newSiteTreeNode.children().get(0).sortChildren(); + newSiteTreeNode.children().get(1).sortChildren(); + newSiteTreeNode.children().get(2).sortChildren(); + //Validating the tree // subFolder1-1-1 (has 2 asset) Assertions.assertEquals(2, newSiteTreeNode.children().get(0).children().get(0).children().get(0).assets().size()); @@ -320,7 +326,6 @@ void Test_Push_New_Site() throws IOException { * * @throws IOException if an I/O error occurs */ - @Disabled("Test is intermittently failing.") @Test void Test_Push_Modified_Data() throws IOException { @@ -457,6 +462,16 @@ void Test_Push_Modified_Data() throws IOException { ); var updatedTreeNode = updatedResults.getRight(); + Assertions.assertEquals(5, updatedTreeNode.children().size()); + + // Sorting the children to make the test deterministic + updatedTreeNode.sortChildren(); + updatedTreeNode.children().get(0).sortChildren(); + updatedTreeNode.children().get(1).sortChildren(); + updatedTreeNode.children().get(2).sortChildren(); + updatedTreeNode.children().get(3).sortChildren(); + updatedTreeNode.children().get(4).sortChildren(); + //Validating the tree // subFolder1-1-1 (has 2 asset) Assertions.assertEquals(2, updatedTreeNode.children().get(0).children().get(0).children().get(0).assets().size()); @@ -492,7 +507,6 @@ void Test_Push_Modified_Data() throws IOException { * If the real intend if really removing the folder remotely. The folder needs to me removed also from the "working" tree nodes branch * @throws IOException */ - @Disabled("Test is intermittently failing.") @Test void Test_Delete_Folder() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceIT.java index 2f877b004613..036d1d8a494a 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/api/client/files/traversal/RemoteTraversalServiceIT.java @@ -15,7 +15,6 @@ import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @QuarkusTest @@ -74,7 +73,6 @@ void Test_Not_Found() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Folders_Check() throws IOException { @@ -97,9 +95,17 @@ void Test_Folders_Check() throws IOException { // ============================ //Validating the tree // ============================ + // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -244,7 +250,6 @@ void Test_Folders_Depth_Zero() throws IOException { Assertions.assertEquals(0, treeNode.children().get(2).children().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include() throws IOException { @@ -273,6 +278,13 @@ void Test_Include() throws IOException { Assertions.assertFalse(child.folder().implicitGlobInclude()); } + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); for (var child : treeNode.children().get(0).children()) { @@ -316,7 +328,6 @@ void Test_Include() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).children().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include2() throws IOException { @@ -345,6 +356,13 @@ void Test_Include2() throws IOException { Assertions.assertFalse(child.folder().implicitGlobInclude()); } + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); Assertions.assertTrue( @@ -392,7 +410,6 @@ void Test_Include2() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).children().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include3() throws IOException { @@ -418,6 +435,13 @@ void Test_Include3() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + Assertions.assertTrue( treeNode.children().get(0).folder().explicitGlobInclude()); Assertions.assertTrue( @@ -438,7 +462,6 @@ void Test_Include3() throws IOException { } } - @Disabled("Test is intermittently failing.") @Test void Test_Include_Assets() throws IOException { @@ -464,6 +487,13 @@ void Test_Include_Assets() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -497,7 +527,6 @@ void Test_Include_Assets() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include_Assets2() throws IOException { @@ -523,6 +552,13 @@ void Test_Include_Assets2() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -556,7 +592,6 @@ void Test_Include_Assets2() throws IOException { Assertions.assertEquals(1, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include_Assets3() throws IOException { @@ -582,6 +617,13 @@ void Test_Include_Assets3() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -615,7 +657,6 @@ void Test_Include_Assets3() throws IOException { Assertions.assertEquals(1, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include_Assets4() throws IOException { @@ -641,6 +682,13 @@ void Test_Include_Assets4() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -674,7 +722,6 @@ void Test_Include_Assets4() throws IOException { Assertions.assertEquals(1, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Include_Assets5() throws IOException { @@ -700,6 +747,13 @@ void Test_Include_Assets5() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -733,7 +787,6 @@ void Test_Include_Assets5() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude() throws IOException { @@ -762,6 +815,13 @@ void Test_Exclude() throws IOException { Assertions.assertTrue(child.folder().implicitGlobInclude()); } + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); for (var child : treeNode.children().get(0).children()) { @@ -805,7 +865,6 @@ void Test_Exclude() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).children().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude2() throws IOException { @@ -834,6 +893,13 @@ void Test_Exclude2() throws IOException { Assertions.assertTrue(child.folder().implicitGlobInclude()); } + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); Assertions.assertTrue( @@ -881,7 +947,6 @@ void Test_Exclude2() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).children().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude3() throws IOException { @@ -907,6 +972,13 @@ void Test_Exclude3() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + Assertions.assertTrue( treeNode.children().get(0).folder().explicitGlobExclude()); Assertions.assertFalse( @@ -925,7 +997,6 @@ void Test_Exclude3() throws IOException { } } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude_Assets() throws IOException { @@ -951,6 +1022,13 @@ void Test_Exclude_Assets() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -984,7 +1062,6 @@ void Test_Exclude_Assets() throws IOException { Assertions.assertEquals(1, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude_Assets2() throws IOException { @@ -1010,6 +1087,13 @@ void Test_Exclude_Assets2() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -1043,7 +1127,6 @@ void Test_Exclude_Assets2() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude_Assets3() throws IOException { @@ -1069,6 +1152,13 @@ void Test_Exclude_Assets3() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 @@ -1102,7 +1192,6 @@ void Test_Exclude_Assets3() throws IOException { Assertions.assertEquals(0, treeNode.children().get(3).assets().size()); } - @Disabled("Test is intermittently failing.") @Test void Test_Exclude_Assets4() throws IOException { @@ -1128,6 +1217,13 @@ void Test_Exclude_Assets4() throws IOException { // Root Assertions.assertEquals(4, treeNode.children().size()); + // Sorting the children to make the test deterministic + treeNode.sortChildren(); + treeNode.children().get(0).sortChildren(); + treeNode.children().get(1).sortChildren(); + treeNode.children().get(2).sortChildren(); + treeNode.children().get(3).sortChildren(); + // Folder1 Assertions.assertEquals(3, treeNode.children().get(0).children().size()); // SubFolder1-1 diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/PullCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/PullCommandIT.java index 11db8de57238..c2cfa2f5f5a8 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/PullCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/PullCommandIT.java @@ -25,7 +25,6 @@ import javax.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.InjectMocks; @@ -81,7 +80,6 @@ void setUp() throws IOException { /** * This test checks for a simple pull situation where everything should work as expected. */ - @Disabled("Test is intermittently failing.") @Test void testSimplePull() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/contenttype/ContentTypeCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/contenttype/ContentTypeCommandIT.java index 47dbf9f960f5..c35e8ae87e0b 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/contenttype/ContentTypeCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/contenttype/ContentTypeCommandIT.java @@ -40,7 +40,6 @@ import javax.ws.rs.NotFoundException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.wildfly.common.Assert; @@ -272,7 +271,6 @@ void Test_Command_Content_Type_Pull_Checking_JSON_DotCMS_Type() throws IOExcepti * * @throws IOException if there is an error reading the YAML content type file */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Content_Type_Pull_Checking_YAML_DotCMS_Type() throws IOException { @@ -348,7 +346,6 @@ void Test_Command_Content_Filter_Option() { * * @throws IOException */ - @Disabled("This test is index dependent therefore there's a chance to see it fail from time to time") @Test void Test_Push_New_Content_Type_From_File_Then_Remove() throws IOException { @@ -431,7 +428,6 @@ void Test_Push_New_Content_Type_From_File_Then_Remove() throws IOException { * * @throws IOException */ - @Disabled("Test is intermittently failing.") @Test void Test_Pull_Same_Content_Type_Multiple_Times() throws IOException { // Create a temporal folder for the workspace @@ -469,7 +465,6 @@ void Test_Pull_Same_Content_Type_Multiple_Times() throws IOException { * folder, checking the content types are properly add, updated and removed on the remote * server. */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Content_Type_Folder_Push() throws IOException { @@ -666,7 +661,6 @@ private String createContentType(Workspace workspace, boolean asFile) throws IOE * * @throws IOException if there is an error pulling the content types */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Content_Type_Pull_Pull_All_Default_Format() throws IOException { @@ -749,7 +743,6 @@ void Test_Command_Content_Type_Pull_Pull_All_Default_Format() throws IOException * * @throws IOException if there is an error pulling the content types */ - @Disabled("Test is intermittently failing.") @Test @Order(13) void Test_Command_Content_Type_Pull_Pull_All_YAML_Format() throws IOException { @@ -833,7 +826,6 @@ void Test_Command_Content_Type_Pull_Pull_All_YAML_Format() throws IOException { * * @throws IOException if there is an error pulling the content types */ - @Disabled("Test is intermittently failing.") @Test @Order(14) void Test_Command_Content_Type_Pull_Pull_All_Twice() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesLsCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesLsCommandIT.java index 454137e2e8e8..c657a5357c38 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesLsCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesLsCommandIT.java @@ -11,7 +11,6 @@ import javax.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import picocli.CommandLine; import picocli.CommandLine.ExitCode; @@ -43,7 +42,6 @@ void Test_Command_Files_Ls_Option_Invalid_Protocol() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Valid_Protocol() { @@ -98,7 +96,6 @@ void Test_Command_Files_Ls_Option_Exclude_Empty2() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Glob_Exclude_Folders() { @@ -113,7 +110,6 @@ void Test_Command_Files_Ls_Option_Glob_Exclude_Folders() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Glob_Exclude_Folders2() { @@ -142,7 +138,6 @@ void Test_Command_Files_Ls_Option_Glob_Exclude_Folders3() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Glob_Exclude_Folders4() { @@ -283,7 +278,6 @@ void Test_Command_Files_Ls_Option_Glob_Include_Folders() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Glob_Include_Folders2() { @@ -396,7 +390,6 @@ void Test_Command_Files_Ls_Option_Glob_Include_Assets3() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Ls_Option_Glob_Include_Assets4() { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesPullCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesPullCommandIT.java index df18a99acf2a..79ce33f8a8e5 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesPullCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesPullCommandIT.java @@ -11,7 +11,6 @@ import javax.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import picocli.CommandLine; @@ -75,7 +74,6 @@ void Test_Command_Files_Pull_Option_Invalid_Protocol() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Pull_Option_Valid_Protocol() throws IOException { @@ -114,7 +112,6 @@ void Test_Command_Files_Pull_Option_Valid_Protocol2() throws IOException { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Pull_Option_Preserve() throws IOException { @@ -172,7 +169,6 @@ void Test_Command_Files_Pull_Option_Include_Empty() throws IOException { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Pull_Option_Include_Empty2() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesTreeCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesTreeCommandIT.java index 1c6fd0d39664..d3714410ff5e 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesTreeCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/files/FilesTreeCommandIT.java @@ -11,7 +11,6 @@ import javax.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import picocli.CommandLine; import picocli.CommandLine.ExitCode; @@ -112,7 +111,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Folders() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Exclude_Folders2() { @@ -141,7 +139,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Folders3() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Exclude_Folders4() { @@ -184,7 +181,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Folders_Missing_Parameter2() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Exclude_Assets() { @@ -199,7 +195,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Assets() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Exclude_Assets2() { @@ -228,7 +223,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Assets3() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Exclude_Assets4() { @@ -271,7 +265,6 @@ void Test_Command_Files_Tree_Option_Glob_Exclude_Assets_Missing_Parameter2() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Include_Folders() { @@ -300,7 +293,6 @@ void Test_Command_Files_Tree_Option_Glob_Include_Folders2() { } } - @Disabled("Test is intermittently failing.") @Test void Test_Command_Files_Tree_Option_Glob_Include_Folders3() { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/language/LanguageCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/language/LanguageCommandIT.java index 0d77f0306962..d7059fb6815c 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/language/LanguageCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/language/LanguageCommandIT.java @@ -26,7 +26,6 @@ import javax.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import picocli.CommandLine; import picocli.CommandLine.ExitCode; @@ -132,7 +131,6 @@ void Test_Command_Language_Pull_By_IsoCode() throws IOException { * * @throws IOException if there is an error reading the JSON language file */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Pull_By_IsoCode_Checking_JSON_DotCMS_Type() throws IOException { @@ -299,7 +297,6 @@ void Test_Command_Language_Push_byIsoCodeWithoutCountry() throws IOException { * A new language with iso code "it-IT" will be created.
* Expected Result: The language returned should be Italian */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Push_byFile_JSON() throws IOException { @@ -349,7 +346,6 @@ void Test_Command_Language_Push_byFile_JSON() throws IOException { *

* Expected Result: The language returned should be Italian */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Push_byFile_JSON_Checking_Auto_Update() throws IOException { @@ -419,7 +415,6 @@ void Test_Command_Language_Push_byFile_JSON_Checking_Auto_Update() throws IOExce * A new language with iso code "it-IT" will be created.
* Expected Result: The language returned should be Italian */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Push_byFile_YAML() throws IOException { @@ -538,7 +533,6 @@ void Test_Command_Language_Remove_byId() throws IOException { * Expected result: The WorkspaceManager should be able to create and destroy a workspace * @throws IOException */ - @Disabled("Test is intermittently failing.") @Test void Test_Pull_Same_Language_Multiple_Times() throws IOException { final Workspace workspace = workspaceManager.getOrCreate(Path.of("")); @@ -572,7 +566,6 @@ void Test_Pull_Same_Language_Multiple_Times() throws IOException { * This tests will test the functionality of the language push command when pushing a folder, * checking that the languages are properly add, updated and removed on the remote server. */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Folder_Push() throws IOException { @@ -876,7 +869,6 @@ void Test_Command_Language_Pull_Pull_All_Default_Format() throws IOException { * * @throws IOException if there is an error pulling the languages */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Pull_Pull_All_YAML_Format() throws IOException { @@ -1001,7 +993,6 @@ void Test_Command_Language_Pull_Pull_All_YAML_Format() throws IOException { * * @throws IOException if there is an error pulling the languages */ - @Disabled("Test is intermittently failing.") @Test void Test_Command_Language_Pull_Pull_All_Twice() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/site/SiteCommandIT.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/site/SiteCommandIT.java index 89b6ae7f9b64..716bdbe4985b 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/site/SiteCommandIT.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/command/site/SiteCommandIT.java @@ -31,7 +31,6 @@ import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.wildfly.common.Assert; @@ -226,7 +225,6 @@ void Test_Command_Create_Then_Pull_Then_Push() throws IOException { * * @throws IOException */ - @Disabled("Test is intermittently failing.") @Test @Order(6) void Test_Create_From_File_via_Push() throws IOException { @@ -427,7 +425,6 @@ void Test_Command_Site_List_All() { * This tests will test the functionality of the site push command when pushing a folder, * checking the sites are properly add, updated and removed on the remote server. */ - @Disabled("Test is intermittently failing.") @Test @Order(11) void Test_Command_Site_Folder_Push() throws IOException { @@ -600,7 +597,6 @@ void Test_Command_Site_Folder_Push() throws IOException { * * @throws IOException if there is an error pulling the sites */ - @Disabled("Test is intermittently failing.") @Test @Order(12) void Test_Command_Site_Pull_Pull_All_Default_Format() throws IOException { @@ -692,7 +688,6 @@ void Test_Command_Site_Pull_Pull_All_Default_Format() throws IOException { * * @throws IOException if there is an error pulling the sites */ - @Disabled("Test is intermittently failing.") @Test @Order(13) void Test_Command_Site_Pull_Pull_All_YAML_Format() throws IOException { @@ -785,7 +780,6 @@ void Test_Command_Site_Pull_Pull_All_YAML_Format() throws IOException { * * @throws IOException if there is an error pulling the sites */ - @Disabled("Test is intermittently failing.") @Test @Order(14) void Test_Command_Site_Pull_Pull_All_Twice() throws IOException { @@ -880,7 +874,6 @@ void Test_Command_Site_Pull_Pull_All_Twice() throws IOException { * Given scenario: Create a new site using a file and the push command, then verify the site * descriptor was updated with the proper identifier. */ - @Disabled("Test is intermittently failing.") @Test @Order(15) void Test_Create_From_File_via_Push_Checking_Auto_Update() throws IOException { @@ -930,7 +923,6 @@ void Test_Create_From_File_via_Push_Checking_Auto_Update() throws IOException { * Given scenario: Create a new site using a file and the push command disabling the auto * update, then verify the site descriptor was not updated. */ - @Disabled("Test is intermittently failing.") @Test @Order(16) void Test_Create_From_File_via_Push_With_Auto_Update_Disabled() throws IOException { @@ -1082,7 +1074,6 @@ void Test_Default_Site_Change() throws IOException { * * @throws IOException if there is an error creating the temporary folder or writing to files */ - @Disabled("Test is intermittently failing.") @Test @Order(18) void Test_Archive_Site() throws IOException { diff --git a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/common/FilesTestHelperService.java b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/common/FilesTestHelperService.java index 517971fef5de..33a8bc131528 100644 --- a/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/common/FilesTestHelperService.java +++ b/tools/dotcms-cli/cli/src/test/java/com/dotcms/cli/common/FilesTestHelperService.java @@ -152,7 +152,7 @@ public String createSite() { final SiteAPI siteAPI = clientFactory.getClient(SiteAPI.class); // Creating a new test site - final String newSiteName = String.format("site-%d", System.currentTimeMillis()); + final String newSiteName = String.format("site-%s", UUID.randomUUID()); CreateUpdateSiteRequest newSiteRequest = CreateUpdateSiteRequest.builder() .siteName(newSiteName).build(); ResponseEntityView createSiteResponse = siteAPI.create(newSiteRequest);