feat: add direct source code to clean up empty folders instead of depending on additional plugin

CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 0.4.0
+
+- Remove `--remove-empty-folder` option. Artifactory provides corresponding built-in functionality already
+- Change the `delete_empty_folder` rule to not depend on an external plugin, but directly delete files from this script
+
+## 0.3.4
+
+* Previous versions do not yet have a changelog

README.md

@@ -69,11 +69,6 @@ artifactory-cleanup --destroy --user user --password password --artifactory-serv
 # debug run - only print founded artifacts. it do not delete
 artifactory-cleanup --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py
 
-# Clean up empty folder
-# --remove-empty-folder
-# You need to use the plugin https://github.com/jfrog/artifactory-user-plugins/tree/master/cleanup/deleteEmptyDirs to delete empty folders
-artifactory-cleanup --remove-empty-folder --user user --password password --artifactory-server https://repo.example.com/artifactory
-
 # Debug run only for policytestname. Find any *policytestname*
 # debug run - only print founded artifacts. it do not delete
 artifactory-cleanup --policy-name policytestname --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py

artifactory_cleanup/artifactorycleanup.py

@@ -10,7 +10,6 @@
 from requests.auth import HTTPBasicAuth
 from artifactory_cleanup.context_managers import get_context_managers
 from artifactory_cleanup.rules.base import CleanupPolicy
-from artifactory_cleanup.rules.delete import delete_empty_folder
 
 
 requests.packages.urllib3.disable_warnings()
@@ -61,10 +60,6 @@ class ArtifactoryCleanup(cli.Application):
         mandatory=False,
     )
 
-    _remove_empty_folder = cli.Flag(
-        "--remove-empty-folder", help="Cleaning up empty folders in local repositories"
-    )
-
     _days_in_future = cli.SwitchAttr(
         "--days-in-future",
         help="Simulate future behaviour",
@@ -83,21 +78,13 @@ def _destroy_or_verbose(self):
     def main(self):
         # remove trailing slash
         self._artifactory_server = self._artifactory_server.rstrip("/")
-        if self._remove_empty_folder:
-            rules = [
-                CleanupPolicy(
-                    "Cleaning up empty folders in local repositories",
-                    delete_empty_folder(),
-                )
-            ]
-        else:
-            try:
-                self._config = self._config.replace(".py", "")
-                sys.path.append(".")
-                rules = getattr(importlib.import_module(self._config), "RULES")
-            except ImportError as error:
-                print("Error: {}".format(error))
-                exit(1)
+        try:
+            self._config = self._config.replace(".py", "")
+            sys.path.append(".")
+            rules = getattr(importlib.import_module(self._config), "RULES")
+        except ImportError as error:
+            print("Error: {}".format(error))
+            exit(1)
 
         self._destroy_or_verbose()
 

artifactory_cleanup/rules/delete.py

@@ -1,6 +1,10 @@
 from datetime import timedelta
 
 from artifactory_cleanup.rules.base import Rule
+from artifactory_cleanup.rules.utils import (
+    artifacts_list_to_tree,
+    folder_artifacts_without_children,
+)
 
 
 class delete_older_than(Rule):
@@ -92,37 +96,15 @@ class delete_empty_folder(Rule):
     """
 
     def _aql_add_filter(self, aql_query_list):
-        update_dict = {
-            "repo": {
-                "$match": "deleteEmptyFolder",
-            }
-        }
-        aql_query_list.append(update_dict)
+        # Get list of all files and folders
+        all_files_dict = {"path": {"$match": "**"}, "type": {"$eq": "any"}}
+        aql_query_list.append(all_files_dict)
         return aql_query_list
 
     def _filter_result(self, result_artifact):
-        r = self.artifactory_session.get(
-            "{}/api/repositories?type=local".format(self.artifactory_server)
-        )
-        r.raise_for_status()
-        repositories = r.json()
-
-        for count, repository in enumerate(repositories, start=1):
-            if repository["packageType"] == "GitLfs":
-                # GitLfs should be handled by the jfrog cli: https://jfrog.com/blog/clean-up-your-git-lfs-repositories-with-jfrog-cli/
-                print(
-                    f"Skipping '{repository['key']}' because it is a Git LFS repository"
-                )
-                continue
-
-            url = "{}/api/plugins/execute/deleteEmptyDirsPlugin?params=paths={}".format(
-                self.artifactory_server, repository["key"]
-            )
-
-            print(
-                f"Deleting empty folders for '{repository['key']}' - {count} of {len(repositories)}"
-            )
-            r = self.artifactory_session.post(url)
-            r.raise_for_status()
-
-        return []
+
+        artifact_tree = artifacts_list_to_tree(result_artifact)
+
+        # Now we have a dict with all folders and files
+        # An empty folder is represented by not having any children
+        return list(folder_artifacts_without_children(artifact_tree))

artifactory_cleanup/rules/utils.py

@@ -0,0 +1,102 @@
+from collections import defaultdict, deque
+from typing import Dict, List
+
+
+def artifacts_list_to_tree(list_of_artifacts: List):
+    """
+    Convert a list of artifacts to a dict representing the directory tree.
+    Each entry name corresponds to the folder or file name. And has two subnodes 'children' and
+    'data'. 'children' is recursively again the list of files/folder within that folder.
+    'data' contains the artifact data returned by artifactory.
+
+    Major idea based on https://stackoverflow.com/a/58917078
+    """
+
+    def nested_dict():
+        """
+        Creates a default dictionary where each value is another default dictionary.
+        """
+        return defaultdict(nested_dict)
+
+    def default_to_regular(d):
+        """
+        Converts defaultdicts of defaultdicts to dict of dicts.
+        """
+        if isinstance(d, defaultdict):
+            d = {k: default_to_regular(v) for k, v in d.items()}
+        return d
+
+    new_path_dict = nested_dict()
+    for artifact in list_of_artifacts:
+        parts = artifact["path"].split("/")
+        if parts:
+            marcher = new_path_dict
+            for key in parts:
+                # We need the repo for the root level folders. They are not in the
+                # artifacts list
+                marcher[key]["data"] = {"repo": artifact["repo"]}
+                marcher = marcher[key]["children"]
+            marcher[artifact["name"]]["data"] = artifact
+    artifact_tree = default_to_regular(new_path_dict)
+    # Artifactory also returns the directory itself. We need to remove it from the list
+    # since that tree branch has no children assigned
+    if "." in artifact_tree:
+        del artifact_tree["."]
+    return artifact_tree
+
+
+def folder_artifacts_without_children(artifacts_tree: Dict, path=""):
+    """
+    Takes the artifacts tree and returns the list of artifacts which are folders
+    and do not have any children.
+
+    If folder1 has only folder2 as a child, and folder2 is empty, the list only contains
+    folder1. I.e., empty folders are also recursively propagated back.
+
+    The input tree will be modified and empty folders will be deleted from the tree.
+
+    """
+
+    # use a deque instead of a list. it's faster to add elements there
+    empty_folder_artifacts = deque()
+
+    def _add_to_del_list(name: str):
+        """
+        Add element with name to empty folder list and remove it from the tree
+        """
+        empty_folder_artifacts.append(artifacts_tree[name]["data"])
+        # Also delete the item from the children list to recursively delete folders
+        # upwards
+        del artifacts_tree[name]
+
+    # Use list(item.keys()) here so that we can delete items while iterating over the
+    # dict.
+    for artifact_name in list(artifacts_tree.keys()):
+        tree_entry = artifacts_tree[artifact_name]
+        if "type" in tree_entry["data"] and tree_entry["data"]["type"] == "file":
+            continue
+        if not "path" in tree_entry["data"]:
+            # Set the path and name for root folders which were not explicitly in the
+            # artifacts list
+            tree_entry["data"]["path"] = path
+            tree_entry["data"]["name"] = artifact_name
+        if not "children" in tree_entry or len(tree_entry["children"]) == 0:
+            # This an empty folder
+            _add_to_del_list(artifact_name)
+        else:
+            artifacts = folder_artifacts_without_children(
+                tree_entry["children"],
+                path=path + "/" + artifact_name if len(path) > 0 else artifact_name,
+            )
+            # Additional check needed here because the recursive call may
+            # delete additional children.
+            # And here we want to check again if all children would be deleted.
+            # Then also delete this.
+            if len(tree_entry["children"]) == 0:
+                # just delete the whole folder since all children are empty
+                _add_to_del_list(artifact_name)
+            else:
+                # add all empty folder children to the list
+                empty_folder_artifacts.extend(artifacts)
+
+    return empty_folder_artifacts

docs/RULES/README.md

@@ -8,7 +8,7 @@
 | `delete_without_downloads()` | Deletes artifacts that have never been downloaded (DownloadCount=0). Better to use with `delete_older_than` rule |
 | `delete_older_than_n_days_without_downloads(days=N)` | Deletes artifacts that are older than N days and have not been downloaded |
 | `delete_not_used_since(days=N)` | Delete artifacts that were downloaded, but for a long time. N days passed. Or not downloaded at all from the moment of creation and it's been N days |
-| `delete_empty_folder()` | Clean up empty folders in local repositories. A special rule that runs separately on all repositories. Refers to [deleteEmptyDirs](https://github.com/jfrog/artifactory-user-plugins/tree/master/cleanup/deleteEmptyDirs) plugin |
+| `delete_empty_folder()` | Clean up empty folders in given repository list |
 | `keep_latest_nupkg_n_version(count=N)` | Leaves N nupkg (adds `*.nupkg` filter) in release feature builds |
 | `keep_latest_n_file(count=N)` | Leaves the last (by creation time) files in the amount of N pieces. WITHOUT accounting subfolders |
 | `keep_latest_n_file_in_folder(count=N)` | Leaves the last (by creation time) files in the number of N pieces in each folder |

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="artifactory-cleanup",
-    version="0.3.4",
+    version="0.4.0",
     description="Rules and cleanup policies for Artifactory",
     long_description=long_description,
     long_description_content_type="text/markdown",