Cleanup stale files

contentcuration/contentcuration/decorators.py

@@ -1,8 +1,8 @@
+from contextlib import ContextDecorator
+
 from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import render
 
-from contentcuration.models import Channel
 
 ACCEPTED_BROWSERS = settings.HEALTH_CHECK_BROWSERS + settings.SUPPORTED_BROWSERS
 
@@ -35,48 +35,6 @@ def wrap(request, *args, **kwargs):
     return wrap
 
 
-def can_access_channel(function):
-    def wrap(request, *args, **kwargs):
-        try:
-            channel = Channel.objects.get(pk=kwargs["channel_id"])
-        except ObjectDoesNotExist:
-            return render(request, "channel_not_found.html")
-
-        if (
-            channel.public
-            or channel.editors.filter(id=request.user.id).exists()
-            or channel.viewers.filter(id=request.user.id).exists()
-            or (not request.user.is_anonymous and request.user.is_admin)
-        ):
-            return function(request, *args, **kwargs)
-
-        return render(request, "channel_not_found.html", status=404)
-
-    wrap.__doc__ = function.__doc__
-    wrap.__name__ = function.__name__
-    return wrap
-
-
-def can_edit_channel(function):
-    def wrap(request, *args, **kwargs):
-        try:
-            channel = Channel.objects.get(pk=kwargs["channel_id"], deleted=False)
-
-            if (
-                not channel.editors.filter(id=request.user.id).exists()
-                and not request.user.is_admin
-            ):
-                return render(request, "unauthorized.html", status=403)
-
-            return function(request, *args, **kwargs)
-        except ObjectDoesNotExist:
-            return render(request, "channel_not_found.html")
-
-    wrap.__doc__ = function.__doc__
-    wrap.__name__ = function.__name__
-    return wrap
-
-
 # NOTE: This approach only works because of a seeming bug in Django
 # https://code.djangoproject.com/ticket/15855
 # Essentially, what happens is that this decorator sets the Vary headers
@@ -104,3 +62,31 @@ def wrap(request, *args, **kwargs):
         return response
 
     return wrap
+
+
+class DelayUserStorageCalculation(ContextDecorator):
+    """
+    Decorator class that will dedupe and delay requests to enqueue storage calculation tasks for users
+    until after the wrapped function has exited
+    """
+    depth = 0
+    queue = []
+
+    @property
+    def is_active(self):
+        return self.depth > 0
+
+    def __enter__(self):
+        self.depth += 1
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        from contentcuration.utils.user import calculate_user_storage
+        self.depth -= 1
+        if not self.is_active:
+            user_ids = set(self.queue)
+            self.queue = []
+            for user_id in user_ids:
+                calculate_user_storage(user_id)
+
+
+delay_user_storage_calculation = DelayUserStorageCalculation()

contentcuration/contentcuration/management/commands/garbage_collect.py

@@ -11,6 +11,7 @@
 from contentcuration.utils.garbage_collect import clean_up_contentnodes
 from contentcuration.utils.garbage_collect import clean_up_deleted_chefs
 from contentcuration.utils.garbage_collect import clean_up_feature_flags
+from contentcuration.utils.garbage_collect import clean_up_stale_files
 from contentcuration.utils.garbage_collect import clean_up_tasks
 
 
@@ -33,5 +34,7 @@ def handle(self, *args, **options):
         clean_up_deleted_chefs()
         logging.info("Cleaning up feature flags")
         clean_up_feature_flags()
+        logging.info("Cleaning up stale file objects")
+        clean_up_stale_files()
         logging.info("Cleaning up tasks")
         clean_up_tasks()

contentcuration/contentcuration/tests/utils/test_user.py → contentcuration/contentcuration/tests/test_decorators.py

@@ -1,11 +1,11 @@
 import mock
 
+from contentcuration.decorators import delay_user_storage_calculation
 from contentcuration.tests.base import StudioTestCase
 from contentcuration.utils.user import calculate_user_storage
-from contentcuration.utils.user import delay_user_storage_calculation
 
 
-class UserUtilsTestCase(StudioTestCase):
+class DecoratorsTestCase(StudioTestCase):
     @mock.patch("contentcuration.utils.user.calculate_user_storage_task")
     def test_delay_storage_calculation(self, mock_task):
         @delay_user_storage_calculation

contentcuration/contentcuration/tests/utils/test_garbage_collect.py

@@ -1,4 +1,5 @@
 import json
+import random
 import uuid
 from datetime import datetime
 from datetime import timedelta
@@ -11,6 +12,8 @@
 from django.core.files.storage import default_storage
 from django.urls import reverse_lazy
 from le_utils.constants import content_kinds
+from le_utils.constants import file_formats
+from le_utils.constants import format_presets
 
 from contentcuration import models as cc
 from contentcuration.api import activate_channel
@@ -23,6 +26,7 @@
 from contentcuration.utils.garbage_collect import clean_up_contentnodes
 from contentcuration.utils.garbage_collect import clean_up_deleted_chefs
 from contentcuration.utils.garbage_collect import clean_up_feature_flags
+from contentcuration.utils.garbage_collect import clean_up_stale_files
 from contentcuration.utils.garbage_collect import clean_up_tasks
 from contentcuration.utils.garbage_collect import get_deleted_chefs_root
 from contentcuration.views.internal import api_commit_channel
@@ -396,3 +400,56 @@ def test_recent_task(self):
             TaskResult.objects.get(task_id=self.recent_task.task_id)
         except TaskResult.DoesNotExist:
             self.fail("Task was removed")
+
+
+TWO_DAYS_AGO = datetime.now() - timedelta(days=2)
+
+
+def _create_stale_file(user, modified_date):
+    checksum = '%32x' % random.getrandbits(16 * 8)
+    file = File(
+        file_size=5,
+        checksum=checksum,
+        original_filename="SomeAudio.mp3",
+        file_on_disk=f"{checksum}.mp3",
+        file_format_id=file_formats.MP3,
+        preset_id=format_presets.AUDIO,
+        uploaded_by=user,
+        duration=365,
+    )
+    file.save()
+
+    # Manually setting last modified date of file
+    File.objects.filter(id=file.id).update(modified=modified_date)
+
+    return file
+
+
+class CleanupStaleFilesTestCase(StudioTestCase):
+
+    def setUp(self):
+        user = self.admin_user
+
+        # create test stale files
+        self.file_to_be_deleted = _create_stale_file(user, THREE_MONTHS_AGO)
+        self.file_to_keep = _create_stale_file(user, TWO_DAYS_AGO)
+
+        # run
+        clean_up_stale_files()
+
+    def test_deletes_correct_files(self):
+        """
+        Function should only get files where ContentNode, Assessmentitem, and SlideshowSlide are null
+        and modified date is older than the date specified
+        """
+        with self.assertRaises(File.DoesNotExist):
+            File.objects.get(id=self.file_to_be_deleted.id)
+
+    def test_doesnt_delete_files_with_modified_date_less_than_date_specified(self):
+        """
+        Function should not delete files where the modified date is more recent than the date specified
+        """
+        try:
+            File.objects.get(id=self.file_to_keep.id)
+        except File.DoesNotExist:
+            self.fail("File was deleted")

contentcuration/contentcuration/utils/garbage_collect.py

@@ -7,7 +7,9 @@
 
 from celery import states
 from django.conf import settings
+from django.db.models import OuterRef
 from django.db.models.expressions import CombinedExpression
+from django.db.models.expressions import Exists
 from django.db.models.expressions import F
 from django.db.models.expressions import Value
 from django.db.models.signals import post_delete
@@ -16,6 +18,7 @@
 
 from contentcuration.constants import feature_flags
 from contentcuration.db.models.functions import JSONObjectKeys
+from contentcuration.decorators import delay_user_storage_calculation
 from contentcuration.models import ContentNode
 from contentcuration.models import File
 from contentcuration.models import TaskResult
@@ -145,3 +148,20 @@ def clean_up_tasks():
         count, _ = TaskResult.objects.filter(date_done__lt=date_cutoff, status__in=states.READY_STATES).delete()
 
     logging.info("Deleted {} completed task(s) from the task table".format(count))
+
+
+@delay_user_storage_calculation
+def clean_up_stale_files(last_modified=now() - datetime.timedelta(days=30)):
+    """
+    Clean up files that aren't attached to any ContentNode, AssessmentItem, or SlideshowSlide and where
+    the modified date is older than `last_modified`
+    """
+
+    files_to_clean_up = File.objects.filter(
+        contentnode__isnull=True, assessment_item__isnull=True, slideshow_slide__isnull=True, modified__lt=last_modified
+    )
+
+    # Uses a subset of Python’s array-slicing syntax to limit deletion to 100000 files per function run
+    files_deleted, _ = File.objects.filter(Exists(files_to_clean_up.filter(pk=OuterRef('pk'))[:100000])).delete()
+
+    logging.info("Files with a modified date older than {} should be deleted. Deleted {} file(s).".format(last_modified, files_deleted))

contentcuration/contentcuration/utils/publish.py

@@ -40,14 +40,14 @@
 from past.utils import old_div
 
 from contentcuration import models as ccmodels
+from contentcuration.decorators import delay_user_storage_calculation
 from contentcuration.statistics import record_publish_stats
 from contentcuration.utils.cache import delete_public_channel_cache_keys
 from contentcuration.utils.files import create_thumbnail_from_base64
 from contentcuration.utils.files import get_thumbnail_encoding
 from contentcuration.utils.parser import extract_value
 from contentcuration.utils.parser import load_json_string
 from contentcuration.utils.sentry import report_exception
-from contentcuration.utils.user import delay_user_storage_calculation
 
 
 logmodule.basicConfig()

contentcuration/contentcuration/utils/user.py

@@ -1,39 +1,12 @@
 import logging
-from contextlib import ContextDecorator
 
 from contentcuration.tasks import calculate_user_storage_task
 
 
-class DelayUserStorageCalculation(ContextDecorator):
-    """
-    Decorator class that will dedupe and delay requests to enqueue storage calculation tasks for users
-    until after the wrapped function has exited
-    """
-    depth = 0
-    queue = []
-
-    @property
-    def is_active(self):
-        return self.depth > 0
-
-    def __enter__(self):
-        self.depth += 1
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.depth -= 1
-        if not self.is_active:
-            user_ids = set(self.queue)
-            self.queue = []
-            for user_id in user_ids:
-                calculate_user_storage(user_id)
-
-
-delay_user_storage_calculation = DelayUserStorageCalculation()
-
-
 def calculate_user_storage(user_id):
     """TODO: Perhaps move this to User model to avoid unnecessary User lookups"""
     from contentcuration.models import User
+    from contentcuration.decorators import delay_user_storage_calculation
 
     if delay_user_storage_calculation.is_active:
         delay_user_storage_calculation.queue.append(user_id)

contentcuration/contentcuration/viewsets/sync/base.py

@@ -2,7 +2,7 @@
 
 from search.viewsets.savedsearch import SavedSearchViewSet
 
-from contentcuration.utils.user import delay_user_storage_calculation
+from contentcuration.decorators import delay_user_storage_calculation
 from contentcuration.viewsets.assessmentitem import AssessmentItemViewSet
 from contentcuration.viewsets.bookmark import BookmarkViewSet
 from contentcuration.viewsets.channel import ChannelViewSet