CONCD-552 Merge Main into Feature branch

.pre-commit-config.yaml

@@ -1,25 +1,25 @@
 exclude: '.*/vendor/.*'
 repos:
     - repo: https://github.com/adamchainz/django-upgrade
-      rev: 1.12.0
+      rev: 1.15.0
       hooks:
           - id: django-upgrade
             args: [--target-version, '3.2']
     - repo: https://github.com/charliermarsh/ruff-pre-commit
-      rev: v0.0.261
+      rev: v0.1.0
       hooks:
           - id: ruff
             args: [--fix, --exit-non-zero-on-fix]
     - repo: https://github.com/rtts/djhtml
-      rev: 'v1.5.2'
+      rev: '3.0.6'
       hooks:
           - id: djhtml
     - repo: https://github.com/ambv/black
-      rev: 23.3.0
+      rev: 23.10.0
       hooks:
           - id: black
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.5.0
       hooks:
           - id: check-added-large-files
             args: ['--maxkb=128']
@@ -49,12 +49,12 @@ repos:
             exclude: fixtures/.*
           - id: trailing-whitespace
     - repo: https://github.com/pre-commit/mirrors-prettier
-      rev: v2.7.1
+      rev: v3.0.3
       hooks:
           - id: prettier
             files: \.(css|less|scss|ts|tsx|graphql|gql|json|js|jsx|md|yaml|yml)$
     - repo: https://github.com/pre-commit/mirrors-eslint
-      rev: v8.32.0
+      rev: v8.51.0
       hooks:
           - id: eslint
             additional_dependencies:
@@ -72,11 +72,11 @@ repos:
                 - 'stylelint-config-recommended@^3.0.0'
                 - 'stylelint-value-no-unknown-custom-properties@^4.0.0'
     - repo: https://github.com/aws-cloudformation/cfn-python-lint
-      rev: v0.72.9
+      rev: v0.82.2
       hooks:
           - id: cfn-python-lint
             files: cloudformation/.*\.(json|yml|yaml)$
     - repo: https://github.com/PyCQA/bandit
-      rev: 1.7.4
+      rev: 1.7.5
       hooks:
           - id: bandit

Pipfile

@@ -49,6 +49,7 @@ weasyprint = "*"
 tesseract = "==0.1.3"
 pytesseract = "*"
 django-redis = "*"
+locust = "*"
 
 [dev-packages]
 invoke = "*"

cloudformation/add_cloudflare_ips_to_sgs.py

@@ -12,8 +12,12 @@
 
 EC2_CLIENT = boto3.client("ec2")
 
-CLOUDFLARE_IPV4 = requests.get("https://www.cloudflare.com/ips-v4").text.splitlines()
-CLOUDFLARE_IPV6 = requests.get("https://www.cloudflare.com/ips-v6").text.splitlines()
+CLOUDFLARE_IPV4 = requests.get(
+    "https://www.cloudflare.com/ips-v4", timeout=30
+).text.splitlines()
+CLOUDFLARE_IPV6 = requests.get(
+    "https://www.cloudflare.com/ips-v6", timeout=30
+).text.splitlines()
 
 
 def add_ingess_rules_for_group(sg_id, existing_permissions):

concordia/forms.py

@@ -86,6 +86,11 @@ def confirm_login_allowed(self, user):
             raise forms.ValidationError(inactive_message, code="inactive")
 
 
+class UserNameForm(forms.Form):
+    first_name = forms.CharField(label="", required=False)
+    last_name = forms.CharField(label="", required=False)
+
+
 class UserProfileForm(forms.Form):
     email = forms.EmailField(label="", required=True)
 

concordia/management/commands/import_site_reports.py

@@ -21,8 +21,8 @@ def handle(self, *, csv_file, **options):
             reader = csv.reader(csv_file, delimiter=",")
             header = reader.__next__()
             for row in reader:
-                site_report_data = {key: value for key, value in zip(header, row)}
-                site_report = dict()
+                site_report_data = dict(zip(header, row, strict=True))
+                site_report = {}
 
                 for key in site_report_data:
                     if site_report_data[key] != "":

concordia/migrations/0085_auto_20231016_1432.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.22 on 2023-10-16 18:32
+
+from django.db import migrations, models
+
+import concordia.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("concordia", "0084_rename_review_actions_sitereport_daily_review_actions"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="asset",
+            name="storage_image",
+            field=models.ImageField(
+                max_length=255, upload_to=concordia.models.Asset.get_storage_path
+            ),
+        ),
+    ]

concordia/migrations/0087_merge_0085_auto_20231016_1432_0086_auto_20230911_1112.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.20 on 2023-10-31 17:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("concordia", "0085_auto_20231016_1432"),
+        ("concordia", "0086_auto_20230911_1112"),
+    ]
+
+    operations = []

concordia/models.py

@@ -440,6 +440,17 @@ def add_contribution_counts(self):
 
 
 class Asset(MetricsModelMixin("asset"), models.Model):
+    def get_storage_path(self, filename):
+        s3_relative_path = "/".join(
+            [
+                self.item.project.campaign.slug,
+                self.item.project.slug,
+                self.item.item_id,
+            ]
+        )
+        filename = self.media_url
+        return os.path.join(s3_relative_path, filename)
+
     objects = AssetQuerySet.as_manager()
 
     item = models.ForeignKey(Item, on_delete=models.CASCADE)
@@ -478,6 +489,8 @@ class Asset(MetricsModelMixin("asset"), models.Model):
 
     difficulty = models.PositiveIntegerField(default=0, blank=True, null=True)
 
+    storage_image = models.ImageField(upload_to=get_storage_path, max_length=255)
+
     class Meta:
         unique_together = (("slug", "item"),)
         indexes = [
@@ -491,6 +504,13 @@ class Meta:
     def __str__(self):
         return self.title
 
+    def save(self, *args, **kwargs):
+        # This ensures all 'required' fields really are required
+        # even when creating objects programmatically. Particularly,
+        # we want to make sure we don't end up with an empty storage_image
+        self.full_clean()
+        super().save(*args, **kwargs)
+
     def get_absolute_url(self):
         return reverse(
             "transcriptions:asset-detail",
@@ -505,21 +525,6 @@ def get_absolute_url(self):
     def latest_transcription(self):
         return self.transcription_set.order_by("-pk").first()
 
-    def get_storage_path(self, filename):
-        s3_relative_path = "/".join(
-            [
-                self.item.project.campaign.slug,
-                self.item.project.slug,
-                self.item.item_id,
-            ]
-        )
-        filename = self.media_url
-        return os.path.join(s3_relative_path, filename)
-
-    storage_image = models.ImageField(
-        upload_to=get_storage_path, max_length=255, blank=True, null=True
-    )
-
     def get_ocr_transcript(self):
         return pytesseract.image_to_string(Image.open(self.storage_image))
 

concordia/secrets.py

@@ -22,11 +22,13 @@ def get_secret(secret_name):
         get_secret_value_response = client.get_secret_value(SecretId=secret_name)
     except ClientError as e:
         if e.response["Error"]["Code"] == "ResourceNotFoundException":
-            raise Exception("The requested secret " + secret_name + " was not found")
+            raise Exception(
+                "The requested secret " + secret_name + " was not found"
+            ) from e
         elif e.response["Error"]["Code"] == "InvalidRequestException":
-            raise Exception("The request was invalid due to:", e)
+            raise Exception("The request was invalid due to:", e) from e
         elif e.response["Error"]["Code"] == "InvalidParameterException":
-            raise Exception("The request had invalid params:", e)
+            raise Exception("The request had invalid params:", e) from e
     else:
         # Decrypted secret using the associated KMS CMK Depending on whether the
         # secret was a string or binary, one of these fields will be populated

concordia/signals/handlers.py

@@ -87,6 +87,7 @@ def add_user_to_newsletter(sender, user, request, **kwargs):
 
 @receiver(post_save, sender=Transcription)
 def update_asset_status(sender, *, instance, **kwargs):
+    logger.info("update_asset_status for %s", instance.id)
     new_status = TranscriptionStatus.IN_PROGRESS
 
     if instance.rejected:
@@ -96,11 +97,22 @@ def update_asset_status(sender, *, instance, **kwargs):
     elif instance.submitted:
         new_status = TranscriptionStatus.SUBMITTED
 
-    instance.asset.transcription_status = new_status
-    instance.asset.full_clean()
-    instance.asset.save()
+    asset = instance.asset
+    logger.info(
+        "Updating asset status for %s (%s) from %s to %s",
+        asset,
+        asset.id,
+        asset.transcription_status,
+        new_status,
+    )
+
+    asset.transcription_status = new_status
+    asset.full_clean()
+    asset.save()
+
+    logger.info("Status for %s (%s) updated", asset, asset.id)
 
-    calculate_difficulty_values(Asset.objects.filter(pk=instance.asset.pk))
+    calculate_difficulty_values(Asset.objects.filter(pk=asset.pk))
 
 
 @receiver(post_save, sender=Asset)

concordia/signals/signals.py

@@ -1,8 +1,5 @@
 import django.dispatch
 
-# signals notify channels of asset reservation - kwargs from handlers
-# ["asset_pk", "reservation_token"]
-
 reservation_obtained = django.dispatch.Signal()
 
 reservation_released = django.dispatch.Signal()

concordia/static/admin/editor-preview.js

@@ -26,17 +26,17 @@
                 var frameDocument = this.contentDocument;
                 frameDocument.open();
                 frameDocument.write(
-                    '<html><body><main>Loading…</main></body></html>'
+                    '<html><body><main>Loading…</main></body></html>',
                 );
                 frameDocument.close();
 
                 var previewTemplate = document.querySelector(
-                    'template#preview-head'
+                    'template#preview-head',
                 ).content;
 
                 previewTemplate.childNodes.forEach((node) => {
-                    frameDocument.head.appendChild(
-                        frameDocument.importNode(node, true)
+                    frameDocument.head.append(
+                        frameDocument.importNode(node, true),
                     );
                 });
 
@@ -90,7 +90,7 @@
                 $formRow.find('.errornote').remove();
 
                 editorLineWidgets.forEach((widget) =>
-                    editor.removeLineWidget(widget)
+                    editor.removeLineWidget(widget),
                 );
 
                 try {
@@ -110,21 +110,19 @@
                     lineWarning.style.whiteSpace = 'nowrap';
                     lineWarning.style.overflow = 'hidden';
 
-                    var icon = lineWarning.appendChild(
-                        document.createElement('span')
+                    var icon = lineWarning.append(
+                        document.createElement('span'),
                     );
                     icon.style.marginRight = '1rem';
                     icon.innerHTML = '⚠️';
-                    lineWarning.appendChild(
-                        document.createTextNode(error.message)
-                    );
+                    lineWarning.append(document.createTextNode(error.message));
 
                     editorLineWidgets.push(
                         editor.addLineWidget(
                             error.loc.start.line - 1,
                             lineWarning,
-                            {coverGutter: false, noHScroll: true}
-                        )
+                            {coverGutter: false, noHScroll: true},
+                        ),
                     );
                 }
             });