Merge pull request #388 from DDMAL/stag

Merge staging branch into develop

django/cantusdb_project/articles/models.py

@@ -1,22 +1,15 @@
-from django.conf import settings
+from django.contrib.auth import get_user_model
 from django.db import models
 from django.urls import reverse
-from main_app.models import Indexer
 from django_quill.fields import QuillField
 
 class Article(models.Model):
     title = models.CharField(max_length=255)
     body = QuillField() # rich text field
-
     author = models.ForeignKey(
-        Indexer,
+        get_user_model(),
         on_delete=models.CASCADE,
     )
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-    )
-
     date_created = models.DateTimeField(
         help_text="The date this article was created",
         null=True,

django/cantusdb_project/articles/tests/test_articles.py

@@ -1,40 +1,26 @@
 from django.test import TestCase
-from django.contrib.auth import get_user_model
 from django.urls import reverse
-
 from articles.models import Article
-from users.models import User
-from main_app.models import Indexer
 from main_app.tests.make_fakes import (
     make_fake_text,
-    make_fake_indexer,
+    make_fake_user,
 )
 
-# run with `python -Wa manage.py test articles.tests.test_articles.py`
+# run with `python -Wa manage.py test articles.tests.test_articles`
 # the -Wa flag tells Python to display deprecation warnings
 
-def make_fake_user():
-    fake_email = "{f}@{g}.com".format(
-        f=make_fake_text(12),
-        g=make_fake_text(12),
-    )
-    user = get_user_model().objects.create(email=fake_email)
-    return user
-
 def make_fake_article(user=None):
     if user is None:
-        user=make_fake_user()
+        user = make_fake_user()
     article = Article.objects.create(
         title=make_fake_text(max_size=12),
-        author=make_fake_indexer(),
-        created_by=user,
+        author=make_fake_user(),
     )
     return article
 
 
 class ArticleListViewTest(TestCase):
     def setUp(self):
-        fake_user = make_fake_user()
         for i in range(10):
             make_fake_article()
 

django/cantusdb_project/main_app/admin.py

@@ -28,9 +28,6 @@ class FeastAdmin(BaseModelAdmin):
 class GenreAdmin(BaseModelAdmin):
     pass
 
-class IndexerAdmin(BaseModelAdmin):
-    pass
-
 class NotationAdmin(BaseModelAdmin):
     pass
 
@@ -59,7 +56,6 @@ class SourceAdmin(BaseModelAdmin):
 admin.site.register(Chant, ChantAdmin)
 admin.site.register(Feast, FeastAdmin)
 admin.site.register(Genre, GenreAdmin)
-admin.site.register(Indexer, IndexerAdmin)
 admin.site.register(Notation, NotationAdmin)
 admin.site.register(Office, OfficeAdmin)
 admin.site.register(Provenance, ProvenanceAdmin)

django/cantusdb_project/main_app/forms.py

@@ -1,5 +1,5 @@
 from django import forms
-from .models import Chant, Office, Genre, Feast, Source, RismSiglum, Provenance, Century, Indexer, Sequence
+from .models import Chant, Office, Genre, Feast, Source, RismSiglum, Provenance, Century, Sequence
 from .widgets import (TextInputWidget,
                 VolpianoInputWidget,
                 TextAreaWidget,
@@ -216,7 +216,7 @@ class Meta:
     current_editors.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
 
     melodies_entered_by = forms.ModelMultipleChoiceField(
-        queryset=Indexer.objects.all().order_by("family_name"), required=False
+        queryset=get_user_model().objects.all().order_by("full_name"), required=False
     )
     melodies_entered_by.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
 
@@ -439,7 +439,7 @@ class Meta:
     current_editors.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
 
     melodies_entered_by = forms.ModelMultipleChoiceField(
-        queryset=Indexer.objects.all().order_by("family_name"), required=False
+        queryset=get_user_model().objects.all().order_by("full_name"), required=False
     )
     melodies_entered_by.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
 

django/cantusdb_project/main_app/management/commands/merge_indexer_user.py

@@ -0,0 +1,55 @@
+from operator import index
+from django.core.management.base import BaseCommand
+from main_app.models import Indexer
+from users.models import User
+from faker import Faker
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        faker = Faker()
+        users = User.objects.all()
+        indexers = Indexer.objects.all()
+
+        for user in users:
+            # set all users to be non-indexer first
+            # those listed as indexers on old Cantus will have this adjusted to True
+            user.is_indexer = False
+            if user.first_name or user.last_name:
+                user.full_name = f"{user.first_name.strip()} {user.last_name.strip()}"
+            else:
+                user.full_name = "Anonymous User"
+            user.save()
+
+        for indexer in indexers:
+            indexer_full_name = f"{indexer.given_name} {indexer.family_name}"
+            print(indexer_full_name)
+            homonymous_users = User.objects.filter(full_name__iexact=indexer_full_name)
+            # if the indexer also exists as a user
+            if homonymous_users:
+                assert homonymous_users.count() == 1
+                homonymous_user = homonymous_users.get()
+                print(f"homonymous: {homonymous_user.full_name}")
+                # keep the user as it is (merge the indexer into existing user)
+                # and store the ID of its indexer object
+                homonymous_user.old_indexer_id = indexer.id
+                homonymous_user.is_indexer = True
+                homonymous_user.save()
+            # if the indexer doesn't exist as a user
+            else:
+                # create a new user with the indexer information
+                User.objects.create(
+                    institution=indexer.institution,
+                    city=indexer.city,
+                    country=indexer.country,
+                    full_name=indexer_full_name,
+                    # assign random email to dummy users
+                    email=f"{faker.lexify('????????')}@fakeemail.com",
+                    # leave the password empty for dummy users
+                    # the password can't be empty in login form, so they can't log in
+                    password="",
+                    old_indexer_id = indexer.id,
+                    is_indexer=True,
+                )

django/cantusdb_project/main_app/management/commands/replace_indexer_fields.py

@@ -0,0 +1,40 @@
+from main_app.models import Source
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+# This command is intended as a temporary approach to 
+# populate the user-related fields using the indexer-related fields. 
+# The goal is to eventually remove the Indexer model and point all fields to User. 
+# After running this command, the indexer-related fields can be removed from the Source model.
+# The user-related fields can then be renamed as needed. 
+# Run with `python manage.py replace_indexer_fields`.
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        sources = Source.objects.all()
+        for source in sources:
+            print(source.id)
+
+            inventoried_by = source.inventoried_by.all()
+            for indexer in inventoried_by:
+                user = get_user_model().objects.get(old_indexer_id=indexer.id)
+                source.inventoried_by_u.add(user)
+
+            full_text_entered_by = source.full_text_entered_by.all()
+            for indexer in full_text_entered_by:
+                user = get_user_model().objects.get(old_indexer_id=indexer.id)
+                source.full_text_entered_by_u.add(user)
+
+            melodies_entered_by = source.melodies_entered_by.all()
+            for indexer in melodies_entered_by:
+                user = get_user_model().objects.get(old_indexer_id=indexer.id)
+                source.melodies_entered_by_u.add(user)
+
+            proofreaders = source.proofreaders.all()
+            for indexer in proofreaders:
+                user = get_user_model().objects.get(old_indexer_id=indexer.id)
+                source.proofreaders_u.add(user)
+
+            other_editors = source.other_editors.all()
+            for indexer in other_editors:
+                user = get_user_model().objects.get(old_indexer_id=indexer.id)
+                source.other_editors_u.add(user)

django/cantusdb_project/main_app/management/commands/sync_indexers.py

@@ -1,10 +1,10 @@
-from main_app.models import Indexer
 from django.core.management.base import BaseCommand
+from django.contrib.auth import get_user_model
 import requests, json
+from faker import Faker
 
 INDEXER_ID_FILE = "indexer_list.txt"
 
-
 def get_id_list(file_path):
     indexer_list = []
     file = open(file_path, "r")
@@ -14,8 +14,8 @@ def get_id_list(file_path):
     file.close()
     return indexer_list
 
-
 def get_new_indexer(indexer_id):
+    # use json-export to get indexer information
     url = f"http://cantus.uwaterloo.ca/json-node/{indexer_id}"
     response = requests.get(url)
     json_response = json.loads(response.content)
@@ -40,44 +40,45 @@ def get_new_indexer(indexer_id):
     else:
         country = None
 
-    indexer, created = Indexer.objects.update_or_create(
-        id=indexer_id,
-        defaults={
-            "given_name": first_name,
-            "family_name": family_name,
-            "institution": institution,
-            "city": city,
-            "country": country,
-        },
-    )
-    if created:
-        print(f"created indexer {indexer_id}")
-
-
-def remove_extra():
-    waterloo_ids = get_id_list(INDEXER_ID_FILE)
-    our_ids = list(Indexer.objects.all().values_list("id", flat=True))
-    our_ids = [str(id) for id in our_ids]
-    waterloo_ids = set(waterloo_ids)
-    print(f"Our count: {len(our_ids)}")
-    print(f"Waterloo count: {len(waterloo_ids)}")
-    extra_ids = [id for id in our_ids if id not in waterloo_ids]
-    for id in extra_ids:
-        Indexer.objects.get(id=id).delete()
-        print(f"Extra item removed: {id}")
+    # check whether the current indexer has a user entry of the same name
+    indexer_full_name = f"{first_name} {family_name}"
+    print(f"{indexer_id} {indexer_full_name}")
+    homonymous_users = get_user_model().objects.filter(full_name__iexact=indexer_full_name)
+    # if the indexer also exists as a user
+    if homonymous_users:
+        assert homonymous_users.count() == 1
+        homonymous_user = homonymous_users.get()
+        print(f"homonymous: {homonymous_user.full_name}")
+        # keep the user as it is (merge the indexer into existing user)
+        # and store the ID of its indexer object
+        homonymous_user.old_indexer_id = indexer_id
+        homonymous_user.is_indexer = True
+        homonymous_user.save()
+    # if the indexer doesn't exist as a user
+    else:
+        faker = Faker()
+        # create a new user with the indexer information
+        get_user_model().objects.create(
+            institution=institution,
+            city=city,
+            country=country,
+            full_name=indexer_full_name,
+            # assign random email to dummy users
+            email=f"{faker.lexify('????????')}@fakeemail.com",
+            # leave the password empty for dummy users
+            # the password can't be empty in login form, so they can't log in
+            password="",
+            old_indexer_id = indexer_id,
+            is_indexer=True,
+        )
 
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
-        parser.add_argument(
-            "--remove_extra",
-            action="store_true",
-            help="add this flag to remove the indexers in our database that are no longer present in waterloo database",
-        )
+        pass
 
     def handle(self, *args, **options):
         indexer_list = get_id_list(INDEXER_ID_FILE)
         for id in indexer_list:
             get_new_indexer(id)
-        if options["remove_extra"]:
-            remove_extra()
+

django/cantusdb_project/main_app/management/commands/sync_sources.py

@@ -1,7 +1,6 @@
 from main_app.models import (
     Source,
     Century,
-    Indexer,
     Notation,
     Provenance,
     RismSiglum,
@@ -40,7 +39,7 @@ def get_indexers(json_response, field_name):
     if json_response[field_name]:
         for und in json_response[field_name]["und"]:
             indexer_id = und["nid"]
-            indexer = Indexer.objects.get(id=indexer_id)
+            indexer = get_user_model().objects.get(old_indexer_id=indexer_id)
             indexers.append(indexer)
     return indexers
 
@@ -268,7 +267,8 @@ def get_new_source(source_id):
     # True for published, False for unpublished
     published = status_published and source_status_published
 
-    # current editors are User, the other "people" fields are Indexer
+    # current editors are User, the other "people" fields were originally Indexer, 
+    # but we harmonize them to User anyways
     current_editors = []
     try:
         current_editors_entries = json_response["field_editors"]["und"]

django/cantusdb_project/main_app/models/__init__.py

@@ -3,7 +3,6 @@
 from main_app.models.chant import Chant
 from main_app.models.feast import Feast
 from main_app.models.genre import Genre
-from main_app.models.indexer import Indexer
 from main_app.models.notation import Notation
 from main_app.models.office import Office
 from main_app.models.provenance import Provenance

django/cantusdb_project/main_app/models/chant.py

@@ -1,9 +1,5 @@
-from django.contrib.postgres.search import SearchVectorField
-from django.db import models
 from django.db.models.query import QuerySet
 from main_app.models.base_chant import BaseChant
-from users.models import User
-
 
 class Chant(BaseChant):
     """The model for chants