From 5541acb776eff9c992783947a741ffacb8d14141 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Tue, 2 Aug 2022 15:17:36 +0200 Subject: [PATCH 1/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/forms.py | 16 +++++++--------- geonode/base/views.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index facd16a388f..e8ab3d80686 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -337,16 +337,14 @@ def _define_choicefield(self, item, required, tname, lang): @staticmethod def _get_thesauro_keyword_label(item, lang): - qs_local = [] - qs_non_local = [("", "------")] - for key in ThesaurusKeyword.objects.filter(thesaurus_id=item.id): - label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang) - if label.exists(): - qs_local.append((label.get().keyword.id, label.get().label)) - else: - qs_non_local.append((key.id, key.alt_label)) + keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") + + qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label")) + qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label")) - return qs_non_local + qs_local + return qs_local + [("", "-------")] + qs_non_local @staticmethod def _get_thesauro_title_label(item, lang): diff --git a/geonode/base/views.py b/geonode/base/views.py index f105f9a3e22..2602abe95a9 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -35,6 +35,7 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.urls import reverse +from django.utils.translation import get_language_from_request, get_supported_language_variant # Geonode dependencies from geonode.maps.models import Map @@ -301,23 +302,28 @@ def get_results(self, context): class ThesaurusAvailable(autocomplete.Select2QuerySetView): def get_queryset(self): + tid = self.request.GET.get("sysid") - lang = self.request.GET.get("lang") - qs_local = [] - qs_non_local = [] - for key in ThesaurusKeyword.objects.filter(thesaurus_id=tid): - label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang) - if self.q: - label = label.filter(label__icontains=self.q) - if label.exists(): - qs_local.append(label.get()) - else: - if self.q in key.alt_label: - qs_non_local.append(key) - elif not self.q: - qs_non_local.append(key) - - return qs_non_local + qs_local + + # help to get real language string + requested_lang = get_language_from_request(self.request) + supported_lang = get_supported_language_variant(requested_lang, strict=True) + if "_" in supported_lang: + supported_lang = supported_lang.split("_")[0] + elif "-" in supported_lang: + supported_lang = supported_lang.split("-")[0] + + keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=supported_lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") + + qs = ThesaurusKeywordLabel.objects.filter(lang=supported_lang, keyword_id__in=keyword_id_for_given_thesaurus) + if self.q: + qs = qs.filter(label__istartswith=self.q) + + qs_local = list(qs) + qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids)) + return qs_local + qs_non_local def get_results(self, context): return [ From c83f969347bc48d9d746b570177c30e9aa68fdba Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 4 Aug 2022 10:06:36 +0200 Subject: [PATCH 2/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index e8ab3d80686..6024a9f5be4 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -344,7 +344,7 @@ def _get_thesauro_keyword_label(item, lang): qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label")) qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label")) - return qs_local + [("", "-------")] + qs_non_local + return [("", "-------")] + qs_local + [("", "-------")] + qs_non_local @staticmethod def _get_thesauro_title_label(item, lang): From 8fb9a87ae487001eb6c86951a15f01a8ba77685f Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Thu, 4 Aug 2022 13:43:30 +0200 Subject: [PATCH 3/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/forms.py | 5 +++++ geonode/base/views.py | 17 ++++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 6024a9f5be4..8871da567d2 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -40,6 +40,8 @@ from taggit.forms import TagField from tinymce.widgets import TinyMCE from django.contrib.admin.utils import flatten +from django.utils.translation import get_language_info + from geonode.base.enumerations import ALL_LANGUAGES from geonode.base.models import (HierarchicalKeyword, License, Region, ResourceBase, Thesaurus, @@ -337,6 +339,9 @@ def _define_choicefield(self, item, required, tname, lang): @staticmethod def _get_thesauro_keyword_label(item, lang): + # clean language string comming from thesaurus database + lang = get_language_info(lang)['code'] + keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item) qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") diff --git a/geonode/base/views.py b/geonode/base/views.py index 2602abe95a9..2adc2e0bf4c 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -35,7 +35,7 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.urls import reverse -from django.utils.translation import get_language_from_request, get_supported_language_variant +from django.utils.translation import get_language_info, get_language_from_request # Geonode dependencies from geonode.maps.models import Map @@ -304,20 +304,15 @@ class ThesaurusAvailable(autocomplete.Select2QuerySetView): def get_queryset(self): tid = self.request.GET.get("sysid") - - # help to get real language string - requested_lang = get_language_from_request(self.request) - supported_lang = get_supported_language_variant(requested_lang, strict=True) - if "_" in supported_lang: - supported_lang = supported_lang.split("_")[0] - elif "-" in supported_lang: - supported_lang = supported_lang.split("-")[0] + # clean language string comming from thesaurus database + lang = get_language_from_request(self.request) + lang = get_language_info(lang)['code'] keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid) - qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=supported_lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") - qs = ThesaurusKeywordLabel.objects.filter(lang=supported_lang, keyword_id__in=keyword_id_for_given_thesaurus) + qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus) if self.q: qs = qs.filter(label__istartswith=self.q) From 2b0d5e2fee28ad67ba817e5b0349f10893b25866 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 8 Aug 2022 11:12:23 +0200 Subject: [PATCH 4/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/forms.py | 24 +++++++++++++++++------- geonode/base/tests.py | 22 +++++++++++++++++++++- geonode/base/views.py | 16 +++++++++------- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 8871da567d2..9ecd0bebedc 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -40,7 +40,7 @@ from taggit.forms import TagField from tinymce.widgets import TinyMCE from django.contrib.admin.utils import flatten -from django.utils.translation import get_language_info +from django.utils.translation import get_language from geonode.base.enumerations import ALL_LANGUAGES from geonode.base.models import (HierarchicalKeyword, @@ -48,11 +48,10 @@ ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel, TopicCategory) from geonode.base.widgets import TaggitSelect2Custom +from geonode.base.fields import MultiThesauriField from geonode.documents.models import Document from geonode.layers.models import Dataset -from django.utils.translation import get_language -from .fields import MultiThesauriField -from geonode.base.utils import validate_extra_metadata +from geonode.base.utils import validate_extra_metadata, remove_country_from_lanugecode logger = logging.getLogger(__name__) @@ -292,7 +291,14 @@ class Meta: ) +THESAURUS_RESULT_LIST_SEPERATOR = ("", "-------") + + class ThesaurusAvailableForm(forms.Form): + + # seperator at beginning of thesaurus search result and between + # results found in local language and alt label + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) lang = get_language() @@ -339,17 +345,21 @@ def _define_choicefield(self, item, required, tname, lang): @staticmethod def _get_thesauro_keyword_label(item, lang): - # clean language string comming from thesaurus database - lang = get_language_info(lang)['code'] keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + if len(qs_keyword_ids) == 0: + lang = remove_country_from_lanugecode(lang) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label")) qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label")) - return [("", "-------")] + qs_local + [("", "-------")] + qs_non_local + return [THESAURUS_RESULT_LIST_SEPERATOR] + qs_local + [THESAURUS_RESULT_LIST_SEPERATOR] + qs_non_local @staticmethod def _get_thesauro_title_label(item, lang): diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 60f9d0dbf25..0de335ae304 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -57,6 +57,7 @@ from geonode.storage.manager import storage_manager from django.test import Client, TestCase, override_settings, SimpleTestCase from django.shortcuts import reverse +from django.utils import translation from geonode.base.middleware import ReadOnlyMiddleware, MaintenanceMiddleware from geonode.base.templatetags.base_tags import get_visibile_resources, facets @@ -72,7 +73,7 @@ from django.core.files import File from django.core.management import call_command from django.core.management.base import CommandError -from geonode.base.forms import ThesaurusAvailableForm +from geonode.base.forms import ThesaurusAvailableForm, THESAURUS_RESULT_LIST_SEPERATOR test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) @@ -922,6 +923,25 @@ def test_will_return_thesaurus_with_the_defaul_order_as_0(self): # will check if the second element of the tuple is the thesaurus_id = 1 self.assertEqual(fields[1][0], '2') + def test_get_thesuro_key_label_with_cmd_language_code(self): + # in python test language code look like 'en' this test checks if key label result function + # returns correct results + t = Thesaurus.objects.get(identifier='inspire-theme') + tid = 1 + translation.activate("en") + t_available_form = ThesaurusAvailableForm(data={"1": tid}) + results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language()) + self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR) + + def test_get_thesuro_key_label_with_browser_language_code(self): + # in browser scenario language does not look like "it", but rather include coutry code + # like "it-it" this test checks if _get_thesauro_keyword_label can handle this + tid = 1 + translation.activate("en-us") + t_available_form = ThesaurusAvailableForm(data={"1": tid}) + results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language()) + self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR) + class TestFacets(TestCase): diff --git a/geonode/base/views.py b/geonode/base/views.py index 2adc2e0bf4c..7f49a08c20f 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -35,7 +35,7 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.urls import reverse -from django.utils.translation import get_language_info, get_language_from_request +from django.utils.translation import get_language # Geonode dependencies from geonode.maps.models import Map @@ -48,7 +48,7 @@ from geonode.resource.manager import resource_manager from geonode.security.utils import get_visible_resources from geonode.notifications_helper import send_notification -from geonode.base.utils import OwnerRightsRequestViewUtils +from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_lanugecode from geonode.base.forms import UserAndGroupPermissionsForm from geonode.base.forms import ( @@ -304,14 +304,16 @@ class ThesaurusAvailable(autocomplete.Select2QuerySetView): def get_queryset(self): tid = self.request.GET.get("sysid") - # clean language string comming from thesaurus database - lang = get_language_from_request(self.request) - lang = get_language_info(lang)['code'] - + lang = get_language() keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid) + + # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") - not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") + if len(qs_keyword_ids) == 0: + lang = remove_country_from_lanugecode(lang) + qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id") + not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id") qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus) if self.q: qs = qs.filter(label__istartswith=self.q) From 9dc6969515c763c716f41c777ab27af03114afb6 Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 8 Aug 2022 11:14:57 +0200 Subject: [PATCH 5/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/geonode/base/utils.py b/geonode/base/utils.py index 07a8ea18d17..9abac5bbe72 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -189,3 +189,16 @@ def validate_extra_metadata(data, instance): raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}") # conerted because in this case, we can store a well formated json instead of the user input return data + + +@staticmethod +def remove_country_from_lanugecode(language: str): + """ Remove country code (us) from language name (en-us) + >>> remove_country_from_lanugecode("en-us") + 'en' + """ + if "-" not in language: + return language + + lang, _, _ = language.lower().partition("-") + return lang From 6151ceb4855c71c93f0828f6815b5acb956c7e9f Mon Sep 17 00:00:00 2001 From: mwallschlaeger Date: Mon, 8 Aug 2022 12:58:15 +0200 Subject: [PATCH 6/6] [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow --- geonode/base/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 0de335ae304..7be78f2ea6a 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -926,7 +926,6 @@ def test_will_return_thesaurus_with_the_defaul_order_as_0(self): def test_get_thesuro_key_label_with_cmd_language_code(self): # in python test language code look like 'en' this test checks if key label result function # returns correct results - t = Thesaurus.objects.get(identifier='inspire-theme') tid = 1 translation.activate("en") t_available_form = ThesaurusAvailableForm(data={"1": tid})