Merge remote-tracking branch 'origin/master' into geosolutions-master

.circleci/config.yml

@@ -128,7 +128,7 @@ workflows:
           name: geonode_test_suite
           load_docker_cache: true
           save_docker_cache: false
-          test_suite: ./test.sh $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))") geonode.thumbs.tests
+          test_suite: ./test.sh $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))") geonode.catalogue.backends.tests geonode.thumbs.tests
           requires:
             - geonode_test_suite_smoke
       - build:

.env

@@ -173,3 +173,18 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK
 # GeoNode APIs
 API_LOCKDOWN=False
 TASTYPIE_APIKEY=
+
+# CELERY
+
+# expressed in KB
+# CELERY__MAX_MEMORY_PER_CHILD="200000"
+# ## 
+# Note right autoscale value must coincide with worker concurrency value
+# CELERY__AUTOSCALE_VALUES="1,4" 
+# CELERY__WORKER_CONCURRENCY="4"
+# ##
+# CELERY__OPTS="--without-gossip --without-mingle -Ofair -B -E"
+# CELERY__BEAT_SCHEDULE="/mnt/volumes/statics/celerybeat-schedule"
+# CELERY__LOG_LEVEL="INFO"
+# CELERY__LOG_FILE="/var/log/celery.log"
+# CELERY__WORKER_NAME="worker1@%h"

Dockerfile

@@ -47,6 +47,9 @@ RUN chmod +x /usr/src/geonode/tasks.py \
 COPY celery.sh /usr/bin/celery-commands
 RUN chmod +x /usr/bin/celery-commands
 
+COPY celery-cmd /usr/bin/celery-cmd
+RUN chmod +x /usr/bin/celery-cmd
+
 # Prepraing dependencies
 RUN apt-get update && apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils
 # RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj

celery-cmd

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# A configurable celery command.
+# Luca Pasquali <[email protected]>
+CELERY_BIN=${CELERY_BIN:-"$(which celery||echo celery)"}
+CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"}
+CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker.state"}
+# expressed in KB
+CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"}
+CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"2,4"}
+CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"}
+CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"}
+CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"/mnt/volumes/statics/celerybeat-schedule"}
+CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"INFO"}
+CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"}
+CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"}
+CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"}
+
+$CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \
+    --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \
+    --statedb=$CELERY__STATE_DB -s $CELERY__BEAT_SCHEDULE \
+    --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \
+    --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD

docker-compose-test.yml

@@ -1,14 +1,11 @@
 version: '3.4'
 services:
 
-# Common Django template for Geonode, Celery services below
+# Common Django template for GeoNode and Celery services below
 x-common-django:
   &default-common-django
   image: geonode/geonode:latest
   restart: on-failure
-  build:
-    context: ./
-    dockerfile: Dockerfile
   env_file:
     - .env_test
   volumes:
@@ -24,6 +21,9 @@ services:
   # Our custom django application. It includes Geonode.
   django:
     << : *default-common-django
+    build:
+      context: ./
+      dockerfile: Dockerfile
     container_name: django4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
   # Celery worker that executes celery tasks created by Django.
   celery:
     << : *default-common-django
+    image: geonode/geonode:latest
     container_name: celery4${COMPOSE_PROJECT_NAME}
+    depends_on:
+      - django
     environment:
       - IS_CELERY=True
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
-    command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
+    command: "celery-cmd"
 
   # Nginx is serving django static and media files and proxies to django and geonode
   geonode:
@@ -110,7 +113,7 @@ services:
 
   # PostGIS database.
   db:
-    # use geonode official postgis 11 image
+    # use geonode official postgis 13 image
     image: geonode/postgis:13
     container_name: db4${COMPOSE_PROJECT_NAME}
     env_file:

docker-compose.yml

@@ -1,14 +1,11 @@
 version: '3.4'
 services:
 
-# Common Django template for Geonode, Celery services below
+# Common Django template for GeoNode and Celery services below
 x-common-django:
   &default-common-django
   image: geonode/geonode:latest
   restart: on-failure
-  build:
-    context: ./
-    dockerfile: Dockerfile
   env_file:
     - .env
   volumes:
@@ -24,6 +21,9 @@ services:
   # Our custom django application. It includes Geonode.
   django:
     << : *default-common-django
+    build:
+      context: ./
+      dockerfile: Dockerfile
     container_name: django4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/"
@@ -39,11 +39,14 @@ services:
   # Celery worker that executes celery tasks created by Django.
   celery:
     << : *default-common-django
+    image: geonode/geonode:latest
     container_name: celery4${COMPOSE_PROJECT_NAME}
+    depends_on:
+      - django
     environment:
       - IS_CELERY=True
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
-    command: "celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
+    command: "celery-cmd"
 
   # Nginx is serving django static and media files and proxies to django and geonode
   geonode:

geonode/api/api.py

@@ -323,6 +323,7 @@ class Meta:
         include_resource_uri = False
         filtering = {'slug': ALL,
                      'name': ALL}
+        ordering = ['name']
         authorization = ApiLockdownAuthorization()
 
     def apply_filters(self, request, applicable_filters):

geonode/api/tests.py

@@ -590,7 +590,7 @@ def test_will_return_keywords_for_the_selected_thesaurus_if_exists(self):
         url = f"{self.list_url}?thesaurus=inspire-theme"
         resp = self.api_client.get(url)
         self.assertValidJSONResponse(resp)
-        self.assertEqual(resp.json()["meta"]["total_count"], 34)
+        self.assertEqual(resp.json()["meta"]["total_count"], 36)
 
     def test_will_return_empty_if_the_alt_label_does_not_exists(self):
         url = f"{self.list_url}?alt_label=invalid-alt_label"
@@ -618,8 +618,8 @@ def test_will_return_expected_keyword_label_for_existing_lang(self, lang):
         resp = self.api_client.get(url)
         # the german translations exists, for the other labels, the alt_label will be used
         expected_labels = [
-            "ac", "Adressen", "af", "am", "au", "br", "bu",
-            "cp", "ef", "el", "er", "ge", "gg", "gn", "hb", "hh",
+            "", "ac", "Adressen", "af", "am", "au", "br", "bu",
+            "cp", "ef", "el", "er", "foo_keyword", "ge", "gg", "gn", "hb", "hh",
             "hy", "lc", "lu", "mf", "mr", "nz", "of", "oi", "pd",
             "pf", "ps", "rs", "sd", "so", "sr", "su", "tn", "us"
         ]
@@ -634,10 +634,10 @@ def test_will_return_default_keyword_label_for_not_existing_lang(self, lang):
         resp = self.api_client.get(url)
         # no translations exists, the alt_label will be used for all keywords
         expected_labels = [
-            "ac", "ad", "af", "am", "au", "br", "bu",
-            "cp", "ef", "el", "er", "ge", "gg", "gn", "hb", "hh",
+            "", "ac", "ad", "af", "am", "au", "br", "bu",
+            "cp", "ef", "el", "er", "foo_keyword", "ge", "gg", "gn", "hb", "hh",
             "hy", "lc", "lu", "mf", "mr", "nz", "of", "oi", "pd",
-            "pf", "ps", "rs", "sd", "so", "sr", "su", "tn", "us"
+            "pf", "ps", "rs", "sd", "so", "sr", "su", "tn", "us",
         ]
         actual_labels = [x["alt_label"] for x in self.deserialize(resp)["objects"]]
         self.assertValidJSONResponse(resp)

geonode/base/api/permissions.py

@@ -18,12 +18,13 @@
 #
 #########################################################################
 import logging
-from django.conf import settings
 from django.contrib.auth import get_user_model
 
 from rest_framework import permissions
 from rest_framework.filters import BaseFilterBackend
 
+from geonode.security.utils import get_resources_with_perms
+
 logger = logging.getLogger(__name__)
 
 
@@ -151,9 +152,6 @@ def filter_queryset(self, request, queryset, view):
         # We want to defer this import until runtime, rather than import-time.
         # See https://github.com/encode/django-rest-framework/issues/4608
         # (Also see #1624 for why we need to make this import explicitly)
-        from guardian.shortcuts import get_objects_for_user
-        from geonode.base.models import ResourceBase
-        from geonode.security.utils import get_visible_resources
 
         user = request.user
         # perm_format = '%(app_label)s.view_%(model_name)s'
@@ -162,22 +160,7 @@ def filter_queryset(self, request, queryset, view):
         #     'model_name': queryset.model._meta.model_name,
         # }
 
-        if settings.SKIP_PERMS_FILTER:
-            resources = ResourceBase.objects.all()
-        else:
-            resources = get_objects_for_user(
-                user,
-                'base.view_resourcebase',
-                **self.shortcut_kwargs
-            )
-        logger.debug(f" user: {user} -- resources: {resources}")
-
-        obj_with_perms = get_visible_resources(
-            resources,
-            user,
-            admin_approval_required=settings.ADMIN_MODERATE_UPLOADS,
-            unpublished_not_visible=settings.RESOURCE_PUBLISHING,
-            private_groups_not_visibile=settings.GROUP_PRIVATE_RESOURCES)
+        obj_with_perms = get_resources_with_perms(user, shortcut_kwargs=self.shortcut_kwargs)
         logger.debug(f" user: {user} -- obj_with_perms: {obj_with_perms}")
 
         return queryset.filter(id__in=obj_with_perms.values('id'))