chore: enhance performance of loading products (#2385)

backend/application/core/api/serializers_product.py

@@ -2,6 +2,7 @@
 from typing import Optional
 
 from rest_framework.serializers import (
+    IntegerField,
     ModelSerializer,
     SerializerMethodField,
     ValidationError,
@@ -36,69 +37,30 @@
     get_product_authorization_group_member,
     get_product_member,
 )
+from application.core.services.product import (
+    get_product_group_license_count,
+    get_product_group_observation_count,
+)
 from application.core.services.risk_acceptance_expiry import (
     calculate_risk_acceptance_expiry_date,
 )
-from application.core.types import Assessment_Status, Status
+from application.core.types import Assessment_Status, Severity, Status
 from application.issue_tracker.types import Issue_Tracker
 from application.licenses.models import License_Component
+from application.licenses.types import License_Policy_Evaluation_Result
 from application.rules.models import Rule
 from application.rules.types import Rule_Status
 
 
 class ProductCoreSerializer(ModelSerializer):
-    open_critical_observation_count = SerializerMethodField()
-    open_high_observation_count = SerializerMethodField()
-    open_medium_observation_count = SerializerMethodField()
-    open_low_observation_count = SerializerMethodField()
-    open_none_observation_count = SerializerMethodField()
-    open_unknown_observation_count = SerializerMethodField()
-    forbidden_licenses_count = SerializerMethodField()
-    review_required_licenses_count = SerializerMethodField()
-    unknown_licenses_count = SerializerMethodField()
-    allowed_licenses_count = SerializerMethodField()
-    ignored_licenses_count = SerializerMethodField()
     permissions = SerializerMethodField()
 
-    class Meta:
-        model = Product
-        fields = "__all__"
-
-    def get_open_critical_observation_count(self, obj: Product) -> int:
-        return obj.open_critical_observation_count
-
-    def get_open_high_observation_count(self, obj: Product) -> int:
-        return obj.open_high_observation_count
-
-    def get_open_medium_observation_count(self, obj: Product) -> int:
-        return obj.open_medium_observation_count
-
-    def get_open_low_observation_count(self, obj: Product) -> int:
-        return obj.open_low_observation_count
-
-    def get_open_none_observation_count(self, obj: Product) -> int:
-        return obj.open_none_observation_count
-
-    def get_open_unknown_observation_count(self, obj: Product) -> int:
-        return obj.open_unknown_observation_count
-
     def get_permissions(self, obj: Product) -> list[Permissions]:
         return get_permissions_for_role(get_highest_user_role(obj))
 
-    def get_forbidden_licenses_count(self, obj: Product) -> int:
-        return obj.forbidden_licenses_count
-
-    def get_review_required_licenses_count(self, obj: Product) -> int:
-        return obj.review_required_licenses_count
-
-    def get_unknown_licenses_count(self, obj: Product) -> int:
-        return obj.unknown_licenses_count
-
-    def get_allowed_licenses_count(self, obj: Product) -> int:
-        return obj.allowed_licenses_count
-
-    def get_ignored_licenses_count(self, obj: Product) -> int:
-        return obj.ignored_licenses_count
+    class Meta:
+        model = Product
+        fields = "__all__"
 
     def validate(self, attrs: dict):
         if attrs.get("repository_branch_housekeeping_active"):
@@ -133,9 +95,74 @@ def validate(self, attrs: dict):
 
 
 class ProductGroupSerializer(ProductCoreSerializer):
+    open_critical_observation_count = SerializerMethodField()
+    open_high_observation_count = SerializerMethodField()
+    open_medium_observation_count = SerializerMethodField()
+    open_low_observation_count = SerializerMethodField()
+    open_none_observation_count = SerializerMethodField()
+    open_unknown_observation_count = SerializerMethodField()
+    forbidden_licenses_count = SerializerMethodField()
+    review_required_licenses_count = SerializerMethodField()
+    unknown_licenses_count = SerializerMethodField()
+    allowed_licenses_count = SerializerMethodField()
+    ignored_licenses_count = SerializerMethodField()
     products_count = SerializerMethodField()
     product_rule_approvals = SerializerMethodField()
 
+    def get_open_critical_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_CRITICAL)
+
+    def get_open_high_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_HIGH)
+
+    def get_open_medium_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_MEDIUM)
+
+    def get_open_low_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_LOW)
+
+    def get_open_none_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_NONE)
+
+    def get_open_unknown_observation_count(self, obj: Product) -> int:
+        return get_product_group_observation_count(obj, Severity.SEVERITY_UNKNOWN)
+
+    def get_forbidden_licenses_count(self, obj: Product) -> int:
+        return get_product_group_license_count(
+            obj, License_Policy_Evaluation_Result.RESULT_FORBIDDEN
+        )
+
+    def get_review_required_licenses_count(self, obj: Product) -> int:
+        return get_product_group_license_count(
+            obj, License_Policy_Evaluation_Result.RESULT_REVIEW_REQUIRED
+        )
+
+    def get_unknown_licenses_count(self, obj: Product) -> int:
+        return get_product_group_license_count(
+            obj, License_Policy_Evaluation_Result.RESULT_UNKNOWN
+        )
+
+    def get_allowed_licenses_count(self, obj: Product) -> int:
+        return get_product_group_license_count(
+            obj, License_Policy_Evaluation_Result.RESULT_ALLOWED
+        )
+
+    def get_ignored_licenses_count(self, obj: Product) -> int:
+        return get_product_group_license_count(
+            obj, License_Policy_Evaluation_Result.RESULT_IGNORED
+        )
+
+    def get_products_count(self, obj: Product) -> int:
+        return obj.products.count()
+
+    def get_product_rule_approvals(self, obj: Product) -> int:
+        if not obj.product_rules_need_approval:
+            return 0
+
+        return Rule.objects.filter(
+            product=obj, approval_status=Rule_Status.RULE_STATUS_NEEDS_APPROVAL
+        ).count()
+
     class Meta:
         model = Product
         fields = [
@@ -177,17 +204,6 @@ class Meta:
             "ignored_licenses_count",
         ]
 
-    def get_products_count(self, obj: Product) -> int:
-        return obj.products.count()
-
-    def get_product_rule_approvals(self, obj: Product) -> int:
-        if not obj.product_rules_need_approval:
-            return 0
-
-        return Rule.objects.filter(
-            product=obj, approval_status=Rule_Status.RULE_STATUS_NEEDS_APPROVAL
-        ).count()
-
     def create(self, validated_data: dict) -> Product:
         product_group = super().create(validated_data)
         product_group.is_product_group = True
@@ -212,6 +228,18 @@ class ProductSerializer(
     ProductCoreSerializer
 ):  # pylint: disable=too-many-public-methods
     # all these methods are needed
+    open_critical_observation_count = IntegerField(read_only=True)
+    open_high_observation_count = IntegerField(read_only=True)
+    open_medium_observation_count = IntegerField(read_only=True)
+    open_low_observation_count = IntegerField(read_only=True)
+    open_none_observation_count = IntegerField(read_only=True)
+    open_unknown_observation_count = IntegerField(read_only=True)
+    forbidden_licenses_count = IntegerField(read_only=True)
+    review_required_licenses_count = IntegerField(read_only=True)
+    unknown_licenses_count = IntegerField(read_only=True)
+    allowed_licenses_count = IntegerField(read_only=True)
+    ignored_licenses_count = IntegerField(read_only=True)
+
     product_group_name = SerializerMethodField()
     product_group_repository_branch_housekeeping_active = SerializerMethodField()
     product_group_security_gate_active = SerializerMethodField()

backend/application/core/api/views.py

@@ -164,7 +164,7 @@ class ProductViewSet(ModelViewSet):
 
     def get_queryset(self):
         return (
-            get_products(is_product_group=False)
+            get_products(is_product_group=False, with_annotations=True)
             .select_related("product_group")
             .select_related("repository_default_branch")
         )

backend/application/core/models.py

@@ -141,171 +141,6 @@ class Meta:
     def __str__(self):
         return self.name
 
-    @property
-    def open_critical_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_critical_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_CRITICAL,
-        ).count()
-
-    @property
-    def open_high_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_high_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_HIGH,
-        ).count()
-
-    @property
-    def open_medium_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_medium_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_MEDIUM,
-        ).count()
-
-    @property
-    def open_low_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_low_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_LOW,
-        ).count()
-
-    @property
-    def open_none_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_none_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_NONE,
-        ).count()
-
-    @property
-    def open_unknown_observation_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.open_unknown_observation_count
-            return count
-
-        return Observation.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            current_status=Status.STATUS_OPEN,
-            current_severity=Severity.SEVERITY_UNKNOWN,
-        ).count()
-
-    @property
-    def forbidden_licenses_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.forbidden_licenses_count
-            return count
-
-        License_Component = apps.get_model("licenses", "License_Component")
-        return License_Component.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            evaluation_result=License_Policy_Evaluation_Result.RESULT_FORBIDDEN,
-        ).count()
-
-    @property
-    def review_required_licenses_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.review_required_licenses_count
-            return count
-
-        License_Component = apps.get_model("licenses", "License_Component")
-        return License_Component.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            evaluation_result=License_Policy_Evaluation_Result.RESULT_REVIEW_REQUIRED,
-        ).count()
-
-    @property
-    def unknown_licenses_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.unknown_licenses_count
-            return count
-
-        License_Component = apps.get_model("licenses", "License_Component")
-        return License_Component.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            evaluation_result=License_Policy_Evaluation_Result.RESULT_UNKNOWN,
-        ).count()
-
-    @property
-    def allowed_licenses_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.allowed_licenses_count
-            return count
-
-        License_Component = apps.get_model("licenses", "License_Component")
-        return License_Component.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            evaluation_result=License_Policy_Evaluation_Result.RESULT_ALLOWED,
-        ).count()
-
-    @property
-    def ignored_licenses_count(self):
-        if self.is_product_group:
-            count = 0
-            for product in Product.objects.filter(product_group=self):
-                count += product.ignored_licenses_count
-            return count
-
-        License_Component = apps.get_model("licenses", "License_Component")
-        return License_Component.objects.filter(
-            product=self,
-            branch=self.repository_default_branch,
-            evaluation_result=License_Policy_Evaluation_Result.RESULT_IGNORED,
-        ).count()
-
 
 class Branch(Model):
     product = ForeignKey(Product, on_delete=CASCADE)