From ad2d857c6f2fdda2fe6dd26d6776f071fcb63264 Mon Sep 17 00:00:00 2001 From: Prowler Bot Date: Fri, 17 Jan 2025 14:02:59 +0100 Subject: [PATCH] feat(findings): add /findings/metadata to retrieve dynamic filters information (#6586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Víctor Fernández Poyatos --- api/src/backend/api/specs/v1.yaml | 613 ++++++++++++++++++++---- api/src/backend/api/tests/test_views.py | 38 +- api/src/backend/api/v1/serializers.py | 13 + api/src/backend/api/v1/views.py | 52 +- 4 files changed, 604 insertions(+), 112 deletions(-) diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index e274a0c9e11..966f09a6ec1 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.1.0 + version: 1.2.0 description: |- Prowler API specification. @@ -1167,12 +1167,439 @@ paths: - Finding security: - jwtAuth: [] + deprecated: true responses: - '201': + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/FindingDynamicFilterResponse' + description: '' + /api/v1/findings/metadata: + get: + operationId: findings_metadata_retrieve + description: Fetch unique metadata values from a set of findings. This is useful + for dynamic filtering. + summary: Retrieve metadata values from findings + parameters: + - in: query + name: fields[findings-metadata] + schema: + type: array + items: + type: string + enum: + - services + - regions + - resource_types + - tags + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[check_id] + schema: + type: string + - in: query + name: filter[check_id__icontains] + schema: + type: string + - in: query + name: filter[check_id__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[delta] + schema: + type: string + nullable: true + enum: + - changed + - new + description: |- + * `new` - New + * `changed` - Changed + - in: query + name: filter[delta__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[id] + schema: + type: string + format: uuid + - in: query + name: filter[id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[impact] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[impact__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_name] + schema: + type: string + - in: query + name: filter[resource_name__icontains] + schema: + type: string + - in: query + name: filter[resource_name__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_type] + schema: + type: string + - in: query + name: filter[resource_type__icontains] + schema: + type: string + - in: query + name: filter[resource_type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resource_uid] + schema: + type: string + - in: query + name: filter[resource_uid__icontains] + schema: + type: string + - in: query + name: filter[resource_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[resources] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[severity] + schema: + type: string + enum: + - critical + - high + - informational + - low + - medium + description: |- + * `critical` - Critical + * `high` - High + * `medium` - Medium + * `low` - Low + * `informational` - Informational + - in: query + name: filter[severity__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[status] + schema: + type: string + enum: + - FAIL + - MANUAL + - MUTED + - PASS + description: |- + * `FAIL` - Fail + * `PASS` - Pass + * `MANUAL` - Manual + * `MUTED` - Muted + - in: query + name: filter[status__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - status + - -status + - severity + - -severity + - check_id + - -check_id + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Finding + security: + - jwtAuth: [] + responses: + '200': content: application/vnd.api+json: schema: - $ref: '#/components/schemas/OpenApiResponseResponse' + $ref: '#/components/schemas/FindingMetadataResponse' description: '' /api/v1/invitations/accept: post: @@ -2948,9 +3375,7 @@ paths: - name - manage_users - manage_account - - manage_billing - manage_providers - - manage_integrations - manage_scans - permission_state - unlimited_visibility @@ -3068,12 +3493,8 @@ paths: - -manage_users - manage_account - -manage_account - - manage_billing - - -manage_billing - manage_providers - -manage_providers - - manage_integrations - - -manage_integrations - manage_scans - -manage_scans - permission_state @@ -3147,9 +3568,7 @@ paths: - name - manage_users - manage_account - - manage_billing - manage_providers - - manage_integrations - manage_scans - permission_state - unlimited_visibility @@ -5458,6 +5877,92 @@ components: readOnly: true required: - scan + FindingDynamicFilter: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/FindingDynamicFilterTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + services: + type: array + items: + type: string + regions: + type: array + items: + type: string + required: + - services + - regions + FindingDynamicFilterResponse: + type: object + properties: + data: + $ref: '#/components/schemas/FindingDynamicFilter' + required: + - data + FindingDynamicFilterTypeEnum: + type: string + enum: + - finding-dynamic-filters + FindingMetadata: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/FindingMetadataTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + services: + type: array + items: + type: string + regions: + type: array + items: + type: string + resource_types: + type: array + items: + type: string + tags: + description: Tags are described as key-value pairs. + required: + - services + - regions + - resource_types + - tags + FindingMetadataResponse: + type: object + properties: + data: + $ref: '#/components/schemas/FindingMetadata' + required: + - data + FindingMetadataTypeEnum: + type: string + enum: + - findings-metadata FindingResponse: type: object properties: @@ -5902,8 +6407,6 @@ components: - data description: A related resource object from type roles title: roles - required: - - roles InvitationUpdateResponse: type: object properties: @@ -5915,7 +6418,6 @@ components: type: object required: - type - - id additionalProperties: false properties: type: @@ -5924,9 +6426,6 @@ components: description: The [type](https://jsonapi.org/format/#document-resource-object-identification) member is used to describe resource objects that share common attributes and relationships. - id: - type: string - format: uuid attributes: type: object properties: @@ -6437,8 +6936,6 @@ components: - data description: A related resource object from type roles title: roles - required: - - roles required: - data PatchedProviderGroupMembershipRequest: @@ -6850,12 +7347,8 @@ components: type: boolean manage_account: type: boolean - manage_billing: - type: boolean manage_providers: type: boolean - manage_integrations: - type: boolean manage_scans: type: boolean permission_state: @@ -7131,37 +7624,6 @@ components: required: - name - email - relationships: - type: object - properties: - roles: - type: object - properties: - data: - type: array - items: - type: object - properties: - id: - type: string - format: uuid - title: Resource Identifier - description: The identifier of the related object. - type: - type: string - enum: - - roles - title: Resource Type Name - description: The [type](https://jsonapi.org/format/#document-resource-object-identification) - member is used to describe resource objects that share - common attributes and relationships. - required: - - id - - type - required: - - data - description: A related resource object from type roles - title: roles required: - data Provider: @@ -8537,12 +8999,8 @@ components: type: boolean manage_account: type: boolean - manage_billing: - type: boolean manage_providers: type: boolean - manage_integrations: - type: boolean manage_scans: type: boolean permission_state: @@ -8670,12 +9128,8 @@ components: type: boolean manage_account: type: boolean - manage_billing: - type: boolean manage_providers: type: boolean - manage_integrations: - type: boolean manage_scans: type: boolean permission_state: @@ -8808,12 +9262,8 @@ components: type: boolean manage_account: type: boolean - manage_billing: - type: boolean manage_providers: type: boolean - manage_integrations: - type: boolean manage_scans: type: boolean permission_state: @@ -9877,37 +10327,6 @@ components: required: - name - email - relationships: - type: object - properties: - roles: - type: object - properties: - data: - type: array - items: - type: object - properties: - id: - type: string - format: uuid - title: Resource Identifier - description: The identifier of the related object. - type: - type: string - enum: - - roles - title: Resource Type Name - description: The [type](https://jsonapi.org/format/#document-resource-object-identification) - member is used to describe resource objects that share common - attributes and relationships. - required: - - id - - type - required: - - data - description: A related resource object from type roles - title: roles UserUpdateResponse: type: object properties: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 581e0d0e979..e6f19f92412 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2582,30 +2582,34 @@ def test_findings_invalid_retrieve(self, authenticated_client): ) assert response.status_code == status.HTTP_404_NOT_FOUND - def test_findings_services_regions_retrieve( - self, authenticated_client, findings_fixture - ): + def test_findings_metadata_retrieve(self, authenticated_client, findings_fixture): finding_1, *_ = findings_fixture response = authenticated_client.get( - reverse("finding-findings_services_regions"), + reverse("finding-metadata"), {"filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d")}, ) data = response.json() expected_services = {"ec2", "s3"} expected_regions = {"eu-west-1", "us-east-1"} + expected_tags = {"key": "value", "key2": "value2"} + expected_resource_types = {"prowler-test"} - assert data["data"]["type"] == "finding-dynamic-filters" + assert data["data"]["type"] == "findings-metadata" assert data["data"]["id"] is None assert set(data["data"]["attributes"]["services"]) == expected_services assert set(data["data"]["attributes"]["regions"]) == expected_regions + assert ( + set(data["data"]["attributes"]["resource_types"]) == expected_resource_types + ) + assert data["data"]["attributes"]["tags"] == expected_tags - def test_findings_services_regions_severity_retrieve( + def test_findings_metadata_severity_retrieve( self, authenticated_client, findings_fixture ): finding_1, *_ = findings_fixture response = authenticated_client.get( - reverse("finding-findings_services_regions"), + reverse("finding-metadata"), { "filter[severity__in]": ["low", "medium"], "filter[inserted_at]": finding_1.updated_at.strftime("%Y-%m-%d"), @@ -2615,26 +2619,34 @@ def test_findings_services_regions_severity_retrieve( expected_services = {"s3"} expected_regions = {"eu-west-1"} + expected_tags = {"key": "value", "key2": "value2"} + expected_resource_types = {"prowler-test"} - assert data["data"]["type"] == "finding-dynamic-filters" + assert data["data"]["type"] == "findings-metadata" assert data["data"]["id"] is None assert set(data["data"]["attributes"]["services"]) == expected_services assert set(data["data"]["attributes"]["regions"]) == expected_regions + assert ( + set(data["data"]["attributes"]["resource_types"]) == expected_resource_types + ) + assert data["data"]["attributes"]["tags"] == expected_tags - def test_findings_services_regions_future_date(self, authenticated_client): + def test_findings_metadata_future_date(self, authenticated_client): response = authenticated_client.get( - reverse("finding-findings_services_regions"), + reverse("finding-metadata"), {"filter[inserted_at]": "2048-01-01"}, ) data = response.json() - assert data["data"]["type"] == "finding-dynamic-filters" + assert data["data"]["type"] == "findings-metadata" assert data["data"]["id"] is None assert data["data"]["attributes"]["services"] == [] assert data["data"]["attributes"]["regions"] == [] + assert data["data"]["attributes"]["tags"] == {} + assert data["data"]["attributes"]["resource_types"] == [] - def test_findings_services_regions_invalid_date(self, authenticated_client): + def test_findings_metadata_invalid_date(self, authenticated_client): response = authenticated_client.get( - reverse("finding-findings_services_regions"), + reverse("finding-metadata"), {"filter[inserted_at]": "2048-01-011"}, ) assert response.json() == { diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 80f4f14eb53..a47cb270358 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -917,6 +917,7 @@ class Meta: } +# To be removed when the related endpoint is removed as well class FindingDynamicFilterSerializer(serializers.Serializer): services = serializers.ListField(child=serializers.CharField(), allow_empty=True) regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) @@ -925,6 +926,18 @@ class Meta: resource_name = "finding-dynamic-filters" +class FindingMetadataSerializer(serializers.Serializer): + services = serializers.ListField(child=serializers.CharField(), allow_empty=True) + regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) + resource_types = serializers.ListField( + child=serializers.CharField(), allow_empty=True + ) + tags = serializers.JSONField(help_text="Tags are described as key-value pairs.") + + class Meta: + resource_name = "findings-metadata" + + # Provider secrets class BaseWriteProviderSecretSerializer(BaseWriteSerializer): @staticmethod diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 17419fa634a..d166a8d15a7 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -4,6 +4,7 @@ from django.contrib.postgres.search import SearchQuery from django.db import transaction from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum +from django.db.models.functions import JSONObject from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control @@ -87,6 +88,7 @@ ComplianceOverviewFullSerializer, ComplianceOverviewSerializer, FindingDynamicFilterSerializer, + FindingMetadataSerializer, FindingSerializer, InvitationAcceptSerializer, InvitationCreateSerializer, @@ -192,7 +194,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.1.1" + spectacular_settings.VERSION = "1.2.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -1274,7 +1276,13 @@ def get_queryset(self): tags=["Finding"], summary="Retrieve the services and regions that are impacted by findings", description="Fetch services and regions affected in findings.", - responses={201: OpenApiResponse(response=MembershipSerializer)}, + filters=True, + deprecated=True, + ), + metadata=extend_schema( + tags=["Finding"], + summary="Retrieve metadata values from findings", + description="Fetch unique metadata values from a set of findings. This is useful for dynamic filtering.", filters=True, ), ) @@ -1308,6 +1316,8 @@ class FindingViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "findings_services_regions": return FindingDynamicFilterSerializer + elif self.action == "metadata": + return FindingMetadataSerializer return super().get_serializer_class() @@ -1376,6 +1386,44 @@ def findings_services_regions(self, request): return Response(data=serializer.data, status=status.HTTP_200_OK) + @action(detail=False, methods=["get"], url_name="metadata") + def metadata(self, request): + queryset = self.get_queryset() + filtered_queryset = self.filter_queryset(queryset) + + result = filtered_queryset.aggregate( + services=ArrayAgg("resources__service", flat=True, distinct=True), + regions=ArrayAgg("resources__region", flat=True, distinct=True), + tags=ArrayAgg( + JSONObject( + key=F("resources__tags__key"), value=F("resources__tags__value") + ), + distinct=True, + filter=Q(resources__tags__key__isnull=False), + ), + resource_types=ArrayAgg("resources__type", flat=True, distinct=True), + ) + if result["services"] is None: + result["services"] = [] + if result["regions"] is None: + result["regions"] = [] + if result["regions"] is None: + result["regions"] = [] + if result["resource_types"] is None: + result["resource_types"] = [] + if result["tags"] is None: + result["tags"] = [] + + result["tags"] = {t["key"]: t["value"] for t in result["tags"]} + + serializer = self.get_serializer( + data=result, + ) + + serializer.is_valid(raise_exception=True) + + return Response(data=serializer.data, status=status.HTTP_200_OK) + @extend_schema_view( list=extend_schema(