refactor: improve API response

app/api.py

@@ -55,7 +55,7 @@ def autocomplete(request: AutocompleteRequest):
         )
         .execute()
     )
-    resp = response.create_response(results, request)
+    resp = response.create_response(results, request.response_fields)
     return resp
 
 
@@ -166,6 +166,7 @@ def search(
     langs: Annotated[list[str] | None, Query()] = None,
     add_english: bool = True,
     num_results: int = 10,
+    projection: Annotated[list[str] | None, Query()] = None,
 ):
     if langs is None:
         langs = {"en"}
@@ -178,7 +179,7 @@ def search(
     query = query_utils.build_search_query(q, langs, num_results, CONFIG)
     logger.info("query:\n%s", json.dumps(query.to_dict(), indent=4))
     results = query.execute()
-    return results.to_dict()
+    return response.create_response(results, projection)
 
 
 @app.post("/advanced-search")
@@ -194,5 +195,4 @@ def search(request: AdvancedSearchRequest):
         )
 
     results = create_search_query(request).execute()
-    resp = response.create_response(results, request)
-    return resp
+    return response.create_response(results, request.response_fields)

app/config.py

@@ -43,11 +43,11 @@ class FieldConfig(BaseModel):
     is_last_modified_field: bool = False
 
     @model_validator(mode="after")
-    def multi_should_be_used_for_keyword_type_only(self):
+    def multi_should_be_used_for_selected_type_only(self):
         """Validator that checks that `multi` flag is only True for fields
-        with type `keyword`."""
-        if self.type not in (FieldType.keyword, FieldType.text) and self.multi:
-            raise ValueError("multi=True should only be used with keyword type")
+        with specific types."""
+        if self.type not in (FieldType.keyword, FieldType.text, FieldType.double, FieldType.date) and self.multi:
+            raise ValueError(f"multi=True is not compatible with type={self.type}")
         return self
 
     @model_validator(mode="after")

app/dsl.py

@@ -1,5 +1,5 @@
 from typing import Iterable
-from elasticsearch_dsl import Double, Keyword, Object, Text, analyzer
+from elasticsearch_dsl import Date, Double, Keyword, Object, Text, analyzer
 
 from app.config import FieldConfig, FieldType
 from app.utils.analyzers import ANALYZER_LANG_MAPPING
@@ -11,12 +11,14 @@ def generate_dsl_field(field: FieldConfig, supported_lang: Iterable[str]):
             lang: Text(analyzer=analyzer(ANALYZER_LANG_MAPPING.get(lang, "standard")))
             for lang in supported_lang
         }
-        return Object(dynamic=False, properties=properties)
+        return Object(required=field.required, dynamic=False, properties=properties)
     elif field.type == FieldType.keyword:
         return Keyword(required=field.required, multi=field.multi)
     elif field.type == FieldType.text:
         return Text(required=field.required, multi=field.multi)
     elif field.type == FieldType.double:
-        return Double()
+        return Double(required=field.required, multi=field.multi)
+    elif field.type == FieldType.date:
+        return Date(required=field.required, multi=field.multi)
     else:
-        raise ValueError("unsupported field type: %s" % field.type)
+        raise ValueError(f"unsupported field type: {field.type}")

app/models/product.py

@@ -183,3 +183,4 @@ def save(self, **kwargs):
     nutrition_grades = _generate_dsl_field(FIELD_BY_NAME["nutrition_grades"])
     ecoscore_grade = _generate_dsl_field(FIELD_BY_NAME["ecoscore_grade"])
     nova_groups = _generate_dsl_field(FIELD_BY_NAME["nova_groups"])
+    last_modified_t = _generate_dsl_field(FIELD_BY_NAME["last_modified_t"])

app/models/request.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import datetime
-from typing import List, Optional, Set
+from typing import List
 
 from pydantic import BaseModel
 

app/utils/response.py

@@ -1,104 +1,63 @@
-from __future__ import annotations
-
-import json
-from functools import lru_cache
-
 from app.models.request import SearchBase
+from app.types import JSONType
 
 
-@lru_cache(maxsize=None)
-def get_json_schema():
-    with open("app/product.schema.json") as json_file:
-        return json.load(json_file)
-
+def create_response(es_results, projection: set[str] | None = None):
+    return [convert_es_result(r, projection) for r in es_results]
 
-def create_response(es_results, request: SearchBase):
-    resp = [convert_es_result(r, request) for r in es_results]
-    return resp
 
-
-def convert_es_result(es_result, request: SearchBase):
+def convert_es_result(es_result, projection: set[str] | None = None):
     if not es_result:
         return None
 
-    result_dict = es_result.to_dict()
-    result_dict = add_images_urls_to_product(result_dict)
+    result_dict = add_images_urls_to_product(es_result.to_dict())
 
     # Trim fields as needed
-    if request.response_fields:
-        trimmed_result_dict = {}
-        for response_field in request.response_fields:
-            if response_field in result_dict:
-                trimmed_result_dict[response_field] = result_dict[response_field]
-
-        result_dict = trimmed_result_dict
+    if projection is not None:
+        return dict((k, v) for k, v in result_dict.items() if k in projection)
 
     return result_dict
 
 
-def add_images_urls_to_product(product):
+def add_images_urls_to_product(product: JSONType):
     # Python copy of the code from https://github.com/openfoodfacts/openfoodfacts-server/blob/b297ed858d526332649562cdec5f1d36be184984/lib/ProductOpener/Display.pm#L10128
     code = product["code"]
 
     for image_type in ["front", "ingredients", "nutrition", "packaging"]:
         display_ids = []
         lc = product.get("lc")
         if lc:
-            display_ids.append("{}_{}".format(image_type, lc))
+            display_ids.append(f"{image_type}_{lc}")
 
         display_ids.append(image_type)
+        base_url = "https://images.openfoodfacts.org/images/products/"
 
         for display_id in display_ids:
-            if (
-                product.get("images")
-                and product["images"].get(display_id)
-                and product["images"][display_id].get("sizes")
-            ):
+            images = product.get("images", {})
+            if display_id in images and images[display_id].get("sizes"):
+                rev_id = product["images"][display_id]["rev"]
                 product[
-                    "image_{}_url".format(image_type)
-                ] = "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                    code,
-                    display_id,
-                    product["images"][display_id].get("rev"),
-                    400,
-                )
+                    f"image_{image_type}_url"
+                ] = f"{base_url}{code}/{display_id}.{rev_id}.400.jpg"
                 product[
-                    "image_{}_small_url".format(image_type)
-                ] = "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                    code,
-                    display_id,
-                    product["images"][display_id].get("rev"),
-                    200,
-                )
+                    f"image_{image_type}_small_url"
+                ] = f"{base_url}{code}/{display_id}.{rev_id}.200.jpg"
                 product[
-                    "image_{}_thumb_url".format(image_type)
-                ] = "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                    code,
-                    display_id,
-                    product["images"][display_id].get("rev"),
-                    100,
-                )
+                    f"image_{image_type}_thumb_url"
+                ] = f"{base_url}{code}/{display_id}.{rev_id}.100.jpg"
 
                 if image_type == "front":
-                    product["image_url"] = product[
-                        "image_{}_url".format(
-                            image_type,
-                        )
-                    ]
+                    product["image_url"] = product[f"image_{image_type}_url"]
                     product["image_small_url"] = product[
-                        "image_{}_small_url".format(
-                            image_type,
-                        )
+                        f"image_{image_type}_small_url"
                     ]
                     product["image_thumb_url"] = product[
-                        "image_{}_thumb_url".format(
-                            image_type,
-                        )
+                        f"image_{image_type}_thumb_url"
                     ]
 
         if product.get("languages_codes"):
             for language_code in product["languages_codes"]:
-                image_id = "{}_{}".format(image_type, language_code)
+                image_id = f"{image_type}_{language_code}"
                 if (
                     product.get("images")
                     and product["images"].get(image_id)
@@ -110,28 +69,13 @@ def add_images_urls_to_product(product):
                         {
                             image_type: {
                                 "display": {
-                                    language_code: "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                                        code,
-                                        image_id,
-                                        product["images"][image_id].get("rev"),
-                                        400,
-                                    ),
+                                    language_code: f"{base_url}{code}/{image_id}.{rev_id}.400.jpg"
                                 },
                                 "small": {
-                                    language_code: "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                                        code,
-                                        image_id,
-                                        product["images"][image_id].get("rev"),
-                                        200,
-                                    ),
+                                    language_code: f"{base_url}{code}/{image_id}.{rev_id}.200.jpg"
                                 },
                                 "thumb": {
-                                    language_code: "https://images.openfoodfacts.org/images/products/{}/{}.{}.{}.jpg".format(
-                                        code,
-                                        image_id,
-                                        product["images"][image_id].get("rev"),
-                                        100,
-                                    ),
+                                    language_code: f"{base_url}{code}/{image_id}.{rev_id}.100.jpg"
                                 },
                             },
                         }