Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/gn synthese query taxons filters #3087

Merged
merged 3 commits into from
Jun 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 23 additions & 11 deletions backend/geonature/core/gn_synthese/utils/query_select_sqla.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,11 @@ def filter_taxonomy(self):
self.query = self.query.where(getattr(Taxref, colname).in_(value))

if colname.startswith("taxhub_attribut"):
# Test si la valeur n'est pas une liste transformation
# de value en liste pour utiliser le filtre IN
if not type(value) is list:
value = [value]

self.add_join(Taxref, Taxref.cd_nom, self.model.cd_nom)
taxhub_id_attr = colname[16:]
aliased_cor_taxon_attr[taxhub_id_attr] = aliased(CorTaxonAttribut)
Expand Down Expand Up @@ -313,7 +318,7 @@ def filter_taxonomy(self):
protection_status_value += value

if protection_status_value or red_list_filters:
self.build_bdc_status_pr_nb_lateral_join(protection_status_value, red_list_filters)
self.build_bdc_status_filters(protection_status_value, red_list_filters)
# remove attributes taxhub from filters
self.filters = {
colname: value
Expand Down Expand Up @@ -531,17 +536,17 @@ def filter_query_all_filters(self, user, permissions):
self.apply_all_filters(user, permissions)
return self.build_query()

def build_bdc_status_pr_nb_lateral_join(self, protection_status_value, red_list_filters):
def build_bdc_status_filters(self, protection_status_value, red_list_filters):
"""
Create subquery for bdc_status filters

Objectif : filtrer les données ayant :
- les statuts du type demandé par l'utilisateur
- les status s'appliquent bien sur la zone géographique de la donnée (c-a-d le département)

Idée de façon à limiter le nombre de sous reqêtes,
Idée de façon à limiter le nombre de sous requêtes (le nombre de status demandé ne dégrade pas les performances),
la liste des status selectionnés par l'utilisateur s'appliquant à l'observation est
aggrégée de façon à tester le nombre puis jointé sur le département de la donnée
aggrégée de façon à tester le nombre puis jointer sur le département de la donnée
"""
# Ajout de la table taxref si non ajouté
self.add_join(Taxref, Taxref.cd_nom, self.model.cd_nom)
Expand All @@ -559,11 +564,13 @@ def build_bdc_status_pr_nb_lateral_join(self, protection_status_value, red_list_

# Creation requête CTE : taxon, zone d'application départementale des textes
# pour les taxons répondant aux critères de selection
bdc_status_cte = (
bdc_status_by_type_cte = (
select(
TaxrefBdcStatutTaxon.cd_ref,
func.array_agg(bdc_statut_cor_text_area.c.id_area).label("ids_area"),
bdc_statut_cor_text_area.c.id_area,
TaxrefBdcStatutText.cd_type_statut,
)
.distinct()
.select_from(
TaxrefBdcStatutTaxon.__table__.join(
TaxrefBdcStatutCorTextValues,
Expand Down Expand Up @@ -601,15 +608,20 @@ def build_bdc_status_pr_nb_lateral_join(self, protection_status_value, red_list_
TaxrefBdcStatutText.cd_type_statut.in_(protection_status_value)
)

bdc_status_cte = bdc_status_cte.where(or_(*bdc_status_filters))
bdc_status_by_type_cte = bdc_status_by_type_cte.where(or_(*bdc_status_filters))
bdc_status_by_type_cte = bdc_status_by_type_cte.cte(name="status_by_type")

# group by de façon à ne selectionner que les taxons
# qui ont les textes selectionnés par l'utilisateurs
bdc_status_cte = bdc_status_cte.group_by(TaxrefBdcStatutTaxon.cd_ref).having(
func.count(distinct(TaxrefBdcStatutText.cd_type_statut))
# qui ont l'ensemble des textes selectionnés par l'utilisateur
# c-a-d dont le nombre de cd_type_statut correspond au nombre demandé
bdc_status_cte = select(
bdc_status_by_type_cte.c.cd_ref,
func.array_agg(bdc_status_by_type_cte.c.id_area).label("ids_area"),
)
bdc_status_cte = bdc_status_cte.group_by(bdc_status_by_type_cte.c.cd_ref).having(
func.count(distinct(bdc_status_by_type_cte.c.cd_type_statut))
== (len(protection_status_value) + len(red_list_filters))
)

bdc_status_cte = bdc_status_cte.cte(name="status")

# Jointure sur le taxon
Expand Down
3 changes: 2 additions & 1 deletion backend/geonature/tests/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,8 @@ def synthese_data(app, users, datasets, source, sources_modules):
data = {}
with db.session.begin_nested():
for name, cd_nom, point, ds, comment_description, source_m in [
("obs1", 713776, point1, datasets["own_dataset"], "obs1", sources_modules[0]),
# Donnnées de gypaète : possède des statuts de protection nationale
("obs1", 2852, point1, datasets["own_dataset"], "obs1", sources_modules[0]),
("obs2", 212, point2, datasets["own_dataset"], "obs2", sources_modules[0]),
("obs3", 2497, point3, datasets["own_dataset"], "obs3", sources_modules[1]),
("p1_af1", 713776, point1, datasets["belong_af_1"], "p1_af1", sources_modules[1]),
Expand Down
6 changes: 5 additions & 1 deletion backend/geonature/tests/test_synthese.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,8 @@ def test_get_observations_for_web(self, app, users, synthese_data, taxon_attribu
# test status protection
filters = {"protections_protection_status": ["PN"]}
r = self.client.get(url, json=filters)
# doit au moins contenir une donnée de gypaète (protection nationale)
assert len(r.json["features"]) >= 1
assert r.status_code == 200
# test status protection and znieff
filters = {"protections_protection_status": ["PN"], "znief_protection_status": True}
Expand Down Expand Up @@ -1253,7 +1255,9 @@ def test_get_taxa_count_id_dataset(self, synthese_data, users, datasets, unexist
response = self.client.get(url_for(url), query_string={"id_dataset": id_dataset})
response_empty = self.client.get(url_for(url), query_string={"id_dataset": unexisted_id})

assert response.json == len(set(synt.cd_nom for synt in synthese_data.values()))
assert response.json == len(
set(synt.cd_nom for synt in synthese_data.values() if synt.id_dataset == id_dataset)
)
assert response_empty.json == 0

def test_get_observation_count(self, synthese_data, users):
Expand Down
8 changes: 7 additions & 1 deletion config/test_config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,10 @@ REF_LAYERS=[
]

[SYNTHESE]
AREA_AGGREGATION_TYPE = "M5"
AREA_AGGREGATION_TYPE = "M5"
STATUS_FILTERS = [
{ "id" = "protections", "show" = true, "display_name" = "Taxons protégés", "status_types" = ["PN", "PR", "PD"] },
{ "id" = "regulations", "show" = true, "display_name" = "Taxons réglementés", "status_types" = ["REGLII", "REGL", "REGLSO"] },
{ "id" = "invasive", "show" = true, "display_name" = "Espèces envahissantes", "status_types" = ["REGLLUTTE"] },
{ "id" = "znief", "show" = true, "display_name" = "Espèces déterminantes ZNIEFF", "status_types" = ["ZDET"] },
]
Loading