Add region parameters for taxref import migrate (#521)

* Add taxref_region parameters for taxref import migrate
PnX-SI · Aug 8, 2024 · 2db7e5f · 2db7e5f
1 parent e03b724
commit 2db7e5f
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 30 deletions.
diff --git a/apptax/taxonomie/commands/migrate_taxref/commands_v17.py b/apptax/taxonomie/commands/migrate_taxref/commands_v17.py
@@ -73,11 +73,17 @@ def test_changes_detection(keep_cdnom):
 @click.option("--keep-oldtaxref", is_flag=True)
 @click.option("--keep-oldbdc", is_flag=True)
 @click.option("--keep-cdnom", is_flag=True)
+@click.option("--taxref-region", type=str)
 @click.option("--script_predetection", type=click.Path(exists=True))
 @click.option("--script_postdetection", type=click.Path(exists=True))
 @with_appcontext
 def apply_changes(
-    keep_oldtaxref, keep_oldbdc, keep_cdnom, script_predetection, script_postdetection
+    keep_oldtaxref,
+    keep_oldbdc,
+    keep_cdnom,
+    taxref_region,
+    script_predetection,
+    script_postdetection,
 ):
     """Procédure de migration de taxref vers la version 17
          Application des changements import des données dans les tables taxref et bdc_status
@@ -114,7 +120,7 @@ def apply_changes(
                 "3.2_alter_taxref_data.sql",
             )
         )
-        db.session.execute(query, {"keep_cd_nom": keep_cdnom})
+        db.session.execute(query, {"keep_cd_nom": keep_cdnom, "taxref_region": taxref_region})
         db.session.commit()
         logger.info("it's done")
     except Exception as e:

diff --git a/.../taxonomie/commands/migrate_taxref/data/specific_taxref_v15_v16/3.2_alter_taxref_data.sql b/.../taxonomie/commands/migrate_taxref/data/specific_taxref_v15_v16/3.2_alter_taxref_data.sql
@@ -20,12 +20,9 @@ ALTER TABLE taxonomie.cor_nom_liste DROP CONSTRAINT cor_nom_listes_taxref_fkey;
 ------------------------------------------------
 ------------------------------------------------
 
--- CORRECTION
-UPDATE taxonomie.import_taxref SET fr = NULL WHERE fr='';
-
 -- UPDATE EXISTING CD_NOM
 UPDATE taxonomie.taxref t
-   SET id_statut = fr, id_habitat = it.habitat::int, id_rang = it.rang, regne = it.regne, phylum = it.phylum,
+   SET id_habitat = it.habitat::int, id_rang = it.rang, regne = it.regne, phylum = it.phylum,
        classe = it.classe, ordre = it.ordre, famille = it.famille, cd_taxsup = it.cd_taxsup,
        cd_sup = it.cd_sup, cd_ref = it.cd_ref,
        lb_nom = it.lb_nom, lb_auteur = it.lb_auteur, nom_complet = it.nom_complet,
@@ -38,11 +35,11 @@ WHERE it.cd_nom  = t.cd_nom;
 
 -- ADD NEW CD_NOM
 INSERT INTO taxonomie.taxref(
-            cd_nom, id_statut, id_habitat, id_rang, regne, phylum, classe,
+            cd_nom, id_habitat, id_rang, regne, phylum, classe,
             ordre, famille, cd_taxsup, cd_sup, cd_ref, lb_nom, lb_auteur,
             nom_complet, nom_complet_html, nom_valide, nom_vern, nom_vern_eng,
             group1_inpn, group2_inpn, sous_famille, tribu, url, group3_inpn)
-SELECT it.cd_nom, it.fr, it.habitat::int, it.rang, it.regne, it.phylum, it.classe,
+SELECT it.cd_nom,it.habitat::int, it.rang, it.regne, it.phylum, it.classe,
     it.ordre, it.famille, it.cd_taxsup, it.cd_sup, it.cd_ref, it.lb_nom, it.lb_auteur,
     it.nom_complet, it.nom_complet_html, it.nom_valide, it.nom_vern, it.nom_vern_eng,
     it.group1_inpn, it.group2_inpn, it.sous_famille, it.tribu, it.url, it.group3_inpn
@@ -51,13 +48,36 @@ LEFT OUTER JOIN taxonomie.taxref t
 ON it.cd_nom = t.cd_nom
 WHERE t.cd_nom IS NULL;
 
+-- Regional Status
+
+DO $$ BEGIN   
+   IF :taxref_region = 'gf' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.gf, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'mar' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.mar, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'gua' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.gua, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'sm' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.sm, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'sb' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.sb, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'spm' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.spm, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'may' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.may, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'epa' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.epa, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'reu' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.reu, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'sa' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.sa, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'ta' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.ta, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'taaf' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.taaf, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'pf' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.pf, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'nc' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.nc, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'wf' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.wf, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSIF :taxref_region = 'cli' THEN UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.cli, '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+   ELSE  UPDATE taxonomie.taxref t SET id_statut = NULLIF(it.fr,  '') FROM taxonomie.import_taxref it WHERE it.cd_nom  = t.cd_nom;
+END IF;
+END $$;
+
 -- DELETE MISSING CD_NOM if not keep_cdnom is specify
 DO $$ BEGIN
     IF  :keep_cd_nom = FALSE THEN
         DELETE FROM taxonomie.taxref
         WHERE cd_nom IN (
           SELECT cd_nom
-          FROM taxonomie.cdnom_disparu
+         FROM taxonomie.cdnom_disparu
         );
 
     END IF;

diff --git a/apptax/taxonomie/commands/taxref.py b/apptax/taxonomie/commands/taxref.py
@@ -57,11 +57,13 @@ def delete():
         "taxonomie.bdc_statut_cor_text_values",
         "taxonomie.bdc_statut_text",
         "taxonomie.bdc_statut_type",
-        "taxonomie.bib_noms",
+        "taxonomie.cor_nom_liste",
+        "taxonomie.cor_taxon_attribut",
         "taxonomie.taxref",
         "taxonomie.bib_taxref_statuts",
         "taxonomie.bib_taxref_rangs",
         "taxonomie.bib_taxref_habitats",
+        "taxonomie.t_meta_taxref",
     ]
     with click.progressbar(
         length=len(tables), label="Delete from table", item_show_func=lambda t: t, show_eta=False

diff --git a/apptax/taxonomie/commands/taxref_v15_v16.py b/apptax/taxonomie/commands/taxref_v15_v16.py
@@ -55,7 +55,7 @@ def import_bdc_statuts_v17(logger):
     )
 
 
-def import_taxref(logger, num_version, taxref_archive_name, taxref_file_name):
+def import_taxref(logger, num_version, taxref_archive_name, taxref_file_name, taxref_region="fr"):
     with open_remote_file(base_url, taxref_archive_name, open_fct=ZipFile) as archive:
         with archive.open("habitats_note.csv") as f:
             logger.info(f"Insert TAXREF v{num_version} habitats…")
@@ -114,7 +114,7 @@ def import_taxref(logger, num_version, taxref_archive_name, taxref_file_name):
                 ),
                 source_cols=(
                     "cd_nom::int",
-                    "NULLIF(fr, '') as id_statut",
+                    f"NULLIF({taxref_region}, '') as id_statut",
                     "habitat::int as id_habitat",
                     "rang as id_rang",
                     "regne",
@@ -146,15 +146,17 @@ def import_taxref(logger, num_version, taxref_archive_name, taxref_file_name):
 
 @click.command()
 @click.option("--skip-bdc-statuts", is_flag=True, help="Skip import of BDC Statuts")
+@click.option("--taxref-region", type=str, default="fr", help="Taxref region : column status")
 @with_appcontext
-def import_v15(skip_bdc_statuts):
+def import_v15(skip_bdc_statuts, taxref_region):
     logger = logging.getLogger()
 
     import_taxref(
         logger,
         num_version="15",
         taxref_archive_name="TAXREF_v15_2021.zip",
         taxref_file_name="TAXREFv15.txt",
+        taxref_region=taxref_region,
     )
 
     if not skip_bdc_statuts:
@@ -171,15 +173,17 @@ def import_v15(skip_bdc_statuts):
 
 @click.command()
 @click.option("--skip-bdc-statuts", is_flag=True, help="Skip import of BDC Statuts")
+@click.option("--taxref-region", type=str, default="fr", help="Taxref region : column status")
 @with_appcontext
-def import_v16(skip_bdc_statuts):
+def import_v16(skip_bdc_statuts, taxref_region):
     logger = logging.getLogger()
 
     import_taxref(
         logger,
         num_version="16",
         taxref_archive_name="TAXREF_v16_2022.zip",
         taxref_file_name="TAXREFv16.txt",
+        taxref_region=taxref_region,
     )
 
     if not skip_bdc_statuts:
@@ -196,15 +200,17 @@ def import_v16(skip_bdc_statuts):
 
 @click.command()
 @click.option("--skip-bdc-statuts", is_flag=True, help="Skip import of BDC Statuts")
+@click.option("--taxref-region", type=str, default="fr", help="Taxref region : column status")
 @with_appcontext
-def import_v17(skip_bdc_statuts):
+def import_v17(skip_bdc_statuts, taxref_region):
     logger = logging.getLogger()
 
     import_taxref(
         logger,
         num_version="17",
         taxref_archive_name="TAXREF_v17_2024.zip",
         taxref_file_name="TAXREFv17.txt",
+        taxref_region=taxref_region,
     )
 
     if not skip_bdc_statuts:

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -1,5 +1,7 @@
 2.0.0 (unrelease)
 ===================
+**🚀 Nouveautés**
+ - ajout d'un paramètre `taxref_region` qui permet de spécifier le nom de la colonne à utiliser pour peupler la colonne `id_statut` de `taxref`
 
 **⚠️ Notes de version**
 - Les données de la table "bib_nom" on été sauvegardées dans une liste nommée "Save bib_nom".  Le champs "nom_français" ainsi que "commentaire" de cette table ne sont pas conservés dans la version 2.0.0 (ils n'étaient plus utilisés dans les recherche de taxons depuis plusieurs versions).
@@ -10,6 +12,7 @@
 - La configuration est maintenant gérée dans le fichier `config/taxhub_config.toml` (#517)
 - L'image Docker ne sera plus générée pour les raisons suivantes : l'intégration de l'interface de TaxHub dans celle de GeoNature, l'image ne fonctionne pas en `standalone` (#519)
 
+
 1.14.1 (2024-05-23)
 ===================
 **🚀 Nouveautés**

diff --git a/install_db.sh b/install_db.sh
@@ -17,7 +17,7 @@ nano settings.ini
 
 #include user config = settings.ini
 . settings.ini
-
+ 
 #get app path
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 LOG_FILE=$DIR/install_db.log
@@ -103,7 +103,7 @@ then
     flask db upgrade ref_geo_fr_departments@head
     flask db autoupgrade
 
-    flask taxref import-v17
+    flask taxref import-v17 --taxref-region=${taxref_region:-fr}
 
     if $insert_geonatureatlas_data
     then

diff --git a/settings.ini.sample b/settings.ini.sample
@@ -7,6 +7,9 @@ app_name=taxhub
 # set to dev for development installation
 mode=prod
 
+
+# Nom de la colonne région à importer dans la colonne id_statut de taxref
+taxref_region=fr
 #######################
 ### Python settings ###
 #######################
@@ -36,19 +39,6 @@ user_pg=geonatuser
 # Database owner password
 user_pg_pass=monpassachanger
 
-### Users schema localisation
-# Possible values : 'local' or 'foreign'
-# 'local' means that the 'utilisateurs' schema has to be installed in TaxHub database.
-# 'foreign' means that the 'utilisateurs' schema is in another database. An FDW schema will be created on it.
-# Nota : foreign requires UsersHub to be already installed
-users_schema=local
-
-### UsersHub relation settings - Required if users_schema=foreign
-usershub_host=localhost
-usershub_db=referentielsdb
-usershub_port=5432
-usershub_user=usershubuser
-usershub_pass=monpassachanger
 
 ###############
 ### Options ###