fix(import) fix synthese progress bar and include one in occhab import (

#3349)
PnX-SI · Feb 3, 2025 · 30195be · 30195be
1 parent 9c792e4
commit 30195be
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 18 deletions.
diff --git a/backend/geonature/core/gn_synthese/imports/actions.py b/backend/geonature/core/gn_synthese/imports/actions.py
@@ -102,7 +102,7 @@ def update_batch_progress(batch, step):
             start = 0.1
             end = 0.4
             step_count = 8
-            progress = start + ((batch + 1) / batch_count) * (step / step_count) * (end - start)
+            progress = start + ((batch + step / step_count) / batch_count) * (end - start)
             task.update_state(state="PROGRESS", meta={"progress": progress})
 
         source_cols = [
@@ -367,11 +367,24 @@ def import_data_to_destination(imprt: TImports) -> None:
             "id_import",
             "last_action",
         ]
+        batch_size = current_app.config["IMPORT"]["INSERT_BATCH_SIZE"]
+        batch_count = ceil(imprt.source_count / batch_size)
+        for batch in range(batch_count):
+            min_line_no = batch * batch_size
+            max_line_no = (batch + 1) * batch_size
+            insert_stmt = sa.insert(Synthese).from_select(
+                names=names,
+                select=select_stmt.filter(
+                    transient_table.c["line_no"] >= min_line_no,
+                    transient_table.c["line_no"] < max_line_no,
+                ),
+            )
+            db.session.execute(insert_stmt)
+            yield (batch + 1) / batch_count
         insert_stmt = sa.insert(Synthese).from_select(
             names=names,
             select=select_stmt,
         )
-        db.session.execute(insert_stmt)
 
         # TODO: Improve this
         imprt.statistics = {

diff --git a/backend/geonature/core/imports/config_schema.py b/backend/geonature/core/imports/config_schema.py
@@ -122,3 +122,4 @@ class ImportConfigSchema(Schema):
     EXPORT_REPORT_PDF_FILENAME = fields.String(
         load_default="import_{id_import}_{date_create_import}_report.pdf"
     )
+    INSERT_BATCH_SIZE = fields.Integer(load_default=1000)
diff --git a/backend/geonature/core/imports/tasks.py b/backend/geonature/core/imports/tasks.py
@@ -78,7 +78,8 @@ def do_import_in_destination(self, import_id):
     transient_table = imprt.destination.get_transient_table()
 
     # Copy valid transient data to destination
-    imprt.destination.actions.import_data_to_destination(imprt)
+    for progress in imprt.destination.actions.import_data_to_destination(imprt):
+        self.update_state(state="PROGRESS", meta={"progress": progress})
 
     count_entities = 0
     entities = db.session.scalars(

diff --git a/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py b/contrib/gn_module_occhab/backend/gn_module_occhab/imports/actions.py
@@ -1,3 +1,4 @@
+from math import ceil
 from flask import current_app
 from werkzeug.exceptions import Conflict
 
@@ -516,12 +517,22 @@ def import_data_to_destination(imprt: TImports) -> None:
                 .where(transient_table.c.id_station.is_not(None))
             )
             destination_table = entity.get_destination_table()
-            insert_stmt = sa.insert(destination_table).from_select(
-                names=names,
-                select=select_stmt,
-            )
-            r = db.session.execute(insert_stmt)
-            imprt.statistics.update({f"{entity.code}_count": r.rowcount})
+            batch_size = current_app.config["IMPORT"]["INSERT_BATCH_SIZE"]
+            batch_count = ceil(imprt.source_count / batch_size)
+            row_count = 0
+            for batch in range(batch_count):
+                min_line_no = batch * batch_size
+                max_line_no = (batch + 1) * batch_size
+                insert_stmt = sa.insert(destination_table).from_select(
+                    names=names,
+                    select=select_stmt.filter(
+                        transient_table.c["line_no"] >= min_line_no,
+                        transient_table.c["line_no"] < max_line_no,
+                    ),
+                )
+                row_count += db.session.execute(insert_stmt).rowcount
+                yield (batch + 1) / batch_count
+            imprt.statistics.update({f"{entity.code}_count": row_count})
 
     @staticmethod
     def remove_data_from_destination(imprt: TImports) -> None:

diff --git a/.../src/app/modules/imports/components/import_process/import-step/import-step.component.html b/.../src/app/modules/imports/components/import_process/import-step/import-step.component.html
@@ -26,10 +26,7 @@ <h4 class="card-header">
       <div class="d-flex justify-content-center align-items-center">
         <div class="text-controls d-flex align-items-center flex-column pb-2">
           <p>Contrôles en cours sur les données</p>
-          <mat-progress-bar
-            class="progress-bar"
-            [value]="progress"
-          ></mat-progress-bar>
+          <mat-progress-bar [value]="progress"></mat-progress-bar>
         </div>
       </div>
     </div>
@@ -142,11 +139,7 @@ <h5 class="card-title mb-0">
         class="d-flex flex-column justify-content-center align-items-center w-100 mb-3"
       >
         <p>Import des données en cours</p>
-        <mat-spinner
-          [color]="color"
-          [diameter]="50"
-          class="upload-spinner"
-        ></mat-spinner>
+        <mat-progress-bar [value]="progress"></mat-progress-bar>
       </div>
 
       <div