feat: standardise following by non visual QA check TDE-450 (#90)

* feat: run standardising following by non_visual_qa

* fix: typing

* chore: remove unused function

* chore: remove unused and duplicated code

scripts/non_visual_qa.py

@@ -122,11 +122,10 @@ def run(self) -> None:
                 self.add_error(error_type="srs", error_message=f"not checked: {str(gee)}")
 
 
-def main() -> None:
+def non_visual_qa(files: List[str]) -> None:
     start_time = time_in_ms()
-    source = parse_source()
 
-    get_log().info("non_visual_qa_start", source=source)
+    get_log().info("non_visual_qa_start", source=files)
 
     # Get srs
     gdalsrsinfo_command = ["gdalsrsinfo", "-o", "wkt", "EPSG:2193"]
@@ -137,7 +136,7 @@ def main() -> None:
         )
     srs = gdalsrsinfo_result.stdout
 
-    for file in source:
+    for file in files:
         if not is_tiff(file):
             get_log().trace("non_visual_qa_file_not_tiff_skipped", file=file)
             continue
@@ -149,7 +148,12 @@ def main() -> None:
         else:
             get_log().info("non_visual_qa_passed", file=file_check.path)
 
-    get_log().info("non_visual_qa_end", source=source, duration=time_in_ms() - start_time)
+    get_log().info("non_visual_qa_end", source=files, duration=time_in_ms() - start_time)
+
+
+def main() -> None:
+    source = parse_source()
+    non_visual_qa(source)
 
 
 if __name__ == "__main__":

scripts/standardise_validate.py

@@ -0,0 +1,13 @@
+from scripts.cli.cli_helper import parse_source
+from scripts.non_visual_qa import non_visual_qa
+from scripts.standardising import standardising
+
+
+def main() -> None:
+    source = parse_source()
+    standardised_files = standardising(source)
+    non_visual_qa(standardised_files)
+
+
+if __name__ == "__main__":
+    main()

scripts/standardising.py

@@ -1,4 +1,5 @@
 import os
+from typing import List
 
 from linz_logger import get_log
 
@@ -8,62 +9,73 @@
 from scripts.gdal.gdal_helper import run_gdal
 from scripts.logging.time_helper import time_in_ms
 
-start_time = time_in_ms()
 
+def standardising(files: List[str]) -> List[str]:
+    start_time = time_in_ms()
+    output_folder = "/tmp/"
+    output_files = []
 
-source = parse_source()
+    get_log().info("standardising_start", source=files)
 
-get_log().info("standardising_start", source=source)
+    for file in files:
+        if not is_tiff(file):
+            get_log().info("standardising_file_not_tiff_skipped", file=file)
+            continue
 
-gdal_env = os.environ.copy()
+        _, src_file_path = parse_path(file)
+        standardized_file_name = f"standardized_{get_file_name_from_path(src_file_path)}"
+        tmp_file_path = os.path.join(output_folder, standardized_file_name)
+        command = [
+            "gdal_translate",
+            "-q",
+            "-scale",
+            "0",
+            "255",
+            "0",
+            "254",
+            "-a_srs",
+            "EPSG:2193",
+            "-a_nodata",
+            "255",
+            "-b",
+            "1",
+            "-b",
+            "2",
+            "-b",
+            "3",
+            "-of",
+            "COG",
+            "-co",
+            "compress=lzw",
+            "-co",
+            "num_threads=all_cpus",
+            "-co",
+            "predictor=2",
+            "-co",
+            "overview_compress=webp",
+            "-co",
+            "bigtiff=yes",
+            "-co",
+            "overview_resampling=lanczos",
+            "-co",
+            "blocksize=512",
+            "-co",
+            "overview_quality=90",
+            "-co",
+            "sparse_ok=true",
+        ]
+        run_gdal(command, input_file=file, output_file=tmp_file_path)
+        output_files.append(tmp_file_path)
 
-for file in source:
-    if not is_tiff(file):
-        get_log().trace("standardising_file_not_tiff_skipped", file=file)
-        continue
+    get_log().info("standardising_end", source=files, duration=time_in_ms() - start_time)
 
-    s3_path = parse_path(file)
-    standardized_file_name = f"standardized_{get_file_name_from_path(s3_path.key)}"
-    tmp_file_path = os.path.join("/tmp/", standardized_file_name)
-    command = [
-        "gdal_translate",
-        "-q",
-        "-scale",
-        "0",
-        "255",
-        "0",
-        "254",
-        "-a_srs",
-        "EPSG:2193",
-        "-a_nodata",
-        "255",
-        "-b",
-        "1",
-        "-b",
-        "2",
-        "-b",
-        "3",
-        "-of",
-        "COG",
-        "-co",
-        "compress=lzw",
-        "-co",
-        "num_threads=all_cpus",
-        "-co",
-        "predictor=2",
-        "-co",
-        "overview_compress=webp",
-        "-co",
-        "bigtiff=yes",
-        "-co",
-        "overview_resampling=lanczos",
-        "-co",
-        "blocksize=512",
-        "-co",
-        "overview_quality=90",
-        "-co",
-        "sparse_ok=true",
-    ]
-    run_gdal(command, input_file=file, output_file=tmp_file_path)
+    return output_files
 
-    get_log().info("standardising_end", source=source, duration=time_in_ms() - start_time)
+
+def main() -> None:
+    source = parse_source()
+    standardising(source)
+
+
+if __name__ == "__main__":
+    main()