feat: switch between two gdal presets "lzw" or "webp"

scripts/gdal/gdal_preset.py

@@ -0,0 +1,78 @@
+from typing import List
+
+from linz_logger import get_log
+
+GDAL_PRESET_LZW = [
+    "gdal_translate",
+    "-q",
+    "-scale",
+    "0",
+    "255",
+    "0",
+    "254",
+    "-a_srs",
+    "EPSG:2193",
+    "-a_nodata",
+    "255",
+    "-b",
+    "1",
+    "-b",
+    "2",
+    "-b",
+    "3",
+    "-of",
+    "COG",
+    "-co",
+    "compress=lzw",
+    "-co",
+    "num_threads=all_cpus",
+    "-co",
+    "predictor=2",
+    "-co",
+    "overview_compress=webp",
+    "-co",
+    "bigtiff=yes",
+    "-co",
+    "overview_resampling=lanczos",
+    "-co",
+    "blocksize=512",
+    "-co",
+    "overview_quality=90",
+    "-co",
+    "sparse_ok=true",
+]
+
+GDAL_PRESET_WEBP = [
+    "gdal_translate",
+    "-a_srs",
+    "EPSG:2193",
+    "-of",
+    "COG",
+    "-co",
+    "compress=webp",
+    "-co",
+    "num_threads=all_cpus",
+    "-co",
+    "quality=100",
+    "-co",
+    "overview_compress=webp",
+    "-co",
+    "bigtiff=yes",
+    "-co",
+    "overview_resampling=lanczos",
+    "-co",
+    "blocksize=512",
+    "-co",
+    "overview_quality=90",
+    "-co",
+    "sparse_ok=true",
+]
+
+
+def get_gdal_command(preset: str) -> List[str]:
+    get_log().info("gdal_preset", preset=preset)
+    if preset == "lzw":
+        return GDAL_PRESET_LZW
+    if preset == "webp":
+        return GDAL_PRESET_WEBP
+    raise Exception(f"Unknown GDAL preset: {preset}")

scripts/standardise_validate.py

@@ -10,7 +10,7 @@ def main() -> None:
     source = parse_source()
     if is_argo():
         concurrency = 4
-    standardised_files = start_standardising(source, concurrency)
+    standardised_files = start_standardising(source, "lzw", concurrency)
     if standardised_files:
         non_visual_qa(standardised_files)
     else:

scripts/standardising.py

@@ -1,17 +1,20 @@
+import argparse
 import os
+from functools import partial
 from multiprocessing import Pool
 from typing import List
 
 from linz_logger import get_log
 
 from scripts.aws.aws_helper import parse_path
-from scripts.cli.cli_helper import is_argo, parse_source
+from scripts.cli.cli_helper import format_source, is_argo
 from scripts.files.files_helper import get_file_name_from_path, is_tiff
 from scripts.gdal.gdal_helper import run_gdal
+from scripts.gdal.gdal_preset import get_gdal_command
 from scripts.logging.time_helper import time_in_ms
 
 
-def start_standardising(files: List[str], concurrency: int) -> List[str]:
+def start_standardising(files: List[str], preset: str, concurrency: int) -> List[str]:
     start_time = time_in_ms()
     tiff_files = []
     output_files = []
@@ -25,7 +28,7 @@ def start_standardising(files: List[str], concurrency: int) -> List[str]:
             get_log().info("standardising_file_not_tiff_skipped", file=file)
 
     with Pool(concurrency) as p:
-        output_files = p.map(standardising, tiff_files)
+        output_files = p.map(partial(standardising, preset=preset), tiff_files)
         p.close()
         p.join()
 
@@ -34,7 +37,7 @@ def start_standardising(files: List[str], concurrency: int) -> List[str]:
     return output_files
 
 
-def standardising(file: str) -> str:
+def standardising(file: str, preset: str) -> str:
     output_folder = "/tmp/"
 
     get_log().info("standardising_start", source=file)
@@ -43,56 +46,25 @@ def standardising(file: str) -> str:
     standardized_file_name = f"{get_file_name_from_path(src_file_path)}.tiff"
     tmp_file_path = os.path.join(output_folder, standardized_file_name)
 
-    command = [
-        "gdal_translate",
-        "-q",
-        "-scale",
-        "0",
-        "255",
-        "0",
-        "254",
-        "-a_srs",
-        "EPSG:2193",
-        "-a_nodata",
-        "255",
-        "-b",
-        "1",
-        "-b",
-        "2",
-        "-b",
-        "3",
-        "-of",
-        "COG",
-        "-co",
-        "compress=lzw",
-        "-co",
-        "num_threads=all_cpus",
-        "-co",
-        "predictor=2",
-        "-co",
-        "overview_compress=webp",
-        "-co",
-        "bigtiff=yes",
-        "-co",
-        "overview_resampling=lanczos",
-        "-co",
-        "blocksize=512",
-        "-co",
-        "overview_quality=90",
-        "-co",
-        "sparse_ok=true",
-    ]
+    command = get_gdal_command(preset)
     run_gdal(command, input_file=file, output_file=tmp_file_path)
 
     return tmp_file_path
 
 
 def main() -> None:
     concurrency: int = 1
-    source = parse_source()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--preset", dest="preset", required=False, default="lzw")
+    parser.add_argument("--source", dest="source", nargs="+", required=True)
+    arguments = parser.parse_args()
+
+    source = format_source(arguments.source)
+
     if is_argo():
         concurrency = 4
-    start_standardising(source, concurrency)
+
+    start_standardising(source, arguments.preset, concurrency)
 
 
 if __name__ == "__main__":