Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: standardise following by non visual QA check TDE-450 #90

Merged
merged 4 commits into from
Aug 17, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: run standardising following by non_visual_qa
paulfouquet committed Aug 15, 2022
commit 910655b6d6d1ffae08342117cada9e9ff45655ce
14 changes: 9 additions & 5 deletions scripts/non_visual_qa.py
Original file line number Diff line number Diff line change
@@ -122,11 +122,10 @@ def run(self) -> None:
self.add_error(error_type="srs", error_message=f"not checked: {str(gee)}")


def main() -> None:
def non_visual_qa(files: List[str]) -> None:
start_time = time_in_ms()
source = parse_source()

get_log().info("non_visual_qa_start", source=source)
get_log().info("non_visual_qa_start", source=files)

# Get srs
gdalsrsinfo_command = ["gdalsrsinfo", "-o", "wkt", "EPSG:2193"]
@@ -137,7 +136,7 @@ def main() -> None:
)
srs = gdalsrsinfo_result.stdout

for file in source:
for file in files:
if not is_tiff(file):
get_log().trace("non_visual_qa_file_not_tiff_skipped", file=file)
continue
@@ -149,7 +148,12 @@ def main() -> None:
else:
get_log().info("non_visual_qa_passed", file=file_check.path)

get_log().info("non_visual_qa_end", source=source, duration=time_in_ms() - start_time)
get_log().info("non_visual_qa_end", source=files, duration=time_in_ms() - start_time)


def main() -> None:
source = parse_source()
non_visual_qa(source)


if __name__ == "__main__":
19 changes: 19 additions & 0 deletions scripts/standardise_validate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from typing import List

from scripts.cli.cli_helper import parse_source
from scripts.non_visual_qa import non_visual_qa
from scripts.standardising import standardising


def standardise_validate(files: List[str]) -> None:
standardising(files)


def main() -> None:
source = parse_source()
standardised_files = standardising(source)
non_visual_qa(standardised_files)


if __name__ == "__main__":
main()
116 changes: 64 additions & 52 deletions scripts/standardising.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
from typing import List

from linz_logger import get_log

@@ -8,62 +9,73 @@
from scripts.gdal.gdal_helper import run_gdal
from scripts.logging.time_helper import time_in_ms

start_time = time_in_ms()

def standardising(files: List[str]) -> str:
start_time = time_in_ms()
output_folder = "/tmp/"
output_files = []

source = parse_source()
get_log().info("standardising_start", source=files)

get_log().info("standardising_start", source=source)
for file in files:
if not is_tiff(file):
get_log().info("standardising_file_not_tiff_skipped", file=file)
continue

gdal_env = os.environ.copy()
_, src_file_path = parse_path(file)
standardized_file_name = f"standardized_{get_file_name_from_path(src_file_path)}"
tmp_file_path = os.path.join(output_folder, standardized_file_name)
command = [
"gdal_translate",
"-q",
"-scale",
"0",
"255",
"0",
"254",
"-a_srs",
"EPSG:2193",
"-a_nodata",
"255",
"-b",
"1",
"-b",
"2",
"-b",
"3",
"-of",
"COG",
"-co",
"compress=lzw",
"-co",
"num_threads=all_cpus",
"-co",
"predictor=2",
"-co",
"overview_compress=webp",
"-co",
"bigtiff=yes",
"-co",
"overview_resampling=lanczos",
"-co",
"blocksize=512",
"-co",
"overview_quality=90",
"-co",
"sparse_ok=true",
]
run_gdal(command, input_file=file, output_file=tmp_file_path)
output_files.append(tmp_file_path)

for file in source:
if not is_tiff(file):
get_log().trace("standardising_file_not_tiff_skipped", file=file)
continue
get_log().info("standardising_end", source=files, duration=time_in_ms() - start_time)

s3_path = parse_path(file)
standardized_file_name = f"standardized_{get_file_name_from_path(s3_path.key)}"
tmp_file_path = os.path.join("/tmp/", standardized_file_name)
command = [
"gdal_translate",
"-q",
"-scale",
"0",
"255",
"0",
"254",
"-a_srs",
"EPSG:2193",
"-a_nodata",
"255",
"-b",
"1",
"-b",
"2",
"-b",
"3",
"-of",
"COG",
"-co",
"compress=lzw",
"-co",
"num_threads=all_cpus",
"-co",
"predictor=2",
"-co",
"overview_compress=webp",
"-co",
"bigtiff=yes",
"-co",
"overview_resampling=lanczos",
"-co",
"blocksize=512",
"-co",
"overview_quality=90",
"-co",
"sparse_ok=true",
]
run_gdal(command, input_file=file, output_file=tmp_file_path)
return output_files

get_log().info("standardising_end", source=source, duration=time_in_ms() - start_time)

def main() -> None:
source = parse_source()
standardising(source)


if __name__ == "__main__":
main()