Merge pull request #8 from NASA-IMPACT/fix/testing

Update testing environment.  And include metadata validation.

Dockerfile

@@ -0,0 +1,17 @@
+FROM osgeo/gdal:ubuntu-full-3.0.3
+
+# Required for click with Python 3.6
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update
+RUN apt-get install python3-pip python3-venv git -y
+
+RUN pip3 install tox tox-venv
+RUN pip3 install --upgrade setuptools
+COPY ./ ./hls-browse_imagery_creator
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["cd hls-browse_imagery_creator && tox -r"]
+
+

README.md

@@ -1,25 +1,26 @@
 # hls-browse_imagery
 ## Create and merge GIBS browse imagery for HLS products.
 
-Requirements - Requires a system installation of [gdal](https://github.com/OSGeo/gdal)
+Requirements - Requires a system installation of [gdal](https://github.com/OSGeo/gdal) with Python bindings.
 
-Installation
+### Installation
 ```bash
-pip install .
+$ pip install .
 ```
 
-Example Usage
+### Example Usage
 ```bash
-granule_to_gibs inputdir outputdir HLS.S30.T01LAH.2020097T222759.v1.5
+$ granule_to_gibs inputdir outputdir HLS.S30.T01LAH.2020097T222759.v1.5
 ```
 ```bash
-create_gibs_tile inputdir HLS.S30.T01LAH.2020097.v1.5.tiff T01LAH
+$ create_gibs_tile inputdir HLS.S30.2020097.320071.v1.5 320071 
 ```
+The create_gibs_tile command returns the gibs tile name with the count of sub tiles appended to the file name.
 ```bash
-generate_metadata inputdir HLS.S30.T01LAH.2020097.v1.5.xml T01LAH HLS.S30.T01LAH.2020097T222759.v1.5 2020097
+$ generate_gibs_metadata inputdir HLS.S30.2020097.320071.v1.5.xml HLS.S30.2020097.320071.v1.5.tiff  2020097
 ```
 
-Run Tests on Python 3.7 
+### Run tests in container
 ```bash
-tox
+docker build -t hls-browse_imagery . && docker run hls-browse_imagery
 ```

hls_browse_imagery_creator/generate_metadata.py → hls_browse_imagery_creator/create_gibs_metadata.py

@@ -4,6 +4,9 @@
 import glob
 import click
 from osgeo import gdal
+from lxml import etree
+from pkg_resources import resource_stream
+from io import BytesIO
 
 
 @click.command()
@@ -12,7 +15,7 @@
 @click.argument("gibsid", type=click.STRING)
 @click.argument("mergefilename", type=click.STRING)
 @click.argument("day", type=click.STRING)
-def generate_metadata(inputdir, outputfile, gibsid, mergefilename, day):
+def create_gibs_metadata(inputdir, outputfile, gibsid, mergefilename, day):
     pattern = "*_" + gibsid + ".tif"
     globpath = os.path.join(inputdir, pattern)
     files = glob.glob(globpath)
@@ -25,17 +28,21 @@ def generate_metadata(inputdir, outputfile, gibsid, mergefilename, day):
         tiff = None
     metadata = {}
     metadata["ProviderProductId"] = mergefilename
-    metadata["PartialId"] = gibsid
-    metadata["DataStartDateTime"] = min(start_dates)
-    metadata["DataEndDateTime"] = max(end_dates)
     metadata["ProductionDateTime"] = datetime.datetime.utcnow().strftime(
         "%Y-%m-%dT%H:%M:%S.%fZ"
     )
+    metadata["DataStartDateTime"] = min(start_dates)
+    metadata["DataEndDateTime"] = max(end_dates)
     metadata["DataDay"] = day
-
+    metadata["PartialId"] = gibsid
+    schema_file = resource_stream("hls_browse_imagery_creator",
+                                  "data/schema/ImageMetadata_v1.2.xsd")
+    xml = dicttoxml.dicttoxml(
+        {"ImageryMetadata": metadata}, root=False, attr_type=False
+    )
+    xmlschema_doc = etree.parse(schema_file)
+    xmlschema = etree.XMLSchema(xmlschema_doc)
+    doc = etree.parse(BytesIO(xml))
+    xmlschema.assertValid(doc)
     with open(outputfile, "wb") as meta_file:
-        meta_file.write(
-            dicttoxml.dicttoxml(
-                {"ImageryMetadata": metadata}, root=False, attr_type=False
-            )
-        )
+        meta_file.write(xml)

hls_browse_imagery_creator/create_gibs_tile.py

@@ -6,15 +6,17 @@
 
 @click.command()
 @click.argument("inputdir", type=click.Path(exists=True,))
-@click.argument("outputfile", type=click.Path(writable=True,))
+@click.argument("outputbasename", type=click.STRING)
 @click.argument("gibsid", type=click.STRING)
-def create_gibs_tile(inputdir, outputfile, gibsid):
+def create_gibs_tile(inputdir, outputbasename, gibsid):
     pattern = "*_" + gibsid + ".tif"
     globpath = os.path.join(inputdir, pattern)
     files = glob.glob(globpath)
+    outputfile = outputbasename + "_" + str(len(files)) + ".tif"
     vrt_options = gdal.BuildVRTOptions(resampleAlg="cubic",)
     vrt = gdal.BuildVRT("", files, options=vrt_options)
     driver = gdal.GetDriverByName("GTiff")
     output = driver.CreateCopy(outputfile, vrt, options=["TILED=YES", "COMPRESS=LZW"])
     vrt = None  # Close out the vrt
     output = None  # Needed to flush the tiff to disk
+    click.echo(outputfile, nl=False)

hls_browse_imagery_creator/data/schema/ImageMetadata_v1.2.xsd

@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+  <xs:element name="ImageryMetadata">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element type="xs:string" name="ProviderProductId"/>
+        <xs:element type="xs:dateTime" name="ProductionDateTime"/>
+        <xs:element type="xs:dateTime" name="DataStartDateTime"/>
+        <xs:element type="xs:dateTime" name="DataEndDateTime"/>
+
+        <xs:choice>
+          <xs:element name="DataDay">
+           <xs:simpleType>
+              <xs:restriction base="xs:string">
+                <xs:pattern value="\d{4}\d{3}" />
+              </xs:restriction>
+            </xs:simpleType>
+          </xs:element>
+          <xs:element type="xs:dateTime" name="DataDateTime"/>
+        </xs:choice>
+
+        <xs:element type="xs:string" name="PartialId" minOccurs="0"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+</xs:schema>

hls_browse_imagery_creator/utils/create_GIBS_grid.py

@@ -1,4 +1,3 @@
-import numpy
 import json
 
 from collections import OrderedDict

setup.py

@@ -5,14 +5,21 @@
     version="0.1",
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,
-    install_requires=["numpy", "gdal", "xmltodict", "click", "dicttoxml"],
-    extras_require={"testing": ["pytest"]},
-    package_data={"hls_browse_imagery_creator": ["data/*.json"]},
-    setup_requires=["flake8"],
+    install_requires=[
+        "numpy",
+        "gdal",
+        "xmltodict",
+        "click",
+        "dicttoxml",
+        "lxml"
+    ],
+    extras_require={"test": ["pytest", "flake8"]},
+    package_data={"hls_browse_imagery_creator": ["data/*.json",
+                                                 "data/schema/*.xml"]},
     entry_points="""
         [console_scripts]
         granule_to_gibs=hls_browse_imagery_creator.granule_to_gibs:granule_to_gibs
         create_gibs_tile=hls_browse_imagery_creator.create_gibs_tile:create_gibs_tile
-        generate_metadata=hls_browse_imagery_creator.generate_metadata:generate_metadata
+        create_gibs_metadata=hls_browse_imagery_creator.create_gibs_metadata:create_gibs_metadata
     """,
 )

tests/test_hls_browse_imagery.py

@@ -0,0 +1,46 @@
+import os
+from click.testing import CliRunner
+from hls_browse_imagery_creator.granule_to_gibs import granule_to_gibs
+from hls_browse_imagery_creator.create_gibs_tile import create_gibs_tile
+from hls_browse_imagery_creator.create_gibs_metadata import create_gibs_metadata
+
+
+current_dir = os.path.dirname(__file__)
+data_dir = os.path.join(current_dir, "data")
+output_dir = os.path.join(current_dir, "output")
+merge_dir = os.path.join(current_dir, "merge")
+os.mkdir(output_dir)
+os.mkdir(merge_dir)
+basename = "HLS.S30.T01LAH.2020097T222759.v1.5"
+gibsid = "320071"
+gibstilebasename = os.path.join(merge_dir, "HLS.S30.2020097.320071.v1.5")
+gibstile = os.path.join(merge_dir, "HLS.S30.2020097.320071.v1.5_1.tif")
+
+
+def test_granule_to_gibs():
+    runner = CliRunner()
+    result = runner.invoke(granule_to_gibs, [data_dir, output_dir, basename])
+    print(result.exception)
+    assert result.exit_code == 0
+
+
+def test_create_gibs_tile():
+    runner = CliRunner()
+    result = runner.invoke(create_gibs_tile, [output_dir, gibstilebasename, gibsid])
+    print(result.exception)
+    assert result.exit_code == 0
+    assert result.stdout == gibstile
+
+
+def test_create_gibs_metadata():
+    gibsmetadata = os.path.join(data_dir, "HLS.S30.2020097.320071.v1.5.xml")
+    runner = CliRunner()
+    result = runner.invoke(create_gibs_metadata, [
+        output_dir,
+        gibsmetadata,
+        gibsid,
+        gibstile,
+        "2020097"
+    ])
+    print(result.exception)
+    assert result.exit_code == 0

tox.ini

@@ -1,11 +1,24 @@
-[tox]
-envlist = py37
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
 
-[testenv]
-extras = testing
-commands= pytest
+[tox]
+envlist = py36
 
-# Lint
 [flake8]
-exclude = .git,__pycache__,docs/source/conf.py,old,build,dist
+ignore = D203
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,site-packages,
+  venv,bin,.eggs,./hls_browse_imagery_creator/create_gibs_tile.py
+max-complexity = 12
 max-line-length = 90
+
+[testenv]
+basepython = python3.6
+envdir = venv
+sitepackages = True
+extras =
+  test
+commands =
+    flake8
+    pytest