Release and distribute workflows

.github/workflows/distribute.yml

@@ -0,0 +1,37 @@
+name: Distribute to PyPI
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  distribute:
+    runs-on: ubuntu-latest
+    outputs:
+      SDIST_VERSION: ${{ steps.build.outputs.version }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools build "setuptools-scm[toml]"
+
+      - name: Build distribuion
+        id: build
+        run: |
+          git fetch origin +refs/tags/*:refs/tags/*
+          export SDIST_VERSION=$(python -m setuptools_scm)
+          echo "::set-output name=version::${SDIST_VERSION}"
+          python -m build
+
+      - name: upload to PyPI.org
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/labeled-pr.yml

@@ -0,0 +1,15 @@
+name: Is PR labeled?
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+    branches:
+      - main
+
+jobs:
+  call-labeled-pr-check-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]

.github/workflows/release-github.yml

@@ -0,0 +1,16 @@
+name: Create Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  call-release-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    with:
+      release_prefix: tile-stitcher
+      develop_branch: dev
+      sync_pr_label: team-bot
+    secrets:
+      USER_TOKEN: ${{ secrets.OPERA_PST_GITHUB_TOKEN }}

.github/workflows/static-analysis.yml

@@ -10,3 +10,6 @@ jobs:
 
   call-secrets-analysis-workflow:
     uses: ASFHyP3/actions/.github/workflows/[email protected]
+
+  call-ruff-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]

CHANGELOG.md

@@ -7,4 +7,10 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [0.0.1]
-* First release with ESA worldcover 2020 and 2021; Pekel water occurence 2021; S1 coherence from 2020; and Hansen annual mosaics. See readme.
+* First release of `tile_stitcher` with following datasets (See readme for links):
+    - ESA 10m worldcover 2020 and 2021; 
+    - Pekel 30m water occurence 2021; 
+    - S1 coherence from 2020;  
+    - Hansen annual mosaics 2000, 2013 - present
+    - Cop 100m Landcover 2015-2019
+* Includes workflows for static analysis, release to PyPI, and integration testing.

environment.yml

@@ -23,3 +23,4 @@ dependencies:
  - flake8-import-order
  - pytest
  - pytest-cov
+ - ruff

pyproject.toml

@@ -5,10 +5,12 @@ build-backend = 'setuptools.build_meta'
 [project]
 name = 'tile-stitcher'
 requires-python = '>=3.10'
-authors = [{name='Alexander Handwerger, Charlie Marshak, and OPERA Project Science Team'}]
+authors = [
+    { name = 'Alexander Handwerger, Charlie Marshak, and OPERA Project Science Team' },
+]
 description = 'Create rasters from global tiles'
-license = {text = 'Apache 2.0'}
-classifiers=[
+license = { text = 'Apache 2.0' }
+classifiers = [
     'Intended Audience :: Science/Research',
     'License :: OSI Approved :: Apache 2.0',
     'Natural Language :: English',
@@ -30,7 +32,7 @@ dependencies = [
     "numpy",
     "pandas",
     "pandera",
-    "shapely"
+    "shapely",
 ]
 
 [project.optional-dependencies]
@@ -56,9 +58,29 @@ include = ['tile_stitcher']
 exclude = ['notebooks*', 'tests*']
 
 [tool.setuptools.dynamic]
-readme = {file = ['README.md'], content-type = 'text/markdown'}
+readme = { file = ['README.md'], content-type = 'text/markdown' }
 
 [tool.setuptools_scm]
 
 [tool.ruff]
-line-length = 120
+line-length = 120
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".eggs",
+    ".git",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "build",
+    "dist",
+    "site-packages",
+]
+indent-width = 4
+
+[tool.ruff.format]
+quote-style = "single"

tests/test_stitch_api.py

@@ -1,8 +1,12 @@
 import pytest
 
 from tile_stitcher import get_raster_from_tiles
-from tile_stitcher.stitcher import (COP_100_YEARS, HANSEN_MOSAIC_YEARS,
-                                    S1_TEMPORAL_BASELINE_DAYS, SEASONS)
+from tile_stitcher.stitcher import (
+    COP_100_YEARS,
+    HANSEN_MOSAIC_YEARS,
+    S1_TEMPORAL_BASELINE_DAYS,
+    SEASONS,
+)
 
 
 def test_esa_world_cover():
@@ -20,34 +24,30 @@ def test_pekel_water_occ():
     assert len(X.shape) == 3
 
 
-@pytest.mark.parametrize("year",
-                         [HANSEN_MOSAIC_YEARS[k] for k in [0, 2, 4, 6, 10]]
-                         )
+@pytest.mark.parametrize('year', [HANSEN_MOSAIC_YEARS[k] for k in [0, 2, 4, 6, 10]])
 def test_hansen_datasets(year):
     # Note only getting 1 tile - these are large datasets!
     bounds = [-120.45, 34.85, -120.15, 34.95]
-    X, _ = get_raster_from_tiles(bounds,
-                                 tile_shortname='hansen_annual_mosaic',
-                                 year=year)
+    X, _ = get_raster_from_tiles(bounds, tile_shortname='hansen_annual_mosaic', year=year)
     assert len(X.shape) == 3
 
 
-@pytest.mark.parametrize("season", SEASONS)
-@pytest.mark.parametrize("temporal_baseline_days", S1_TEMPORAL_BASELINE_DAYS)
+@pytest.mark.parametrize('season', SEASONS)
+@pytest.mark.parametrize('temporal_baseline_days', S1_TEMPORAL_BASELINE_DAYS)
 def test_coherence_dataset(season, temporal_baseline_days):
     # Note only getting 1 tile
     bounds = [-120.45, 34.85, -120.15, 34.95]
-    X, _ = get_raster_from_tiles(bounds,
-                                 tile_shortname='s1_coherence_2020',
-                                 season=season,
-                                 temporal_baseline_days=temporal_baseline_days)
+    X, _ = get_raster_from_tiles(
+        bounds,
+        tile_shortname='s1_coherence_2020',
+        season=season,
+        temporal_baseline_days=temporal_baseline_days,
+    )
     assert len(X.shape) == 3
 
 
-@pytest.mark.parametrize("year", COP_100_YEARS)
+@pytest.mark.parametrize('year', COP_100_YEARS)
 def test_cop100_dataset(year: int):
     bounds = [-120.45, 34.85, -120.15, 34.95]
-    X, _ = get_raster_from_tiles(bounds,
-                                 tile_shortname='cop_100_lulc_discrete',
-                                 year=year)
+    X, _ = get_raster_from_tiles(bounds, tile_shortname='cop_100_lulc_discrete', year=year)
     assert len(X.shape) == 3

tile_stitcher/__init__.py

@@ -6,9 +6,12 @@
     __version__ = version(__name__)
 except PackageNotFoundError:
     __version__ = None
-    warnings.warn('package is not installed!\n'
-                  'Install in editable/develop mode via (from the top of this repo):\n'
-                  '   python -m pip install -e .\n', RuntimeWarning)
+    warnings.warn(
+        'package is not installed!\n'
+        'Install in editable/develop mode via (from the top of this repo):\n'
+        '   python -m pip install -e .\n',
+        RuntimeWarning,
+    )
 
 
 from .stitcher import DATASET_SHORTNAMES, get_raster_from_tiles

tile_stitcher/stitcher.py

@@ -18,7 +18,7 @@
     'esa_world_cover_2021': 'esa_world_cover_2021.geojson.zip',
     'hansen_annual_mosaic': 'hansen_landsat_mosaic_2022.geojson.zip',
     's1_coherence_2020': 's1_coherence_2020.geojson.zip',
-    'cop_100_lulc_discrete': 'cop_100m_lulc_discrete_classes.geojson.zip'
+    'cop_100_lulc_discrete': 'cop_100m_lulc_discrete_classes.geojson.zip',
 }
 DATASET_SHORTNAMES = list(GEOJSON_DICT.keys())
 
@@ -42,20 +42,23 @@ def get_all_tile_data(tile_key: str) -> gpd.GeoDataFrame:
 
 
 @lru_cache
-def get_tile_data(tile_key: str,
-                  year: int = None,
-                  season: str = None,
-                  temporal_baseline_days: int = None) -> gpd.GeoDataFrame:
+def get_tile_data(
+    tile_key: str,
+    year: int = None,
+    season: str = None,
+    temporal_baseline_days: int = None,
+) -> gpd.GeoDataFrame:
     # Because tile data is cached - we need to copy it.
     df_tiles = get_all_tile_data(tile_key).copy()
 
-    if (year is not None):
+    if year is not None:
         if tile_key not in DATASETS_WITH_YEAR:
-            raise NotImplementedError('Year is only supported '
-                                      f'with {DATASETS_WITH_YEAR}')
+            raise NotImplementedError('Year is only supported ' f'with {DATASETS_WITH_YEAR}')
         if tile_key == 'hansen_annual_mosaic':
+
             def update_hansen_landsat_mosaic_url_p(url):
                 return update_hansen_landsat_mosaic_url(url, year)
+
             df_tiles.url = df_tiles.url.map(update_hansen_landsat_mosaic_url_p)
         if tile_key == 'cop_100_lulc_discrete':
             df_tiles = df_tiles[df_tiles.year == year].reset_index(drop=True)
@@ -65,8 +68,7 @@ def update_hansen_landsat_mosaic_url_p(url):
 
     if tile_key == 's1_coherence_2020':
         if any([var is None for var in [temporal_baseline_days, season]]):
-            raise ValueError(f'{tile_key} requires season and temporal baseline '
-                             'to be specified')
+            raise ValueError(f'{tile_key} requires season and temporal baseline ' 'to be specified')
         if season not in SEASONS:
             raise ValueError(f'season keyword must be in {", ".join(SEASONS)}')
         if temporal_baseline_days not in S1_TEMPORAL_BASELINE_DAYS:
@@ -88,30 +90,27 @@ def update_hansen_landsat_mosaic_url(url: str, year: int):
     elif year <= 2015:
         # Gets the "last_00N_040W.tif" portion of the url
         url_end = url[-17:]
-        url_updated = ('https://storage.googleapis.com/earthenginepartners-hansen/'
-                       f'GFC{year}/Hansen_GFC{year}_{url_end}')
+        url_updated = (
+            'https://storage.googleapis.com/earthenginepartners-hansen/' f'GFC{year}/Hansen_GFC{year}_{url_end}'
+        )
     else:
         year_diff = CURRENT_HANSEN_YEAR - year
         version_updated = CURRENT_HANSEN_VERSION - year_diff
-        url_updated = url.replace(str(CURRENT_HANSEN_YEAR),
-                                  str(year))
-        url_updated = url_updated.replace(f'v1.{CURRENT_HANSEN_VERSION}',
-                                          f'v1.{version_updated}')
+        url_updated = url.replace(str(CURRENT_HANSEN_YEAR), str(year))
+        url_updated = url_updated.replace(f'v1.{CURRENT_HANSEN_VERSION}', f'v1.{version_updated}')
 
     return url_updated
 
 
-def get_urls_from_tile_df(extent: list[float],
-                          df_tiles: gpd.GeoDataFrame) -> list[str]:
+def get_urls_from_tile_df(extent: list[float], df_tiles: gpd.GeoDataFrame) -> list[str]:
     bbox = box(*extent)
     ind_inter = df_tiles.geometry.intersects(bbox)
     df_subset = df_tiles[ind_inter].reset_index(drop=True)
     urls = df_subset.url.tolist()
     return urls
 
 
-def get_additional_tile_metadata(urls: list[str],
-                                 max_tile_tries: int = 10) -> dict:
+def get_additional_tile_metadata(urls: list[str], max_tile_tries: int = 10) -> dict:
     """Some tile sets may have missing data when they should not. Ideally we
     can remove said tiles from dataframe. However, in the case of Hansen
     mosiacs, these errors seem to be year-to-year e.g. 2017 where the upper left
@@ -125,48 +124,47 @@ def get_additional_tile_metadata(urls: list[str],
             with rasterio.open(url) as ds:
                 tags = ds.tags()
                 try:
-                    cmap = {k+1: ds.colormap(k+1) for k in range(ds.count)}
+                    cmap = {k + 1: ds.colormap(k + 1) for k in range(ds.count)}
                 # no colormap in existing dataset yields a ValueError in Rasterio
                 except ValueError:
                     cmap = {}
-            metadata = {'tags': tags,
-                        'colormap': cmap}
+            metadata = {'tags': tags, 'colormap': cmap}
             break
         # When dataset does not exist with given url
         except RasterioIOError:
             continue
     return metadata
 
 
-def get_raster_from_tiles(extent: list[float],
-                          tile_shortname: str = None,
-                          df_tiles: gpd.GeoDataFrame = None,
-                          year: int = None,
-                          season: str = None,
-                          temporal_baseline_days: int = None) -> tuple:
-
+def get_raster_from_tiles(
+    extent: list[float],
+    tile_shortname: str = None,
+    df_tiles: gpd.GeoDataFrame = None,
+    year: int = None,
+    season: str = None,
+    temporal_baseline_days: int = None,
+) -> tuple:
     if (tile_shortname is None) and (df_tiles is None):
-        raise ValueError('Either "tile_shortname" or '
-                         '"df_tiles" must be provided')
+        raise ValueError('Either "tile_shortname" or "df_tiles" must be provided')
 
     if (tile_shortname is not None) and (df_tiles is not None):
-        raise ValueError('"tile_shortname" and "df_tiles" cannot '
-                         'both be provided')
+        raise ValueError('"tile_shortname" and "df_tiles" cannot both be provided')
 
     if isinstance(tile_shortname, str):
-        df_tiles = get_tile_data(tile_shortname,
-                                 year=year,
-                                 temporal_baseline_days=temporal_baseline_days,
-                                 season=season)
+        df_tiles = get_tile_data(
+            tile_shortname,
+            year=year,
+            temporal_baseline_days=temporal_baseline_days,
+            season=season,
+        )
 
     df_tiles = TILE_SCHEMA.validate(df_tiles)
 
     urls = df_tiles.url.tolist()
     tile_metadata = get_additional_tile_metadata(urls)
 
     urls_subset = get_urls_from_tile_df(extent, df_tiles)
-    X_merged, p_merged = merge_tile_datasets_within_extent(urls_subset,
-                                                           extent)
+    X_merged, p_merged = merge_tile_datasets_within_extent(urls_subset, extent)
 
     # Are stored in the profile for provenance
     p_merged.update(**tile_metadata)