From a45590d5322009827e294eb4bb3525d17ad7b0e9 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 13:48:30 +0200 Subject: [PATCH 01/30] refactor: remove _optional module from utils --- srai/{utils => }/_optional.py | 0 srai/embedders/gtfs2vec/embedder.py | 2 +- srai/embedders/gtfs2vec/model.py | 2 +- srai/embedders/hex2vec/embedder.py | 2 +- srai/embedders/hex2vec/model.py | 2 +- srai/embedders/hex2vec/neighbour_dataset.py | 2 +- srai/embedders/highway2vec/embedder.py | 2 +- srai/embedders/highway2vec/model.py | 2 +- srai/loaders/gtfs_loader.py | 2 +- srai/loaders/osm_loaders/osm_online_loader.py | 7 ++----- srai/loaders/osm_loaders/osm_pbf_loader.py | 7 ++----- srai/loaders/osm_loaders/osm_tile_loader.py | 2 +- srai/loaders/osm_way_loader/osm_way_loader.py | 2 +- srai/plotting/folium_wrapper.py | 2 +- srai/plotting/plotly_wrapper.py | 2 +- srai/regionalizers/administrative_boundary_regionalizer.py | 2 +- srai/regionalizers/voronoi_regionalizer.py | 2 +- tests/miscellaneous/test_optional_dependencies.py | 2 +- 18 files changed, 19 insertions(+), 25 deletions(-) rename srai/{utils => }/_optional.py (100%) diff --git a/srai/utils/_optional.py b/srai/_optional.py similarity index 100% rename from srai/utils/_optional.py rename to srai/_optional.py diff --git a/srai/embedders/gtfs2vec/embedder.py b/srai/embedders/gtfs2vec/embedder.py index 5629e8b2..29a8a635 100644 --- a/srai/embedders/gtfs2vec/embedder.py +++ b/srai/embedders/gtfs2vec/embedder.py @@ -17,11 +17,11 @@ import numpy as np import pandas as pd +from srai._optional import import_optional_dependencies from srai.embedders import Embedder, ModelT from srai.embedders.gtfs2vec.model import GTFS2VecModel from srai.exceptions import ModelNotFitException from srai.loaders.gtfs_loader import GTFS2VEC_DIRECTIONS_PREFIX, GTFS2VEC_TRIPS_PREFIX -from srai.utils._optional import import_optional_dependencies class GTFS2VecEmbedder(Embedder): diff --git a/srai/embedders/gtfs2vec/model.py b/srai/embedders/gtfs2vec/model.py index dc793c66..08bb9744 100644 --- a/srai/embedders/gtfs2vec/model.py +++ b/srai/embedders/gtfs2vec/model.py @@ -8,8 +8,8 @@ """ from typing import TYPE_CHECKING, Any +from srai._optional import import_optional_dependencies from srai.embedders import Model -from srai.utils._optional import import_optional_dependencies if TYPE_CHECKING: # pragma: no cover import torch diff --git a/srai/embedders/hex2vec/embedder.py b/srai/embedders/hex2vec/embedder.py index d85fdd74..7d79c58d 100644 --- a/srai/embedders/hex2vec/embedder.py +++ b/srai/embedders/hex2vec/embedder.py @@ -14,12 +14,12 @@ import numpy as np import pandas as pd +from srai._optional import import_optional_dependencies from srai.embedders import CountEmbedder, ModelT from srai.embedders.hex2vec.model import Hex2VecModel from srai.embedders.hex2vec.neighbour_dataset import NeighbourDataset from srai.exceptions import ModelNotFitException from srai.neighbourhoods import Neighbourhood -from srai.utils._optional import import_optional_dependencies T = TypeVar("T") diff --git a/srai/embedders/hex2vec/model.py b/srai/embedders/hex2vec/model.py index 8b8aca25..33df4db0 100644 --- a/srai/embedders/hex2vec/model.py +++ b/srai/embedders/hex2vec/model.py @@ -8,8 +8,8 @@ """ from typing import TYPE_CHECKING, List, Tuple +from srai._optional import import_optional_dependencies from srai.embedders import Model -from srai.utils._optional import import_optional_dependencies if TYPE_CHECKING: # pragma: no cover import torch diff --git a/srai/embedders/hex2vec/neighbour_dataset.py b/srai/embedders/hex2vec/neighbour_dataset.py index 8e269dc1..96d38d54 100644 --- a/srai/embedders/hex2vec/neighbour_dataset.py +++ b/srai/embedders/hex2vec/neighbour_dataset.py @@ -13,8 +13,8 @@ import pandas as pd from tqdm import tqdm +from srai._optional import import_optional_dependencies from srai.neighbourhoods import Neighbourhood -from srai.utils._optional import import_optional_dependencies if TYPE_CHECKING: # pragma: no cover import torch diff --git a/srai/embedders/highway2vec/embedder.py b/srai/embedders/highway2vec/embedder.py index cf6d665b..15bcb8eb 100644 --- a/srai/embedders/highway2vec/embedder.py +++ b/srai/embedders/highway2vec/embedder.py @@ -13,9 +13,9 @@ import geopandas as gpd import pandas as pd +from srai._optional import import_optional_dependencies from srai.embedders import Embedder, ModelT from srai.exceptions import ModelNotFitException -from srai.utils._optional import import_optional_dependencies from .model import Highway2VecModel diff --git a/srai/embedders/highway2vec/model.py b/srai/embedders/highway2vec/model.py index 5f1b4a32..31c0dccb 100644 --- a/srai/embedders/highway2vec/model.py +++ b/srai/embedders/highway2vec/model.py @@ -8,8 +8,8 @@ """ from typing import TYPE_CHECKING +from srai._optional import import_optional_dependencies from srai.embedders import Model -from srai.utils._optional import import_optional_dependencies if TYPE_CHECKING: # pragma: no cover import torch diff --git a/srai/loaders/gtfs_loader.py b/srai/loaders/gtfs_loader.py index 8656152a..8c12e7ef 100644 --- a/srai/loaders/gtfs_loader.py +++ b/srai/loaders/gtfs_loader.py @@ -17,9 +17,9 @@ import pandas as pd from shapely.geometry import Point +from srai._optional import import_optional_dependencies from srai.constants import GEOMETRY_COLUMN, WGS84_CRS from srai.loaders import Loader -from srai.utils._optional import import_optional_dependencies if TYPE_CHECKING: # pragma: no cover from gtfs_kit import Feed diff --git a/srai/loaders/osm_loaders/osm_online_loader.py b/srai/loaders/osm_loaders/osm_online_loader.py index efe6eeb1..d5b31958 100644 --- a/srai/loaders/osm_loaders/osm_online_loader.py +++ b/srai/loaders/osm_loaders/osm_online_loader.py @@ -11,13 +11,10 @@ from functional import seq from tqdm import tqdm +from srai._optional import import_optional_dependencies from srai.constants import FEATURES_INDEX, GEOMETRY_COLUMN, WGS84_CRS from srai.loaders.osm_loaders._base import OSMLoader -from srai.loaders.osm_loaders.filters._typing import ( - grouped_osm_tags_type, - osm_tags_type, -) -from srai.utils._optional import import_optional_dependencies +from srai.loaders.osm_loaders.filters._typing import grouped_osm_tags_type, osm_tags_type class OSMOnlineLoader(OSMLoader): diff --git a/srai/loaders/osm_loaders/osm_pbf_loader.py b/srai/loaders/osm_loaders/osm_pbf_loader.py index b22f9e40..c6de8fa4 100644 --- a/srai/loaders/osm_loaders/osm_pbf_loader.py +++ b/srai/loaders/osm_loaders/osm_pbf_loader.py @@ -9,13 +9,10 @@ import geopandas as gpd import pandas as pd +from srai._optional import import_optional_dependencies from srai.constants import FEATURES_INDEX, GEOMETRY_COLUMN, WGS84_CRS from srai.loaders.osm_loaders._base import OSMLoader -from srai.loaders.osm_loaders.filters._typing import ( - grouped_osm_tags_type, - osm_tags_type, -) -from srai.utils._optional import import_optional_dependencies +from srai.loaders.osm_loaders.filters._typing import grouped_osm_tags_type, osm_tags_type class OSMPbfLoader(OSMLoader): diff --git a/srai/loaders/osm_loaders/osm_tile_loader.py b/srai/loaders/osm_loaders/osm_tile_loader.py index 36c777df..d42a3d78 100644 --- a/srai/loaders/osm_loaders/osm_tile_loader.py +++ b/srai/loaders/osm_loaders/osm_tile_loader.py @@ -12,8 +12,8 @@ import pandas as pd import requests +from srai._optional import import_optional_dependencies from srai.regionalizers.slippy_map_regionalizer import SlippyMapRegionalizer -from srai.utils._optional import import_optional_dependencies from .osm_tile_data_collector import ( DataCollector, diff --git a/srai/loaders/osm_way_loader/osm_way_loader.py b/srai/loaders/osm_way_loader/osm_way_loader.py index 6af725ad..88615311 100644 --- a/srai/loaders/osm_way_loader/osm_way_loader.py +++ b/srai/loaders/osm_way_loader/osm_way_loader.py @@ -14,10 +14,10 @@ from functional import seq from tqdm.auto import tqdm +from srai._optional import import_optional_dependencies from srai.constants import FEATURES_INDEX, GEOMETRY_COLUMN, WGS84_CRS from srai.exceptions import LoadedDataIsEmptyException from srai.loaders import Loader -from srai.utils._optional import import_optional_dependencies from . import constants diff --git a/srai/plotting/folium_wrapper.py b/srai/plotting/folium_wrapper.py index 329e504a..08285743 100644 --- a/srai/plotting/folium_wrapper.py +++ b/srai/plotting/folium_wrapper.py @@ -14,10 +14,10 @@ import pandas as pd import plotly.express as px +from srai._optional import import_optional_dependencies from srai.constants import REGIONS_INDEX from srai.neighbourhoods import Neighbourhood from srai.neighbourhoods._base import IndexType -from srai.utils._optional import import_optional_dependencies import_optional_dependencies(dependency_group="plotting", modules=["folium", "plotly"]) diff --git a/srai/plotting/plotly_wrapper.py b/srai/plotting/plotly_wrapper.py index 63b2e7ac..fe47d13d 100644 --- a/srai/plotting/plotly_wrapper.py +++ b/srai/plotting/plotly_wrapper.py @@ -11,10 +11,10 @@ import plotly.graph_objs as go from shapely.geometry import Point +from srai._optional import import_optional_dependencies from srai.constants import REGIONS_INDEX, WGS84_CRS from srai.neighbourhoods import Neighbourhood from srai.neighbourhoods._base import IndexType -from srai.utils._optional import import_optional_dependencies import_optional_dependencies(dependency_group="plotting", modules=["plotly"]) diff --git a/srai/regionalizers/administrative_boundary_regionalizer.py b/srai/regionalizers/administrative_boundary_regionalizer.py index 5472b8e2..522f6ae1 100644 --- a/srai/regionalizers/administrative_boundary_regionalizer.py +++ b/srai/regionalizers/administrative_boundary_regionalizer.py @@ -14,10 +14,10 @@ from shapely.validation import make_valid from tqdm import tqdm +from srai._optional import import_optional_dependencies from srai.constants import GEOMETRY_COLUMN, REGIONS_INDEX, WGS84_CRS from srai.regionalizers import Regionalizer from srai.utils import flatten_geometry_series -from srai.utils._optional import import_optional_dependencies class AdministrativeBoundaryRegionalizer(Regionalizer): diff --git a/srai/regionalizers/voronoi_regionalizer.py b/srai/regionalizers/voronoi_regionalizer.py index 01e8fd4c..6d8b38b6 100644 --- a/srai/regionalizers/voronoi_regionalizer.py +++ b/srai/regionalizers/voronoi_regionalizer.py @@ -9,9 +9,9 @@ import geopandas as gpd from shapely.geometry import Point, box +from srai._optional import import_optional_dependencies from srai.constants import GEOMETRY_COLUMN, REGIONS_INDEX, WGS84_CRS from srai.regionalizers import Regionalizer -from srai.utils._optional import import_optional_dependencies class VoronoiRegionalizer(Regionalizer): diff --git a/tests/miscellaneous/test_optional_dependencies.py b/tests/miscellaneous/test_optional_dependencies.py index 5b5a6eed..68a576b3 100644 --- a/tests/miscellaneous/test_optional_dependencies.py +++ b/tests/miscellaneous/test_optional_dependencies.py @@ -7,8 +7,8 @@ import pytest from shapely.geometry import box +from srai._optional import ImportErrorHandle, import_optional_dependency from srai.constants import GEOMETRY_COLUMN, REGIONS_INDEX, WGS84_CRS -from srai.utils._optional import ImportErrorHandle, import_optional_dependency @pytest.fixture # type: ignore From 71d1b26555ffa2ec3eadff159fe0da3ea820198d Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 13:49:48 +0200 Subject: [PATCH 02/30] refactor: remove _typing module from utils --- srai/{utils/typing.py => _typing.py} | 0 srai/loaders/osm_loaders/_base.py | 2 +- srai/loaders/osm_loaders/filters/_typing.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename srai/{utils/typing.py => _typing.py} (100%) diff --git a/srai/utils/typing.py b/srai/_typing.py similarity index 100% rename from srai/utils/typing.py rename to srai/_typing.py diff --git a/srai/loaders/osm_loaders/_base.py b/srai/loaders/osm_loaders/_base.py index 13c53178..3d3b49be 100644 --- a/srai/loaders/osm_loaders/_base.py +++ b/srai/loaders/osm_loaders/_base.py @@ -8,13 +8,13 @@ import pandas as pd from tqdm import tqdm +from srai._typing import is_expected_type from srai.loaders import Loader from srai.loaders.osm_loaders.filters._typing import ( grouped_osm_tags_type, merge_grouped_osm_tags_type, osm_tags_type, ) -from srai.utils.typing import is_expected_type class OSMLoader(Loader, abc.ABC): diff --git a/srai/loaders/osm_loaders/filters/_typing.py b/srai/loaders/osm_loaders/filters/_typing.py index ad6fdaf4..6d7dd096 100644 --- a/srai/loaders/osm_loaders/filters/_typing.py +++ b/srai/loaders/osm_loaders/filters/_typing.py @@ -1,7 +1,7 @@ """Module contains a dedicated type alias for OSM tags filter.""" from typing import Dict, List, Union, cast -from srai.utils.typing import is_expected_type +from srai._typing import is_expected_type osm_tags_type = Dict[str, Union[List[str], str, bool]] From f2d70bc4075de7e653356c708dc7ba5582bf49a8 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 14:02:38 +0200 Subject: [PATCH 03/30] refactor: remove download module from utils --- README.md | 4 ++-- docs/README.md | 4 ++-- examples/loaders/gtfs_loader.ipynb | 5 ++--- srai/loaders/__init__.py | 2 ++ srai/{utils => loaders}/download.py | 0 srai/loaders/osm_loaders/pbf_file_downloader.py | 3 ++- srai/utils/__init__.py | 2 -- 7 files changed, 10 insertions(+), 10 deletions(-) rename srai/{utils => loaders}/download.py (100%) diff --git a/README.md b/README.md index 3203ea3d..e09218c8 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,8 @@ To extract features from GTFS use `GTFSLoader`. It will extract trip count and a ```python from pathlib import Path -from srai.loaders import GTFSLoader -from srai.utils import geocode_to_region_gdf, download_file +from srai.loaders import GTFSLoader, download_file +from srai.utils import geocode_to_region_gdf from srai.plotting import plot_regions area = geocode_to_region_gdf("Vienna, Austria") diff --git a/docs/README.md b/docs/README.md index 2e975f59..900d18cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -142,8 +142,8 @@ To extract features from GTFS use `GTFSLoader`. It will extract trip count and a ```python from pathlib import Path -from srai.loaders import GTFSLoader -from srai.utils import geocode_to_region_gdf, download_file +from srai.loaders import GTFSLoader, download_file +from srai.utils import geocode_to_region_gdf from srai.plotting import plot_regions area = geocode_to_region_gdf("Vienna, Austria") diff --git a/examples/loaders/gtfs_loader.ipynb b/examples/loaders/gtfs_loader.ipynb index 1d4cac08..b22dceb2 100644 --- a/examples/loaders/gtfs_loader.ipynb +++ b/examples/loaders/gtfs_loader.ipynb @@ -15,12 +15,11 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "from srai.loaders import GTFSLoader\n", + "from srai.loaders import GTFSLoader, download_file\n", "import gtfs_kit as gk\n", "import geopandas as gpd\n", "from shapely.geometry import Point\n", - "from srai.constants import WGS84_CRS\n", - "from srai.utils import download_file" + "from srai.constants import WGS84_CRS" ] }, { diff --git a/srai/loaders/__init__.py b/srai/loaders/__init__.py index a6da2fdd..941793b7 100644 --- a/srai/loaders/__init__.py +++ b/srai/loaders/__init__.py @@ -7,6 +7,7 @@ """ from ._base import Loader +from .download import download_file from .geoparquet_loader import GeoparquetLoader from .gtfs_loader import GTFSLoader from .osm_loaders import OSMLoader, OSMOnlineLoader, OSMPbfLoader, OSMTileLoader @@ -22,4 +23,5 @@ "OSMPbfLoader", "OSMTileLoader", "OSMNetworkType", + "download_file", ] diff --git a/srai/utils/download.py b/srai/loaders/download.py similarity index 100% rename from srai/utils/download.py rename to srai/loaders/download.py diff --git a/srai/loaders/osm_loaders/pbf_file_downloader.py b/srai/loaders/osm_loaders/pbf_file_downloader.py index d6184926..05b0a733 100644 --- a/srai/loaders/osm_loaders/pbf_file_downloader.py +++ b/srai/loaders/osm_loaders/pbf_file_downloader.py @@ -19,7 +19,8 @@ from tqdm import tqdm from srai.constants import WGS84_CRS -from srai.utils import buffer_geometry, download_file, flatten_geometry, remove_interiors +from srai.loaders import download_file +from srai.utils import buffer_geometry, flatten_geometry, remove_interiors class PbfFileDownloader: diff --git a/srai/utils/__init__.py b/srai/utils/__init__.py index 1bb34dda..5c11a613 100644 --- a/srai/utils/__init__.py +++ b/srai/utils/__init__.py @@ -4,13 +4,11 @@ Those are either used internally by other modules, or can be used to simplify spatial data processing. """ -from .download import download_file from .geocode import geocode_to_region_gdf from .geometry import buffer_geometry, flatten_geometry, flatten_geometry_series, remove_interiors from .merge import merge_disjointed_gdf_geometries, merge_disjointed_polygons __all__ = [ - "download_file", "geocode_to_region_gdf", "buffer_geometry", "flatten_geometry", From a80d47c2568c6a8b3a30ab961083fedc4d0460a2 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 14:53:33 +0200 Subject: [PATCH 04/30] refactor: remove geocode module from utils --- README.md | 15 ++++++--------- docs/README.md | 15 ++++++--------- .../embedders/contextual_count_embedder.ipynb | 2 +- examples/embedders/hex2vec_embedder.ipynb | 3 +-- examples/embedders/highway2vec_embedder.ipynb | 2 +- examples/embedders/load_and_save.ipynb | 3 +-- examples/loaders/osm_online_loader.ipynb | 2 +- examples/loaders/osm_pbf_loader.ipynb | 3 ++- examples/loaders/osm_tile_loader.ipynb | 2 +- examples/loaders/osm_way_loader.ipynb | 2 +- .../neighbourhoods/adjacency_neighbourhood.ipynb | 7 +++++-- examples/neighbourhoods/h3_neighbourhood.ipynb | 3 +-- .../administrative_boundary_regionalizer.ipynb | 5 ++--- examples/regionalizers/voronoi_regionalizer.ipynb | 5 ++--- srai/regionalizers/__init__.py | 2 ++ srai/{utils => regionalizers}/geocode.py | 0 srai/utils/__init__.py | 2 -- tests/embedders/hex2vec/generation.py | 2 +- 18 files changed, 34 insertions(+), 41 deletions(-) rename srai/{utils => regionalizers}/geocode.py (100%) diff --git a/README.md b/README.md index e09218c8..0eb2e40a 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,8 @@ Example with `OSMOnlineLoader`: ```python from srai.loaders import OSMOnlineLoader -from srai.utils import geocode_to_region_gdf from srai.plotting import plot_regions +from srai.regionalizers import geocode_to_region_gdf query = {"leisure": "park"} area = geocode_to_region_gdf("Wrocław, Poland") @@ -119,8 +119,8 @@ Road network downloading is a special case of OSM data downloading. To download ```python from srai.loaders import OSMNetworkType, OSMWayLoader -from srai.utils import geocode_to_region_gdf from srai.plotting import plot_regions +from srai.regionalizers import geocode_to_region_gdf area = geocode_to_region_gdf("Utrecht, Netherlands") loader = OSMWayLoader(OSMNetworkType.BIKE) @@ -143,8 +143,8 @@ To extract features from GTFS use `GTFSLoader`. It will extract trip count and a from pathlib import Path from srai.loaders import GTFSLoader, download_file -from srai.utils import geocode_to_region_gdf from srai.plotting import plot_regions +from srai.regionalizers import geocode_to_region_gdf area = geocode_to_region_gdf("Vienna, Austria") gtfs_file = Path("vienna_gtfs.zip") @@ -173,8 +173,7 @@ Regionalization is a process of dividing a given area into smaller regions. This Example: ```python -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf area = geocode_to_region_gdf("Berlin, Germany") regionalizer = H3Regionalizer(resolution=7) @@ -206,8 +205,7 @@ from srai.embedders import CountEmbedder from srai.joiners import IntersectionJoiner from srai.loaders import OSMOnlineLoader from srai.plotting import plot_regions, plot_numeric_data -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf loader = OSMOnlineLoader() regionalizer = H3Regionalizer(resolution=9) @@ -238,8 +236,7 @@ from srai.joiners import IntersectionJoiner from srai.loaders import OSMPbfLoader from srai.loaders.osm_loaders.filters import HEX2VEC_FILTER from srai.neighbourhoods.h3_neighbourhood import H3Neighbourhood -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf from srai.plotting import plot_regions, plot_numeric_data loader = OSMPbfLoader() diff --git a/docs/README.md b/docs/README.md index 900d18cd..777a6436 100644 --- a/docs/README.md +++ b/docs/README.md @@ -97,7 +97,7 @@ Example with `OSMOnlineLoader`: ```python from srai.loaders import OSMOnlineLoader -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import geocode_to_region_gdf from srai.plotting import plot_regions query = {"leisure": "park"} @@ -119,7 +119,7 @@ Road network downloading is a special case of OSM data downloading. To download ```python from srai.loaders import OSMNetworkType, OSMWayLoader -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import geocode_to_region_gdf from srai.plotting import plot_regions area = geocode_to_region_gdf("Utrecht, Netherlands") @@ -143,7 +143,7 @@ To extract features from GTFS use `GTFSLoader`. It will extract trip count and a from pathlib import Path from srai.loaders import GTFSLoader, download_file -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import geocode_to_region_gdf from srai.plotting import plot_regions area = geocode_to_region_gdf("Vienna, Austria") @@ -173,8 +173,7 @@ Regionalization is a process of dividing a given area into smaller regions. This Example: ```python -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf area = geocode_to_region_gdf("Berlin, Germany") regionalizer = H3Regionalizer(resolution=7) @@ -206,8 +205,7 @@ from srai.embedders import CountEmbedder from srai.joiners import IntersectionJoiner from srai.loaders import OSMOnlineLoader from srai.plotting import plot_regions, plot_numeric_data -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf loader = OSMOnlineLoader() regionalizer = H3Regionalizer(resolution=9) @@ -238,8 +236,7 @@ from srai.joiners import IntersectionJoiner from srai.loaders import OSMPbfLoader from srai.loaders.osm_loaders.filters import HEX2VEC_FILTER from srai.neighbourhoods.h3_neighbourhood import H3Neighbourhood -from srai.regionalizers import H3Regionalizer -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf from srai.plotting import plot_regions, plot_numeric_data loader = OSMPbfLoader() diff --git a/examples/embedders/contextual_count_embedder.ipynb b/examples/embedders/contextual_count_embedder.ipynb index dbc77a4f..e2dd5425 100644 --- a/examples/embedders/contextual_count_embedder.ipynb +++ b/examples/embedders/contextual_count_embedder.ipynb @@ -32,7 +32,7 @@ "metadata": {}, "outputs": [], "source": [ - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import geocode_to_region_gdf\n", "\n", "area_gdf = geocode_to_region_gdf(\"Lisboa, PT\")\n", "plot_regions(area_gdf)" diff --git a/examples/embedders/hex2vec_embedder.ipynb b/examples/embedders/hex2vec_embedder.ipynb index 311e6fc2..5e8bfac5 100644 --- a/examples/embedders/hex2vec_embedder.ipynb +++ b/examples/embedders/hex2vec_embedder.ipynb @@ -10,8 +10,7 @@ "from srai.joiners import IntersectionJoiner\n", "from srai.loaders import OSMOnlineLoader\n", "from srai.neighbourhoods import H3Neighbourhood\n", - "from srai.regionalizers import H3Regionalizer\n", - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf\n", "from srai.plotting import plot_regions, plot_numeric_data\n", "from pytorch_lightning import seed_everything" ] diff --git a/examples/embedders/highway2vec_embedder.ipynb b/examples/embedders/highway2vec_embedder.ipynb index 8b6dfbaf..1b70309f 100644 --- a/examples/embedders/highway2vec_embedder.ipynb +++ b/examples/embedders/highway2vec_embedder.ipynb @@ -32,7 +32,7 @@ "metadata": {}, "outputs": [], "source": [ - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import geocode_to_region_gdf\n", "\n", "area_gdf = geocode_to_region_gdf(\"Wrocław, PL\")\n", "plot_regions(area_gdf, tiles_style=\"CartoDB positron\")" diff --git a/examples/embedders/load_and_save.ipynb b/examples/embedders/load_and_save.ipynb index 0784bdb0..c04047c8 100644 --- a/examples/embedders/load_and_save.ipynb +++ b/examples/embedders/load_and_save.ipynb @@ -10,8 +10,7 @@ "from srai.joiners import IntersectionJoiner\n", "from srai.loaders import OSMOnlineLoader\n", "from srai.neighbourhoods import H3Neighbourhood\n", - "from srai.regionalizers import H3Regionalizer\n", - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf\n", "from srai.plotting import plot_regions, plot_numeric_data\n", "from pytorch_lightning import seed_everything" ] diff --git a/examples/loaders/osm_online_loader.ipynb b/examples/loaders/osm_online_loader.ipynb index b96f02a8..30e1982d 100644 --- a/examples/loaders/osm_online_loader.ipynb +++ b/examples/loaders/osm_online_loader.ipynb @@ -17,7 +17,7 @@ "from srai.loaders.osm_loaders.filters.popular import get_popular_tags\n", "from srai.loaders.osm_loaders.filters import GEOFABRIK_LAYERS, HEX2VEC_FILTER\n", "from srai.loaders.osm_loaders import OSMOnlineLoader\n", - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import geocode_to_region_gdf\n", "from srai.plotting.folium_wrapper import plot_regions\n", "from functional import seq" ] diff --git a/examples/loaders/osm_pbf_loader.ipynb b/examples/loaders/osm_pbf_loader.ipynb index ad923990..aafba38b 100644 --- a/examples/loaders/osm_pbf_loader.ipynb +++ b/examples/loaders/osm_pbf_loader.ipynb @@ -24,7 +24,8 @@ "from srai.loaders.osm_loaders.filters.popular import get_popular_tags\n", "from srai.loaders.osm_loaders import OSMPbfLoader\n", "from srai.constants import REGIONS_INDEX, WGS84_CRS\n", - "from srai.utils import buffer_geometry, geocode_to_region_gdf\n", + "from srai.regionalizers import geocode_to_region_gdf\n", + "from srai.utils import buffer_geometry\n", "\n", "from shapely.geometry import Point, box\n", "import geopandas as gpd" diff --git a/examples/loaders/osm_tile_loader.ipynb b/examples/loaders/osm_tile_loader.ipynb index 2161559a..b497086b 100644 --- a/examples/loaders/osm_tile_loader.ipynb +++ b/examples/loaders/osm_tile_loader.ipynb @@ -15,7 +15,7 @@ "outputs": [], "source": [ "from srai.loaders.osm_loaders import OSMTileLoader\n", - "from srai.utils import geocode_to_region_gdf\n", + "from srai.regionalizers import geocode_to_region_gdf\n", "\n", "ZOOM = 9" ] diff --git a/examples/loaders/osm_way_loader.ipynb b/examples/loaders/osm_way_loader.ipynb index ae2140f3..87f320aa 100644 --- a/examples/loaders/osm_way_loader.ipynb +++ b/examples/loaders/osm_way_loader.ipynb @@ -20,7 +20,7 @@ "from srai.loaders import OSMNetworkType, OSMWayLoader\n", "from srai.constants import WGS84_CRS, REGIONS_INDEX\n", "from srai.plotting.folium_wrapper import plot_regions\n", - "from srai.utils import geocode_to_region_gdf" + "from srai.regionalizers import geocode_to_region_gdf" ] }, { diff --git a/examples/neighbourhoods/adjacency_neighbourhood.ipynb b/examples/neighbourhoods/adjacency_neighbourhood.ipynb index 59632e10..88cc76da 100644 --- a/examples/neighbourhoods/adjacency_neighbourhood.ipynb +++ b/examples/neighbourhoods/adjacency_neighbourhood.ipynb @@ -12,8 +12,11 @@ "\n", "from srai.constants import WGS84_CRS\n", "from srai.neighbourhoods import AdjacencyNeighbourhood\n", - "from srai.regionalizers import AdministrativeBoundaryRegionalizer, VoronoiRegionalizer\n", - "from srai.utils.geocode import geocode_to_region_gdf\n", + "from srai.regionalizers import (\n", + " AdministrativeBoundaryRegionalizer,\n", + " VoronoiRegionalizer,\n", + " geocode_to_region_gdf,\n", + ")\n", "from srai.plotting.folium_wrapper import plot_regions, plot_neighbours, plot_all_neighbourhood" ] }, diff --git a/examples/neighbourhoods/h3_neighbourhood.ipynb b/examples/neighbourhoods/h3_neighbourhood.ipynb index eb1874e5..a8157c34 100644 --- a/examples/neighbourhoods/h3_neighbourhood.ipynb +++ b/examples/neighbourhoods/h3_neighbourhood.ipynb @@ -7,8 +7,7 @@ "outputs": [], "source": [ "from srai.neighbourhoods import H3Neighbourhood\n", - "from srai.regionalizers import H3Regionalizer\n", - "from srai.utils.geocode import geocode_to_region_gdf\n", + "from srai.regionalizers import H3Regionalizer, geocode_to_region_gdf\n", "from srai.plotting.folium_wrapper import plot_neighbours, plot_all_neighbourhood" ] }, diff --git a/examples/regionalizers/administrative_boundary_regionalizer.ipynb b/examples/regionalizers/administrative_boundary_regionalizer.ipynb index e5c43a7a..c9629f01 100644 --- a/examples/regionalizers/administrative_boundary_regionalizer.ipynb +++ b/examples/regionalizers/administrative_boundary_regionalizer.ipynb @@ -10,9 +10,8 @@ "import plotly.express as px\n", "from shapely.geometry import Point, box\n", "\n", - "from srai.regionalizers import AdministrativeBoundaryRegionalizer\n", - "from srai.plotting.folium_wrapper import plot_regions\n", - "from srai.utils import geocode_to_region_gdf" + "from srai.regionalizers import AdministrativeBoundaryRegionalizer, geocode_to_region_gdf\n", + "from srai.plotting.folium_wrapper import plot_regions" ] }, { diff --git a/examples/regionalizers/voronoi_regionalizer.ipynb b/examples/regionalizers/voronoi_regionalizer.ipynb index f866ad7d..385ea796 100644 --- a/examples/regionalizers/voronoi_regionalizer.ipynb +++ b/examples/regionalizers/voronoi_regionalizer.ipynb @@ -11,10 +11,9 @@ "import plotly.express as px\n", "from shapely.geometry import Point\n", "\n", - "from srai.regionalizers import VoronoiRegionalizer\n", + "from srai.regionalizers import VoronoiRegionalizer, geocode_to_region_gdf\n", "from srai.constants import WGS84_CRS\n", - "from srai.plotting.folium_wrapper import plot_regions\n", - "from srai.utils import geocode_to_region_gdf" + "from srai.plotting.folium_wrapper import plot_regions" ] }, { diff --git a/srai/regionalizers/__init__.py b/srai/regionalizers/__init__.py index 7c75041f..d373a401 100644 --- a/srai/regionalizers/__init__.py +++ b/srai/regionalizers/__init__.py @@ -9,6 +9,7 @@ from ._base import Regionalizer from .administrative_boundary_regionalizer import AdministrativeBoundaryRegionalizer +from .geocode import geocode_to_region_gdf from .h3_regionalizer import H3Regionalizer from .s2_regionalizer import S2Regionalizer from .slippy_map_regionalizer import SlippyMapRegionalizer @@ -21,4 +22,5 @@ "S2Regionalizer", "VoronoiRegionalizer", "SlippyMapRegionalizer", + "geocode_to_region_gdf", ] diff --git a/srai/utils/geocode.py b/srai/regionalizers/geocode.py similarity index 100% rename from srai/utils/geocode.py rename to srai/regionalizers/geocode.py diff --git a/srai/utils/__init__.py b/srai/utils/__init__.py index 5c11a613..83fbf5c5 100644 --- a/srai/utils/__init__.py +++ b/srai/utils/__init__.py @@ -4,12 +4,10 @@ Those are either used internally by other modules, or can be used to simplify spatial data processing. """ -from .geocode import geocode_to_region_gdf from .geometry import buffer_geometry, flatten_geometry, flatten_geometry_series, remove_interiors from .merge import merge_disjointed_gdf_geometries, merge_disjointed_polygons __all__ = [ - "geocode_to_region_gdf", "buffer_geometry", "flatten_geometry", "flatten_geometry_series", diff --git a/tests/embedders/hex2vec/generation.py b/tests/embedders/hex2vec/generation.py index 9bc28ffa..605cc48e 100644 --- a/tests/embedders/hex2vec/generation.py +++ b/tests/embedders/hex2vec/generation.py @@ -13,7 +13,7 @@ from srai.loaders.osm_loaders import OSMPbfLoader from srai.loaders.osm_loaders.filters import osm_tags_type from srai.neighbourhoods import H3Neighbourhood -from srai.utils import geocode_to_region_gdf +from srai.regionalizers import geocode_to_region_gdf from tests.embedders.hex2vec.constants import ENCODER_SIZES, TRAINER_KWARGS From 14e3f88bd0e5221237e29a351729ca11f400d62d Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 14:56:21 +0200 Subject: [PATCH 05/30] refactor: remove _pytorch_stubs module from utils --- srai/embedders/_base.py | 2 +- srai/{utils => embedders}/_pytorch_stubs.py | 0 srai/embedders/hex2vec/neighbour_dataset.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename srai/{utils => embedders}/_pytorch_stubs.py (100%) diff --git a/srai/embedders/_base.py b/srai/embedders/_base.py index e99c077c..0ac30abc 100644 --- a/srai/embedders/_base.py +++ b/srai/embedders/_base.py @@ -13,7 +13,7 @@ from pytorch_lightning import LightningModule except ImportError: - from srai.utils._pytorch_stubs import LightningModule + from srai.embedders._pytorch_stubs import LightningModule class Model(LightningModule): # type: ignore diff --git a/srai/utils/_pytorch_stubs.py b/srai/embedders/_pytorch_stubs.py similarity index 100% rename from srai/utils/_pytorch_stubs.py rename to srai/embedders/_pytorch_stubs.py diff --git a/srai/embedders/hex2vec/neighbour_dataset.py b/srai/embedders/hex2vec/neighbour_dataset.py index 96d38d54..88e796e3 100644 --- a/srai/embedders/hex2vec/neighbour_dataset.py +++ b/srai/embedders/hex2vec/neighbour_dataset.py @@ -23,7 +23,7 @@ from torch.utils.data import Dataset except ImportError: - from srai.utils._pytorch_stubs import Dataset + from srai.embedders._pytorch_stubs import Dataset T = TypeVar("T") From 93b56e402bf327afa43facd5bfb8d2c6cca2a885 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 15:52:33 +0200 Subject: [PATCH 06/30] refactor: move merge module to geometry --- srai/utils/__init__.py | 3 -- srai/utils/geometry.py | 40 +++++++++++++++++- srai/utils/merge.py | 42 ------------------- ...st_administrative_boundary_regionalizer.py | 2 +- .../test_voronoi_regionalizer.py | 2 +- 5 files changed, 40 insertions(+), 49 deletions(-) delete mode 100644 srai/utils/merge.py diff --git a/srai/utils/__init__.py b/srai/utils/__init__.py index 83fbf5c5..948b277b 100644 --- a/srai/utils/__init__.py +++ b/srai/utils/__init__.py @@ -5,13 +5,10 @@ processing. """ from .geometry import buffer_geometry, flatten_geometry, flatten_geometry_series, remove_interiors -from .merge import merge_disjointed_gdf_geometries, merge_disjointed_polygons __all__ = [ "buffer_geometry", "flatten_geometry", "flatten_geometry_series", "remove_interiors", - "merge_disjointed_polygons", - "merge_disjointed_gdf_geometries", ] diff --git a/srai/utils/geometry.py b/srai/utils/geometry.py index eb96ae44..aea07bb0 100644 --- a/srai/utils/geometry.py +++ b/srai/utils/geometry.py @@ -1,10 +1,10 @@ """Utility geometry operations functions.""" -from typing import List +from typing import List, Union import geopandas as gpd import pyproj from functional import seq -from shapely.geometry import Polygon +from shapely.geometry import MultiPolygon, Polygon from shapely.geometry.base import BaseGeometry, BaseMultipartGeometry from shapely.ops import transform as shapely_transform @@ -71,3 +71,39 @@ def buffer_geometry(geometry: BaseGeometry, meters: float) -> BaseGeometry: bufferred_projected_geometry = projected_geometry.buffer(meters) return shapely_transform(aeqd_to_wgs84, bufferred_projected_geometry) + + +def merge_disjointed_polygons(polygons: List[Union[Polygon, MultiPolygon]]) -> MultiPolygon: + """ + Merges all polygons into a single MultiPolygon. + + Input polygons are expected to be disjointed. + + Args: + polygons (List[Union[Polygon, MultiPolygon]]): List of polygons to merge + + Returns: + MultiPolygon: Merged polygon + """ + single_polygons = [] + for geom in polygons: + if isinstance(geom, Polygon): + single_polygons.append(geom) + else: + single_polygons.extend(geom.geoms) + return MultiPolygon(single_polygons) + + +def merge_disjointed_gdf_geometries(gdf: gpd.GeoDataFrame) -> MultiPolygon: + """ + Merges geometries from a GeoDataFrame into a single MultiPolygon. + + Input geometries are expected to be disjointed. + + Args: + gdf (gpd.GeoDataFrame): GeoDataFrame with geometries to merge. + + Returns: + MultiPolygon: Merged polygon + """ + return merge_disjointed_polygons(list(gdf.geometry)) diff --git a/srai/utils/merge.py b/srai/utils/merge.py deleted file mode 100644 index 41ab5512..00000000 --- a/srai/utils/merge.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Utility function for merging Shapely polygons.""" - -from typing import List, Union - -import geopandas as gpd -from shapely.geometry import MultiPolygon, Polygon - - -def merge_disjointed_polygons(polygons: List[Union[Polygon, MultiPolygon]]) -> MultiPolygon: - """ - Merges all polygons into a single MultiPolygon. - - Input polygons are expected to be disjointed. - - Args: - polygons (List[Union[Polygon, MultiPolygon]]): List of polygons to merge - - Returns: - MultiPolygon: Merged polygon - """ - single_polygons = [] - for geom in polygons: - if type(geom) is Polygon: - single_polygons.append(geom) - else: - single_polygons.extend(geom.geoms) - return MultiPolygon(single_polygons) - - -def merge_disjointed_gdf_geometries(gdf: gpd.GeoDataFrame) -> MultiPolygon: - """ - Merges geometries from a GeoDataFrame into a single MultiPolygon. - - Input geometries are expected to be disjointed. - - Args: - gdf (gpd.GeoDataFrame): GeoDataFrame with geometries to merge. - - Returns: - MultiPolygon: Merged polygon - """ - return merge_disjointed_polygons(list(gdf.geometry)) diff --git a/tests/regionalizers/test_administrative_boundary_regionalizer.py b/tests/regionalizers/test_administrative_boundary_regionalizer.py index 4592440c..3335816f 100644 --- a/tests/regionalizers/test_administrative_boundary_regionalizer.py +++ b/tests/regionalizers/test_administrative_boundary_regionalizer.py @@ -10,7 +10,7 @@ from srai.constants import GEOMETRY_COLUMN, WGS84_CRS from srai.regionalizers import AdministrativeBoundaryRegionalizer -from srai.utils import merge_disjointed_gdf_geometries +from srai.utils.geometry import merge_disjointed_gdf_geometries bbox = box(minx=-180, maxx=180, miny=-90, maxy=90) bbox_gdf = gpd.GeoDataFrame({GEOMETRY_COLUMN: [bbox]}, crs=WGS84_CRS) diff --git a/tests/regionalizers/test_voronoi_regionalizer.py b/tests/regionalizers/test_voronoi_regionalizer.py index 20118a4e..11a716c5 100644 --- a/tests/regionalizers/test_voronoi_regionalizer.py +++ b/tests/regionalizers/test_voronoi_regionalizer.py @@ -9,7 +9,7 @@ from srai.constants import GEOMETRY_COLUMN, WGS84_CRS from srai.regionalizers import VoronoiRegionalizer from srai.regionalizers._spherical_voronoi import generate_voronoi_regions -from srai.utils import merge_disjointed_gdf_geometries +from srai.utils.geometry import merge_disjointed_gdf_geometries def test_empty_gdf_attribute_error(gdf_empty: gpd.GeoDataFrame) -> None: From 5c7db9e21f239e6dccf247404c517670d4e21c81 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 15:56:47 +0200 Subject: [PATCH 07/30] refactor: remove geometry module from utils --- examples/loaders/osm_pbf_loader.ipynb | 2 +- srai/{utils => }/geometry.py | 0 srai/loaders/osm_loaders/pbf_file_downloader.py | 2 +- .../administrative_boundary_regionalizer.py | 2 +- srai/utils/__init__.py | 14 -------------- .../test_administrative_boundary_regionalizer.py | 2 +- tests/regionalizers/test_voronoi_regionalizer.py | 2 +- 7 files changed, 5 insertions(+), 19 deletions(-) rename srai/{utils => }/geometry.py (100%) delete mode 100644 srai/utils/__init__.py diff --git a/examples/loaders/osm_pbf_loader.ipynb b/examples/loaders/osm_pbf_loader.ipynb index aafba38b..cc3c6175 100644 --- a/examples/loaders/osm_pbf_loader.ipynb +++ b/examples/loaders/osm_pbf_loader.ipynb @@ -25,7 +25,7 @@ "from srai.loaders.osm_loaders import OSMPbfLoader\n", "from srai.constants import REGIONS_INDEX, WGS84_CRS\n", "from srai.regionalizers import geocode_to_region_gdf\n", - "from srai.utils import buffer_geometry\n", + "from srai.geometry import buffer_geometry\n", "\n", "from shapely.geometry import Point, box\n", "import geopandas as gpd" diff --git a/srai/utils/geometry.py b/srai/geometry.py similarity index 100% rename from srai/utils/geometry.py rename to srai/geometry.py diff --git a/srai/loaders/osm_loaders/pbf_file_downloader.py b/srai/loaders/osm_loaders/pbf_file_downloader.py index 05b0a733..695f3441 100644 --- a/srai/loaders/osm_loaders/pbf_file_downloader.py +++ b/srai/loaders/osm_loaders/pbf_file_downloader.py @@ -19,8 +19,8 @@ from tqdm import tqdm from srai.constants import WGS84_CRS +from srai.geometry import buffer_geometry, flatten_geometry, remove_interiors from srai.loaders import download_file -from srai.utils import buffer_geometry, flatten_geometry, remove_interiors class PbfFileDownloader: diff --git a/srai/regionalizers/administrative_boundary_regionalizer.py b/srai/regionalizers/administrative_boundary_regionalizer.py index 522f6ae1..1f8b6bc6 100644 --- a/srai/regionalizers/administrative_boundary_regionalizer.py +++ b/srai/regionalizers/administrative_boundary_regionalizer.py @@ -16,8 +16,8 @@ from srai._optional import import_optional_dependencies from srai.constants import GEOMETRY_COLUMN, REGIONS_INDEX, WGS84_CRS +from srai.geometry import flatten_geometry_series from srai.regionalizers import Regionalizer -from srai.utils import flatten_geometry_series class AdministrativeBoundaryRegionalizer(Regionalizer): diff --git a/srai/utils/__init__.py b/srai/utils/__init__.py deleted file mode 100644 index 948b277b..00000000 --- a/srai/utils/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Module containing different utility functions. - -Those are either used internally by other modules, or can be used to simplify spatial data -processing. -""" -from .geometry import buffer_geometry, flatten_geometry, flatten_geometry_series, remove_interiors - -__all__ = [ - "buffer_geometry", - "flatten_geometry", - "flatten_geometry_series", - "remove_interiors", -] diff --git a/tests/regionalizers/test_administrative_boundary_regionalizer.py b/tests/regionalizers/test_administrative_boundary_regionalizer.py index 3335816f..fdd264ad 100644 --- a/tests/regionalizers/test_administrative_boundary_regionalizer.py +++ b/tests/regionalizers/test_administrative_boundary_regionalizer.py @@ -9,8 +9,8 @@ from shapely.geometry import Point, box from srai.constants import GEOMETRY_COLUMN, WGS84_CRS +from srai.geometry import merge_disjointed_gdf_geometries from srai.regionalizers import AdministrativeBoundaryRegionalizer -from srai.utils.geometry import merge_disjointed_gdf_geometries bbox = box(minx=-180, maxx=180, miny=-90, maxy=90) bbox_gdf = gpd.GeoDataFrame({GEOMETRY_COLUMN: [bbox]}, crs=WGS84_CRS) diff --git a/tests/regionalizers/test_voronoi_regionalizer.py b/tests/regionalizers/test_voronoi_regionalizer.py index 11a716c5..dc0e155e 100644 --- a/tests/regionalizers/test_voronoi_regionalizer.py +++ b/tests/regionalizers/test_voronoi_regionalizer.py @@ -7,9 +7,9 @@ from shapely.geometry import Point, Polygon from srai.constants import GEOMETRY_COLUMN, WGS84_CRS +from srai.geometry import merge_disjointed_gdf_geometries from srai.regionalizers import VoronoiRegionalizer from srai.regionalizers._spherical_voronoi import generate_voronoi_regions -from srai.utils.geometry import merge_disjointed_gdf_geometries def test_empty_gdf_attribute_error(gdf_empty: gpd.GeoDataFrame) -> None: From a8ce08c46ccffa3d273d46b64afa59fbab171fdc Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 20 Jul 2023 16:03:36 +0200 Subject: [PATCH 08/30] chore: changed changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e90cf25..195092e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactored H3Regionalizer to be faster using [h3ronpy](https://github.com/nmandery/h3ronpy) library [#311](https://github.com/srai-lab/srai/issues/311) - BREAKING! Renamed NetworkType to OSMNetworkType and made it importable directly from `srai.loaders` [#227](https://github.com/srai-lab/srai/issues/227) +- BREAKING! Removed `utils` module [#128](https://github.com/srai-lab/srai/issues/128) + - `srai.utils._optional` moved to `srai._optional` + - `srai.utils._pytorch_stubs` moved to `srai.embedders._pytorch_stubs` + - `srai.utils.download` moved to `srai.loaders.download` (and can be imported with `from srai.loaders import download_file`) + - `srai.utils.geocode` moved to `srai.regionalizers.geocode` (and can be imported with `from srai.regionalizers import geocode_to_region_gdf`) + - `srai.utils.geometry` and `srai.utils.merge` moved to `srai.geometry` + - `srai.utils.typing` moved to `srai._typing` ### Deprecated From 0d2059bb932c2f69ac0fde83c67b3e2d9aec2da9 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Fri, 21 Jul 2023 00:43:05 +0200 Subject: [PATCH 09/30] feat: add h3 helper module --- CHANGELOG.md | 2 + srai/h3.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 srai/h3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 195092e9..0d76cbcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `srai.h3` module with functions for translating list of h3 cells into shapely polygons and calculating local ij coordinates. + ### Changed - Refactored H3Regionalizer to be faster using [h3ronpy](https://github.com/nmandery/h3ronpy) library [#311](https://github.com/srai-lab/srai/issues/311) diff --git a/srai/h3.py b/srai/h3.py new file mode 100644 index 00000000..de89e0b7 --- /dev/null +++ b/srai/h3.py @@ -0,0 +1,118 @@ +"""Utility H3 related functions.""" + +from collections.abc import Iterable +from typing import List, Literal, Tuple, Union, overload + +import geopandas as gpd +import h3 +import numpy as np +import numpy.typing as npt +from h3ronpy.arrow.vector import cells_to_wkb_polygons, wkb_to_cells +from shapely.geometry import Polygon +from shapely.geometry.base import BaseGeometry + +from srai.constants import WGS84_CRS + + +def shapely_geometry_to_h3( + geometry: Union[BaseGeometry, Iterable[BaseGeometry], gpd.GeoSeries], + h3_resolution: int, + buffer: bool, +) -> List[str]: + """TODO.""" + wkb = [] + if isinstance(geometry, Iterable): + wkb = [sub_geometry.wkb for sub_geometry in geometry] + elif isinstance(geometry, gpd.GeoSeries): + wkb = geometry.to_wkb() + else: + wkb = [geometry.wkb] + + h3_indexes = wkb_to_cells( + wkb, resolution=h3_resolution, all_intersecting=buffer, flatten=True + ).unique() + + return [h3.int_to_str(h3_index) for h3_index in h3_indexes.tolist()] + + +def h3_to_geoseries(h3_index: Union[int, str, Iterable[Union[int, str]]]) -> gpd.GeoSeries: + """TODO.""" + if isinstance(h3_index, (str, int)): + return h3_to_geoseries([h3_index]) + else: + h3_int_indexes = ( + h3_cell if isinstance(h3_cell, int) else h3.str_to_int(h3_cell) for h3_cell in h3_index + ) + return gpd.GeoSeries.from_wkb(cells_to_wkb_polygons(h3_int_indexes), crs=WGS84_CRS) + + +@overload +def h3_to_shapely_geometry(h3_index: Union[int, str]) -> Polygon: + ... + + +@overload +def h3_to_shapely_geometry(h3_index: Iterable[Union[int, str]]) -> List[Polygon]: + ... + + +def h3_to_shapely_geometry( + h3_index: Union[int, str, Iterable[Union[int, str]]] +) -> Union[Polygon, List[Polygon]]: + """TODO.""" + if isinstance(h3_index, (str, int)): + coords = h3.cell_to_boundary(h3_index, geo_json=True) + return Polygon(coords) + else: + return h3_to_geoseries(h3_index).values.tolist() + + +@overload +def get_local_ij_index(origin_index: str, h3_index: str) -> Tuple[int, int]: + ... + + +@overload +def get_local_ij_index( + origin_index: str, h3_index: List[str], return_as_numpy: Literal[False] +) -> List[Tuple[int, int]]: + ... + + +@overload +def get_local_ij_index( + origin_index: str, h3_index: List[str], return_as_numpy: Literal[True] +) -> npt.NDArray[np.int8]: + ... + + +def get_local_ij_index( + origin_index: str, h3_index: Union[str, List[str]], return_as_numpy: bool = False +) -> Union[Tuple[int, int], List[Tuple[int, int]], npt.NDArray[np.int8]]: + """ + Calculate the local H3 ij index based on provided origin index. + + Wraps H3's cell_to_local_ij function and centers returned coordinates + around provided origin cell. + + Args: + origin_index (str): H3 index of the origin region. + h3_index (Union[str, List[str]]): H3 index of the second region or list of regions. + return_as_numpy (bool, optional): Flag whether to return calculated indexes as a Numpy array + or a list of tuples. + + Returns: + Union[Tuple[int, int], List[Tuple[int, int]], npt.NDArray[np.int8]]: The local ij index of + the second region (or regions) with respect to the first one. + """ + origin_coords = h3.cell_to_local_ij(origin_index, origin_index) + if isinstance(h3_index, str): + ijs = h3.cell_to_local_ij(origin_index, h3_index) + return (origin_coords[0] - ijs[0], origin_coords[1] - ijs[1]) + ijs = np.array([h3.cell_to_local_ij(origin_index, h3_cell) for h3_cell in h3_index]) + local_ijs = np.array(origin_coords) - ijs + + if not return_as_numpy: + local_ijs = [(coords[0], coords[1]) for coords in local_ijs] + + return local_ijs From 7c888f33f47313e52e13c4fd05907b12f5ba9329 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Fri, 21 Jul 2023 00:56:09 +0200 Subject: [PATCH 10/30] refactor: simplified imports aftere merge --- srai/loaders/osm_loaders/osm_online_loader.py | 2 +- srai/loaders/osm_loaders/pbf_file_handler.py | 2 +- tests/loaders/osm_loaders/filters/test_merge_filter_types.py | 2 +- tests/loaders/osm_loaders/test_osm_online_loader.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/srai/loaders/osm_loaders/osm_online_loader.py b/srai/loaders/osm_loaders/osm_online_loader.py index 08908d83..5df71f46 100644 --- a/srai/loaders/osm_loaders/osm_online_loader.py +++ b/srai/loaders/osm_loaders/osm_online_loader.py @@ -14,7 +14,7 @@ from srai._optional import import_optional_dependencies from srai.constants import FEATURES_INDEX, GEOMETRY_COLUMN, WGS84_CRS from srai.loaders.osm_loaders._base import OSMLoader -from srai.loaders.osm_loaders.filters._typing import GroupedOsmTagsFilter, OsmTagsFilter +from srai.loaders.osm_loaders.filters import GroupedOsmTagsFilter, OsmTagsFilter class OSMOnlineLoader(OSMLoader): diff --git a/srai/loaders/osm_loaders/pbf_file_handler.py b/srai/loaders/osm_loaders/pbf_file_handler.py index bd58bc01..0461470c 100644 --- a/srai/loaders/osm_loaders/pbf_file_handler.py +++ b/srai/loaders/osm_loaders/pbf_file_handler.py @@ -16,7 +16,7 @@ from tqdm import tqdm from srai.constants import FEATURES_INDEX, WGS84_CRS -from srai.loaders.osm_loaders.filters._typing import OsmTagsFilter +from srai.loaders.osm_loaders.filters import OsmTagsFilter if TYPE_CHECKING: import os diff --git a/tests/loaders/osm_loaders/filters/test_merge_filter_types.py b/tests/loaders/osm_loaders/filters/test_merge_filter_types.py index 448306ca..fb0a336c 100644 --- a/tests/loaders/osm_loaders/filters/test_merge_filter_types.py +++ b/tests/loaders/osm_loaders/filters/test_merge_filter_types.py @@ -5,7 +5,7 @@ import pytest -from srai.loaders.osm_loaders.filters._typing import OsmTagsFilter, merge_grouped_osm_tags_filter +from srai.loaders.osm_loaders.filters import OsmTagsFilter, merge_grouped_osm_tags_filter ut = TestCase() diff --git a/tests/loaders/osm_loaders/test_osm_online_loader.py b/tests/loaders/osm_loaders/test_osm_online_loader.py index 4d4853cc..56bed6f5 100644 --- a/tests/loaders/osm_loaders/test_osm_online_loader.py +++ b/tests/loaders/osm_loaders/test_osm_online_loader.py @@ -7,7 +7,7 @@ from srai.constants import WGS84_CRS from srai.loaders.osm_loaders import OSMOnlineLoader -from srai.loaders.osm_loaders.filters._typing import OsmTagsFilter +from srai.loaders.osm_loaders.filters import OsmTagsFilter if TYPE_CHECKING: from shapely.geometry import Polygon From 52609a7ab207473bba32ad50c8fffcdf671e3665 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Fri, 21 Jul 2023 09:09:48 +0200 Subject: [PATCH 11/30] refactor: apply refurb suggestion --- srai/h3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index de89e0b7..ce1117e8 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -63,8 +63,7 @@ def h3_to_shapely_geometry( if isinstance(h3_index, (str, int)): coords = h3.cell_to_boundary(h3_index, geo_json=True) return Polygon(coords) - else: - return h3_to_geoseries(h3_index).values.tolist() + return h3_to_geoseries(h3_index).values.tolist() @overload From becdb717e816bdf10be31f8badd4fe6fdf574512 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Fri, 21 Jul 2023 13:45:14 +0200 Subject: [PATCH 12/30] chore: add docstrings and change H3Regionalizer --- srai/h3.py | 59 +++++++++++++++++++++++---- srai/regionalizers/h3_regionalizer.py | 22 +++++----- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index ce1117e8..bd0f6b68 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -11,20 +11,43 @@ from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry -from srai.constants import WGS84_CRS +from srai.constants import GEOMETRY_COLUMN, WGS84_CRS def shapely_geometry_to_h3( - geometry: Union[BaseGeometry, Iterable[BaseGeometry], gpd.GeoSeries], + geometry: Union[BaseGeometry, Iterable[BaseGeometry], gpd.GeoSeries, gpd.GeoDataFrame], h3_resolution: int, - buffer: bool, + buffer: bool = True, ) -> List[str]: - """TODO.""" + """ + Convert Shapely geometry to H3 indexes. + + Args: + geometry (Union[BaseGeometry, Iterable[BaseGeometry], GeoSeries, GeoDataFrame]): + Shapely geometry to be converted. + h3_resolution (int): H3 resolution of the cells. See [1] for a full comparison. + buffer (bool, optional): Whether to fully cover the geometries with + H3 Cells (visible on the borders). Defaults to True. + + Returns: + List[str]: List of H3 indexes that cover a given geometry. + + Raises: + ValueError: If resolution is not between 0 and 15. + + References: + 1. https://h3geo.org/docs/core-library/restable/ + """ + if not (0 <= h3_resolution <= 15): + raise ValueError(f"Resolution {h3_resolution} is not between 0 and 15.") + wkb = [] - if isinstance(geometry, Iterable): - wkb = [sub_geometry.wkb for sub_geometry in geometry] - elif isinstance(geometry, gpd.GeoSeries): + if isinstance(geometry, gpd.GeoSeries): wkb = geometry.to_wkb() + elif isinstance(geometry, gpd.GeoDataFrame): + wkb = geometry[GEOMETRY_COLUMN].to_wkb() + elif isinstance(geometry, Iterable): + wkb = [sub_geometry.wkb for sub_geometry in geometry] else: wkb = [geometry.wkb] @@ -36,7 +59,16 @@ def shapely_geometry_to_h3( def h3_to_geoseries(h3_index: Union[int, str, Iterable[Union[int, str]]]) -> gpd.GeoSeries: - """TODO.""" + """ + Convert H3 index to GeoPandas GeoSeries. + + Args: + h3_index (Union[int, str, Iterable[Union[int, str]]]): H3 index (or list of indexes) + to be converted. + + Returns: + GeoSeries: Geometries as GeoSeries with default CRS applied. + """ if isinstance(h3_index, (str, int)): return h3_to_geoseries([h3_index]) else: @@ -59,7 +91,16 @@ def h3_to_shapely_geometry(h3_index: Iterable[Union[int, str]]) -> List[Polygon] def h3_to_shapely_geometry( h3_index: Union[int, str, Iterable[Union[int, str]]] ) -> Union[Polygon, List[Polygon]]: - """TODO.""" + """ + Convert H3 index to Shapely polygon. + + Args: + h3_index (Union[int, str, Iterable[Union[int, str]]]): H3 index (or list of indexes) + to be converted. + + Returns: + Union[Polygon, List[Polygon]]: Converted polygon (or list of polygons). + """ if isinstance(h3_index, (str, int)): coords = h3.cell_to_boundary(h3_index, geo_json=True) return Polygon(coords) diff --git a/srai/regionalizers/h3_regionalizer.py b/srai/regionalizers/h3_regionalizer.py index 855c96b9..ac917150 100644 --- a/srai/regionalizers/h3_regionalizer.py +++ b/srai/regionalizers/h3_regionalizer.py @@ -15,10 +15,9 @@ import geopandas as gpd -import h3 -from h3ronpy.arrow.vector import cells_to_wkb_polygons, wkb_to_cells from srai.constants import GEOMETRY_COLUMN, REGIONS_INDEX, WGS84_CRS +from srai.h3 import h3_to_geoseries, shapely_geometry_to_h3 from srai.regionalizers import Regionalizer @@ -71,15 +70,18 @@ def transform(self, gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: gdf_exploded = self._explode_multipolygons(gdf_wgs84) - h3_indexes = wkb_to_cells( - gdf_exploded[GEOMETRY_COLUMN].to_wkb(), - resolution=self.resolution, - all_intersecting=self.buffer, - flatten=True, - ).unique() + h3_indexes = list( + set( + shapely_geometry_to_h3( + gdf_exploded[GEOMETRY_COLUMN], + h3_resolution=self.resolution, + buffer=self.buffer, + ) + ) + ) gdf_h3 = gpd.GeoDataFrame( - data={REGIONS_INDEX: [h3.int_to_str(h3_index) for h3_index in h3_indexes.tolist()]}, - geometry=gpd.GeoSeries.from_wkb(cells_to_wkb_polygons(h3_indexes)), + data={REGIONS_INDEX: h3_indexes}, + geometry=h3_to_geoseries(h3_indexes), crs=WGS84_CRS, ).set_index(REGIONS_INDEX) From d89ee70d70d897fbfa92994b42f68311ecb12c86 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Fri, 21 Jul 2023 15:42:20 +0200 Subject: [PATCH 13/30] chore: add h3 tests files --- tests/h3/test_ij_coordinates.py | 19 +++++++++++++++++++ tests/h3/test_shapely_conversion.py | 1 + 2 files changed, 20 insertions(+) create mode 100644 tests/h3/test_ij_coordinates.py create mode 100644 tests/h3/test_shapely_conversion.py diff --git a/tests/h3/test_ij_coordinates.py b/tests/h3/test_ij_coordinates.py new file mode 100644 index 00000000..07b993f4 --- /dev/null +++ b/tests/h3/test_ij_coordinates.py @@ -0,0 +1,19 @@ +"""H3 IJ coordinates tests.""" +# import sys +# from contextlib import nullcontext as does_not_raise +# from typing import Any, List + +# import geopandas as gpd +# import pytest + +# from srai.h3 import get_local_ij_index + +# def _get_random_point() + +# def test_self_ok(mocker: MockerFixture, gtfs_validation_ok: pd.DataFrame) -> None: +# """Test checks if GTFSLoader returns no errors.""" +# feed_mock = mocker.MagicMock() +# feed_mock.configure_mock(**{"validate.return_value": gtfs_validation_ok}) + +# loader = GTFSLoader() +# loader._validate_feed(feed_mock) diff --git a/tests/h3/test_shapely_conversion.py b/tests/h3/test_shapely_conversion.py new file mode 100644 index 00000000..1dbced97 --- /dev/null +++ b/tests/h3/test_shapely_conversion.py @@ -0,0 +1 @@ +"""H3 shapely conversion tests.""" From 71a0378b0d2466060b4d8c2048c66ae391f443a3 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 27 Jul 2023 20:58:06 +0200 Subject: [PATCH 14/30] chore: change iterable import --- srai/h3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index bd0f6b68..f6b899c7 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -1,7 +1,6 @@ """Utility H3 related functions.""" -from collections.abc import Iterable -from typing import List, Literal, Tuple, Union, overload +from typing import Iterable, List, Literal, Tuple, Union, overload import geopandas as gpd import h3 From 63940b4b8c24b5fab4d2f7add85ef0ede65d061a Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Thu, 27 Jul 2023 23:54:29 +0200 Subject: [PATCH 15/30] test: add h3 ij tests --- srai/h3.py | 9 +++ tests/h3/test_ij_coordinates.py | 113 ++++++++++++++++++++++++++++---- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index f6b899c7..2dadd513 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -125,6 +125,15 @@ def get_local_ij_index( ... +# Last fallback needed as per documentation: +# https://mypy.readthedocs.io/en/stable/literal_types.html#literal-types +@overload +def get_local_ij_index( + origin_index: str, h3_index: List[str], return_as_numpy: bool +) -> Union[List[Tuple[int, int]], npt.NDArray[np.int8]]: + ... + + def get_local_ij_index( origin_index: str, h3_index: Union[str, List[str]], return_as_numpy: bool = False ) -> Union[Tuple[int, int], List[Tuple[int, int]], npt.NDArray[np.int8]]: diff --git a/tests/h3/test_ij_coordinates.py b/tests/h3/test_ij_coordinates.py index 07b993f4..8ed34dbe 100644 --- a/tests/h3/test_ij_coordinates.py +++ b/tests/h3/test_ij_coordinates.py @@ -1,19 +1,106 @@ """H3 IJ coordinates tests.""" -# import sys -# from contextlib import nullcontext as does_not_raise -# from typing import Any, List -# import geopandas as gpd -# import pytest +from typing import List -# from srai.h3 import get_local_ij_index +import h3 +import numpy as np +import pytest -# def _get_random_point() +from srai.h3 import get_local_ij_index -# def test_self_ok(mocker: MockerFixture, gtfs_validation_ok: pd.DataFrame) -> None: -# """Test checks if GTFSLoader returns no errors.""" -# feed_mock = mocker.MagicMock() -# feed_mock.configure_mock(**{"validate.return_value": gtfs_validation_ok}) -# loader = GTFSLoader() -# loader._validate_feed(feed_mock) +@pytest.mark.parametrize( + "h3_origin", + [ + "891e2040d4bffff", + "871e20400ffffff", + "821f77fffffffff", + "81743ffffffffff", + ], +) # type: ignore +def test_self_ok(h3_origin: str) -> None: + """Test checks if self coordinates are in origin.""" + coordinates = get_local_ij_index(origin_index=h3_origin, h3_index=h3_origin) + assert coordinates == (0, 0) + + +@pytest.mark.parametrize( + "h3_origin, h3_cell", + [ + ("871f53c93ffffff", "871f53c91ffffff"), + ("861fae207ffffff", "861fae22fffffff"), + ("81597ffffffffff", "813fbffffffffff"), + ("84be185ffffffff", "84be181ffffffff"), + ], +) # type: ignore +def test_string_ok(h3_origin: str, h3_cell: str) -> None: + """Test checks if pairs are in right orientation.""" + coordinates = get_local_ij_index(origin_index=h3_origin, h3_index=h3_cell) + assert coordinates == (0, 1) + + +@pytest.mark.parametrize( + "h3_origin, h3_cells", + [ + ( + "892a100d6d3ffff", + [ + "892a100896fffff", + "892a100d6d7ffff", + "892a100d6c3ffff", + "892a100d6dbffff", + "892a1008ba7ffff", + "892a100896bffff", + ], + ), + ( + "86195da4fffffff", + [ + "86194ad37ffffff", + "86194ad17ffffff", + "86194ada7ffffff", + "86195da5fffffff", + "86195da47ffffff", + "86195da6fffffff", + ], + ), + ( + "8a1e24aa5637fff", + [ + "8a1e24aa5627fff", + "8a1e24aa5607fff", + "8a1e24aa5617fff", + "8a1e24aa578ffff", + "8a1e24aa57affff", + "8a1e24aa571ffff", + ], + ), + ], +) # type: ignore +@pytest.mark.parametrize("return_as_numpy", [True, False]) # type: ignore +def test_list_ok(h3_origin: str, h3_cells: List[str], return_as_numpy: bool) -> None: + """Test checks if lists are parsed correctly.""" + coordinates = get_local_ij_index( + origin_index=h3_origin, h3_index=h3_cells, return_as_numpy=return_as_numpy + ) + + expected_coordinates = [(0, 1), (1, 1), (1, 0), (0, -1), (-1, -1), (-1, 0)] + + if return_as_numpy: + np.testing.assert_array_equal(coordinates, expected_coordinates) + else: + assert coordinates == expected_coordinates + + +@pytest.mark.parametrize( + "h3_origin, h3_cell", + [ + ("83a75dfffffffff", "83a791fffffffff"), + ("84a605bffffffff", "84a6021ffffffff"), + ("836200fffffffff", "837400fffffffff"), + ], +) # type: ignore +def test_pentagon_error(h3_origin: str, h3_cell: str) -> None: + """Test checks if method fails over pentagon pairs.""" + with pytest.raises(h3._cy.error_system.H3FailedError): + get_local_ij_index(origin_index=h3_origin, h3_index=h3_cell) From b33eea8f12d10e2c830a19583049b7ba9114bd70 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 21:42:26 +0200 Subject: [PATCH 16/30] test: added new tests for h3 module --- pdm.lock | 14 +-- pyproject.toml | 2 +- tests/h3/conftest.py | 24 +++++ tests/h3/test_shapely_conversion.py | 89 ++++++++++++++++++ tests/regionalizers/conftest.py | 141 +--------------------------- tests/regionalizers/fixtures.py | 141 ++++++++++++++++++++++++++++ 6 files changed, 263 insertions(+), 148 deletions(-) create mode 100644 tests/h3/conftest.py create mode 100644 tests/regionalizers/fixtures.py diff --git a/pdm.lock b/pdm.lock index 0f1d6dd4..f33d1a8f 100644 --- a/pdm.lock +++ b/pdm.lock @@ -529,7 +529,7 @@ summary = "Hierarchical hexagonal geospatial indexing system" [[package]] name = "h3ronpy" -version = "0.17.2" +version = "0.17.4" summary = "Data science toolkit for the H3 geospatial grid" dependencies = [ "Shapely>=1.7", @@ -2360,7 +2360,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "4.2" groups = ["default", "all", "dev", "docs", "gtfs", "license", "lint", "osm", "performance", "plotting", "test", "torch", "visualization", "voronoi"] -content_hash = "sha256:fc6f33de0a32db546b016e11ef714b82da4b979d79ab0c31f90599fdd4f6a78f" +content_hash = "sha256:bf7006156f874b71f59d10d18a20457833beb6e8dacd6ac8c8afb13123453a19" [metadata.files] "aiohttp 3.8.5" = [ @@ -3111,11 +3111,11 @@ content_hash = "sha256:fc6f33de0a32db546b016e11ef714b82da4b979d79ab0c31f90599fdd {url = "https://files.pythonhosted.org/packages/fd/30/ce35791eac7efce2ff6458a94f1a936c8631418e0a0d2a8bf30fdd3ee51b/h3-4.0.0b2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e952df0562581c2a564fed48d5910822963307756de93157ba5a5f8535b42044"}, {url = "https://files.pythonhosted.org/packages/fe/aa/9a97f3f866f79232a07f30bc838bb48e46b12ccf9e40b052720db7fd70ee/h3-4.0.0b2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0489e8892472b9514825d480544bac854862ce5cf94c7f497936053565787954"}, ] -"h3ronpy 0.17.2" = [ - {url = "https://files.pythonhosted.org/packages/0f/87/bd9b72f3e127732e9e84f6ab14971c64d537b08344f80163a7ed3e76c410/h3ronpy-0.17.2.tar.gz", hash = "sha256:b0b1adb5c5709a1a63c6d6bcc20706a29fa7263e4d0d9d255c0bc9a0f0b26b76"}, - {url = "https://files.pythonhosted.org/packages/24/2d/8eeee30f3894940c0d47d8a3d85005fb9a8fc8e93fdbdbc0a702f23085e4/h3ronpy-0.17.2-cp38-abi3-macosx_10_7_x86_64.whl", hash = "sha256:43bc39f43f4d5342ace553296d23bee9d3ea3b10c0850ab9d01d2373a28bcdea"}, - {url = "https://files.pythonhosted.org/packages/90/23/991ba93c9f8333bd2cf3d3333bc85336cfc189a20ff9c2c8e7464e492ae3/h3ronpy-0.17.2-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:14a5b5e14a7d54e06534fc94a11aa22792114dfab9a09e4a4c4ee19fe94e5c56"}, - {url = "https://files.pythonhosted.org/packages/f2/f3/d107af14a392b0bae1fecab3c8105caa7ca817320bd9be0c528901b25106/h3ronpy-0.17.2-cp38-abi3-win_amd64.whl", hash = "sha256:622e325abe3b276ed264d855a3c9ff76ab5cac037a92961f4c4853e45973a9fa"}, +"h3ronpy 0.17.4" = [ + {url = "https://files.pythonhosted.org/packages/84/f1/7f41961ad818f9b49fb16e68bc704218f03ad65beef48b96aefc4c0aea54/h3ronpy-0.17.4-cp38-abi3-win_amd64.whl", hash = "sha256:bcdd460dcf69cdfb03aca5e0d8599e1fe670d9b40e7a69133996d474cde5a742"}, + {url = "https://files.pythonhosted.org/packages/ad/bf/6feca53dd0797fd83e117b8547c1cba41e46782f824d2b1b2b57dac84bd4/h3ronpy-0.17.4.tar.gz", hash = "sha256:1a341d82122366fbbaef29550449d7c67854ad351f15be02e8d600b8cb810fed"}, + {url = "https://files.pythonhosted.org/packages/c0/e3/3362e56e1b6aeb6e21dd8d2a2c0f8ecb48a8132f39772c6f3478f95e5019/h3ronpy-0.17.4-cp38-abi3-macosx_10_7_x86_64.whl", hash = "sha256:82b761c2febd840779866d1370a8860fcf0457c3c6c94d296f8abe31e2028cf0"}, + {url = "https://files.pythonhosted.org/packages/cb/bc/1ebeee02871998301835da8ad1d26e15bf514e89e70179118ee5ac5036ed/h3ronpy-0.17.4-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:dbc6f272b48d0824f742be89eae59a13e3e6f3932db40f39f8c268cea6d2787a"}, ] "haversine 2.8.0" = [ {url = "https://files.pythonhosted.org/packages/b9/6b/0a774af6a2eea772aa99e5fbc7af7711eba02ff0dee3e71838c1b5926ef5/haversine-2.8.0.tar.gz", hash = "sha256:cca39afd2ae5f1e6ed9231b332395bb8afb2e0a64edf70c238c176492e60c150"}, diff --git a/pyproject.toml b/pyproject.toml index c2261caa..d7c015bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "s2", "typeguard", "requests", - "h3ronpy", + "h3ronpy>=0.17.4", ] requires-python = ">=3.8" readme = "README.md" diff --git a/tests/h3/conftest.py b/tests/h3/conftest.py new file mode 100644 index 00000000..935b6dea --- /dev/null +++ b/tests/h3/conftest.py @@ -0,0 +1,24 @@ +"""Conftest for H3 tests.""" +from typing import List + +import geopandas as gpd +import pytest +from shapely.geometry import Point + +from srai.constants import WGS84_CRS + +pytest_plugins = ["tests.regionalizers.fixtures", "tests.regionalizers.test_h3_regionalizer"] + + +@pytest.fixture # type: ignore +def gdf_single_point() -> gpd.GeoDataFrame: + """Get the point case.""" + return gpd.GeoDataFrame(geometry=[Point(17.9261, 50.6696)], crs=WGS84_CRS) + + +@pytest.fixture # type: ignore +def expected_point_h3_index() -> List[str]: + """Get expected h3 index for the point case.""" + return [ + "8a1e23c44b5ffff", + ] diff --git a/tests/h3/test_shapely_conversion.py b/tests/h3/test_shapely_conversion.py index 1dbced97..b5c51715 100644 --- a/tests/h3/test_shapely_conversion.py +++ b/tests/h3/test_shapely_conversion.py @@ -1 +1,90 @@ """H3 shapely conversion tests.""" + +from typing import Any, Callable, List +from unittest import TestCase + +import geopandas as gpd +import pytest +from shapely.geometry.base import BaseGeometry + +from srai.constants import GEOMETRY_COLUMN +from srai.h3 import shapely_geometry_to_h3 +from tests.regionalizers.test_h3_regionalizer import H3_RESOLUTION + +ut = TestCase() + + +def _gdf_noop(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoDataFrame: + return gdf_fixture + + +def _gdf_to_geoseries(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoSeries: + return gdf_fixture[GEOMETRY_COLUMN] + + +def _gdf_to_geometry_list(gdf_fixture: gpd.GeoDataFrame) -> List[BaseGeometry]: + return list(gdf_fixture[GEOMETRY_COLUMN]) + + +def _gdf_to_single_geometry(gdf_fixture: gpd.GeoDataFrame) -> BaseGeometry: + return gdf_fixture[GEOMETRY_COLUMN][0] + + +@pytest.mark.parametrize( + "geometry_fixture, resolution, expected_h3_cells_fixture", + [ + ("gdf_single_point", 10, "expected_point_h3_index"), + ("gdf_multipolygon", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), + ("gdf_polygons", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), + ], +) # type: ignore +@pytest.mark.parametrize( + "geometry_parser_function", + [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +) # type: ignore +def test_shapely_geometry_to_h3_unbuffered( + geometry_fixture: str, + resolution: int, + expected_h3_cells_fixture: str, + geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], + request: pytest.FixtureRequest, +) -> None: + """Test checks if conversion from shapely to h3 works.""" + geometry = request.getfixturevalue(geometry_fixture) + expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + + parsed_geometry = geometry_parser_function(geometry) + h3_cells = shapely_geometry_to_h3( + geometry=parsed_geometry, h3_resolution=resolution, buffer=False + ) + ut.assertCountEqual(h3_cells, expected_h3_cells) + + +@pytest.mark.parametrize( + "geometry_fixture, resolution, expected_h3_cells_fixture", + [ + ("gdf_single_point", 10, "expected_point_h3_index"), + ("gdf_multipolygon", H3_RESOLUTION, "expected_h3_indexes"), + ("gdf_polygons", H3_RESOLUTION, "expected_h3_indexes"), + ], +) # type: ignore +@pytest.mark.parametrize( + "geometry_parser_function", + [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +) # type: ignore +def test_shapely_geometry_to_h3_buffered( + geometry_fixture: str, + resolution: int, + expected_h3_cells_fixture: str, + geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], + request: pytest.FixtureRequest, +) -> None: + """Test checks if conversion from shapely to h3 with buffer works.""" + geometry = request.getfixturevalue(geometry_fixture) + expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + + parsed_geometry = geometry_parser_function(geometry) + h3_cells = shapely_geometry_to_h3( + geometry=parsed_geometry, h3_resolution=resolution, buffer=True + ) + ut.assertCountEqual(h3_cells, expected_h3_cells) diff --git a/tests/regionalizers/conftest.py b/tests/regionalizers/conftest.py index 47b2b22d..d7e08430 100644 --- a/tests/regionalizers/conftest.py +++ b/tests/regionalizers/conftest.py @@ -1,141 +1,2 @@ """Conftest for Regionalizers.""" - -from typing import List - -import geopandas as gpd -import pytest -from shapely import geometry - -from srai.constants import GEOMETRY_COLUMN, WGS84_CRS - - -@pytest.fixture # type: ignore -def gdf_empty() -> gpd.GeoDataFrame: - """Get empty GeoDataFrame.""" - return gpd.GeoDataFrame() - - -@pytest.fixture # type: ignore -def gdf_no_crs() -> gpd.GeoDataFrame: - """Get GeoDataFrame with no crs.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.Polygon( - shell=[ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ] - ) - ] - ) - - -@pytest.fixture # type: ignore -def gdf_polygons() -> gpd.GeoDataFrame: - """Get GeoDataFrame with two polygons.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.Polygon( - shell=[ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ], - holes=[ - [ - (0.8, 0.9), - (0.9, 0.55), - (0.8, 0.3), - (0.5, 0.4), - ] - ], - ), - geometry.Polygon(shell=[(-0.25, 0), (0.25, 0), (0, 0.2)]), - ], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def gdf_multipolygon() -> gpd.GeoDataFrame: - """Get GeoDataFrame with multipolygon.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.MultiPolygon( - [ - ( - [ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ], - ( - [ - [ - (0.8, 0.9), - (0.9, 0.55), - (0.8, 0.3), - (0.5, 0.4), - ] - ] - ), - ), - ( - [(-0.25, 0), (0.25, 0), (0, 0.2)], - (), - ), - ] - ) - ], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def earth_poles() -> List[geometry.Point]: - """Get 6 Earth poles.""" - return [ - geometry.Point(0, 0), - geometry.Point(90, 0), - geometry.Point(180, 0), - geometry.Point(-90, 0), - geometry.Point(0, 90), - geometry.Point(0, -90), - ] - - -@pytest.fixture # type: ignore -def gdf_earth_poles(earth_poles) -> gpd.GeoDataFrame: - """Get GeoDataFrame with 6 Earth poles.""" - return gpd.GeoDataFrame( - {GEOMETRY_COLUMN: earth_poles}, - index=[1, 2, 3, 4, 5, 6], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def gdf_poland() -> gpd.GeoDataFrame: - """Get Poland GeoDataFrame.""" - world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) - area = world[world.name == "Poland"] - return area - - -@pytest.fixture # type: ignore -def earth_bbox() -> geometry.Polygon: - """Get full bounding box GeoDataFrame.""" - return geometry.box(minx=-180, maxx=180, miny=-90, maxy=90) - - -@pytest.fixture # type: ignore -def gdf_earth_bbox(earth_bbox) -> gpd.GeoDataFrame: - """Get full bounding box GeoDataFrame.""" - return gpd.GeoDataFrame({GEOMETRY_COLUMN: [earth_bbox]}, crs=WGS84_CRS) +pytest_plugins = ["tests.regionalizers.fixtures"] diff --git a/tests/regionalizers/fixtures.py b/tests/regionalizers/fixtures.py new file mode 100644 index 00000000..9320f1bf --- /dev/null +++ b/tests/regionalizers/fixtures.py @@ -0,0 +1,141 @@ +"""Fixtures for Regionalizers.""" + +from typing import List + +import geopandas as gpd +import pytest +from shapely import geometry + +from srai.constants import GEOMETRY_COLUMN, WGS84_CRS + + +@pytest.fixture # type: ignore +def gdf_empty() -> gpd.GeoDataFrame: + """Get empty GeoDataFrame.""" + return gpd.GeoDataFrame() + + +@pytest.fixture # type: ignore +def gdf_no_crs() -> gpd.GeoDataFrame: + """Get GeoDataFrame with no crs.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.Polygon( + shell=[ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ] + ) + ] + ) + + +@pytest.fixture # type: ignore +def gdf_polygons() -> gpd.GeoDataFrame: + """Get GeoDataFrame with two polygons.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.Polygon( + shell=[ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + holes=[ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ], + ), + geometry.Polygon(shell=[(-0.25, 0), (0.25, 0), (0, 0.2)]), + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def gdf_multipolygon() -> gpd.GeoDataFrame: + """Get GeoDataFrame with multipolygon.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.MultiPolygon( + [ + ( + [ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + ( + [ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ] + ), + ), + ( + [(-0.25, 0), (0.25, 0), (0, 0.2)], + (), + ), + ] + ) + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def earth_poles() -> List[geometry.Point]: + """Get 6 Earth poles.""" + return [ + geometry.Point(0, 0), + geometry.Point(90, 0), + geometry.Point(180, 0), + geometry.Point(-90, 0), + geometry.Point(0, 90), + geometry.Point(0, -90), + ] + + +@pytest.fixture # type: ignore +def gdf_earth_poles(earth_poles) -> gpd.GeoDataFrame: + """Get GeoDataFrame with 6 Earth poles.""" + return gpd.GeoDataFrame( + {GEOMETRY_COLUMN: earth_poles}, + index=[1, 2, 3, 4, 5, 6], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def gdf_poland() -> gpd.GeoDataFrame: + """Get Poland GeoDataFrame.""" + world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + area = world[world.name == "Poland"] + return area + + +@pytest.fixture # type: ignore +def earth_bbox() -> geometry.Polygon: + """Get full bounding box GeoDataFrame.""" + return geometry.box(minx=-180, maxx=180, miny=-90, maxy=90) + + +@pytest.fixture # type: ignore +def gdf_earth_bbox(earth_bbox) -> gpd.GeoDataFrame: + """Get full bounding box GeoDataFrame.""" + return gpd.GeoDataFrame({GEOMETRY_COLUMN: [earth_bbox]}, crs=WGS84_CRS) From 644a61be885b8e203fd58bca379b62fbb5b8cce2 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 21:42:54 +0200 Subject: [PATCH 17/30] chore: add future issue comments --- srai/h3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/srai/h3.py b/srai/h3.py index 2dadd513..5caddefc 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -57,6 +57,7 @@ def shapely_geometry_to_h3( return [h3.int_to_str(h3_index) for h3_index in h3_indexes.tolist()] +# TODO: write tests (#322) def h3_to_geoseries(h3_index: Union[int, str, Iterable[Union[int, str]]]) -> gpd.GeoSeries: """ Convert H3 index to GeoPandas GeoSeries. @@ -87,6 +88,7 @@ def h3_to_shapely_geometry(h3_index: Iterable[Union[int, str]]) -> List[Polygon] ... +# TODO: write tests (#322) def h3_to_shapely_geometry( h3_index: Union[int, str, Iterable[Union[int, str]]] ) -> Union[Polygon, List[Polygon]]: From 57eff3f5b85605aa7a9868052fe109371ce90d7a Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 21:48:22 +0200 Subject: [PATCH 18/30] ci: add new pytest flag --- pyproject.toml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d7c015bd..264d87ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,12 +59,7 @@ Changelog = "https://github.com/srai-lab/srai/blob/main/CHANGELOG.md" # add tests # pdm add -G osm -osm = [ - "osmium", - "osmnx", - "overpass", - "pillow", -] +osm = ["osmium", "osmnx", "overpass", "pillow"] # pdm add -G voronoi voronoi = ["pymap3d", "haversine", "scipy", "spherical-geometry"] # pdm add -G gtfs @@ -200,7 +195,7 @@ push = false "CITATION.cff" = ['^version: {version}$'] [tool.pytest.ini_options] -addopts = ["--import-mode=importlib"] +addopts = ["--import-mode=importlib", "--ignore=dbt_packages"] markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] log_cli = true From 8573c5aa5ff9d762a638b2cf94b7b282482808c9 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 21:53:07 +0200 Subject: [PATCH 19/30] ci: add new pytest flag --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5a206f46..fc34bb38 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ groups = test deps = coverage commands = - coverage run --source=srai -m pytest -v --durations=20 tests + coverage run --source=srai -m pytest -v --durations=20 --import-mode=importlib --ignore=dbt_packages tests coverage xml -o coverage.{envname}.xml coverage report -m skip_install = true From 6c2fa2113adeb8a1c374adc27fb8607ea07182e7 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 22:03:53 +0200 Subject: [PATCH 20/30] chore: change conftests for H3 module --- pyproject.toml | 2 +- tests/h3/conftest.py | 93 ++++++++++++++++++++- tests/regionalizers/conftest.py | 143 +++++++++++++++++++++++++++++++- tests/regionalizers/fixtures.py | 141 ------------------------------- tox.ini | 2 +- 5 files changed, 232 insertions(+), 149 deletions(-) delete mode 100644 tests/regionalizers/fixtures.py diff --git a/pyproject.toml b/pyproject.toml index 264d87ff..302763a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,7 +195,7 @@ push = false "CITATION.cff" = ['^version: {version}$'] [tool.pytest.ini_options] -addopts = ["--import-mode=importlib", "--ignore=dbt_packages"] +addopts = ["--import-mode=importlib"] markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] log_cli = true diff --git a/tests/h3/conftest.py b/tests/h3/conftest.py index 935b6dea..dda436de 100644 --- a/tests/h3/conftest.py +++ b/tests/h3/conftest.py @@ -3,17 +3,15 @@ import geopandas as gpd import pytest -from shapely.geometry import Point +from shapely import geometry from srai.constants import WGS84_CRS -pytest_plugins = ["tests.regionalizers.fixtures", "tests.regionalizers.test_h3_regionalizer"] - @pytest.fixture # type: ignore def gdf_single_point() -> gpd.GeoDataFrame: """Get the point case.""" - return gpd.GeoDataFrame(geometry=[Point(17.9261, 50.6696)], crs=WGS84_CRS) + return gpd.GeoDataFrame(geometry=[geometry.Point(17.9261, 50.6696)], crs=WGS84_CRS) @pytest.fixture # type: ignore @@ -22,3 +20,90 @@ def expected_point_h3_index() -> List[str]: return [ "8a1e23c44b5ffff", ] + + +@pytest.fixture # type: ignore +def gdf_polygons() -> gpd.GeoDataFrame: + """Get GeoDataFrame with two polygons.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.Polygon( + shell=[ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + holes=[ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ], + ), + geometry.Polygon(shell=[(-0.25, 0), (0.25, 0), (0, 0.2)]), + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def gdf_multipolygon() -> gpd.GeoDataFrame: + """Get GeoDataFrame with multipolygon.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.MultiPolygon( + [ + ( + [ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + ( + [ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ] + ), + ), + ( + [(-0.25, 0), (0.25, 0), (0, 0.2)], + (), + ), + ] + ) + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def expected_h3_indexes() -> List[str]: + """Get expected h3 indexes.""" + return [ + "837559fffffffff", + "83754efffffffff", + "83754cfffffffff", + "837541fffffffff", + "83755dfffffffff", + "837543fffffffff", + "83754afffffffff", + ] + + +@pytest.fixture # type: ignore +def expected_unbuffered_h3_indexes() -> List[str]: + """Get expected h3 index for the unbuffered case.""" + return [ + "83754efffffffff", + ] diff --git a/tests/regionalizers/conftest.py b/tests/regionalizers/conftest.py index d7e08430..9320f1bf 100644 --- a/tests/regionalizers/conftest.py +++ b/tests/regionalizers/conftest.py @@ -1,2 +1,141 @@ -"""Conftest for Regionalizers.""" -pytest_plugins = ["tests.regionalizers.fixtures"] +"""Fixtures for Regionalizers.""" + +from typing import List + +import geopandas as gpd +import pytest +from shapely import geometry + +from srai.constants import GEOMETRY_COLUMN, WGS84_CRS + + +@pytest.fixture # type: ignore +def gdf_empty() -> gpd.GeoDataFrame: + """Get empty GeoDataFrame.""" + return gpd.GeoDataFrame() + + +@pytest.fixture # type: ignore +def gdf_no_crs() -> gpd.GeoDataFrame: + """Get GeoDataFrame with no crs.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.Polygon( + shell=[ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ] + ) + ] + ) + + +@pytest.fixture # type: ignore +def gdf_polygons() -> gpd.GeoDataFrame: + """Get GeoDataFrame with two polygons.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.Polygon( + shell=[ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + holes=[ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ], + ), + geometry.Polygon(shell=[(-0.25, 0), (0.25, 0), (0, 0.2)]), + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def gdf_multipolygon() -> gpd.GeoDataFrame: + """Get GeoDataFrame with multipolygon.""" + return gpd.GeoDataFrame( + geometry=[ + geometry.MultiPolygon( + [ + ( + [ + (-1, 0), + (0, 0.5), + (1, 0), + (1, 1), + (0, 1), + ], + ( + [ + [ + (0.8, 0.9), + (0.9, 0.55), + (0.8, 0.3), + (0.5, 0.4), + ] + ] + ), + ), + ( + [(-0.25, 0), (0.25, 0), (0, 0.2)], + (), + ), + ] + ) + ], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def earth_poles() -> List[geometry.Point]: + """Get 6 Earth poles.""" + return [ + geometry.Point(0, 0), + geometry.Point(90, 0), + geometry.Point(180, 0), + geometry.Point(-90, 0), + geometry.Point(0, 90), + geometry.Point(0, -90), + ] + + +@pytest.fixture # type: ignore +def gdf_earth_poles(earth_poles) -> gpd.GeoDataFrame: + """Get GeoDataFrame with 6 Earth poles.""" + return gpd.GeoDataFrame( + {GEOMETRY_COLUMN: earth_poles}, + index=[1, 2, 3, 4, 5, 6], + crs=WGS84_CRS, + ) + + +@pytest.fixture # type: ignore +def gdf_poland() -> gpd.GeoDataFrame: + """Get Poland GeoDataFrame.""" + world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + area = world[world.name == "Poland"] + return area + + +@pytest.fixture # type: ignore +def earth_bbox() -> geometry.Polygon: + """Get full bounding box GeoDataFrame.""" + return geometry.box(minx=-180, maxx=180, miny=-90, maxy=90) + + +@pytest.fixture # type: ignore +def gdf_earth_bbox(earth_bbox) -> gpd.GeoDataFrame: + """Get full bounding box GeoDataFrame.""" + return gpd.GeoDataFrame({GEOMETRY_COLUMN: [earth_bbox]}, crs=WGS84_CRS) diff --git a/tests/regionalizers/fixtures.py b/tests/regionalizers/fixtures.py deleted file mode 100644 index 9320f1bf..00000000 --- a/tests/regionalizers/fixtures.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Fixtures for Regionalizers.""" - -from typing import List - -import geopandas as gpd -import pytest -from shapely import geometry - -from srai.constants import GEOMETRY_COLUMN, WGS84_CRS - - -@pytest.fixture # type: ignore -def gdf_empty() -> gpd.GeoDataFrame: - """Get empty GeoDataFrame.""" - return gpd.GeoDataFrame() - - -@pytest.fixture # type: ignore -def gdf_no_crs() -> gpd.GeoDataFrame: - """Get GeoDataFrame with no crs.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.Polygon( - shell=[ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ] - ) - ] - ) - - -@pytest.fixture # type: ignore -def gdf_polygons() -> gpd.GeoDataFrame: - """Get GeoDataFrame with two polygons.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.Polygon( - shell=[ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ], - holes=[ - [ - (0.8, 0.9), - (0.9, 0.55), - (0.8, 0.3), - (0.5, 0.4), - ] - ], - ), - geometry.Polygon(shell=[(-0.25, 0), (0.25, 0), (0, 0.2)]), - ], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def gdf_multipolygon() -> gpd.GeoDataFrame: - """Get GeoDataFrame with multipolygon.""" - return gpd.GeoDataFrame( - geometry=[ - geometry.MultiPolygon( - [ - ( - [ - (-1, 0), - (0, 0.5), - (1, 0), - (1, 1), - (0, 1), - ], - ( - [ - [ - (0.8, 0.9), - (0.9, 0.55), - (0.8, 0.3), - (0.5, 0.4), - ] - ] - ), - ), - ( - [(-0.25, 0), (0.25, 0), (0, 0.2)], - (), - ), - ] - ) - ], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def earth_poles() -> List[geometry.Point]: - """Get 6 Earth poles.""" - return [ - geometry.Point(0, 0), - geometry.Point(90, 0), - geometry.Point(180, 0), - geometry.Point(-90, 0), - geometry.Point(0, 90), - geometry.Point(0, -90), - ] - - -@pytest.fixture # type: ignore -def gdf_earth_poles(earth_poles) -> gpd.GeoDataFrame: - """Get GeoDataFrame with 6 Earth poles.""" - return gpd.GeoDataFrame( - {GEOMETRY_COLUMN: earth_poles}, - index=[1, 2, 3, 4, 5, 6], - crs=WGS84_CRS, - ) - - -@pytest.fixture # type: ignore -def gdf_poland() -> gpd.GeoDataFrame: - """Get Poland GeoDataFrame.""" - world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) - area = world[world.name == "Poland"] - return area - - -@pytest.fixture # type: ignore -def earth_bbox() -> geometry.Polygon: - """Get full bounding box GeoDataFrame.""" - return geometry.box(minx=-180, maxx=180, miny=-90, maxy=90) - - -@pytest.fixture # type: ignore -def gdf_earth_bbox(earth_bbox) -> gpd.GeoDataFrame: - """Get full bounding box GeoDataFrame.""" - return gpd.GeoDataFrame({GEOMETRY_COLUMN: [earth_bbox]}, crs=WGS84_CRS) diff --git a/tox.ini b/tox.ini index fc34bb38..6e7e46eb 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ groups = test deps = coverage commands = - coverage run --source=srai -m pytest -v --durations=20 --import-mode=importlib --ignore=dbt_packages tests + coverage run --source=srai -m pytest -v --durations=20 --import-mode=importlib tests coverage xml -o coverage.{envname}.xml coverage report -m skip_install = true From 6c24dcb89972d7ecfa89b7593a954f3ca10f911d Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 22:21:05 +0200 Subject: [PATCH 21/30] ci: remove pytest flag --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6e7e46eb..5a206f46 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ groups = test deps = coverage commands = - coverage run --source=srai -m pytest -v --durations=20 --import-mode=importlib tests + coverage run --source=srai -m pytest -v --durations=20 tests coverage xml -o coverage.{envname}.xml coverage report -m skip_install = true From 9db168bf714120bd8f689833006ee11eb8d51b5c Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 22:46:11 +0200 Subject: [PATCH 22/30] ci: add verbose flag to tox --- .github/workflows/_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 709b350c..29c52635 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -44,7 +44,7 @@ jobs: restore-keys: | tox-cache-${{ matrix.os }}-${{ matrix.python-version }}- - name: Run tests with tox - run: pdm run tox -e python${{ matrix.python-version }} + run: pdm run tox -e python${{ matrix.python-version }} --verbose - name: Upload coverage to Codecov uses: Wandalen/wretry.action@master with: From 716ee333065bfb01c3475327167683b8dd0082bd Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 22:54:12 +0200 Subject: [PATCH 23/30] ci: check raw pytest output --- .github/workflows/_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 29c52635..b57b6a23 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -43,6 +43,8 @@ jobs: key: tox-cache-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pdm.lock') }} restore-keys: | tox-cache-${{ matrix.os }}-${{ matrix.python-version }}- + - name: Run tests with pytest + run: pdm run pytest -v --durations=20 tests - name: Run tests with tox run: pdm run tox -e python${{ matrix.python-version }} --verbose - name: Upload coverage to Codecov From fd7cf8d1faae9d8c47c62354d87ae4cd9aa4d694 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 22:57:11 +0200 Subject: [PATCH 24/30] ci: check raw pytest output --- .github/workflows/_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index b57b6a23..952436fb 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -30,7 +30,8 @@ jobs: cache: true cache-dependency-path: "**/pdm.lock" - name: Install dependencies - run: pdm install -d -G test --skip=post_install + run: pdm install -dG:all --skip=post_install + # run: pdm install -d -G test --skip=post_install - name: Cache Overpass data uses: actions/cache@v3 with: From bbaf10865fb12a6acb08febe7c364de17b53756e Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 23:13:03 +0200 Subject: [PATCH 25/30] ci: remove macos tests --- .github/workflows/_tests.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 952436fb..6a1d6faa 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -9,9 +9,9 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10"] - include: - - os: macos-latest - python-version: "3.10" + # include: + # - os: macos-latest + # python-version: "3.10" env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} @@ -30,8 +30,7 @@ jobs: cache: true cache-dependency-path: "**/pdm.lock" - name: Install dependencies - run: pdm install -dG:all --skip=post_install - # run: pdm install -d -G test --skip=post_install + run: pdm install -d -G test --skip=post_install - name: Cache Overpass data uses: actions/cache@v3 with: @@ -44,10 +43,8 @@ jobs: key: tox-cache-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pdm.lock') }} restore-keys: | tox-cache-${{ matrix.os }}-${{ matrix.python-version }}- - - name: Run tests with pytest - run: pdm run pytest -v --durations=20 tests - name: Run tests with tox - run: pdm run tox -e python${{ matrix.python-version }} --verbose + run: pdm run tox -e python${{ matrix.python-version }} - name: Upload coverage to Codecov uses: Wandalen/wretry.action@master with: From 642830383a960739d048e1560dab2d000cda9bde Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 23:24:50 +0200 Subject: [PATCH 26/30] ci: test macos once again --- .github/workflows/_tests.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 6a1d6faa..3daeb31b 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -9,9 +9,9 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10"] - # include: - # - os: macos-latest - # python-version: "3.10" + include: + - os: macos-latest + python-version: "3.10" env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} @@ -30,7 +30,15 @@ jobs: cache: true cache-dependency-path: "**/pdm.lock" - name: Install dependencies - run: pdm install -d -G test --skip=post_install + # run: pdm install -d -G test --skip=post_install + run: pdm install -dG:all --skip=post_install + - uses: jannekem/run-python-script-action@v1 + name: Run h3ronpy test + with: + script: | + from srai.h3 import shapely_geometry_to_h3 + from shapely.geometry import Point + shapely_geometry_to_h3(geometry=Point(0,0), h3_resolution=9) - name: Cache Overpass data uses: actions/cache@v3 with: From 438055e139e3506d6f69a41fd1bdb14f734c9a6a Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 23:35:07 +0200 Subject: [PATCH 27/30] ci: test macos once again --- .github/workflows/_tests.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 3daeb31b..5645c2be 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -32,13 +32,8 @@ jobs: - name: Install dependencies # run: pdm install -d -G test --skip=post_install run: pdm install -dG:all --skip=post_install - - uses: jannekem/run-python-script-action@v1 - name: Run h3ronpy test - with: - script: | - from srai.h3 import shapely_geometry_to_h3 - from shapely.geometry import Point - shapely_geometry_to_h3(geometry=Point(0,0), h3_resolution=9) + - name: Run h3ronpy test + run: echo "from srai.h3 import shapely_geometry_to_h3; from shapely.geometry import Point; x = shapely_geometry_to_h3(geometry=Point(0,0), h3_resolution=9); print(x)" | pdm run python - name: Cache Overpass data uses: actions/cache@v3 with: From 145a41e56b60b76eeb0bf60dd1d047f38398ba02 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Mon, 7 Aug 2023 23:46:59 +0200 Subject: [PATCH 28/30] tests: disable h3 tests with faulty macos runner --- .github/workflows/_tests.yml | 9 +- tests/h3/test_shapely_conversion.py | 176 ++++++++++++++-------------- 2 files changed, 91 insertions(+), 94 deletions(-) diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml index 5645c2be..d1b63db6 100644 --- a/.github/workflows/_tests.yml +++ b/.github/workflows/_tests.yml @@ -8,10 +8,10 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] include: - os: macos-latest - python-version: "3.10" + python-version: "3.11" env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} @@ -30,10 +30,7 @@ jobs: cache: true cache-dependency-path: "**/pdm.lock" - name: Install dependencies - # run: pdm install -d -G test --skip=post_install - run: pdm install -dG:all --skip=post_install - - name: Run h3ronpy test - run: echo "from srai.h3 import shapely_geometry_to_h3; from shapely.geometry import Point; x = shapely_geometry_to_h3(geometry=Point(0,0), h3_resolution=9); print(x)" | pdm run python + run: pdm install -d -G test --skip=post_install - name: Cache Overpass data uses: actions/cache@v3 with: diff --git a/tests/h3/test_shapely_conversion.py b/tests/h3/test_shapely_conversion.py index b5c51715..21af6e1a 100644 --- a/tests/h3/test_shapely_conversion.py +++ b/tests/h3/test_shapely_conversion.py @@ -1,90 +1,90 @@ """H3 shapely conversion tests.""" -from typing import Any, Callable, List -from unittest import TestCase - -import geopandas as gpd -import pytest -from shapely.geometry.base import BaseGeometry - -from srai.constants import GEOMETRY_COLUMN -from srai.h3 import shapely_geometry_to_h3 -from tests.regionalizers.test_h3_regionalizer import H3_RESOLUTION - -ut = TestCase() - - -def _gdf_noop(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoDataFrame: - return gdf_fixture - - -def _gdf_to_geoseries(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoSeries: - return gdf_fixture[GEOMETRY_COLUMN] - - -def _gdf_to_geometry_list(gdf_fixture: gpd.GeoDataFrame) -> List[BaseGeometry]: - return list(gdf_fixture[GEOMETRY_COLUMN]) - - -def _gdf_to_single_geometry(gdf_fixture: gpd.GeoDataFrame) -> BaseGeometry: - return gdf_fixture[GEOMETRY_COLUMN][0] - - -@pytest.mark.parametrize( - "geometry_fixture, resolution, expected_h3_cells_fixture", - [ - ("gdf_single_point", 10, "expected_point_h3_index"), - ("gdf_multipolygon", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), - ("gdf_polygons", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), - ], -) # type: ignore -@pytest.mark.parametrize( - "geometry_parser_function", - [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], -) # type: ignore -def test_shapely_geometry_to_h3_unbuffered( - geometry_fixture: str, - resolution: int, - expected_h3_cells_fixture: str, - geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], - request: pytest.FixtureRequest, -) -> None: - """Test checks if conversion from shapely to h3 works.""" - geometry = request.getfixturevalue(geometry_fixture) - expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) - - parsed_geometry = geometry_parser_function(geometry) - h3_cells = shapely_geometry_to_h3( - geometry=parsed_geometry, h3_resolution=resolution, buffer=False - ) - ut.assertCountEqual(h3_cells, expected_h3_cells) - - -@pytest.mark.parametrize( - "geometry_fixture, resolution, expected_h3_cells_fixture", - [ - ("gdf_single_point", 10, "expected_point_h3_index"), - ("gdf_multipolygon", H3_RESOLUTION, "expected_h3_indexes"), - ("gdf_polygons", H3_RESOLUTION, "expected_h3_indexes"), - ], -) # type: ignore -@pytest.mark.parametrize( - "geometry_parser_function", - [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], -) # type: ignore -def test_shapely_geometry_to_h3_buffered( - geometry_fixture: str, - resolution: int, - expected_h3_cells_fixture: str, - geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], - request: pytest.FixtureRequest, -) -> None: - """Test checks if conversion from shapely to h3 with buffer works.""" - geometry = request.getfixturevalue(geometry_fixture) - expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) - - parsed_geometry = geometry_parser_function(geometry) - h3_cells = shapely_geometry_to_h3( - geometry=parsed_geometry, h3_resolution=resolution, buffer=True - ) - ut.assertCountEqual(h3_cells, expected_h3_cells) +# from typing import Any, Callable, List +# from unittest import TestCase + +# import geopandas as gpd +# import pytest +# from shapely.geometry.base import BaseGeometry + +# from srai.constants import GEOMETRY_COLUMN +# from srai.h3 import shapely_geometry_to_h3 +# from tests.regionalizers.test_h3_regionalizer import H3_RESOLUTION + +# ut = TestCase() + + +# def _gdf_noop(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoDataFrame: +# return gdf_fixture + + +# def _gdf_to_geoseries(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoSeries: +# return gdf_fixture[GEOMETRY_COLUMN] + + +# def _gdf_to_geometry_list(gdf_fixture: gpd.GeoDataFrame) -> List[BaseGeometry]: +# return list(gdf_fixture[GEOMETRY_COLUMN]) + + +# def _gdf_to_single_geometry(gdf_fixture: gpd.GeoDataFrame) -> BaseGeometry: +# return gdf_fixture[GEOMETRY_COLUMN][0] + + +# @pytest.mark.parametrize( +# "geometry_fixture, resolution, expected_h3_cells_fixture", +# [ +# ("gdf_single_point", 10, "expected_point_h3_index"), +# ("gdf_multipolygon", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), +# ("gdf_polygons", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), +# ], +# ) # type: ignore +# @pytest.mark.parametrize( +# "geometry_parser_function", +# [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +# ) # type: ignore +# def test_shapely_geometry_to_h3_unbuffered( +# geometry_fixture: str, +# resolution: int, +# expected_h3_cells_fixture: str, +# geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], +# request: pytest.FixtureRequest, +# ) -> None: +# """Test checks if conversion from shapely to h3 works.""" +# geometry = request.getfixturevalue(geometry_fixture) +# expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + +# parsed_geometry = geometry_parser_function(geometry) +# h3_cells = shapely_geometry_to_h3( +# geometry=parsed_geometry, h3_resolution=resolution, buffer=False +# ) +# ut.assertCountEqual(h3_cells, expected_h3_cells) + + +# @pytest.mark.parametrize( +# "geometry_fixture, resolution, expected_h3_cells_fixture", +# [ +# ("gdf_single_point", 10, "expected_point_h3_index"), +# ("gdf_multipolygon", H3_RESOLUTION, "expected_h3_indexes"), +# ("gdf_polygons", H3_RESOLUTION, "expected_h3_indexes"), +# ], +# ) # type: ignore +# @pytest.mark.parametrize( +# "geometry_parser_function", +# [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +# ) # type: ignore +# def test_shapely_geometry_to_h3_buffered( +# geometry_fixture: str, +# resolution: int, +# expected_h3_cells_fixture: str, +# geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], +# request: pytest.FixtureRequest, +# ) -> None: +# """Test checks if conversion from shapely to h3 with buffer works.""" +# geometry = request.getfixturevalue(geometry_fixture) +# expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + +# parsed_geometry = geometry_parser_function(geometry) +# h3_cells = shapely_geometry_to_h3( +# geometry=parsed_geometry, h3_resolution=resolution, buffer=True +# ) +# ut.assertCountEqual(h3_cells, expected_h3_cells) From b8600cfe1ab0082678fd388cca8e76192d7c356b Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Tue, 8 Aug 2023 00:28:45 +0200 Subject: [PATCH 29/30] chore: fix macos h3ronpy error --- srai/h3.py | 68 +++++++++++ tests/h3/test_shapely_conversion.py | 176 ++++++++++++++-------------- 2 files changed, 156 insertions(+), 88 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index 5caddefc..ba5e77c7 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -1,5 +1,6 @@ """Utility H3 related functions.""" +from sys import platform from typing import Iterable, List, Literal, Tuple, Union, overload import geopandas as gpd @@ -11,6 +12,7 @@ from shapely.geometry.base import BaseGeometry from srai.constants import GEOMETRY_COLUMN, WGS84_CRS +from srai.geometry import buffer_geometry def shapely_geometry_to_h3( @@ -40,6 +42,9 @@ def shapely_geometry_to_h3( if not (0 <= h3_resolution <= 15): raise ValueError(f"Resolution {h3_resolution} is not between 0 and 15.") + if _is_macos(): + return _polygon_to_h3_index_raw(geometry, h3_resolution, buffer) + wkb = [] if isinstance(geometry, gpd.GeoSeries): wkb = geometry.to_wkb() @@ -57,6 +62,48 @@ def shapely_geometry_to_h3( return [h3.int_to_str(h3_index) for h3_index in h3_indexes.tolist()] +def _polygon_to_h3_index_raw( + geometry: Union[BaseGeometry, Iterable[BaseGeometry], gpd.GeoSeries, gpd.GeoDataFrame], + h3_resolution: int, + buffer: bool = True, +) -> List[str]: + def _polygon_shapely_to_h3(polygon: Polygon) -> h3.Polygon: + exterior = [coord[::-1] for coord in list(polygon.exterior.coords)] + interiors = [ + [coord[::-1] for coord in list(interior.coords)] for interior in polygon.interiors + ] + return h3.Polygon(exterior, *interiors) + + from functional import seq + + geoseries: gpd.GeoSeries + + if isinstance(geometry, gpd.GeoSeries): + geoseries = geometry + elif isinstance(geometry, gpd.GeoDataFrame): + geoseries = geometry[GEOMETRY_COLUMN] + elif isinstance(geometry, Iterable): + geoseries = gpd.GeoSeries(geometry, crs=WGS84_CRS) + else: + return _polygon_to_h3_index_raw([geometry], h3_resolution, buffer) + + if buffer: + buffer_distance_meters = 2 * h3.average_hexagon_edge_length(h3_resolution, unit="m") + geoseries = geoseries.apply( + lambda polygon: buffer_geometry(polygon, buffer_distance_meters) + ) + + h3_indexes: List[str] = ( + seq(geoseries) + .map(_polygon_shapely_to_h3) + .flat_map(lambda polygon: h3.polygon_to_cells(polygon, h3_resolution)) + .distinct() + .to_list() + ) + + return h3_indexes + + # TODO: write tests (#322) def h3_to_geoseries(h3_index: Union[int, str, Iterable[Union[int, str]]]) -> gpd.GeoSeries: """ @@ -72,12 +119,28 @@ def h3_to_geoseries(h3_index: Union[int, str, Iterable[Union[int, str]]]) -> gpd if isinstance(h3_index, (str, int)): return h3_to_geoseries([h3_index]) else: + if _is_macos(): + return gpd.GeoSeries( + [_h3_index_to_polygon_raw(h3_cell) for h3_cell in h3_index], crs=WGS84_CRS + ) + h3_int_indexes = ( h3_cell if isinstance(h3_cell, int) else h3.str_to_int(h3_cell) for h3_cell in h3_index ) return gpd.GeoSeries.from_wkb(cells_to_wkb_polygons(h3_int_indexes), crs=WGS84_CRS) +def _h3_index_to_polygon_raw(h3_index: Union[int, str]) -> Polygon: + if isinstance(h3_index, int): + h3_index = h3.int_to_str(h3_index) + h3_poly = h3.cells_to_polygons([h3_index])[0] + + return Polygon( + shell=[coord[::-1] for coord in h3_poly.outer], + holes=[[coord[::-1] for coord in hole] for hole in h3_poly.holes], + ) + + @overload def h3_to_shapely_geometry(h3_index: Union[int, str]) -> Polygon: ... @@ -166,3 +229,8 @@ def get_local_ij_index( local_ijs = [(coords[0], coords[1]) for coords in local_ijs] return local_ijs + + +def _is_macos() -> bool: + """Return flag if code is run on OS X.""" + return platform == "darwin" diff --git a/tests/h3/test_shapely_conversion.py b/tests/h3/test_shapely_conversion.py index 21af6e1a..b5c51715 100644 --- a/tests/h3/test_shapely_conversion.py +++ b/tests/h3/test_shapely_conversion.py @@ -1,90 +1,90 @@ """H3 shapely conversion tests.""" -# from typing import Any, Callable, List -# from unittest import TestCase - -# import geopandas as gpd -# import pytest -# from shapely.geometry.base import BaseGeometry - -# from srai.constants import GEOMETRY_COLUMN -# from srai.h3 import shapely_geometry_to_h3 -# from tests.regionalizers.test_h3_regionalizer import H3_RESOLUTION - -# ut = TestCase() - - -# def _gdf_noop(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoDataFrame: -# return gdf_fixture - - -# def _gdf_to_geoseries(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoSeries: -# return gdf_fixture[GEOMETRY_COLUMN] - - -# def _gdf_to_geometry_list(gdf_fixture: gpd.GeoDataFrame) -> List[BaseGeometry]: -# return list(gdf_fixture[GEOMETRY_COLUMN]) - - -# def _gdf_to_single_geometry(gdf_fixture: gpd.GeoDataFrame) -> BaseGeometry: -# return gdf_fixture[GEOMETRY_COLUMN][0] - - -# @pytest.mark.parametrize( -# "geometry_fixture, resolution, expected_h3_cells_fixture", -# [ -# ("gdf_single_point", 10, "expected_point_h3_index"), -# ("gdf_multipolygon", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), -# ("gdf_polygons", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), -# ], -# ) # type: ignore -# @pytest.mark.parametrize( -# "geometry_parser_function", -# [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], -# ) # type: ignore -# def test_shapely_geometry_to_h3_unbuffered( -# geometry_fixture: str, -# resolution: int, -# expected_h3_cells_fixture: str, -# geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], -# request: pytest.FixtureRequest, -# ) -> None: -# """Test checks if conversion from shapely to h3 works.""" -# geometry = request.getfixturevalue(geometry_fixture) -# expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) - -# parsed_geometry = geometry_parser_function(geometry) -# h3_cells = shapely_geometry_to_h3( -# geometry=parsed_geometry, h3_resolution=resolution, buffer=False -# ) -# ut.assertCountEqual(h3_cells, expected_h3_cells) - - -# @pytest.mark.parametrize( -# "geometry_fixture, resolution, expected_h3_cells_fixture", -# [ -# ("gdf_single_point", 10, "expected_point_h3_index"), -# ("gdf_multipolygon", H3_RESOLUTION, "expected_h3_indexes"), -# ("gdf_polygons", H3_RESOLUTION, "expected_h3_indexes"), -# ], -# ) # type: ignore -# @pytest.mark.parametrize( -# "geometry_parser_function", -# [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], -# ) # type: ignore -# def test_shapely_geometry_to_h3_buffered( -# geometry_fixture: str, -# resolution: int, -# expected_h3_cells_fixture: str, -# geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], -# request: pytest.FixtureRequest, -# ) -> None: -# """Test checks if conversion from shapely to h3 with buffer works.""" -# geometry = request.getfixturevalue(geometry_fixture) -# expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) - -# parsed_geometry = geometry_parser_function(geometry) -# h3_cells = shapely_geometry_to_h3( -# geometry=parsed_geometry, h3_resolution=resolution, buffer=True -# ) -# ut.assertCountEqual(h3_cells, expected_h3_cells) +from typing import Any, Callable, List +from unittest import TestCase + +import geopandas as gpd +import pytest +from shapely.geometry.base import BaseGeometry + +from srai.constants import GEOMETRY_COLUMN +from srai.h3 import shapely_geometry_to_h3 +from tests.regionalizers.test_h3_regionalizer import H3_RESOLUTION + +ut = TestCase() + + +def _gdf_noop(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoDataFrame: + return gdf_fixture + + +def _gdf_to_geoseries(gdf_fixture: gpd.GeoDataFrame) -> gpd.GeoSeries: + return gdf_fixture[GEOMETRY_COLUMN] + + +def _gdf_to_geometry_list(gdf_fixture: gpd.GeoDataFrame) -> List[BaseGeometry]: + return list(gdf_fixture[GEOMETRY_COLUMN]) + + +def _gdf_to_single_geometry(gdf_fixture: gpd.GeoDataFrame) -> BaseGeometry: + return gdf_fixture[GEOMETRY_COLUMN][0] + + +@pytest.mark.parametrize( + "geometry_fixture, resolution, expected_h3_cells_fixture", + [ + ("gdf_single_point", 10, "expected_point_h3_index"), + ("gdf_multipolygon", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), + ("gdf_polygons", H3_RESOLUTION, "expected_unbuffered_h3_indexes"), + ], +) # type: ignore +@pytest.mark.parametrize( + "geometry_parser_function", + [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +) # type: ignore +def test_shapely_geometry_to_h3_unbuffered( + geometry_fixture: str, + resolution: int, + expected_h3_cells_fixture: str, + geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], + request: pytest.FixtureRequest, +) -> None: + """Test checks if conversion from shapely to h3 works.""" + geometry = request.getfixturevalue(geometry_fixture) + expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + + parsed_geometry = geometry_parser_function(geometry) + h3_cells = shapely_geometry_to_h3( + geometry=parsed_geometry, h3_resolution=resolution, buffer=False + ) + ut.assertCountEqual(h3_cells, expected_h3_cells) + + +@pytest.mark.parametrize( + "geometry_fixture, resolution, expected_h3_cells_fixture", + [ + ("gdf_single_point", 10, "expected_point_h3_index"), + ("gdf_multipolygon", H3_RESOLUTION, "expected_h3_indexes"), + ("gdf_polygons", H3_RESOLUTION, "expected_h3_indexes"), + ], +) # type: ignore +@pytest.mark.parametrize( + "geometry_parser_function", + [_gdf_noop, _gdf_to_geoseries, _gdf_to_geometry_list], +) # type: ignore +def test_shapely_geometry_to_h3_buffered( + geometry_fixture: str, + resolution: int, + expected_h3_cells_fixture: str, + geometry_parser_function: Callable[[gpd.GeoDataFrame], Any], + request: pytest.FixtureRequest, +) -> None: + """Test checks if conversion from shapely to h3 with buffer works.""" + geometry = request.getfixturevalue(geometry_fixture) + expected_h3_cells = request.getfixturevalue(expected_h3_cells_fixture) + + parsed_geometry = geometry_parser_function(geometry) + h3_cells = shapely_geometry_to_h3( + geometry=parsed_geometry, h3_resolution=resolution, buffer=True + ) + ut.assertCountEqual(h3_cells, expected_h3_cells) From 82052bfcf635fb9eec8691905c039f1c11d3ab59 Mon Sep 17 00:00:00 2001 From: Kamil Raczycki Date: Tue, 8 Aug 2023 00:56:09 +0200 Subject: [PATCH 30/30] chore: fix old h3 implementation --- srai/h3.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/srai/h3.py b/srai/h3.py index ba5e77c7..758d24a6 100644 --- a/srai/h3.py +++ b/srai/h3.py @@ -8,7 +8,7 @@ import numpy as np import numpy.typing as npt from h3ronpy.arrow.vector import cells_to_wkb_polygons, wkb_to_cells -from shapely.geometry import Polygon +from shapely.geometry import Point, Polygon from shapely.geometry.base import BaseGeometry from srai.constants import GEOMETRY_COLUMN, WGS84_CRS @@ -67,12 +67,33 @@ def _polygon_to_h3_index_raw( h3_resolution: int, buffer: bool = True, ) -> List[str]: - def _polygon_shapely_to_h3(polygon: Polygon) -> h3.Polygon: - exterior = [coord[::-1] for coord in list(polygon.exterior.coords)] + def _polygon_shapely_to_h3( + geometry: Union[Point, Polygon], h3_resolution: int, buffer: bool + ) -> List[str]: + if isinstance(geometry, Point): + return [h3.latlng_to_cell(geometry.y, geometry.x, h3_resolution)] + + buffer_distance_meters = 2 * h3.average_hexagon_edge_length(h3_resolution, unit="m") + + buffered_geometry = ( + buffer_geometry(geometry, buffer_distance_meters) if buffer else geometry + ) + + exterior = [coord[::-1] for coord in list(buffered_geometry.exterior.coords)] interiors = [ - [coord[::-1] for coord in list(interior.coords)] for interior in polygon.interiors + [coord[::-1] for coord in list(interior.coords)] + for interior in buffered_geometry.interiors ] - return h3.Polygon(exterior, *interiors) + h3_cells: List[str] = h3.polygon_to_cells(h3.Polygon(exterior, *interiors), h3_resolution) + + if buffer: + h3_cells = [ + h3_cell + for h3_cell in h3_cells + if _h3_index_to_polygon_raw(h3_cell).intersects(geometry) + ] + + return h3_cells from functional import seq @@ -87,16 +108,11 @@ def _polygon_shapely_to_h3(polygon: Polygon) -> h3.Polygon: else: return _polygon_to_h3_index_raw([geometry], h3_resolution, buffer) - if buffer: - buffer_distance_meters = 2 * h3.average_hexagon_edge_length(h3_resolution, unit="m") - geoseries = geoseries.apply( - lambda polygon: buffer_geometry(polygon, buffer_distance_meters) - ) + geoseries = geoseries.explode(ignore_index=True, index_parts=True) h3_indexes: List[str] = ( seq(geoseries) - .map(_polygon_shapely_to_h3) - .flat_map(lambda polygon: h3.polygon_to_cells(polygon, h3_resolution)) + .flat_map(lambda polygon: _polygon_shapely_to_h3(polygon, h3_resolution, buffer)) .distinct() .to_list() )