Arrow/Parquet: support LargeString and LargeBinary for geometry colum…

…ns (read support only)
OSGeo · Oct 26, 2023 · 87b8653 · 87b8653
1 parent e9fb165
commit 87b8653
Show file tree

Hide file tree

Showing 4 changed files with 195 additions and 39 deletions.
diff --git a/autotest/ogr/ogr_parquet.py b/autotest/ogr/ogr_parquet.py
@@ -35,6 +35,7 @@
 import gdaltest
 import ogrtest
 import pytest
+import test_cli_utilities
 
 from osgeo import gdal, ogr, osr
 
@@ -493,7 +494,6 @@ def test_ogr_parquet_1(use_vsi):
 
 
 def test_ogr_parquet_test_ogrsf_test():
-    import test_cli_utilities
 
     if test_cli_utilities.get_test_ogrsf_path() is None:
         pytest.skip()
@@ -511,7 +511,6 @@ def test_ogr_parquet_test_ogrsf_test():
 
 
 def test_ogr_parquet_test_ogrsf_example():
-    import test_cli_utilities
 
     if test_cli_utilities.get_test_ogrsf_path() is None:
         pytest.skip()
@@ -529,7 +528,6 @@ def test_ogr_parquet_test_ogrsf_example():
 
 
 def test_ogr_parquet_test_ogrsf_all_geoms():
-    import test_cli_utilities
 
     if test_cli_utilities.get_test_ogrsf_path() is None:
         pytest.skip()
@@ -3085,3 +3083,29 @@ def test_ogr_parquet_write_arrow_rewind_polygon(tmp_vsimem):
     lyr = ds.GetLayer(0)
     f = lyr.GetNextFeature()
     assert f.GetGeometryRef().ExportToWkt() == "POLYGON ((0 0,1 1,0 1,0 0))"
+
+
+###############################################################################
+
+
+@gdaltest.enable_exceptions()
+@pytest.mark.parametrize(
+    "filename",
+    [
+        "data/parquet/poly_wkb_large_binary.parquet",
+        "data/parquet/poly_wkt_large_string.parquet",
+    ],
+)
+def test_ogr_parquet_read_large_binary_or_string_for_geometry(filename):
+    ds = ogr.Open(filename)
+    lyr = ds.GetLayer(0)
+    f = lyr.GetNextFeature()
+    assert f.GetGeometryRef() is not None
+
+    if test_cli_utilities.get_test_ogrsf_path() is None:
+        ret = gdaltest.runexternal(
+            test_cli_utilities.get_test_ogrsf_path() + " -ro " + filename
+        )
+
+        assert "INFO" in ret
+        assert "ERROR" not in ret
diff --git a/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h b/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h
@@ -93,8 +93,9 @@ class OGRArrowLayer CPL_NON_FINAL
     void ExploreExprNode(const swq_expr_node *poNode);
     bool UseRecordBatchBaseImplementation() const;
 
+    template <typename SourceOffset>
     static struct ArrowArray *
-    CreateWKTArrayFromWKBArray(const struct ArrowArray *sourceArray);
+    CreateWKBArrayFromWKTArray(const struct ArrowArray *sourceArray);
 
     int GetArrowSchemaInternal(struct ArrowSchema *out) const;
 
@@ -116,6 +117,7 @@ class OGRArrowLayer CPL_NON_FINAL
     int m_iBBOXMaxYField = -1;
 
     const arrow::BinaryArray *m_poArrayWKB = nullptr;
+    const arrow::LargeBinaryArray *m_poArrayWKBLarge = nullptr;
     const arrow::Array *m_poArrayBBOX = nullptr;
     const arrow::DoubleArray *m_poArrayMinX = nullptr;
     const arrow::DoubleArray *m_poArrayMinY = nullptr;