diff --git a/.github/workflows/benchmarks/build.sh b/.github/workflows/benchmarks/build.sh new file mode 100755 index 000000000000..a5c82e244e58 --- /dev/null +++ b/.github/workflows/benchmarks/build.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -eu + +CMAKE_ARGS=( + "-DUSE_CCACHE=ON" \ + "-DCMAKE_BUILD_TYPE=Release" \ + "-DCMAKE_INSTALL_PREFIX=/usr" \ + "-DGDAL_USE_TIFF_INTERNAL=ON" \ + "-DGDAL_USE_GEOTIFF_INTERNAL=ON" \ + "-DECW_ROOT=/opt/libecwj2-3.3" \ + "-DMRSID_ROOT=/usr/local" \ + "-DFileGDB_ROOT=/usr/local/FileGDB_API" \ + "-DBUILD_CSHARP_BINDINGS=OFF" \ + "-DBUILD_JAVA_BINDINGS=OFF" \ + "-DGDAL_BUILD_OPTIONAL_DRIVERS=OFF" \ + "-DOGR_BUILD_OPTIONAL_DRIVERS=OFF" \ + "-DOGR_ENABLE_DRIVER_GPKG=ON" \ +) + +cmake ${GDAL_SOURCE_DIR:=..} \ + "${CMAKE_ARGS[@]}" + +make -j$(nproc) + +mkdir old_version +cd old_version +# To be updated with a true reference branch and commit +git clone https://github.com/rouault/gdal +cd gdal +git checkout b880cad693cd6cec0b0c90422cc6430121787ce4 +mkdir build +cd build + +cmake .. \ + "${CMAKE_ARGS[@]}" + +make -j$(nproc) diff --git a/.github/workflows/benchmarks/test.sh b/.github/workflows/benchmarks/test.sh new file mode 100755 index 000000000000..2739e7962874 --- /dev/null +++ b/.github/workflows/benchmarks/test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eu + +BENCHMARK_STORAGE="file:///tmp" + +# Use time.process_time for more reliability on VMs +BENCHMARK_OPTIONS=( + "--benchmark-storage=${BENCHMARK_STORAGE}" \ + "--benchmark-timer=time.process_time" \ +) + +# Dry run to hopefully stabilize later timings +(cd old_version/gdal/build; source ../scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" --capture=no -ra -vv) + +# Run reference (old) build and save its results +(cd old_version/gdal/build; source ../scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" --benchmark-save=ref --capture=no -ra -vv) + +# Run target build and compare its results to the reference one. +# Fail if we get results 20% slower or more. +# Retry if that fails a first time. +BENCHMARK_COMPARE_OPTIONS=( + "--benchmark-compare-fail=min:20%" \ + "--benchmark-compare=0001_ref" \ +) + +(source ${GDAL_SOURCE_DIR:=..}/scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv || (echo "Retrying..."; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv)) diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 8e323fbf6d36..b33cd3a619d6 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -433,7 +433,7 @@ jobs: - name: Install dependency shell: bash -l {0} run: | - conda install --yes --quiet curl libiconv icu python=3.10 swig numpy pytest pytest-env filelock zlib lxml jsonschema + conda install --yes --quiet curl libiconv icu python=3.10 swig numpy pytest pytest-env pytest-benchmark filelock zlib lxml jsonschema # FIXME: remove libnetcdf=4.9.2=nompi_h5902ca5_107 pinning as soon as https://github.com/conda-forge/libnetcdf-feedstock/issues/182 is resolved conda install --yes --quiet proj geos hdf4 hdf5 kealib \ libnetcdf=4.9.2=nompi_h5902ca5_107 openjpeg poppler libtiff libpng xerces-c expat libxml2 kealib json-c \ @@ -520,7 +520,7 @@ jobs: - name: Install dependency shell: bash -l {0} run: | - conda install --yes --quiet proj pytest pytest-env filelock lxml + conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml - name: Configure shell: bash -l {0} run: | @@ -658,7 +658,7 @@ jobs: - name: Install dependency shell: bash -l {0} run: | - conda install --yes --quiet --name gdalenv curl libiconv icu python=3.9 swig numpy pytest pytest-env filelock zlib clcache lxml + conda install --yes --quiet --name gdalenv curl libiconv icu python=3.9 swig numpy pytest pytest-env pytest-benchmark filelock zlib clcache lxml conda install --yes --quiet --name gdalenv -c conda-forge libgdal - name: Configure shell: bash -l {0} diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index 2c6535d3f9f7..b207f0b34552 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -90,6 +90,13 @@ jobs: build_script: build.sh test_script: test.sh + - name: Ubuntu 20.04, benchmarks + id: benchmarks + travis_branch: ubuntu_2004 + container: ubuntu_20.04 + build_script: build.sh + test_script: test.sh + - name: Ubuntu 20.04, Intel compiler id: icc container: icc @@ -270,6 +277,13 @@ jobs: TEST_CMD="ctest -V -j $(nproc)" fi + if test "${{ matrix.id }}" = "benchmarks"; then + if test -f /sys/devices/system/cpu/intel_pstate/no_turbo; then + echo "Disable TurboBoost" + echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo + fi + fi + # For cache mkdir .gdal diff --git a/autotest/CMakeLists.txt b/autotest/CMakeLists.txt index 154891309957..b206319a5b99 100644 --- a/autotest/CMakeLists.txt +++ b/autotest/CMakeLists.txt @@ -99,7 +99,8 @@ endfunction () osr gnm pyscripts - utilities) + utilities + benchmark) if (NOT "${CMAKE_BINARY_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}") if (SKIP_COPYING_AUTOTEST_SUBDIRS) message(STATUS "Skipping copying ${CMAKE_CURRENT_SOURCE_DIR}/${tgt}") diff --git a/autotest/benchmark/conftest.py b/autotest/benchmark/conftest.py new file mode 100755 index 000000000000..f7a2419a8eff --- /dev/null +++ b/autotest/benchmark/conftest.py @@ -0,0 +1,54 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Benchmarking +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import os + +from osgeo import gdal + + +def pytest_report_header(config): + gdal_header_info = "" + + if os.path.exists("/sys/devices/system/cpu/intel_pstate/no_turbo"): + content = open("/sys/devices/system/cpu/intel_pstate/no_turbo", "rb").read() + if content[0] == b"0"[0]: + gdal_header_info += "\n" + gdal_header_info += "WARNING WARNING\n" + gdal_header_info += "---------------\n" + gdal_header_info += "Intel TurboBoost is enabled. Benchmarking results will not be accurate.\n" + gdal_header_info += "Disable TurboBoost with: 'echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo'\n" + gdal_header_info += "---------------\n" + gdal_header_info += "WARNING WARNING\n" + + if "debug" in gdal.VersionInfo(""): + gdal_header_info += "WARNING: Running benchmarks on debug build. Results will not be accurate.\n" + + return gdal_header_info diff --git a/autotest/benchmark/test_gdalwarp.py b/autotest/benchmark/test_gdalwarp.py new file mode 100755 index 000000000000..ba412e6c40fd --- /dev/null +++ b/autotest/benchmark/test_gdalwarp.py @@ -0,0 +1,73 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Benchmarking of gdalwarp +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import gdaltest +import pytest + +from osgeo import gdal, osr + +# Must be set to run the test_XXX functions under the benchmark fixture +pytestmark = pytest.mark.usefixtures("decorate_with_benchmark") + + +@pytest.fixture() +def source_ds_filename(tmp_vsimem): + filename = str(tmp_vsimem / "source.tif") + if "debug" in gdal.VersionInfo(""): + size = 1024 + else: + size = 4096 + ds = gdal.GetDriverByName("GTiff").Create( + filename, size, size, 3, options=["TILED=YES"] + ) + srs = osr.SpatialReference() + srs.ImportFromEPSG(32631) + ds.SetSpatialRef(srs) + ds.SetGeoTransform([400000, 1, 0, 4500000, 0, -1]) + ds.GetRasterBand(1).Fill(1) + ds.GetRasterBand(2).Fill(2) + ds.GetRasterBand(3).Fill(3) + ds = None + return filename + + +@pytest.mark.parametrize("num_threads", ["1", "ALL_CPUS"]) +@pytest.mark.parametrize("resample_alg", ["near", "cubic"]) +def test_gdalwarp(tmp_vsimem, source_ds_filename, num_threads, resample_alg): + filename = str(tmp_vsimem / "test_gdalwarp.tif") + if gdal.VSIStatL(filename): + gdal.Unlink(filename) + with gdaltest.config_option("GDAL_NUM_THREADS", num_threads): + gdal.Warp( + filename, + source_ds_filename, + options=f"-co TILED=YES -r {resample_alg} -t_srs EPSG:4326", + ) diff --git a/autotest/benchmark/test_gtiff.py b/autotest/benchmark/test_gtiff.py new file mode 100755 index 000000000000..920ee658bfad --- /dev/null +++ b/autotest/benchmark/test_gtiff.py @@ -0,0 +1,183 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Benchmarking of GeoTIFF driver +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import array +from threading import Thread + +import gdaltest +import pytest + +from osgeo import gdal + +# Must be set to run the test_XXX functions under the benchmark fixture +pytestmark = pytest.mark.usefixtures("decorate_with_benchmark") + + +def test_gtiff_byte(): + gdal.Open("../gcore/data/byte.tif") + + +def test_gtiff_byte_get_srs(): + ds = gdal.Open("../gcore/data/byte.tif") + ds.GetSpatialRef() + + +@pytest.mark.parametrize("with_optim", [True, False]) +def test_gtiff_multithread_write(with_optim): + num_threads = gdal.GetNumCPUs() + nbands = 1 + compression = "DEFLATE" + buffer_pixel_interleaved = True + width = 2048 + height = 2048 + + nloops = 10 // nbands + data = array.array("B", [i % 255 for i in range(nbands * width * height)]) + + def thread_function(num): + filename = "/vsimem/tmp%d.tif" % num + drv = gdal.GetDriverByName("GTiff") + options = ["TILED=YES", "COMPRESS=" + compression] + for i in range(nloops): + ds = drv.Create(filename, width, height, nbands, options=options) + if not with_optim: + # Calling ReadRaster() disables the cache bypass write optimization + ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) + if nbands > 1: + if buffer_pixel_interleaved: + # Write pixel-interleaved buffer for maximum efficiency + ds.WriteRaster( + 0, + 0, + width, + height, + data, + buf_pixel_space=nbands, + buf_line_space=width * nbands, + buf_band_space=1, + ) + else: + ds.WriteRaster(0, 0, width, height, data) + else: + ds.GetRasterBand(1).WriteRaster(0, 0, width, height, data) + gdal.Unlink(filename) + + with gdaltest.SetCacheMax(width * height * nbands * num_threads): + + # Spawn num_threads running thread_function + threads_array = [] + + for i in range(num_threads): + t = Thread( + target=thread_function, + args=[i], + ) + t.start() + threads_array.append(t) + + for t in threads_array: + t.join() + + +@pytest.fixture() +def source_ds_4096x4096_filename(tmp_vsimem, request): + filename = str(tmp_vsimem / "source.tif") + ds = gdal.GetDriverByName("GTiff").Create( + filename, 4096, 4096, 3, options=request.param + ) + ds.GetRasterBand(1).Fill(1) + ds.GetRasterBand(2).Fill(2) + ds.GetRasterBand(3).Fill(3) + ds = None + return filename + + +@pytest.mark.parametrize( + "source_ds_4096x4096_filename", + [[], ["TILED=YES"]], + indirect=True, + ids=["source_default", "source_tiled"], +) +@pytest.mark.parametrize( + "options", + [ + [], + ["TILED=YES"], + ["TILED=YES", "COMPRESS=LZW"], + ["TILED=YES", "COMPRESS=LZW", "NUM_THREADS=ALL_CPUS"], + ], + ids=["dest_default", "dest_tiled", "dest_tiled_lzw", "dest_tiled_lzw_all_cpus"], +) +def test_gtiff_create_copy(tmp_vsimem, source_ds_4096x4096_filename, options): + filename = str(tmp_vsimem / "source.tif") + src_ds = gdal.Open(source_ds_4096x4096_filename) + gdal.GetDriverByName("GTiff").CreateCopy(filename, src_ds, options=options) + + +@pytest.fixture() +def source_ds_2048x2048_filename(tmp_vsimem, request): + filename = str(tmp_vsimem / "source.tif") + ds = gdal.GetDriverByName("GTiff").Create( + filename, 2048, 2048, 3, options=request.param + ) + ds.GetRasterBand(1).Fill(1) + ds.GetRasterBand(2).Fill(2) + ds.GetRasterBand(3).Fill(3) + ds = None + return filename + + +@pytest.mark.parametrize( + "source_ds_2048x2048_filename", [["TILED=YES"]], indirect=True, ids=["source_tiled"] +) +@pytest.mark.parametrize( + "ovr_alg", + [ + "NEAREST", + "BILINEAR", + "CUBIC", + "CUBICSPLINE", + "LANCZOS", + "AVERAGE", + "RMS", + "MODE", + "GAUSS", + ], +) +def test_gtiff_build_overviews(tmp_vsimem, source_ds_2048x2048_filename, ovr_alg): + filename = str(tmp_vsimem / "source.tif") + f = gdal.VSIFOpenL(source_ds_2048x2048_filename, "rb") + source_data = gdal.VSIFReadL(gdal.VSIStatL(source_ds_2048x2048_filename).size, 1, f) + gdal.VSIFCloseL(f) + gdal.FileFromMemBuffer(filename, source_data) + ds = gdal.Open(filename, gdal.GA_Update) + ds.BuildOverviews(ovr_alg, [2, 4, 8]) + ds.Close() diff --git a/autotest/benchmark/test_ogr2ogr.py b/autotest/benchmark/test_ogr2ogr.py new file mode 100755 index 000000000000..f7f3e2af4c5e --- /dev/null +++ b/autotest/benchmark/test_ogr2ogr.py @@ -0,0 +1,83 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Benchmarking of ogr2ogr +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import pytest + +from osgeo import gdal, ogr, osr + +# Must be set to run the test_XXX functions under the benchmark fixture +pytestmark = [ + pytest.mark.require_driver("GPKG"), + pytest.mark.usefixtures("decorate_with_benchmark"), +] + + +def create_file(filename, numfeatures=50000): + ds = ogr.GetDriverByName("GPKG").CreateDataSource(filename) + srs = osr.SpatialReference() + srs.ImportFromEPSG(32631) + lyr = ds.CreateLayer("test", srs=srs) + for i in range(20): + lyr.CreateField(ogr.FieldDefn(f"field{i}")) + f = ogr.Feature(lyr.GetLayerDefn()) + for i in range(20): + f.SetField(f"field{i}", f"value{i}") + lyr.StartTransaction() + for i in range(numfeatures): + f.SetFID(-1) + g = ogr.Geometry(ogr.wkbPoint) + g.SetPoint_2D(0, 400000 + i, i) + f.SetGeometry(g) + lyr.CreateFeature(f) + lyr.CommitTransaction() + + +@pytest.fixture() +def source_file(tmp_vsimem, request): + filename = str(tmp_vsimem / "source_file.gpkg") + create_file(filename, numfeatures=request.param) + return filename + + +@pytest.mark.parametrize("source_file", [50000], indirect=True) +def test_ogr2ogr(tmp_vsimem, source_file): + filename = str(tmp_vsimem / "test_ogr2ogr.gpkg") + if gdal.VSIStatL(filename): + gdal.Unlink(filename) + gdal.VectorTranslate(filename, source_file) + + +@pytest.mark.parametrize("source_file", [10000], indirect=True) +def test_ogr2ogr_reproject(tmp_vsimem, source_file): + filename = str(tmp_vsimem / "test_ogr2ogr.gpkg") + if gdal.VSIStatL(filename): + gdal.Unlink(filename) + gdal.VectorTranslate(filename, source_file, dstSRS="EPSG:4326", reproject=True) diff --git a/autotest/benchmark/test_ogr_gpkg.py b/autotest/benchmark/test_ogr_gpkg.py new file mode 100755 index 000000000000..fa51060b8137 --- /dev/null +++ b/autotest/benchmark/test_ogr_gpkg.py @@ -0,0 +1,80 @@ +#!/usr/bin/env pytest +# -*- coding: utf-8 -*- +############################################################################### +# $Id$ +# +# Project: GDAL/OGR Test Suite +# Purpose: Benchmarking of GeoPackage driver +# Author: Even Rouault +# +############################################################################### +# Copyright (c) 2023, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +############################################################################### + +import pytest + +from osgeo import ogr + +# Must be set to run the test_XXX functions under the benchmark fixture +pytestmark = [ + pytest.mark.require_driver("GPKG"), + pytest.mark.usefixtures("decorate_with_benchmark"), +] + + +def create_file(filename, numfeatures=50000): + ds = ogr.GetDriverByName("GPKG").CreateDataSource(filename) + lyr = ds.CreateLayer("test") + for i in range(20): + lyr.CreateField(ogr.FieldDefn(f"field{i}")) + f = ogr.Feature(lyr.GetLayerDefn()) + for i in range(20): + f.SetField(f"field{i}", f"value{i}") + lyr.StartTransaction() + for i in range(numfeatures): + f.SetFID(-1) + g = ogr.Geometry(ogr.wkbPoint) + g.SetPoint_2D(0, i, i) + f.SetGeometry(g) + lyr.CreateFeature(f) + lyr.CommitTransaction() + + +def test_ogr_gpkg_create(tmp_vsimem): + filename = str(tmp_vsimem / "test.gpkg") + create_file(filename) + + +@pytest.fixture() +def source_file(tmp_vsimem): + filename = str(tmp_vsimem / "test.gpkg") + create_file(filename) + return filename + + +def test_ogr_gpkg_spatial_index(source_file): + ds = ogr.Open(source_file) + lyr = ds.GetLayer(0) + lyr.SetSpatialFilterRect(1000, 1000, 10000, 10000) + count = 0 + for f in lyr: + count += 1 + assert count == 10000 - 1000 + 1 diff --git a/autotest/conftest.py b/autotest/conftest.py index 7471949a8f7b..7baf54d29867 100755 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -291,3 +291,17 @@ def tmp_vsimem(request): yield path gdal.RmdirRecursive(str(path)) + + +# Fixture to run a test function with pytest_benchmark +@pytest.fixture(scope="function") +def decorate_with_benchmark(request, benchmark): + def run_under_benchmark(f, benchmark): + def test_with_benchmark_fixture(*args, **kwargs): + @benchmark + def do(): + f(*args, **kwargs) + + return test_with_benchmark_fixture + + request.node.obj = run_under_benchmark(request.node.obj, benchmark) diff --git a/autotest/requirements.txt b/autotest/requirements.txt index c884565eb8cb..61697e503875 100644 --- a/autotest/requirements.txt +++ b/autotest/requirements.txt @@ -2,6 +2,7 @@ pytest>=6.0.0 pytest-sugar<=0.9.6; python_version < '3.7' pytest-sugar; python_version >= '3.7' pytest-env +pytest-benchmark lxml jsonschema filelock diff --git a/cmake/template/pytest.ini.in b/cmake/template/pytest.ini.in index 72c62aff93c9..b55a3c534251 100644 --- a/cmake/template/pytest.ini.in +++ b/cmake/template/pytest.ini.in @@ -2,7 +2,7 @@ [pytest] python_files = *.py -testpaths = ogr gcore gdrivers osr alg gnm utilities pyscripts +testpaths = ogr gcore gdrivers osr alg gnm utilities pyscripts benchmark norecursedirs = ogr/data gdrivers/data cpp log_file = @AUTOTEST_LOG_FILE@ log_file_level = INFO diff --git a/gdal.cmake b/gdal.cmake index e221f99b7932..b590b07b347c 100644 --- a/gdal.cmake +++ b/gdal.cmake @@ -546,6 +546,14 @@ endif () set(GDAL_RASTER_FORMAT_SOURCE_DIR "${PROJECT_SOURCE_DIR}/frmts") set(GDAL_VECTOR_FORMAT_SOURCE_DIR "${PROJECT_SOURCE_DIR}/ogr/ogrsf_frmts") +if(OGR_ENABLE_DRIVER_GPKG AND + NOT DEFINED OGR_ENABLE_DRIVER_SQLITE AND + DEFINED OGR_BUILD_OPTIONAL_DRIVERS AND + NOT OGR_BUILD_OPTIONAL_DRIVERS) + message(STATUS "Automatically enabling SQLite driver") + set(OGR_ENABLE_DRIVER_SQLITE ON CACHE BOOL "Set ON to build OGR SQLite driver") +endif() + # We need to forward declare a few OGR drivers because raster formats need them option(OGR_ENABLE_DRIVER_AVC "Set ON to build OGR AVC driver" ${OGR_BUILD_OPTIONAL_DRIVERS}) option(OGR_ENABLE_DRIVER_GML "Set ON to build OGR GML driver" ${OGR_BUILD_OPTIONAL_DRIVERS}) diff --git a/scripts/setdevenv.sh b/scripts/setdevenv.sh index 53a1c5b27968..78c2c0f3637b 100755 --- a/scripts/setdevenv.sh +++ b/scripts/setdevenv.sh @@ -38,30 +38,30 @@ if [[ ! ${PATH} =~ $CUR_DIR/apps ]]; then fi if [[ "$(uname -s)" == "Darwin" ]]; then - if [[ ! "${DYLD_LIBRARY_PATH}" =~ $CUR_DIR ]]; then - export DYLD_LIBRARY_PATH="$CUR_DIR:$DYLD_LIBRARY_PATH" + if [[ ! "${DYLD_LIBRARY_PATH:-}" =~ $CUR_DIR ]]; then + export DYLD_LIBRARY_PATH="$CUR_DIR:${DYLD_LIBRARY_PATH:-}" echo "Setting DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH" fi else - if [[ ! "${LD_LIBRARY_PATH}" =~ $CUR_DIR ]]; then - export LD_LIBRARY_PATH="$CUR_DIR:$LD_LIBRARY_PATH" + if [[ ! "${LD_LIBRARY_PATH:-}" =~ $CUR_DIR ]]; then + export LD_LIBRARY_PATH="$CUR_DIR:${LD_LIBRARY_PATH:-}" echo "Setting LD_LIBRARY_PATH=$LD_LIBRARY_PATH" fi fi -if [[ ! ${GDAL_DRIVER_PATH} =~ $CUR_DIR/gdalplugins ]]; then +if [[ ! ${GDAL_DRIVER_PATH:-} =~ $CUR_DIR/gdalplugins ]]; then export GDAL_DRIVER_PATH="$CUR_DIR/gdalplugins" echo "Setting GDAL_DRIVER_PATH=$GDAL_DRIVER_PATH" fi -if [[ ! "${GDAL_DATA}" =~ $GDAL_ROOT/data ]]; then +if [[ ! "${GDAL_DATA:-}" =~ $GDAL_ROOT/data ]]; then export GDAL_DATA="$GDAL_ROOT/data" echo "Setting GDAL_DATA=$GDAL_DATA" fi GDAL_PYTHONPATH="$CUR_DIR/swig/python" -if [[ ! "${PYTHONPATH}" =~ $GDAL_PYTHONPATH ]]; then - export PYTHONPATH="$GDAL_PYTHONPATH:$PYTHONPATH" +if [[ ! "${PYTHONPATH:-}" =~ $GDAL_PYTHONPATH ]]; then + export PYTHONPATH="$GDAL_PYTHONPATH:${PYTHONPATH:-}" echo "Setting PYTHONPATH=$PYTHONPATH" fi unset GDAL_PYTHONPATH