Merge pull request #8538 from rouault/benchmark

CI: add benchmark test suite

.github/workflows/benchmarks/build.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -eu
+
+CMAKE_ARGS=(
+        "-DUSE_CCACHE=ON" \
+        "-DCMAKE_BUILD_TYPE=Release" \
+        "-DCMAKE_INSTALL_PREFIX=/usr" \
+        "-DGDAL_USE_TIFF_INTERNAL=ON" \
+        "-DGDAL_USE_GEOTIFF_INTERNAL=ON" \
+        "-DECW_ROOT=/opt/libecwj2-3.3" \
+        "-DMRSID_ROOT=/usr/local" \
+        "-DFileGDB_ROOT=/usr/local/FileGDB_API" \
+        "-DBUILD_CSHARP_BINDINGS=OFF" \
+        "-DBUILD_JAVA_BINDINGS=OFF" \
+        "-DGDAL_BUILD_OPTIONAL_DRIVERS=OFF" \
+        "-DOGR_BUILD_OPTIONAL_DRIVERS=OFF" \
+        "-DOGR_ENABLE_DRIVER_GPKG=ON" \
+)
+
+cmake ${GDAL_SOURCE_DIR:=..} \
+    "${CMAKE_ARGS[@]}"
+
+make -j$(nproc)
+
+mkdir old_version
+cd old_version
+# To be updated with a true reference branch and commit
+git clone https://github.com/rouault/gdal
+cd gdal
+git checkout b880cad693cd6cec0b0c90422cc6430121787ce4
+mkdir build
+cd build
+
+cmake .. \
+    "${CMAKE_ARGS[@]}"
+
+make -j$(nproc)

.github/workflows/benchmarks/test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -eu
+
+BENCHMARK_STORAGE="file:///tmp"
+
+# Use time.process_time for more reliability on VMs
+BENCHMARK_OPTIONS=(
+        "--benchmark-storage=${BENCHMARK_STORAGE}" \
+        "--benchmark-timer=time.process_time" \
+)
+
+# Dry run to hopefully stabilize later timings
+(cd old_version/gdal/build; source ../scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" --capture=no -ra -vv)
+
+# Run reference (old) build and save its results
+(cd old_version/gdal/build; source ../scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" --benchmark-save=ref  --capture=no -ra -vv)
+
+# Run target build and compare its results to the reference one.
+# Fail if we get results 20% slower or more.
+# Retry if that fails a first time.
+BENCHMARK_COMPARE_OPTIONS=(
+        "--benchmark-compare-fail=min:20%" \
+        "--benchmark-compare=0001_ref" \
+)
+
+(source ${GDAL_SOURCE_DIR:=..}/scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv || (echo "Retrying..."; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv))

.github/workflows/cmake_builds.yml

@@ -433,7 +433,7 @@ jobs:
     - name: Install dependency
       shell: bash -l {0}
       run: |
-        conda install --yes --quiet curl libiconv icu python=3.10 swig numpy pytest pytest-env filelock zlib lxml jsonschema
+        conda install --yes --quiet curl libiconv icu python=3.10 swig numpy pytest pytest-env pytest-benchmark filelock zlib lxml jsonschema
         # FIXME: remove libnetcdf=4.9.2=nompi_h5902ca5_107 pinning as soon as https://github.com/conda-forge/libnetcdf-feedstock/issues/182 is resolved
         conda install --yes --quiet proj geos hdf4 hdf5 kealib \
             libnetcdf=4.9.2=nompi_h5902ca5_107 openjpeg poppler libtiff libpng xerces-c expat libxml2 kealib json-c \
@@ -520,7 +520,7 @@ jobs:
     - name: Install dependency
       shell: bash -l {0}
       run: |
-        conda install --yes --quiet proj pytest pytest-env filelock lxml
+        conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml
     - name: Configure
       shell: bash -l {0}
       run: |
@@ -658,7 +658,7 @@ jobs:
     - name: Install dependency
       shell: bash -l {0}
       run: |
-        conda install --yes --quiet --name gdalenv curl libiconv icu python=3.9 swig numpy pytest pytest-env filelock zlib clcache lxml
+        conda install --yes --quiet --name gdalenv curl libiconv icu python=3.9 swig numpy pytest pytest-env pytest-benchmark filelock zlib clcache lxml
         conda install --yes --quiet --name gdalenv -c conda-forge libgdal
     - name: Configure
       shell: bash -l {0}

.github/workflows/linux_build.yml

@@ -90,6 +90,13 @@ jobs:
             build_script: build.sh
             test_script: test.sh
 
+          - name: Ubuntu 20.04, benchmarks
+            id: benchmarks
+            travis_branch: ubuntu_2004
+            container: ubuntu_20.04
+            build_script: build.sh
+            test_script: test.sh
+
           - name: Ubuntu 20.04, Intel compiler
             id: icc
             container: icc
@@ -270,6 +277,13 @@ jobs:
             TEST_CMD="ctest -V -j $(nproc)"
           fi
 
+          if test "${{ matrix.id }}" = "benchmarks"; then
+            if test -f /sys/devices/system/cpu/intel_pstate/no_turbo; then
+              echo "Disable TurboBoost"
+              echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
+            fi
+          fi
+
           # For cache
           mkdir .gdal
 

autotest/CMakeLists.txt

@@ -99,7 +99,8 @@ endfunction ()
           osr
           gnm
           pyscripts
-          utilities)
+          utilities
+          benchmark)
     if (NOT "${CMAKE_BINARY_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}")
         if (SKIP_COPYING_AUTOTEST_SUBDIRS)
           message(STATUS "Skipping copying ${CMAKE_CURRENT_SOURCE_DIR}/${tgt}")

autotest/benchmark/conftest.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env pytest
+# -*- coding: utf-8 -*-
+###############################################################################
+# $Id$
+#
+# Project:  GDAL/OGR Test Suite
+# Purpose:  Benchmarking
+# Author:   Even Rouault <even dot rouault at spatialys.com>
+#
+###############################################################################
+# Copyright (c) 2023, Even Rouault <even dot rouault at spatialys.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+###############################################################################
+
+import os
+
+from osgeo import gdal
+
+
+def pytest_report_header(config):
+    gdal_header_info = ""
+
+    if os.path.exists("/sys/devices/system/cpu/intel_pstate/no_turbo"):
+        content = open("/sys/devices/system/cpu/intel_pstate/no_turbo", "rb").read()
+        if content[0] == b"0"[0]:
+            gdal_header_info += "\n"
+            gdal_header_info += "WARNING WARNING\n"
+            gdal_header_info += "---------------\n"
+            gdal_header_info += "Intel TurboBoost is enabled. Benchmarking results will not be accurate.\n"
+            gdal_header_info += "Disable TurboBoost with: 'echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo'\n"
+            gdal_header_info += "---------------\n"
+            gdal_header_info += "WARNING WARNING\n"
+
+    if "debug" in gdal.VersionInfo(""):
+        gdal_header_info += "WARNING: Running benchmarks on debug build. Results will not be accurate.\n"
+
+    return gdal_header_info

autotest/benchmark/test_gdalwarp.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env pytest
+# -*- coding: utf-8 -*-
+###############################################################################
+# $Id$
+#
+# Project:  GDAL/OGR Test Suite
+# Purpose:  Benchmarking of gdalwarp
+# Author:   Even Rouault <even dot rouault at spatialys.com>
+#
+###############################################################################
+# Copyright (c) 2023, Even Rouault <even dot rouault at spatialys.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+###############################################################################
+
+import gdaltest
+import pytest
+
+from osgeo import gdal, osr
+
+# Must be set to run the test_XXX functions under the benchmark fixture
+pytestmark = pytest.mark.usefixtures("decorate_with_benchmark")
+
+
+@pytest.fixture()
+def source_ds_filename(tmp_vsimem):
+    filename = str(tmp_vsimem / "source.tif")
+    if "debug" in gdal.VersionInfo(""):
+        size = 1024
+    else:
+        size = 4096
+    ds = gdal.GetDriverByName("GTiff").Create(
+        filename, size, size, 3, options=["TILED=YES"]
+    )
+    srs = osr.SpatialReference()
+    srs.ImportFromEPSG(32631)
+    ds.SetSpatialRef(srs)
+    ds.SetGeoTransform([400000, 1, 0, 4500000, 0, -1])
+    ds.GetRasterBand(1).Fill(1)
+    ds.GetRasterBand(2).Fill(2)
+    ds.GetRasterBand(3).Fill(3)
+    ds = None
+    return filename
+
+
+@pytest.mark.parametrize("num_threads", ["1", "ALL_CPUS"])
+@pytest.mark.parametrize("resample_alg", ["near", "cubic"])
+def test_gdalwarp(tmp_vsimem, source_ds_filename, num_threads, resample_alg):
+    filename = str(tmp_vsimem / "test_gdalwarp.tif")
+    if gdal.VSIStatL(filename):
+        gdal.Unlink(filename)
+    with gdaltest.config_option("GDAL_NUM_THREADS", num_threads):
+        gdal.Warp(
+            filename,
+            source_ds_filename,
+            options=f"-co TILED=YES -r {resample_alg} -t_srs EPSG:4326",
+        )