benchmarks/test.sh: bump tolerance to 20%

OSGeo · Oct 11, 2023 · b0f6525 · b0f6525
1 parent 974175b
commit b0f6525
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/.github/workflows/benchmarks/test.sh b/.github/workflows/benchmarks/test.sh
@@ -17,6 +17,11 @@ BENCHMARK_OPTIONS=(
 (cd old_version/gdal/build; source ../scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" --benchmark-save=ref  --capture=no -ra -vv)
 
 # Run target build and compare its results to the reference one.
-# Fail if we get results 5% slower or more.
+# Fail if we get results 20% slower or more.
 # Retry if that fails a first time.
-(source ${GDAL_SOURCE_DIR:=..}/scripts/setdevenv.sh; pytest autotest/benchmark --benchmark-compare-fail="min:5%" --benchmark-compare=0001_ref "${BENCHMARK_OPTIONS[@]}" --capture=no -ra -vv || (echo "Retrying..."; pytest autotest/benchmark --benchmark-compare-fail="min:5%" --benchmark-compare=0001_ref "${BENCHMARK_OPTIONS[@]}" --capture=no -ra -vv))
+BENCHMARK_COMPARE_OPTIONS=(
+        "--benchmark-compare-fail=min:20%" \
+        "--benchmark-compare=0001_ref" \
+)
+
+(source ${GDAL_SOURCE_DIR:=..}/scripts/setdevenv.sh; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv || (echo "Retrying..."; pytest autotest/benchmark "${BENCHMARK_OPTIONS[@]}" "${BENCHMARK_COMPARE_OPTIONS[@]}" --capture=no -ra -vv))