Merge pull request OSGeo#9897 from rouault/sanitize_ubuntu_2204

Avoid temporary file byte.tif.aux.xml creation, and fixes for CI ASAN for Ubuntu 22.04

.github/workflows/asan/test.sh

@@ -45,8 +45,6 @@ find -L \
         ! -name ogr_gpsbabel.py `# new-delete-type-mismatch error in gpsbabel binary that we can't suppress` \
         ! -name "__init__.py" \
         ! -path 'ogr/data/*' \
-        ! -name test_gdal_merge.py \
-        ! -name test_gdal_retile.py \
     -print \
     -exec ./pytest_wrapper.sh {} \; \
     | tee ./test-output.txt

.github/workflows/linux_build.yml

@@ -110,8 +110,7 @@ jobs:
             container: ubuntu_22.04
             build_script: build.sh
             test_script: test.sh
-            # We force the host OS to be 20.04 to avoid "AddressSanitizer:DEADLYSIGNAL"
-            os: ubuntu-20.04
+            os: ubuntu-22.04
 
           - name: Ubuntu 20.04, gcc
             id: ubuntu_20.04
@@ -167,6 +166,13 @@ jobs:
           echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV}
           echo "CONTAINER_NAME_FULL=${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER,,}/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN}"  >>${GITHUB_ENV}
 
+      # Work around segfaults in ASan/MSan jobs
+      # Cf https://github.com/libjpeg-turbo/libjpeg-turbo/commit/2dfe6c0fe9e18671105e94f7cbf044d4a1d157e6
+      # and https://github.com/actions/runner-images/issues/9491
+      - name: Set up build
+        run: |
+          sudo sysctl vm.mmap_rnd_bits=28
+
       - name: Checkout
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
 

autotest/conftest.py

@@ -124,6 +124,19 @@ def chdir_to_test_file(request):
     os.chdir(old)
 
 
+@pytest.fixture(autouse=True, scope="function")
+def check_no_unintended_side_car_files(request):
+    """Detect if some tests generate unintended side car files that can cause
+    troubles to other tests.
+    """
+    yield
+
+    filename = os.path.dirname(__file__) + "/gcore/data/byte.tif.aux.xml"
+    if os.path.exists(filename):
+        os.unlink(filename)
+        assert False, f"{filename} exists but should not"
+
+
 def pytest_collection_modifyitems(config, items):
     # skip test with @ptest.mark.require_run_on_demand when RUN_ON_DEMAND is not set
     skip_run_on_demand_not_set = pytest.mark.skip("RUN_ON_DEMAND not set")

autotest/gcore/tiff_read.py

@@ -747,14 +747,12 @@ def test_tiff_12bitjpeg():
 # Test that statistics for TIFF files are stored and correctly read from .aux.xml
 
 
-def test_tiff_read_stats_from_pam():
+def test_tiff_read_stats_from_pam(tmp_path):
 
-    try:
-        os.remove("data/byte.tif.aux.xml")
-    except OSError:
-        pass
+    tmp_tif = str(tmp_path / "byte.tif")
+    shutil.copy("data/byte.tif", tmp_tif)
 
-    ds = gdal.Open("data/byte.tif")
+    ds = gdal.Open(tmp_tif)
     md = ds.GetRasterBand(1).GetMetadata()
     assert "STATISTICS_MINIMUM" not in md, "Unexpected presence of STATISTICS_MINIMUM"
 
@@ -763,22 +761,14 @@ def test_tiff_read_stats_from_pam():
     assert stats[0] == 74.0 and stats[1] == 255.0
 
     ds = None
-    try:
-        os.stat("data/byte.tif.aux.xml")
-    except OSError:
-        pytest.fail("Expected generation of data/byte.tif.aux.xml")
+    assert os.path.exists(tmp_tif + ".aux.xml")
 
-    ds = gdal.Open("data/byte.tif")
+    ds = gdal.Open(tmp_tif)
     # Just read statistics (from PAM) without forcing their computation
     stats = ds.GetRasterBand(1).GetStatistics(0, 0)
     assert stats[0] == 74.0 and stats[1] == 255.0
     ds = None
 
-    try:
-        os.remove("data/byte.tif.aux.xml")
-    except OSError:
-        pass
-
 
 ###############################################################################
 # Test extracting georeferencing from a .TAB file

autotest/gdrivers/hfa.py

@@ -30,6 +30,7 @@
 ###############################################################################
 
 import os
+import shutil
 import struct
 
 import gdaltest
@@ -1237,21 +1238,17 @@ def test_hfa_write_rat():
 # Test STATISTICS creation option
 
 
-def test_hfa_createcopy_statistics():
+def test_hfa_createcopy_statistics(tmp_path):
 
-    tmpAuxXml = "../gcore/data/byte.tif.aux.xml"
-    try:
-        os.remove(tmpAuxXml)
-    except OSError:
-        pass
-    ds_src = gdal.Open("../gcore/data/byte.tif")
+    tmp_tif = str(tmp_path / "byte.tif")
+    shutil.copy("../gcore/data/byte.tif", tmp_tif)
+
+    ds_src = gdal.Open(tmp_tif)
     out_ds = gdal.GetDriverByName("HFA").CreateCopy(
         "/vsimem/byte.img", ds_src, options=["STATISTICS=YES"]
     )
     del out_ds
     ds_src = None
-    if os.path.exists(tmpAuxXml):
-        os.remove(tmpAuxXml)
 
     gdal.Unlink("/vsimem/byte.img.aux.xml")
 

autotest/osr/osr_basic.py

@@ -1840,9 +1840,6 @@ def threaded_function(arg):
 
 def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ok():
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     backup_search_paths = osr.GetPROJSearchPaths()
     # conftest.py set 2 paths: autotest/gcore/tmp/proj_db_tmpdir and autotest/proj_grids
     assert len(backup_search_paths) == 2
@@ -1861,9 +1858,6 @@ def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ok():
 
 def test_Set_PROJ_DATA_config_option_sub_proccess_config_option_ko():
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     backup_search_paths = osr.GetPROJSearchPaths()
     # conftest.py set 2 paths: autotest/gcore/tmp/proj_db_tmpdir and autotest/proj_grids
     assert len(backup_search_paths) == 2

autotest/pyscripts/test_gdal_calc.py

@@ -36,7 +36,6 @@
 from collections import defaultdict
 from copy import copy
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -70,9 +69,6 @@ def script_path():
 
 def test_gdal_calc_help(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_calc", "--help"
     )
@@ -84,9 +80,6 @@ def test_gdal_calc_help(script_path):
 
 def test_gdal_calc_version(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_calc", "--version"
     )

autotest/pyscripts/test_gdal_edit.py

@@ -33,7 +33,6 @@
 import shutil
 import sys
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -141,9 +140,6 @@ def test_gdal_edit_py_1(script_path, tmp_path, read_only):
 
 def test_gdal_edit_py_1b(script_path, tmp_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     filename = str(tmp_path / "test_gdal_edit_py.tif")
     shutil.copy(test_py_scripts.get_data_path("gcore") + "byte.tif", filename)
 

autotest/pyscripts/test_gdal_fillnodata.py

@@ -30,7 +30,6 @@
 
 import struct
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -53,9 +52,6 @@ def script_path():
 
 def test_gdal_fillnodata_help(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_fillnodata", "--help"
     )
@@ -67,9 +63,6 @@ def test_gdal_fillnodata_help(script_path):
 
 def test_gdal_fillnodata_version(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_fillnodata", "--version"
     )

autotest/pyscripts/test_gdal_pansharpen.py

@@ -30,7 +30,6 @@
 ###############################################################################
 
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -76,9 +75,6 @@ def small_world_pan_tif(tmp_path_factory):
 
 def test_gdal_pansharpen_help(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_pansharpen", "--help"
     )

autotest/pyscripts/test_gdal_proximity.py

@@ -29,7 +29,6 @@
 # DEALINGS IN THE SOFTWARE.
 ###############################################################################
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -52,9 +51,6 @@ def script_path():
 
 def test_gdal_proximity_help(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_proximity", "--help"
     )
@@ -66,9 +62,6 @@ def test_gdal_proximity_help(script_path):
 
 def test_gdal_proximity_version(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_proximity", "--version"
     )

autotest/pyscripts/test_gdal_sieve.py

@@ -30,7 +30,6 @@
 ###############################################################################
 
 
-import gdaltest
 import pytest
 import test_py_scripts
 
@@ -53,9 +52,6 @@ def script_path():
 
 def test_gdal_sieve_help(script_path):
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     assert "ERROR" not in test_py_scripts.run_py_script(
         script_path, "gdal_sieve", "--help"
     )

autotest/pyscripts/test_ogrmerge.py

@@ -30,7 +30,6 @@
 
 import sys
 
-import gdaltest
 import pytest
 import test_py_scripts
 from test_py_scripts import samples_path
@@ -246,9 +245,6 @@ def test_ogrmerge_7(script_path, tmp_path):
 
     out_vrt = str(tmp_path / "out.vrt")
 
-    if gdaltest.is_travis_branch("sanitize"):
-        pytest.skip("fails on sanitize for unknown reason")
-
     # No match in -single mode
     test_py_scripts.run_py_script(
         script_path,

autotest/utilities/test_gdal_translate.py

@@ -626,9 +626,7 @@ def test_gdal_translate_23(gdal_translate_path, tmp_path):
 
     shutil.copy("../gcore/data/byte.tif", src_tif)
 
-    gdaltest.runexternal(
-        f"{gdal_translate_path} -stats ../gcore/data/byte.tif {dst_tif}"
-    )
+    gdaltest.runexternal(f"{gdal_translate_path} -stats {src_tif} {dst_tif}")
 
     ds = gdal.Open(dst_tif)
     md = ds.GetRasterBand(1).GetMetadata()

autotest/utilities/test_gdalinfo.py

@@ -569,18 +569,16 @@ def test_gdalinfo_34(gdalinfo_path):
 # Test -hist option
 
 
-def test_gdalinfo_35(gdalinfo_path):
+def test_gdalinfo_35(gdalinfo_path, tmp_path):
 
-    try:
-        os.remove("../gcore/data/byte.tif.aux.xml")
-    except OSError:
-        pass
+    tmp_tif = str(tmp_path / "byte.tif")
+    shutil.copy("../gcore/data/byte.tif", tmp_tif)
 
-    ret = gdaltest.runexternal(gdalinfo_path + " -json ../gcore/data/byte.tif")
+    ret = gdaltest.runexternal(gdalinfo_path + f" -json {tmp_tif}")
     ret = json.loads(ret)
     assert "histogram" not in ret["bands"][0], "did not expect histogram."
 
-    ret = gdaltest.runexternal(gdalinfo_path + " -json -hist ../gcore/data/byte.tif")
+    ret = gdaltest.runexternal(gdalinfo_path + f" -json -hist {tmp_tif}")
     ret = json.loads(ret)
     assert ret["bands"][0]["histogram"]["buckets"] == [
         0,
@@ -841,8 +839,7 @@ def test_gdalinfo_35(gdalinfo_path):
         1,
     ], "did not get expected histogram."
 
-    # We will blow an exception if the file does not exist now!
-    os.remove("../gcore/data/byte.tif.aux.xml")
+    assert os.path.exists(tmp_tif + ".aux.xml")
 
 
 ###############################################################################

autotest/utilities/test_gdalinfo_lib.py

@@ -31,6 +31,7 @@
 
 
 import pathlib
+import shutil
 
 import gdaltest
 import pytest
@@ -110,9 +111,12 @@ def test_gdalinfo_lib_4():
 # Test all options
 
 
-def test_gdalinfo_lib_5():
+def test_gdalinfo_lib_5(tmp_path):
 
-    ds = gdal.Open("../gdrivers/data/byte.tif")
+    tmp_tif = str(tmp_path / "byte.tif")
+    shutil.copy("../gcore/data/byte.tif", tmp_tif)
+
+    ds = gdal.Open(tmp_tif)
 
     ret = gdal.Info(
         ds,
@@ -143,8 +147,6 @@ def test_gdalinfo_lib_5():
 
     ds = None
 
-    gdal.Unlink("../gdrivers/data/byte.tif.aux.xml")
-
 
 ###############################################################################
 # Test command line syntax + dataset as string