From 49f9cfb358e20a81307ddf8669c1b6f8a7c2fb10 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:03:11 +1000 Subject: [PATCH 1/7] Adapt some grass tests to account for formats used in newer GRASS versions --- .../grass_algorithms_imagery_tests.yaml | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml index 1b35814267c7..5af68a3027ad 100644 --- a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml +++ b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml @@ -381,8 +381,13 @@ tests: type: raster results: signaturefile: - type: file name: expected/i.gensig.txt + type: regex + rules: + - '300\s*\n(?:\d+\s*\n)7043\.06\s*\n1\.89772e\+07\s*' + - '1700\s*\n(?:\d+\s*\n)7405\.93\s*\n2\.01921e\+07\s*' + - '6000\s*\n(?:\d+\s*\n)7287\.21\s*\n1\.90221e\+07\s*' + - '7000\s*\n(?:\d+\s*\n)7246\.81\s*\n1\.90226e\+07\s*' - algorithm: grass7:i.gensigset name: GRASS7 i.gensigset @@ -399,8 +404,26 @@ tests: type: raster results: signaturefile: - type: file + type: regex name: expected/i.gensigset.txt + rules: + - 'subclass:\s*\n\s*pi:\s*0\.22802\s*\n\s*means:\s*9764\.5\s*\n\s*covar:\s*\n\s*3\.02234e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.495197\s*\n\s*means:\s*5208\.5\s*\n\s*covar:\s*\n\s*7\.97435e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.118668\s*\n\s*means:\s*997\.979\s*\n\s*covar:\s*\n\s*686411' + - 'subclass:\s*\n\s*pi:\s*0\.158115\s*\n\s*means:\s*13401\s*\n\s*covar:\s*\n\s*702077' + - 'subclass:\s*\n\s*pi:\s*0\.177696\s*\n\s*means:\s*13267\.4\s*\n\s*covar:\s*\n\s*1\.06971e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.172485\s*\n\s*means:\s*3861\.81\s*\n\s*covar:\s*\n\s*2\.35556e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.48383\s*\n\s*means:\s*8696\.48\s*\n\s*covar:\s*\n\s*7\.62681e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.165989\s*\n\s*means:\s*1052\.05\s*\n\s*covar:\s*\n\s*692627' + - 'subclass:\s*\n\s*pi:\s*0\.135816\s*\n\s*means:\s*1158\.36\s*\n\s*covar:\s*\n\s*934515' + - 'subclass:\s*\n\s*pi:\s*0\.187835\s*\n\s*means:\s*11847\.9\s*\n\s*covar:\s*\n\s*1\.74438e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.0829488\s*\n\s*means:\s*14066\.2\s*\n\s*covar:\s*\n\s*268579' + - 'subclass:\s*\n\s*pi:\s*0\.5934\s*\n\s*means:\s*6298\.74\s*\n\s*covar:\s*\n\s*8\.66516e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.278456\s*\n\s*means:\s*3394\.59\s*\n\s*covar:\s*\n\s*2\.33337e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.0849227\s*\n\s*means:\s*13998\.4\s*\n\s*covar:\s*\n\s*256746' + - 'subclass:\s*\n\s*pi:\s*0\.102818\s*\n\s*means:\s*686\.099\s*\n\s*covar:\s*\n\s*379342' + - 'subclass:\s*\n\s*pi:\s*0\.222279\s*\n\s*means:\s*11683\.6\s*\n\s*covar:\s*\n\s*1\.75208e\+06' + - 'subclass:\s*\n\s*pi:\s*0\.311523\s*\n\s*means:\s*7849\.26\s*\n\s*covar:\s*\n\s*3\.21781e\+06' - algorithm: grass7:i.rgb.his name: GRASS7 i.rgb.his From 67a67c30e908bd36c0dfe776c0e98704f502f2aa Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:08:02 +1000 Subject: [PATCH 2/7] Resync grass plugin algorithm test base file --- .../grassprovider/tests/AlgorithmsTestBase.py | 100 +++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/python/plugins/grassprovider/tests/AlgorithmsTestBase.py b/python/plugins/grassprovider/tests/AlgorithmsTestBase.py index 353b44384623..08637ac8186d 100644 --- a/python/plugins/grassprovider/tests/AlgorithmsTestBase.py +++ b/python/plugins/grassprovider/tests/AlgorithmsTestBase.py @@ -34,12 +34,15 @@ from numpy import nan_to_num from copy import deepcopy +from qgis.PyQt.QtCore import QT_VERSION from qgis.core import ( + Qgis, QgsVectorLayer, QgsRasterLayer, QgsCoordinateReferenceSystem, QgsFeatureRequest, QgsMapLayer, + QgsMeshLayer, QgsProject, QgsApplication, QgsProcessingContext, @@ -48,7 +51,6 @@ ) from qgis.analysis import QgsNativeAlgorithms from qgis.testing import _UnexpectedSuccess, QgisTestCase, start_app - from utilities import unitTestDataPath import processing @@ -56,6 +58,10 @@ gdal.UseExceptions() +def GDAL_COMPUTE_VERSION(maj, min, rev): + return (maj) * 1000000 + (min) * 10000 + (rev) * 100 + + def processingTestDataPath(): return os.path.join(os.path.dirname(__file__), "testdata") @@ -73,6 +79,82 @@ def test_algorithms(self): if "tests" in algorithm_tests and algorithm_tests["tests"] is not None: for idx, algtest in enumerate(algorithm_tests["tests"]): + condition = algtest.get("condition") + if condition: + geos_condition = condition.get("geos") + if geos_condition: + less_than_condition = geos_condition.get("less_than") + if less_than_condition: + if Qgis.geosVersionInt() >= less_than_condition: + print( + "!!! Skipping {}, requires GEOS < {}, have version {}".format( + algtest["name"], + less_than_condition, + Qgis.geosVersionInt(), + ) + ) + continue + at_least_condition = geos_condition.get("at_least") + if at_least_condition: + if Qgis.geosVersionInt() < at_least_condition: + print( + "!!! Skipping {}, requires GEOS >= {}, have version {}".format( + algtest["name"], + at_least_condition, + Qgis.geosVersionInt(), + ) + ) + continue + gdal_condition = condition.get("gdal") + if gdal_condition: + less_than_condition = gdal_condition.get("less_than") + if less_than_condition: + if ( + int(gdal.VersionInfo("VERSION_NUM")) + >= less_than_condition + ): + print( + "!!! Skipping {}, requires GDAL < {}, have version {}".format( + algtest["name"], + less_than_condition, + gdal.VersionInfo("VERSION_NUM"), + ) + ) + continue + at_least_condition = gdal_condition.get("at_least") + if at_least_condition: + if ( + int(gdal.VersionInfo("VERSION_NUM")) + < at_least_condition + ): + print( + "!!! Skipping {}, requires GDAL >= {}, have version {}".format( + algtest["name"], + at_least_condition, + gdal.VersionInfo("VERSION_NUM"), + ) + ) + continue + qt_condition = condition.get("qt") + if qt_condition: + less_than_condition = qt_condition.get("less_than") + if less_than_condition: + if QT_VERSION >= less_than_condition: + print( + "!!! Skipping {}, requires Qt < {}, have version {}".format( + algtest["name"], less_than_condition, QT_VERSION + ) + ) + continue + at_least_condition = qt_condition.get("at_least") + if at_least_condition: + if QT_VERSION < at_least_condition: + print( + "!!! Skipping {}, requires Qt >= {}, have version {}".format( + algtest["name"], at_least_condition, QT_VERSION + ) + ) + continue print( 'About to start {} of {}: "{}"'.format( idx, len(algorithm_tests["tests"]), algtest["name"] @@ -136,6 +218,12 @@ def check_algorithm(self, name, defs): # ignore user setting for invalid geometry handling context = QgsProcessingContext() context.setProject(QgsProject.instance()) + if "ellipsoid" in defs: + # depending on the project settings, we can't always rely + # on QgsProject.ellipsoid() returning the same ellipsoid as was + # specified in the test definition. So just force ensure that the + # context's ellipsoid is the desired one + context.setEllipsoid(defs["ellipsoid"]) if "skipInvalid" in defs and defs["skipInvalid"]: context.setInvalidGeometryCheck( @@ -180,7 +268,7 @@ def load_param(self, param, id=None): parameter based on its key `type` and return the appropriate parameter to pass to the algorithm. """ try: - if param["type"] in ("vector", "raster", "table"): + if param["type"] in ("vector", "raster", "table", "mesh"): return self.load_layer(id, param).id() elif param["type"] == "vrtlayers": vals = [] @@ -290,6 +378,8 @@ def load_layer(self, id, param): options = QgsRasterLayer.LayerOptions() options.loadDefaultStyle = False lyr = QgsRasterLayer(filepath, param["name"], "gdal", options) + elif param["type"] == "mesh": + lyr = QgsMeshLayer(filepath, param["name"], "mdal") self.assertTrue( lyr.isValid(), f'Could not load layer "{filepath}" from param {param}' @@ -440,10 +530,16 @@ class GenericAlgorithmsTest(QgisTestCase): @classmethod def setUpClass(cls): start_app() + from processing.core.Processing import Processing + + Processing.initialize() cls.cleanup_paths = [] @classmethod def tearDownClass(cls): + from processing.core.Processing import Processing + + Processing.deinitialize() for path in cls.cleanup_paths: shutil.rmtree(path) From 6aa594dcb6f86c710a9d9111fabdb845e7628361 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:13:35 +1000 Subject: [PATCH 3/7] Allow skipping tests by grass version --- .../grassprovider/tests/AlgorithmsTestBase.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python/plugins/grassprovider/tests/AlgorithmsTestBase.py b/python/plugins/grassprovider/tests/AlgorithmsTestBase.py index 08637ac8186d..1ec9d855666e 100644 --- a/python/plugins/grassprovider/tests/AlgorithmsTestBase.py +++ b/python/plugins/grassprovider/tests/AlgorithmsTestBase.py @@ -52,6 +52,7 @@ from qgis.analysis import QgsNativeAlgorithms from qgis.testing import _UnexpectedSuccess, QgisTestCase, start_app from utilities import unitTestDataPath +from grassprovider.grass_utils import GrassUtils import processing @@ -62,6 +63,11 @@ def GDAL_COMPUTE_VERSION(maj, min, rev): return (maj) * 1000000 + (min) * 10000 + (rev) * 100 +def GRASS_VERSION_INT(version: str) -> int: + parts = version.split(".") + return int(parts[0]) * 1000000 + int(parts[1]) * 10000 + int(parts[2]) * 100 + + def processingTestDataPath(): return os.path.join(os.path.dirname(__file__), "testdata") @@ -81,6 +87,32 @@ def test_algorithms(self): for idx, algtest in enumerate(algorithm_tests["tests"]): condition = algtest.get("condition") if condition: + grass_condition = condition.get("grass") + if grass_condition: + grass_version = GRASS_VERSION_INT(GrassUtils.installedVersion()) + less_than_condition = grass_condition.get("less_than") + if less_than_condition: + if grass_version >= less_than_condition: + print( + "!!! Skipping {}, requires GRASS < {}, have version {}".format( + algtest["name"], + less_than_condition, + grass_version, + ) + ) + continue + at_least_condition = grass_condition.get("at_least") + if at_least_condition: + if grass_version < at_least_condition: + print( + "!!! Skipping {}, requires GRASS >= {}, have version {}".format( + algtest["name"], + at_least_condition, + grass_version, + ) + ) + continue + geos_condition = condition.get("geos") if geos_condition: less_than_condition = geos_condition.get("less_than") From d3df8c13cbf81db2c833873f6f06a0a694f54f49 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:14:19 +1000 Subject: [PATCH 4/7] Only run i.smap , i.maxlik tests on older GRASS Newer ones have the raster paths hardcoded into the signature file, and we can't handle that currently in the test suite --- .../testdata/grass_algorithms_imagery_tests.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml index 5af68a3027ad..a09546ae4031 100644 --- a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml +++ b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml @@ -484,8 +484,15 @@ tests: # - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6 # type: rasterhash # + - algorithm: grass7:i.smap name: GRASS7 i.smap + condition: + grass: + # we can only run this test on older GRASS versions -- newer ones + # have the raster paths hardcoded into the signature file, and we can't handle + # that currently in the test suite + less_than: 8000000 params: GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0' blocksize: 1024 @@ -507,6 +514,12 @@ tests: - algorithm: grass7:i.maxlik name: GRASS7 i.maxlik + condition: + grass: + # we can only run this test on older GRASS versions -- newer ones + # have the raster paths hardcoded into the signature file, and we can't handle + # that currently in the test suite + less_than: 8000000 params: GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0' input: From 176dadfaffd6c283d3907ce8ea1c180bdb672086 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:18:14 +1000 Subject: [PATCH 5/7] Run test on qt 6 --- .ci/test_blocklist_qt6.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/test_blocklist_qt6.txt b/.ci/test_blocklist_qt6.txt index bf8da70ae337..be0af599e830 100644 --- a/.ci/test_blocklist_qt6.txt +++ b/.ci/test_blocklist_qt6.txt @@ -60,6 +60,5 @@ PyQgsServerWFS ProcessingQgisAlgorithmsTestPt2 ProcessingQgisAlgorithmsTestPt4 ProcessingGdalAlgorithmsVectorTest -ProcessingGrassAlgorithmsImageryTest # PyQgsProviderRegistry runs fine locally on Fedora:rawhide but not on CI PyQgsProviderRegistry From 787f8072d1393754c989b783391e49247b060326 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 13:19:55 +1000 Subject: [PATCH 6/7] Enable test which passes now --- .../grass_algorithms_imagery_tests.yaml | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml index a09546ae4031..9587329ca38b 100644 --- a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml +++ b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml @@ -449,41 +449,44 @@ tests: hash: 07578ad38cf948473a519f040acb0235f60a592904ac8ef46aa607e1 type: rasterhash -# Triggers a Python exception on the GRASS version used on the CI - disabling until fixed in GRASS -# - algorithm: grass7:i.pansharpen -# name: GRASS7 i.pansharpen -# params: -# GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0' -# blue: -# name: raster_4class.tif -# type: raster -# green: -# name: raster_5class.tif -# type: raster -# method: 2 -# pan: -# name: float_raster.tif -# type: raster -# red: -# name: raster_6class.tif -# type: raster -# results: -# blueoutput: -# hash: -# - 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17 -# - 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4 -# type: rasterhash -# greenoutput: -# hash: -# - 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed -# - 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6 -# type: rasterhash -# redoutput: -# hash: -# - ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899 -# - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6 -# type: rasterhash -# + - algorithm: grass7:i.pansharpen + name: GRASS7 i.pansharpen + condition: + grass: + # This test used to segfault on older GRASS releases + at_least: 8040000 + params: + GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0' + blue: + name: raster_4class.tif + type: raster + green: + name: raster_5class.tif + type: raster + method: 2 + pan: + name: float_raster.tif + type: raster + red: + name: raster_6class.tif + type: raster + results: + blueoutput: + hash: + - 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17 + - 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4 + type: rasterhash + greenoutput: + hash: + - 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed + - 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6 + type: rasterhash + redoutput: + hash: + - ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899 + - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6 + type: rasterhash + - algorithm: grass7:i.smap name: GRASS7 i.smap From c1809728ffe32d380c501d6f9be9bb01ed45a309 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Wed, 18 Dec 2024 15:04:49 +1000 Subject: [PATCH 7/7] Fix regex --- .../tests/testdata/grass_algorithms_imagery_tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml index 9587329ca38b..d1e86ab17889 100644 --- a/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml +++ b/python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml @@ -384,10 +384,10 @@ tests: name: expected/i.gensig.txt type: regex rules: - - '300\s*\n(?:\d+\s*\n)7043\.06\s*\n1\.89772e\+07\s*' - - '1700\s*\n(?:\d+\s*\n)7405\.93\s*\n2\.01921e\+07\s*' - - '6000\s*\n(?:\d+\s*\n)7287\.21\s*\n1\.90221e\+07\s*' - - '7000\s*\n(?:\d+\s*\n)7246\.81\s*\n1\.90226e\+07\s*' + - '300\s*\n(?:\d+\s*\n)?7043\.06\s*\n1\.89772e\+07\s*' + - '1700\s*\n(?:\d+\s*\n)?7405\.93\s*\n2\.01921e\+07\s*' + - '6000\s*\n(?:\d+\s*\n)?7287\.21\s*\n1\.90221e\+07\s*' + - '7000\s*\n(?:\d+\s*\n)?7246\.81\s*\n1\.90226e\+07\s*' - algorithm: grass7:i.gensigset name: GRASS7 i.gensigset