diff --git a/.conan/test_package/CMakeLists.txt b/.conan/test_package/CMakeLists.txt index 00a6af23df..9a28bd5d8c 100644 --- a/.conan/test_package/CMakeLists.txt +++ b/.conan/test_package/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15) +cmake_minimum_required(VERSION 3.20) project(PackageTest CXX) find_package(Catch2 CONFIG REQUIRED) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c24e1d1c4..8df72e7be4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.20) # detect if Catch is being bundled, # disable testsuite in that case @@ -37,9 +37,7 @@ endif() project(Catch2 VERSION 3.7.1 # CML version placeholder, don't delete LANGUAGES CXX - # HOMEPAGE_URL is not supported until CMake version 3.12, which - # we do not target yet. - # HOMEPAGE_URL "https://github.com/catchorg/Catch2" + HOMEPAGE_URL "https://github.com/catchorg/Catch2" DESCRIPTION "A modern, C++-native, unit test framework." ) @@ -192,12 +190,6 @@ if (NOT_SUBPROJECT) ${PKGCONFIG_INSTALL_DIR} ) - # CPack/CMake started taking the package version from project version 3.12 - # So we need to set the version manually for older CMake versions - if(${CMAKE_VERSION} VERSION_LESS "3.12.0") - set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION}) - endif() - set(CPACK_PACKAGE_CONTACT "https://github.com/catchorg/Catch2/") diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md index ad6ca00455..daeb5f7577 100644 --- a/docs/cmake-integration.md +++ b/docs/cmake-integration.md @@ -81,12 +81,11 @@ to your CMake module path. `Catch.cmake` provides function `catch_discover_tests` to get tests from a target. This function works by running the resulting executable with -`--list-test-names-only` flag, and then parsing the output to find all -existing tests. +`--list-test` flag, and then parsing the output to find all existing tests. #### Usage ```cmake -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.20) project(baz LANGUAGES CXX VERSION 0.0.1) @@ -128,6 +127,8 @@ catch_discover_tests(target [OUTPUT_PREFIX prefix] [OUTPUT_SUFFIX suffix] [DISCOVERY_MODE ] + [SKIP_IS_FAILURE] + [ADD_TAGS_AS_LABELS] ) ``` @@ -211,6 +212,15 @@ execution (useful e.g. in cross-compilation environments). calling ``catch_discover_tests``. This provides a mechanism for globally selecting a preferred test discovery behavior. +* `SKIP_IS_FAILURE` + +Skipped tests will be marked as failed instead. + +* `ADD_TAGS_AS_LABELS` + +Add the tags from tests as labels to CTest. + + ### `ParseAndAddCatchTests.cmake` ⚠ This script is [deprecated](https://github.com/catchorg/Catch2/pull/2120) @@ -229,7 +239,7 @@ parsed are *silently ignored*. #### Usage ```cmake -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.20) project(baz LANGUAGES CXX VERSION 0.0.1) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 4647df1dd9..6dcb762375 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.10 ) +cmake_minimum_required( VERSION 3.20 ) project( Catch2Examples LANGUAGES CXX ) diff --git a/extras/Catch.cmake b/extras/Catch.cmake index cef73def2d..3d93fe20a3 100644 --- a/extras/Catch.cmake +++ b/extras/Catch.cmake @@ -39,6 +39,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. [OUTPUT_SUFFIX suffix] [DISCOVERY_MODE ] [SKIP_IS_FAILURE] + [ADD_TAGS_AS_LABELS] ) ``catch_discover_tests`` sets up a post-build command on the test executable @@ -148,6 +149,9 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. ``SKIP_IS_FAILURE`` Disables skipped test detection. + ``ADD_TAGS_AS_LABELS`` + Adds all test tags as CTest labels. + #]=======================================================================] #------------------------------------------------------------------------------ @@ -155,12 +159,16 @@ function(catch_discover_tests TARGET) cmake_parse_arguments( "" - "SKIP_IS_FAILURE" + "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS" "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE" "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS" ${ARGN} ) + if (${CMAKE_VERSION} VERSION_LESS "3.19") + message(FATAL_ERROR "This script requires JSON support from CMake version 3.19 or greater.") + endif() + if(NOT _WORKING_DIRECTORY) set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") endif() @@ -222,6 +230,7 @@ function(catch_discover_tests TARGET) -D "TEST_DL_PATHS=${_DL_PATHS}" -D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}" -D "CTEST_FILE=${ctest_tests_file}" + -D "ADD_TAGS_AS_LABELS=${_ADD_TAGS_AS_LABELS}" -P "${_CATCH_DISCOVER_TESTS_SCRIPT}" VERBATIM ) @@ -267,6 +276,7 @@ function(catch_discover_tests TARGET) " CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n" " TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n" " TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n" + " ADD_TAGS_AS_LABELS" " [==[" "${_ADD_TAGS_AS_LABELS}" "]==]" "\n" " )" "\n" " endif()" "\n" " include(\"${ctest_tests_file}\")" "\n" @@ -293,22 +303,10 @@ function(catch_discover_tests TARGET) endif() endif() - if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") - # Add discovered tests to directory TEST_INCLUDE_FILES - set_property(DIRECTORY - APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" - ) - else() - # Add discovered tests as directory TEST_INCLUDE_FILE if possible - get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET) - if (NOT ${test_include_file_set}) - set_property(DIRECTORY - PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}" - ) - else() - message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE") - endif() - endif() + # Add discovered tests to directory TEST_INCLUDE_FILES + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) endfunction() diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index 399a839d92..ceaf86523f 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -22,10 +22,11 @@ function(catch_discover_tests_impl) "" "" "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE" - "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS" + "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS" ${ARGN} ) + set(add_tags "${_ADD_TAGS_AS_LABELS}") set(prefix "${_TEST_PREFIX}") set(suffix "${_TEST_SUFFIX}") set(spec ${_TEST_SPEC}) @@ -72,25 +73,19 @@ function(catch_discover_tests_impl) endif() execute_process( - COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --verbosity quiet - OUTPUT_VARIABLE output + COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json + OUTPUT_VARIABLE listing_output RESULT_VARIABLE result WORKING_DIRECTORY "${_TEST_WORKING_DIR}" ) if(NOT ${result} EQUAL 0) message(FATAL_ERROR - "Error running test executable '${_TEST_EXECUTABLE}':\n" + "Error listing tests from executable '${_TEST_EXECUTABLE}':\n" " Result: ${result}\n" - " Output: ${output}\n" + " Output: ${listing_output}\n" ) endif() - # Make sure to escape ; (semicolons) in test names first, because - # that'd break the foreach loop for "Parse output" later and create - # wrongly splitted and thus failing test cases (false positives) - string(REPLACE ";" "\;" output "${output}") - string(REPLACE "\n" ";" output "${output}") - # Prepare reporter if(reporter) set(reporter_arg "--reporter ${reporter}") @@ -110,7 +105,7 @@ function(catch_discover_tests_impl) ) elseif(NOT ${reporter_check_result} EQUAL 0) message(FATAL_ERROR - "Error running test executable '${_TEST_EXECUTABLE}':\n" + "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n" " Result: ${reporter_check_result}\n" " Output: ${reporter_check_output}\n" ) @@ -139,46 +134,86 @@ function(catch_discover_tests_impl) endforeach() endif() - # Parse output - foreach(line ${output}) - set(test "${line}") + # Parse JSON output for list of tests/class names/tags + string(JSON version GET "${listing_output}" "version") + if (NOT version STREQUAL "1") + message(FATAL_ERROR "Unsupported catch output version: '${version}'") + endif() + + # Speed-up reparsing by cutting away unneeded parts of JSON. + string(JSON test_listing GET "${listing_output}" "listings" "tests") + string(JSON num_tests LENGTH "${test_listing}") + # CMake's foreach-RANGE is inclusive, so we have to subtract 1 + math(EXPR num_tests "${num_tests} - 1") + + foreach(idx RANGE ${num_tests}) + string(JSON single_test GET ${test_listing} ${idx}) + string(JSON test_tags GET "${single_test}" "tags") + string(JSON plain_name GET "${single_test}" "name") + # Escape characters in test case names that would be parsed by Catch2 # Note that the \ escaping must happen FIRST! Do not change the order. - set(test_name "${test}") - foreach(char \\ , [ ]) - string(REPLACE ${char} "\\${char}" test_name "${test_name}") + set(escaped_name "${plain_name}") + foreach(char \\ , [ ] ;) + string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}") endforeach(char) # ...add output dir if(output_dir) - string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}") - set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}") + string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}") + set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}") endif() # ...and add to script add_command(add_test - "${prefix}${test}${suffix}" + "${prefix}${plain_name}${suffix}" ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" - "${test_name}" + "${escaped_name}" ${extra_args} "${reporter_arg}" "${output_dir_arg}" ) add_command(set_tests_properties - "${prefix}${test}${suffix}" + "${prefix}${plain_name}${suffix}" PROPERTIES WORKING_DIRECTORY "${_TEST_WORKING_DIR}" ${properties} ) + if (add_tags) + string(JSON num_tags LENGTH "${test_tags}") + math(EXPR num_tags "${num_tags} - 1") + set(tag_list "") + if (num_tags GREATER_EQUAL "0") + foreach(tag_idx RANGE ${num_tags}) + string(JSON a_tag GET "${test_tags}" "${tag_idx}") + # Catch2's tags can contain semicolons, which are list element separators + # in CMake, so we have to escape them. Ideally we could use the [=[...]=] + # syntax for this, but CTest currently keeps the square quotes in the label + # name. So we add 2 backslashes to escape it instead. + # **IMPORTANT**: The number of backslashes depends on how many layers + # of CMake the tag goes. If this script is changed, the + # number of backslashes to escape may change as well. + string(REPLACE ";" "\\;" a_tag "${a_tag}") + list(APPEND tag_list "${a_tag}") + endforeach() + + add_command(set_tests_properties + "${prefix}${plain_name}${suffix}" + PROPERTIES + LABELS "${tag_list}" + ) + endif() + endif(add_tags) + if(environment_modifications) add_command(set_tests_properties - "${prefix}${test}${suffix}" + "${prefix}${plain_name}${suffix}" PROPERTIES ENVIRONMENT_MODIFICATION "${environment_modifications}") endif() - list(APPEND tests "${prefix}${test}${suffix}") + list(APPEND tests "${prefix}${plain_name}${suffix}") endforeach() # Create a list of all discovered tests, which users may use to e.g. set @@ -207,5 +242,6 @@ if(CMAKE_SCRIPT_MODE_FILE) TEST_DL_PATHS ${TEST_DL_PATHS} TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS} CTEST_FILE ${CTEST_FILE} + ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS} ) endif() diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index a7268e54ca..82e71ea6cb 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -2,7 +2,7 @@ # Build extra tests. # -cmake_minimum_required( VERSION 3.10 ) +cmake_minimum_required( VERSION 3.20 ) project( Catch2ExtraTests LANGUAGES CXX ) diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt index 5105cddb64..64eb251f63 100644 --- a/tests/TestScripts/DiscoverTests/CMakeLists.txt +++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.20) project(discover-tests-test LANGUAGES CXX @@ -19,4 +19,11 @@ if (CMAKE_VERSION GREATER_EQUAL 3.27) DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.." ) endif () -catch_discover_tests(tests ${extra_args}) +catch_discover_tests( + tests + ADD_TAGS_AS_LABELS + DISCOVERY_MODE PRE_TEST + ${extra_args} +) + +# DISCOVERY_MODE diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py index 9800674f25..33150caec1 100644 --- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py +++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py @@ -12,6 +12,10 @@ import sys import re import json +from collections import namedtuple +from typing import List + +TestInfo = namedtuple('TestInfo', ['name', 'tags']) cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)') @@ -61,7 +65,7 @@ def build_project(sources_dir, output_base_path, catch2_path): -def get_test_names(build_path): +def get_test_names(build_path: str) -> List[TestInfo]: # For now we assume that Windows builds are done using MSBuild under # Debug configuration. This means that we need to add "Debug" folder # to the path when constructing it. On Linux, we don't add anything. @@ -69,15 +73,23 @@ def get_test_names(build_path): full_path = os.path.join(build_path, config_path, 'tests') - cmd = [full_path, '--reporter', 'xml', '--list-tests'] + cmd = [full_path, '--reporter', 'json', '--list-tests'] result = subprocess.run(cmd, capture_output = True, check = True, text = True) - import xml.etree.ElementTree as ET - root = ET.fromstring(result.stdout) - return [tc.text for tc in root.findall('TestCase/Name')] + test_listing = json.loads(result.stdout) + + assert test_listing['version'] == 1 + + tests = [] + for test in test_listing['listings']['tests']: + test_name = test['name'] + tags = test['tags'] + tests.append(TestInfo(test_name, tags)) + + return tests def get_ctest_listing(build_path): old_path = os.getcwd() @@ -91,20 +103,25 @@ def get_ctest_listing(build_path): os.chdir(old_path) return result.stdout -def extract_tests_from_ctest(ctest_output): +def extract_tests_from_ctest(ctest_output) -> List[TestInfo]: ctest_response = json.loads(ctest_output) tests = ctest_response['tests'] - test_names = [] + test_infos = [] for test in tests: test_command = test['command'] # First part of the command is the binary, second is the filter. # If there are less, registration has failed. If there are more, # registration has changed and the script needs updating. assert len(test_command) == 2 - test_names.append(test_command[1]) test_name = test_command[1] + labels = [] + for prop in test['properties']: + if prop['name'] == 'LABELS': + labels = prop['value'] + + test_infos.append(TestInfo(test_name, labels)) - return test_names + return test_infos def check_DL_PATHS(ctest_output): ctest_response = json.loads(ctest_output) @@ -115,10 +132,14 @@ def check_DL_PATHS(ctest_output): if property['name'] == 'ENVIRONMENT_MODIFICATION': assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}" -def escape_catch2_test_name(name): - for char in ('\\', ',', '[', ']'): - name = name.replace(char, f"\\{char}") - return name +def escape_catch2_test_names(infos: List[TestInfo]): + escaped = [] + for info in infos: + name = info.name + for char in ('\\', ',', '[', ']'): + name = name.replace(char, f"\\{char}") + escaped.append(TestInfo(name, info.tags)) + return escaped if __name__ == '__main__': if len(sys.argv) != 3: @@ -130,7 +151,7 @@ def escape_catch2_test_name(name): build_path = build_project(sources_dir, output_base_path, catch2_path) - catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)] + catch_test_names = escape_catch2_test_names(get_test_names(build_path)) ctest_output = get_ctest_listing(build_path) ctest_test_names = extract_tests_from_ctest(ctest_output) @@ -147,6 +168,7 @@ def escape_catch2_test_name(name): if mismatched: print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!") exit(1) + print(f"{len(catch_test_names)} tests matched") cmake_version = get_cmake_version() if cmake_version >= (3, 27): diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp index aa603df1ac..be533ab69b 100644 --- a/tests/TestScripts/DiscoverTests/register-tests.cpp +++ b/tests/TestScripts/DiscoverTests/register-tests.cpp @@ -14,3 +14,10 @@ TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. " "Really looooooooooooong. Even longer than that. Multiple lines " "worth of test name. Yep, like this." ) {} TEST_CASE( "And now a test case with weird tags.", "[tl;dr][tl;dw][foo,bar]" ) {} +// Also check that we handle tests on class, which have name in output as 'class-name', not 'name'. +class TestCaseFixture { +public: + int m_a; +}; + +TEST_CASE_METHOD(TestCaseFixture, "A test case as method", "[tagstagstags]") {} diff --git a/tools/misc/CMakeLists.txt b/tools/misc/CMakeLists.txt index bf80846cdc..0091ae6f18 100644 --- a/tools/misc/CMakeLists.txt +++ b/tools/misc/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.20) project(CatchCoverageHelper)