Support adding test tags as CTest labels in catch_discover_tests

catchorg · Jan 5, 2025 · e3ec6f2 · e3ec6f2
1 parent b0d0aa4
commit e3ec6f2
Show file tree

Hide file tree

Showing 11 changed files with 148 additions and 76 deletions.
diff --git a/.conan/test_package/CMakeLists.txt b/.conan/test_package/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15)
+cmake_minimum_required(VERSION 3.20)
 project(PackageTest CXX)
 
 find_package(Catch2 CONFIG REQUIRED)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.20)
 
 # detect if Catch is being bundled,
 # disable testsuite in that case
@@ -37,9 +37,7 @@ endif()
 project(Catch2
   VERSION 3.7.1 # CML version placeholder, don't delete
   LANGUAGES CXX
-  # HOMEPAGE_URL is not supported until CMake version 3.12, which
-  # we do not target yet.
-  # HOMEPAGE_URL "https://github.com/catchorg/Catch2"
+  HOMEPAGE_URL "https://github.com/catchorg/Catch2"
   DESCRIPTION "A modern, C++-native, unit test framework."
 )
 
@@ -192,12 +190,6 @@ if (NOT_SUBPROJECT)
         ${PKGCONFIG_INSTALL_DIR}
     )
 
-    # CPack/CMake started taking the package version from project version 3.12
-    # So we need to set the version manually for older CMake versions
-    if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
-        set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION})
-    endif()
-
     set(CPACK_PACKAGE_CONTACT "https://github.com/catchorg/Catch2/")
 
 

diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md
@@ -81,12 +81,11 @@ to your CMake module path.
 
 `Catch.cmake` provides function `catch_discover_tests` to get tests from
 a target. This function works by running the resulting executable with
-`--list-test-names-only` flag, and then parsing the output to find all
-existing tests.
+`--list-test` flag, and then parsing the output to find all existing tests.
 
 #### Usage
 ```cmake
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.20)
 
 project(baz LANGUAGES CXX VERSION 0.0.1)
 
@@ -128,6 +127,8 @@ catch_discover_tests(target
                      [OUTPUT_PREFIX prefix]
                      [OUTPUT_SUFFIX suffix]
                      [DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
+                     [SKIP_IS_FAILURE]
+                     [ADD_TAGS_AS_LABELS]
 )
 ```
 
@@ -211,6 +212,15 @@ execution (useful e.g. in cross-compilation environments).
 calling ``catch_discover_tests``. This provides a mechanism for globally
 selecting a preferred test discovery behavior.
 
+* `SKIP_IS_FAILURE`
+
+Skipped tests will be marked as failed instead.
+
+* `ADD_TAGS_AS_LABELS`
+
+Add the tags from tests as labels to CTest.
+
+
 ### `ParseAndAddCatchTests.cmake`
 
 ⚠ This script is [deprecated](https://github.com/catchorg/Catch2/pull/2120)
@@ -229,7 +239,7 @@ parsed are *silently ignored*.
 #### Usage
 
 ```cmake
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.20)
 
 project(baz LANGUAGES CXX VERSION 0.0.1)
 

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.10 )
+cmake_minimum_required( VERSION 3.20 )
 
 project( Catch2Examples LANGUAGES CXX )
 

diff --git a/extras/Catch.cmake b/extras/Catch.cmake
@@ -39,6 +39,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
                          [OUTPUT_SUFFIX suffix]
                          [DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
                          [SKIP_IS_FAILURE]
+                         [ADD_TAGS_AS_LABELS]
     )
 
   ``catch_discover_tests`` sets up a post-build command on the test executable
@@ -148,19 +149,26 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
   ``SKIP_IS_FAILURE``
     Disables skipped test detection.
 
+  ``ADD_TAGS_AS_LABELS``
+    Adds all test tags as CTest labels.
+
 #]=======================================================================]
 
 #------------------------------------------------------------------------------
 function(catch_discover_tests TARGET)
 
   cmake_parse_arguments(
     ""
-    "SKIP_IS_FAILURE"
+    "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
     "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
     "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
     ${ARGN}
   )
 
+  if (${CMAKE_VERSION} VERSION_LESS "3.19")
+    message(FATAL_ERROR "This script requires JSON support from CMake version 3.19 or greater.")
+  endif()
+
   if(NOT _WORKING_DIRECTORY)
     set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
   endif()
@@ -222,6 +230,7 @@ function(catch_discover_tests TARGET)
               -D "TEST_DL_PATHS=${_DL_PATHS}"
               -D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}"
               -D "CTEST_FILE=${ctest_tests_file}"
+              -D "ADD_TAGS_AS_LABELS=${_ADD_TAGS_AS_LABELS}"
               -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
       VERBATIM
     )
@@ -267,6 +276,7 @@ function(catch_discover_tests TARGET)
       "      CTEST_FILE"             " [==[" "${ctest_tests_file}"        "]==]"   "\n"
       "      TEST_DL_PATHS"          " [==[" "${_DL_PATHS}"               "]==]"   "\n"
       "      TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}"     "]==]"   "\n"
+      "      ADD_TAGS_AS_LABELS"     " [==[" "${_ADD_TAGS_AS_LABELS}"     "]==]"   "\n"
       "    )"                                                                      "\n"
       "  endif()"                                                                  "\n"
       "  include(\"${ctest_tests_file}\")"                                         "\n"
@@ -293,22 +303,10 @@ function(catch_discover_tests TARGET)
     endif()
   endif()
 
-  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0")
-    # Add discovered tests to directory TEST_INCLUDE_FILES
-    set_property(DIRECTORY
-      APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
-    )
-  else()
-    # Add discovered tests as directory TEST_INCLUDE_FILE if possible
-    get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
-    if (NOT ${test_include_file_set})
-      set_property(DIRECTORY
-        PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
-      )
-    else()
-      message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE")
-    endif()
-  endif()
+  # Add discovered tests to directory TEST_INCLUDE_FILES
+  set_property(DIRECTORY
+    APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+  )
 
 endfunction()
 

diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake
@@ -22,10 +22,11 @@ function(catch_discover_tests_impl)
     ""
     ""
     "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
-    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS"
+    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
     ${ARGN}
   )
 
+  set(add_tags "${_ADD_TAGS_AS_LABELS}")
   set(prefix "${_TEST_PREFIX}")
   set(suffix "${_TEST_SUFFIX}")
   set(spec ${_TEST_SPEC})
@@ -72,25 +73,19 @@ function(catch_discover_tests_impl)
   endif()
 
   execute_process(
-    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --verbosity quiet
-    OUTPUT_VARIABLE output
+    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json
+    OUTPUT_VARIABLE listing_output
     RESULT_VARIABLE result
     WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
   )
   if(NOT ${result} EQUAL 0)
     message(FATAL_ERROR
-      "Error running test executable '${_TEST_EXECUTABLE}':\n"
+      "Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
       "  Result: ${result}\n"
-      "  Output: ${output}\n"
+      "  Output: ${listing_output}\n"
     )
   endif()
 
-  # Make sure to escape ; (semicolons) in test names first, because
-  # that'd break the foreach loop for "Parse output" later and create
-  # wrongly splitted and thus failing test cases (false positives)
-  string(REPLACE ";" "\;" output "${output}")
-  string(REPLACE "\n" ";" output "${output}")
-
   # Prepare reporter
   if(reporter)
     set(reporter_arg "--reporter ${reporter}")
@@ -110,7 +105,7 @@ function(catch_discover_tests_impl)
       )
     elseif(NOT ${reporter_check_result} EQUAL 0)
       message(FATAL_ERROR
-        "Error running test executable '${_TEST_EXECUTABLE}':\n"
+        "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
         "  Result: ${reporter_check_result}\n"
         "  Output: ${reporter_check_output}\n"
       )
@@ -139,46 +134,86 @@ function(catch_discover_tests_impl)
     endforeach()
   endif()
 
-  # Parse output
-  foreach(line ${output})
-    set(test "${line}")
+  # Parse JSON output for list of tests/class names/tags
+  string(JSON version GET "${listing_output}" "version")
+  if (NOT version STREQUAL "1")
+    message(FATAL_ERROR "Unsupported catch output version: '${version}'")
+  endif()
+
+  # Speed-up reparsing by cutting away unneeded parts of JSON.
+  string(JSON test_listing GET "${listing_output}" "listings" "tests")
+  string(JSON num_tests LENGTH "${test_listing}")
+  # CMake's foreach-RANGE is inclusive, so we have to subtract 1
+  math(EXPR num_tests "${num_tests} - 1")
+
+  foreach(idx RANGE ${num_tests})
+    string(JSON single_test GET ${test_listing} ${idx})
+    string(JSON test_tags GET "${single_test}" "tags")
+    string(JSON plain_name GET "${single_test}" "name")
+
     # Escape characters in test case names that would be parsed by Catch2
     # Note that the \ escaping must happen FIRST! Do not change the order.
-    set(test_name "${test}")
-    foreach(char \\ , [ ])
-      string(REPLACE ${char} "\\${char}" test_name "${test_name}")
+    set(escaped_name "${plain_name}")
+    foreach(char \\ , [ ] ;)
+      string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
     endforeach(char)
     # ...add output dir
     if(output_dir)
-      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}")
-      set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
+      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
+      set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
     endif()
 
     # ...and add to script
     add_command(add_test
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       ${_TEST_EXECUTOR}
       "${_TEST_EXECUTABLE}"
-      "${test_name}"
+      "${escaped_name}"
       ${extra_args}
       "${reporter_arg}"
       "${output_dir_arg}"
     )
     add_command(set_tests_properties
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       PROPERTIES
       WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
       ${properties}
     )
 
+    if (add_tags)
+      string(JSON num_tags LENGTH "${test_tags}")
+      math(EXPR num_tags "${num_tags} - 1")
+      set(tag_list "")
+      if (num_tags GREATER_EQUAL "0")
+        foreach(tag_idx RANGE ${num_tags})
+          string(JSON a_tag GET "${test_tags}" "${tag_idx}")
+          # Catch2's tags can contain semicolons, which are list element separators
+          # in CMake, so we have to escape them. Ideally we could use the [=[...]=]
+          # syntax for this, but CTest currently keeps the square quotes in the label
+          # name. So we add 2 backslashes to escape it instead.
+          # **IMPORTANT**: The number of backslashes depends on how many layers
+          #                of CMake the tag goes. If this script is changed, the
+          #                number of backslashes to escape may change as well.
+          string(REPLACE ";" "\\;" a_tag "${a_tag}")
+          list(APPEND tag_list "${a_tag}")
+        endforeach()
+
+        add_command(set_tests_properties
+          "${prefix}${plain_name}${suffix}"
+          PROPERTIES
+          LABELS "${tag_list}"
+        )
+      endif()
+    endif(add_tags)
+
     if(environment_modifications)
       add_command(set_tests_properties
-        "${prefix}${test}${suffix}"
+        "${prefix}${plain_name}${suffix}"
         PROPERTIES
         ENVIRONMENT_MODIFICATION "${environment_modifications}")
     endif()
 
-    list(APPEND tests "${prefix}${test}${suffix}")
+    list(APPEND tests "${prefix}${plain_name}${suffix}")
   endforeach()
 
   # Create a list of all discovered tests, which users may use to e.g. set
@@ -207,5 +242,6 @@ if(CMAKE_SCRIPT_MODE_FILE)
     TEST_DL_PATHS ${TEST_DL_PATHS}
     TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
     CTEST_FILE ${CTEST_FILE}
+    ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
   )
 endif()
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
@@ -2,7 +2,7 @@
 # Build extra tests.
 #
 
-cmake_minimum_required( VERSION 3.10 )
+cmake_minimum_required( VERSION 3.20 )
 
 project( Catch2ExtraTests LANGUAGES CXX )
 

diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.20)
 
 project(discover-tests-test
   LANGUAGES CXX
@@ -19,4 +19,11 @@ if (CMAKE_VERSION GREATER_EQUAL 3.27)
             DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
     )
 endif ()
-catch_discover_tests(tests ${extra_args})
+catch_discover_tests(
+  tests
+  ADD_TAGS_AS_LABELS
+  DISCOVERY_MODE PRE_TEST
+  ${extra_args}
+)
+
+# DISCOVERY_MODE <POST_BUILD|PRE_TEST>