Use modern cmake testing functionality including catch integration

2024-11-25 04:41:05 +08:00 · 2018-08-19 19:04:11 +01:00 · 2018-08-19 19:04:11 +01:00 · d3a072fb63
commit d3a072fb63
parent a88e854dc3
8 changed files with 361 additions and 23 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -285,6 +285,7 @@ if(SFML_BUILD_DOC)
    add_subdirectory(doc)
 endif()
 if(SFML_BUILD_TEST_SUITE)
+    enable_testing()
    add_subdirectory(test)
 endif()

--- a/cmake/Catch.cmake
+++ b/cmake/Catch.cmake
@ -0,0 +1,175 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+Catch
+-----
+
+This module defines a function to help use the Catch test framework.
+
+The :command:`catch_discover_tests` discovers tests by asking the compiled test
+executable to enumerate its tests.  This does not require CMake to be re-run
+when tests change.  However, it may not work in a cross-compiling environment,
+and setting test properties is less convenient.
+
+This command is intended to replace use of :command:`add_test` to register
+tests, and will create a separate CTest test for each Catch test case.  Note
+that this is in some cases less efficient, as common set-up and tear-down logic
+cannot be shared by multiple test cases executing in the same instance.
+However, it provides more fine-grained pass/fail information to CTest, which is
+usually considered as more beneficial.  By default, the CTest test name is the
+same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
+
+.. command:: catch_discover_tests
+
+  Automatically add tests with CTest by querying the compiled test executable
+  for available tests::
+
+    catch_discover_tests(target
+                         [TEST_SPEC arg1...]
+                         [EXTRA_ARGS arg1...]
+                         [WORKING_DIRECTORY dir]
+                         [TEST_PREFIX prefix]
+                         [TEST_SUFFIX suffix]
+                         [PROPERTIES name1 value1...]
+                         [TEST_LIST var]
+    )
+
+  ``catch_discover_tests`` sets up a post-build command on the test executable
+  that generates the list of tests by parsing the output from running the test
+  with the ``--list-test-names-only`` argument.  This ensures that the full
+  list of tests is obtained.  Since test discovery occurs at build time, it is
+  not necessary to re-run CMake when the list of tests changes.
+  However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set
+  in order to function in a cross-compiling environment.
+
+  Additionally, setting properties on tests is somewhat less convenient, since
+  the tests are not available at CMake time.  Additional test properties may be
+  assigned to the set of tests as a whole using the ``PROPERTIES`` option.  If
+  more fine-grained test control is needed, custom content may be provided
+  through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES`
+  directory property.  The set of discovered tests is made accessible to such a
+  script via the ``<target>_TESTS`` variable.
+
+  The options are:
+
+  ``target``
+    Specifies the Catch executable, which must be a known CMake executable
+    target.  CMake will substitute the location of the built executable when
+    running the test.
+
+  ``TEST_SPEC arg1...``
+    Specifies test cases, wildcarded test cases, tags and tag expressions to
+    pass to the Catch executable with the ``--list-test-names-only`` argument.
+
+  ``EXTRA_ARGS arg1...``
+    Any extra arguments to pass on the command line to each test case.
+
+  ``WORKING_DIRECTORY dir``
+    Specifies the directory in which to run the discovered test cases.  If this
+    option is not provided, the current binary directory is used.
+
+  ``TEST_PREFIX prefix``
+    Specifies a ``prefix`` to be prepended to the name of each discovered test
+    case.  This can be useful when the same test executable is being used in
+    multiple calls to ``catch_discover_tests()`` but with different
+    ``TEST_SPEC`` or ``EXTRA_ARGS``.
+
+  ``TEST_SUFFIX suffix``
+    Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of
+    every discovered test case.  Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may
+    be specified.
+
+  ``PROPERTIES name1 value1...``
+    Specifies additional properties to be set on all tests discovered by this
+    invocation of ``catch_discover_tests``.
+
+  ``TEST_LIST var``
+    Make the list of tests available in the variable ``var``, rather than the
+    default ``<target>_TESTS``.  This can be useful when the same test
+    executable is being used in multiple calls to ``catch_discover_tests()``.
+    Note that this variable is only available in CTest.
+
+#]=======================================================================]
+
+#------------------------------------------------------------------------------
+function(catch_discover_tests TARGET)
+  cmake_parse_arguments(
+    ""
+    ""
+    "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST"
+    "TEST_SPEC;EXTRA_ARGS;PROPERTIES"
+    ${ARGN}
+  )
+
+  if(NOT _WORKING_DIRECTORY)
+    set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  endif()
+  if(NOT _TEST_LIST)
+    set(_TEST_LIST ${TARGET}_TESTS)
+  endif()
+
+  ## Generate a unique name based on the extra arguments
+  string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}")
+  string(SUBSTRING ${args_hash} 0 7 args_hash)
+
+  # Define rule to generate test list for aforementioned test executable
+  set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_include-${args_hash}.cmake")
+  set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_tests-${args_hash}.cmake")
+  get_property(crosscompiling_emulator
+    TARGET ${TARGET}
+    PROPERTY CROSSCOMPILING_EMULATOR
+  )
+  add_custom_command(
+    TARGET ${TARGET} POST_BUILD
+    BYPRODUCTS "${ctest_tests_file}"
+    COMMAND "${CMAKE_COMMAND}"
+            -D "TEST_TARGET=${TARGET}"
+            -D "TEST_EXECUTABLE=$<TARGET_FILE:${TARGET}>"
+            -D "TEST_EXECUTOR=${crosscompiling_emulator}"
+            -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
+            -D "TEST_SPEC=${_TEST_SPEC}"
+            -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}"
+            -D "TEST_PROPERTIES=${_PROPERTIES}"
+            -D "TEST_PREFIX=${_TEST_PREFIX}"
+            -D "TEST_SUFFIX=${_TEST_SUFFIX}"
+            -D "TEST_LIST=${_TEST_LIST}"
+            -D "CTEST_FILE=${ctest_tests_file}"
+            -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
+    VERBATIM
+  )
+
+  file(WRITE "${ctest_include_file}"
+    "if(EXISTS \"${ctest_tests_file}\")\n"
+    "  include(\"${ctest_tests_file}\")\n"
+    "else()\n"
+    "  add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n"
+    "endif()\n"
+  )
+
+  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") 
+    # Add discovered tests to directory TEST_INCLUDE_FILES
+    set_property(DIRECTORY
+      APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+    )
+  else()
+    # Add discovered tests as directory TEST_INCLUDE_FILE if possible
+    get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
+    if (NOT ${test_include_file_set})
+      set_property(DIRECTORY
+        PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
+      )
+    else()
+      message(FATAL_ERROR
+        "Cannot set more than one TEST_INCLUDE_FILE"
+      )
+    endif()
+  endif()
+
+endfunction()
+
+###############################################################################
+
+set(_CATCH_DISCOVER_TESTS_SCRIPT
+  ${CMAKE_CURRENT_LIST_DIR}/CatchAddTests.cmake
+)
--- a/cmake/CatchAddTests.cmake
+++ b/cmake/CatchAddTests.cmake
@ -0,0 +1,151 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+set(prefix "${TEST_PREFIX}")
+set(suffix "${TEST_SUFFIX}")
+set(spec ${TEST_SPEC})
+set(extra_args ${TEST_EXTRA_ARGS})
+set(properties ${TEST_PROPERTIES})
+set(script)
+set(suite)
+set(tests)
+
+function(add_command NAME)
+  set(_args "")
+  foreach(_arg ${ARGN})
+    if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
+      set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
+    else()
+      set(_args "${_args} ${_arg}")
+    endif()
+  endforeach()
+  set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
+endfunction()
+
+# Run test executable to get list of available tests
+if(NOT EXISTS "${TEST_EXECUTABLE}")
+  message(FATAL_ERROR
+    "Specified test executable '${TEST_EXECUTABLE}' does not exist"
+  )
+endif()
+execute_process(
+  COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only
+  OUTPUT_VARIABLE output
+  RESULT_VARIABLE result
+)
+# Catch --list-test-names-only reports the number of tests, so 0 is... surprising
+if(${result} EQUAL 0)
+  message(WARNING
+    "Test executable '${TEST_EXECUTABLE}' contains no tests!\n"
+  )
+elseif(${result} LESS 0)
+  message(FATAL_ERROR
+    "Error running test executable '${TEST_EXECUTABLE}':\n"
+    "  Result: ${result}\n"
+    "  Output: ${output}\n"
+  )
+endif()
+
+string(REPLACE "\n" ";" output "${output}")
+
+# Parse output
+foreach(line ${output})
+  set(test ${line})
+  # ...and add to script
+  add_command(add_test
+    "${prefix}${test}${suffix}"
+    ${TEST_EXECUTOR}
+    "${TEST_EXECUTABLE}"
+    "${test}"
+    ${extra_args}
+  )
+  add_command(set_tests_properties
+    "${prefix}${test}${suffix}"
+    PROPERTIES
+    WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+    ${properties}
+  )
+  list(APPEND tests "${prefix}${test}${suffix}")
+endforeach()
+
+# Create a list of all discovered tests, which users may use to e.g. set
+# properties on the tests
+add_command(set ${TEST_LIST} ${tests})
+
+# Write CTest script
+file(WRITE "${CTEST_FILE}" "${script}")# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+set(prefix "${TEST_PREFIX}")
+set(suffix "${TEST_SUFFIX}")
+set(spec ${TEST_SPEC})
+set(extra_args ${TEST_EXTRA_ARGS})
+set(properties ${TEST_PROPERTIES})
+set(script)
+set(suite)
+set(tests)
+
+function(add_command NAME)
+  set(_args "")
+  foreach(_arg ${ARGN})
+    if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
+      set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
+    else()
+      set(_args "${_args} ${_arg}")
+    endif()
+  endforeach()
+  set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
+endfunction()
+
+# Run test executable to get list of available tests
+if(NOT EXISTS "${TEST_EXECUTABLE}")
+  message(FATAL_ERROR
+    "Specified test executable '${TEST_EXECUTABLE}' does not exist"
+  )
+endif()
+execute_process(
+  COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only
+  OUTPUT_VARIABLE output
+  RESULT_VARIABLE result
+)
+# Catch --list-test-names-only reports the number of tests, so 0 is... surprising
+if(${result} EQUAL 0)
+  message(WARNING
+    "Test executable '${TEST_EXECUTABLE}' contains no tests!\n"
+  )
+elseif(${result} LESS 0)
+  message(FATAL_ERROR
+    "Error running test executable '${TEST_EXECUTABLE}':\n"
+    "  Result: ${result}\n"
+    "  Output: ${output}\n"
+  )
+endif()
+
+string(REPLACE "\n" ";" output "${output}")
+
+# Parse output
+foreach(line ${output})
+  set(test ${line})
+  # ...and add to script
+  add_command(add_test
+    "${prefix}${test}${suffix}"
+    ${TEST_EXECUTOR}
+    "${TEST_EXECUTABLE}"
+    "${test}"
+    ${extra_args}
+  )
+  add_command(set_tests_properties
+    "${prefix}${test}${suffix}"
+    PROPERTIES
+    WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+    ${properties}
+  )
+  list(APPEND tests "${prefix}${test}${suffix}")
+endforeach()
+
+# Create a list of all discovered tests, which users may use to e.g. set
+# properties on the tests
+add_command(set ${TEST_LIST} ${tests})
+
+# Write CTest script
+file(WRITE "${CTEST_FILE}" "${script}")
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@ -1,3 +1,6 @@
+# Include the extra catch cmake functions
+include("${CMAKE_SOURCE_DIR}/cmake/Catch.cmake")
+
 set(SRCROOT "${PROJECT_SOURCE_DIR}/test/src")

 include_directories("${PROJECT_SOURCE_DIR}/include")
@ -10,14 +13,19 @@ SET(SYSTEM_SRC
    "${SRCROOT}/TestUtilities/System.hpp"
    "${SRCROOT}/TestUtilities/System.cpp"
 )
-SET(SYSTEM_LIB "sfml-system")
+add_executable(systemtest ${SYSTEM_SRC})
+target_link_libraries(systemtest sfml-system)
+catch_discover_tests(systemtest)

 if(SFML_BUILD_WINDOW)
    SET(WINDOW_SRC
+        "${SRCROOT}/Window.cpp"
        "${SRCROOT}/TestUtilities/Window.hpp"
        "${SRCROOT}/TestUtilities/Window.cpp"
    )
-    SET(WINDOW_LIB "sfml-window")
+    add_executable(windowtest ${WINDOW_SRC})
+    target_link_libraries(windowtest sfml-window)
+    catch_discover_tests(windowtest)
 endif()

 if(SFML_BUILD_GRAPHICS)
@ -26,26 +34,17 @@ if(SFML_BUILD_GRAPHICS)
        "${SRCROOT}/TestUtilities/Graphics.hpp"
        "${SRCROOT}/TestUtilities/Graphics.cpp"
    )
-    SET(WINDOW_LIB "sfml-graphics")
+    add_executable(graphicstest ${GRAPHICS_SRC})
+    target_link_libraries(graphicstest sfml-graphics)
+    catch_discover_tests(graphicstest)
 endif()

-SET(SRC
-    "${SRCROOT}/Main.cpp"
-    "${SYSTEM_SRC}"
-    "${WINDOW_SRC}"
-    "${GRAPHICS_SRC}"
-    "${AUDIO_SRC}"
-    "${NETWORK_SRC}"
+# Automatically run the tests at the end of the build
+add_custom_target( runtests ALL
+                   DEPENDS systemtest windowtest graphicstest
 )
-
-# Using sfmltest instead of test because the latter is reserved.
-add_executable(sfmltest ${SRC})
-
-target_link_libraries(sfmltest ${GRAPHICS_LIB} ${WINDOW_LIB} ${AUDIO_LIB} ${NETWORK_LIB} ${SYSTEM_LIB})
-
-add_custom_target(runtests ALL
-    DEPENDS sfmltest
-    WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/test"
-    COMMAND ${CMAKE_COMMAND} -E echo "Running test suite..."
-    COMMAND sfmltest
+add_custom_command(TARGET runtests
+                   COMMENT "Run tests"
+                   POST_BUILD COMMAND ctest ARGS --output-on-failure
+                   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 )
--- a/test/src/Main.cpp
+++ b/test/src/Main.cpp
@ -1,2 +0,0 @@
-#define CATCH_CONFIG_MAIN
-#include <catch.hpp>
--- a/test/src/Rect.cpp
+++ b/test/src/Rect.cpp
@ -1,3 +1,5 @@
+#define CATCH_CONFIG_MAIN
+
 #include <SFML/Graphics/Rect.hpp>
 #include <SFML/System/Vector2.hpp>
 #include "TestUtilities/Graphics.hpp"
--- a/test/src/Vector2.cpp
+++ b/test/src/Vector2.cpp
@ -1,3 +1,5 @@
+#define CATCH_CONFIG_MAIN
+
 #include <SFML/System/Vector2.hpp>
 #include "TestUtilities/System.hpp"

--- a/test/src/Window.cpp
+++ b/test/src/Window.cpp
@ -0,0 +1,10 @@
+#define CATCH_CONFIG_MAIN
+
+#include <SFML/Graphics/Rect.hpp>
+#include <SFML/System/Vector2.hpp>
+#include "TestUtilities/Window.hpp"
+
+TEST_CASE("Window unit test exists", "[window]")
+{
+    CHECK(TRUE);
+}