From d3a072fb63f089edcaf3f3937aaea5b5890ce8b5 Mon Sep 17 00:00:00 2001 From: Jonny Paton Date: Sun, 19 Aug 2018 19:04:11 +0100 Subject: [PATCH] Use modern cmake testing functionality including catch integration --- CMakeLists.txt | 1 + cmake/Catch.cmake | 175 ++++++++++++++++++++++++++++++++++++++ cmake/CatchAddTests.cmake | 151 ++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 41 +++++---- test/src/Main.cpp | 2 - test/src/Rect.cpp | 2 + test/src/Vector2.cpp | 2 + test/src/Window.cpp | 10 +++ 8 files changed, 361 insertions(+), 23 deletions(-) create mode 100644 cmake/Catch.cmake create mode 100644 cmake/CatchAddTests.cmake delete mode 100644 test/src/Main.cpp create mode 100644 test/src/Window.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bbea9886..ff8aa6de4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,6 +285,7 @@ if(SFML_BUILD_DOC) add_subdirectory(doc) endif() if(SFML_BUILD_TEST_SUITE) + enable_testing() add_subdirectory(test) endif() diff --git a/cmake/Catch.cmake b/cmake/Catch.cmake new file mode 100644 index 000000000..5f76e0cde --- /dev/null +++ b/cmake/Catch.cmake @@ -0,0 +1,175 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +Catch +----- + +This module defines a function to help use the Catch test framework. + +The :command:`catch_discover_tests` discovers tests by asking the compiled test +executable to enumerate its tests. This does not require CMake to be re-run +when tests change. However, it may not work in a cross-compiling environment, +and setting test properties is less convenient. + +This command is intended to replace use of :command:`add_test` to register +tests, and will create a separate CTest test for each Catch test case. Note +that this is in some cases less efficient, as common set-up and tear-down logic +cannot be shared by multiple test cases executing in the same instance. +However, it provides more fine-grained pass/fail information to CTest, which is +usually considered as more beneficial. By default, the CTest test name is the +same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. + +.. command:: catch_discover_tests + + Automatically add tests with CTest by querying the compiled test executable + for available tests:: + + catch_discover_tests(target + [TEST_SPEC arg1...] + [EXTRA_ARGS arg1...] + [WORKING_DIRECTORY dir] + [TEST_PREFIX prefix] + [TEST_SUFFIX suffix] + [PROPERTIES name1 value1...] + [TEST_LIST var] + ) + + ``catch_discover_tests`` sets up a post-build command on the test executable + that generates the list of tests by parsing the output from running the test + with the ``--list-test-names-only`` argument. This ensures that the full + list of tests is obtained. Since test discovery occurs at build time, it is + not necessary to re-run CMake when the list of tests changes. + However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set + in order to function in a cross-compiling environment. + + Additionally, setting properties on tests is somewhat less convenient, since + the tests are not available at CMake time. Additional test properties may be + assigned to the set of tests as a whole using the ``PROPERTIES`` option. If + more fine-grained test control is needed, custom content may be provided + through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES` + directory property. The set of discovered tests is made accessible to such a + script via the ``_TESTS`` variable. + + The options are: + + ``target`` + Specifies the Catch executable, which must be a known CMake executable + target. CMake will substitute the location of the built executable when + running the test. + + ``TEST_SPEC arg1...`` + Specifies test cases, wildcarded test cases, tags and tag expressions to + pass to the Catch executable with the ``--list-test-names-only`` argument. + + ``EXTRA_ARGS arg1...`` + Any extra arguments to pass on the command line to each test case. + + ``WORKING_DIRECTORY dir`` + Specifies the directory in which to run the discovered test cases. If this + option is not provided, the current binary directory is used. + + ``TEST_PREFIX prefix`` + Specifies a ``prefix`` to be prepended to the name of each discovered test + case. This can be useful when the same test executable is being used in + multiple calls to ``catch_discover_tests()`` but with different + ``TEST_SPEC`` or ``EXTRA_ARGS``. + + ``TEST_SUFFIX suffix`` + Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of + every discovered test case. Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may + be specified. + + ``PROPERTIES name1 value1...`` + Specifies additional properties to be set on all tests discovered by this + invocation of ``catch_discover_tests``. + + ``TEST_LIST var`` + Make the list of tests available in the variable ``var``, rather than the + default ``_TESTS``. This can be useful when the same test + executable is being used in multiple calls to ``catch_discover_tests()``. + Note that this variable is only available in CTest. + +#]=======================================================================] + +#------------------------------------------------------------------------------ +function(catch_discover_tests TARGET) + cmake_parse_arguments( + "" + "" + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST" + "TEST_SPEC;EXTRA_ARGS;PROPERTIES" + ${ARGN} + ) + + if(NOT _WORKING_DIRECTORY) + set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif() + if(NOT _TEST_LIST) + set(_TEST_LIST ${TARGET}_TESTS) + endif() + + ## Generate a unique name based on the extra arguments + string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}") + string(SUBSTRING ${args_hash} 0 7 args_hash) + + # Define rule to generate test list for aforementioned test executable + set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_include-${args_hash}.cmake") + set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_tests-${args_hash}.cmake") + get_property(crosscompiling_emulator + TARGET ${TARGET} + PROPERTY CROSSCOMPILING_EMULATOR + ) + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_EXECUTABLE=$" + -D "TEST_EXECUTOR=${crosscompiling_emulator}" + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" + -D "TEST_SPEC=${_TEST_SPEC}" + -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" + -D "TEST_PROPERTIES=${_PROPERTIES}" + -D "TEST_PREFIX=${_TEST_PREFIX}" + -D "TEST_SUFFIX=${_TEST_SUFFIX}" + -D "TEST_LIST=${_TEST_LIST}" + -D "CTEST_FILE=${ctest_tests_file}" + -P "${_CATCH_DISCOVER_TESTS_SCRIPT}" + VERBATIM + ) + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n" + "endif()\n" + ) + + if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") + # Add discovered tests to directory TEST_INCLUDE_FILES + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) + else() + # Add discovered tests as directory TEST_INCLUDE_FILE if possible + get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET) + if (NOT ${test_include_file_set}) + set_property(DIRECTORY + PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}" + ) + else() + message(FATAL_ERROR + "Cannot set more than one TEST_INCLUDE_FILE" + ) + endif() + endif() + +endfunction() + +############################################################################### + +set(_CATCH_DISCOVER_TESTS_SCRIPT + ${CMAKE_CURRENT_LIST_DIR}/CatchAddTests.cmake +) \ No newline at end of file diff --git a/cmake/CatchAddTests.cmake b/cmake/CatchAddTests.cmake new file mode 100644 index 000000000..4b855ce28 --- /dev/null +++ b/cmake/CatchAddTests.cmake @@ -0,0 +1,151 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +set(prefix "${TEST_PREFIX}") +set(suffix "${TEST_SUFFIX}") +set(spec ${TEST_SPEC}) +set(extra_args ${TEST_EXTRA_ARGS}) +set(properties ${TEST_PROPERTIES}) +set(script) +set(suite) +set(tests) + +function(add_command NAME) + set(_args "") + foreach(_arg ${ARGN}) + if(_arg MATCHES "[^-./:a-zA-Z0-9_]") + set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument + else() + set(_args "${_args} ${_arg}") + endif() + endforeach() + set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) +endfunction() + +# Run test executable to get list of available tests +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable '${TEST_EXECUTABLE}' does not exist" + ) +endif() +execute_process( + COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only + OUTPUT_VARIABLE output + RESULT_VARIABLE result +) +# Catch --list-test-names-only reports the number of tests, so 0 is... surprising +if(${result} EQUAL 0) + message(WARNING + "Test executable '${TEST_EXECUTABLE}' contains no tests!\n" + ) +elseif(${result} LESS 0) + message(FATAL_ERROR + "Error running test executable '${TEST_EXECUTABLE}':\n" + " Result: ${result}\n" + " Output: ${output}\n" + ) +endif() + +string(REPLACE "\n" ";" output "${output}") + +# Parse output +foreach(line ${output}) + set(test ${line}) + # ...and add to script + add_command(add_test + "${prefix}${test}${suffix}" + ${TEST_EXECUTOR} + "${TEST_EXECUTABLE}" + "${test}" + ${extra_args} + ) + add_command(set_tests_properties + "${prefix}${test}${suffix}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${properties} + ) + list(APPEND tests "${prefix}${test}${suffix}") +endforeach() + +# Create a list of all discovered tests, which users may use to e.g. set +# properties on the tests +add_command(set ${TEST_LIST} ${tests}) + +# Write CTest script +file(WRITE "${CTEST_FILE}" "${script}")# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +set(prefix "${TEST_PREFIX}") +set(suffix "${TEST_SUFFIX}") +set(spec ${TEST_SPEC}) +set(extra_args ${TEST_EXTRA_ARGS}) +set(properties ${TEST_PROPERTIES}) +set(script) +set(suite) +set(tests) + +function(add_command NAME) + set(_args "") + foreach(_arg ${ARGN}) + if(_arg MATCHES "[^-./:a-zA-Z0-9_]") + set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument + else() + set(_args "${_args} ${_arg}") + endif() + endforeach() + set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) +endfunction() + +# Run test executable to get list of available tests +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable '${TEST_EXECUTABLE}' does not exist" + ) +endif() +execute_process( + COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only + OUTPUT_VARIABLE output + RESULT_VARIABLE result +) +# Catch --list-test-names-only reports the number of tests, so 0 is... surprising +if(${result} EQUAL 0) + message(WARNING + "Test executable '${TEST_EXECUTABLE}' contains no tests!\n" + ) +elseif(${result} LESS 0) + message(FATAL_ERROR + "Error running test executable '${TEST_EXECUTABLE}':\n" + " Result: ${result}\n" + " Output: ${output}\n" + ) +endif() + +string(REPLACE "\n" ";" output "${output}") + +# Parse output +foreach(line ${output}) + set(test ${line}) + # ...and add to script + add_command(add_test + "${prefix}${test}${suffix}" + ${TEST_EXECUTOR} + "${TEST_EXECUTABLE}" + "${test}" + ${extra_args} + ) + add_command(set_tests_properties + "${prefix}${test}${suffix}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${properties} + ) + list(APPEND tests "${prefix}${test}${suffix}") +endforeach() + +# Create a list of all discovered tests, which users may use to e.g. set +# properties on the tests +add_command(set ${TEST_LIST} ${tests}) + +# Write CTest script +file(WRITE "${CTEST_FILE}" "${script}") \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bbdcb37bd..73712d36d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,3 +1,6 @@ +# Include the extra catch cmake functions +include("${CMAKE_SOURCE_DIR}/cmake/Catch.cmake") + set(SRCROOT "${PROJECT_SOURCE_DIR}/test/src") include_directories("${PROJECT_SOURCE_DIR}/include") @@ -10,14 +13,19 @@ SET(SYSTEM_SRC "${SRCROOT}/TestUtilities/System.hpp" "${SRCROOT}/TestUtilities/System.cpp" ) -SET(SYSTEM_LIB "sfml-system") +add_executable(systemtest ${SYSTEM_SRC}) +target_link_libraries(systemtest sfml-system) +catch_discover_tests(systemtest) if(SFML_BUILD_WINDOW) SET(WINDOW_SRC + "${SRCROOT}/Window.cpp" "${SRCROOT}/TestUtilities/Window.hpp" "${SRCROOT}/TestUtilities/Window.cpp" ) - SET(WINDOW_LIB "sfml-window") + add_executable(windowtest ${WINDOW_SRC}) + target_link_libraries(windowtest sfml-window) + catch_discover_tests(windowtest) endif() if(SFML_BUILD_GRAPHICS) @@ -26,26 +34,17 @@ if(SFML_BUILD_GRAPHICS) "${SRCROOT}/TestUtilities/Graphics.hpp" "${SRCROOT}/TestUtilities/Graphics.cpp" ) - SET(WINDOW_LIB "sfml-graphics") + add_executable(graphicstest ${GRAPHICS_SRC}) + target_link_libraries(graphicstest sfml-graphics) + catch_discover_tests(graphicstest) endif() -SET(SRC - "${SRCROOT}/Main.cpp" - "${SYSTEM_SRC}" - "${WINDOW_SRC}" - "${GRAPHICS_SRC}" - "${AUDIO_SRC}" - "${NETWORK_SRC}" +# Automatically run the tests at the end of the build +add_custom_target( runtests ALL + DEPENDS systemtest windowtest graphicstest ) - -# Using sfmltest instead of test because the latter is reserved. -add_executable(sfmltest ${SRC}) - -target_link_libraries(sfmltest ${GRAPHICS_LIB} ${WINDOW_LIB} ${AUDIO_LIB} ${NETWORK_LIB} ${SYSTEM_LIB}) - -add_custom_target(runtests ALL - DEPENDS sfmltest - WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/test" - COMMAND ${CMAKE_COMMAND} -E echo "Running test suite..." - COMMAND sfmltest +add_custom_command(TARGET runtests + COMMENT "Run tests" + POST_BUILD COMMAND ctest ARGS --output-on-failure + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) diff --git a/test/src/Main.cpp b/test/src/Main.cpp deleted file mode 100644 index b3143fbb1..000000000 --- a/test/src/Main.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define CATCH_CONFIG_MAIN -#include diff --git a/test/src/Rect.cpp b/test/src/Rect.cpp index 7ac8933ab..e35aa3973 100644 --- a/test/src/Rect.cpp +++ b/test/src/Rect.cpp @@ -1,3 +1,5 @@ +#define CATCH_CONFIG_MAIN + #include #include #include "TestUtilities/Graphics.hpp" diff --git a/test/src/Vector2.cpp b/test/src/Vector2.cpp index 73ae84630..175ebee22 100644 --- a/test/src/Vector2.cpp +++ b/test/src/Vector2.cpp @@ -1,3 +1,5 @@ +#define CATCH_CONFIG_MAIN + #include #include "TestUtilities/System.hpp" diff --git a/test/src/Window.cpp b/test/src/Window.cpp new file mode 100644 index 000000000..e5733626d --- /dev/null +++ b/test/src/Window.cpp @@ -0,0 +1,10 @@ +#define CATCH_CONFIG_MAIN + +#include +#include +#include "TestUtilities/Window.hpp" + +TEST_CASE("Window unit test exists", "[window]") +{ + CHECK(TRUE); +}