From 66339bc09c9470b5a809743459ce5b942288765b Mon Sep 17 00:00:00 2001 From: binary1248 Date: Sun, 6 Feb 2022 05:38:28 +0100 Subject: [PATCH] Added support for coverage reporting. --- .github/.codecov.yml | 31 +++++++++++++++++ .github/workflows/ci.yml | 57 ++++++++++++++++++++++++++++--- CMakeLists.txt | 3 ++ cmake/Macros.cmake | 25 ++++++++++++++ test/Audio/Dummy.cpp | 0 test/CMakeLists.txt | 73 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 .github/.codecov.yml create mode 100644 test/Audio/Dummy.cpp diff --git a/.github/.codecov.yml b/.github/.codecov.yml new file mode 100644 index 00000000..21de6af0 --- /dev/null +++ b/.github/.codecov.yml @@ -0,0 +1,31 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: auto + threshold: 100% + base: auto + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: no + require_head: no + require_base: no + +github_checks: + annotations: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4248ed..9e767572 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,10 +35,23 @@ jobs: - name: Checkout Code uses: actions/checkout@v2 - - name: Install Linux Dependencies + - name: Detect Linux Clang Version if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install libxrandr-dev libxcursor-dev libudev-dev libopenal-dev libflac-dev libvorbis-dev libgl1-mesa-dev libegl1-mesa-dev + run: clang++ --version | sed -n 's/.*version \([0-9]\+\)\..*/clang_version=\1/p' >> $GITHUB_ENV + - name: Install Linux Tools and Dependencies + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install gcovr llvm-${{ env.clang_version }} libxrandr-dev libxcursor-dev libudev-dev libopenal-dev libflac-dev libvorbis-dev libgl1-mesa-dev libegl1-mesa-dev + + - name: Install macOS Tools and Dependencies + if: runner.os == 'macOS' + run: brew install gcovr + + - name: Install OpenCppCoverage and add to PATH + if: runner.os == 'Windows' + run: | + choco install OpenCppCoverage -y + echo "C:\Program Files\OpenCppCoverage" >> $env:GITHUB_PATH - name: Install Android Components if: matrix.platform.name == 'Android' @@ -48,11 +61,47 @@ jobs: wget -nv https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip -P $GITHUB_WORKSPACE unzip -qq -d $GITHUB_WORKSPACE android-ndk-r18b-linux-x86_64.zip - - name: Configure CMake shell: bash - run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install -DSFML_BUILD_EXAMPLES=TRUE -DCMAKE_VERBOSE_MAKEFILE=ON -DSFML_BUILD_TEST_SUITE=TRUE ${{matrix.platform.flags}} ${{matrix.config.flags}} + run: cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install -DSFML_BUILD_EXAMPLES=TRUE -DCMAKE_VERBOSE_MAKEFILE=ON -DSFML_BUILD_TEST_SUITE=TRUE -DSFML_ENABLE_COVERAGE=TRUE ${{matrix.platform.flags}} ${{matrix.config.flags}} - name: Build shell: bash run: cmake --build $GITHUB_WORKSPACE/build --config Release --target install + + - name: Generate OpenCppCoverage Coverage Report + if: matrix.platform.name == 'Windows VS2019' || matrix.platform.name == 'Windows VS2022' + shell: bash + run: cmake --build $GITHUB_WORKSPACE/build --config Debug --target runtests + + - name: Generate GCC Gcov Coverage Report + if: matrix.platform.name == 'Linux GCC' + shell: bash + run: | + cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install -DCMAKE_BUILD_TYPE=Debug -DSFML_BUILD_EXAMPLES=TRUE -DCMAKE_VERBOSE_MAKEFILE=ON -DSFML_BUILD_TEST_SUITE=TRUE -DSFML_ENABLE_COVERAGE=TRUE ${{matrix.platform.flags}} ${{matrix.config.flags}} + cmake --build $GITHUB_WORKSPACE/build --target runtests + gcovr -r $GITHUB_WORKSPACE -x $GITHUB_WORKSPACE/build/coverage.out -s -f 'src/SFML/.*' -f 'include/SFML/.*' $GITHUB_WORKSPACE + + - name: Generate Clang Gcov Coverage Report + if: matrix.platform.name == 'Linux Clang' + shell: bash + run: | + cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install -DCMAKE_BUILD_TYPE=Debug -DSFML_BUILD_EXAMPLES=TRUE -DCMAKE_VERBOSE_MAKEFILE=ON -DSFML_BUILD_TEST_SUITE=TRUE -DSFML_ENABLE_COVERAGE=TRUE ${{matrix.platform.flags}} ${{matrix.config.flags}} + cmake --build $GITHUB_WORKSPACE/build --target runtests + gcovr --gcov-executable="llvm-cov-${{ env.clang_version }} gcov" -r $GITHUB_WORKSPACE -x $GITHUB_WORKSPACE/build/coverage.out -s -f 'src/SFML/.*' -f 'include/SFML/.*' $GITHUB_WORKSPACE + + - name: Generate Apple Clang Gcov Coverage Report + if: matrix.platform.name == 'MacOS XCode' && (matrix.config.name == 'Shared' || matrix.config.name == 'Static') + shell: bash + run: | + cmake -S $GITHUB_WORKSPACE -B $GITHUB_WORKSPACE/build -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install -DCMAKE_BUILD_TYPE=Debug -DSFML_BUILD_EXAMPLES=TRUE -DCMAKE_VERBOSE_MAKEFILE=ON -DSFML_BUILD_TEST_SUITE=TRUE -DSFML_ENABLE_COVERAGE=TRUE ${{matrix.platform.flags}} ${{matrix.config.flags}} + cmake --build $GITHUB_WORKSPACE/build --target runtests + gcovr -r $GITHUB_WORKSPACE -x $GITHUB_WORKSPACE/build/coverage.out -s -f 'src/SFML/.*' -f 'include/SFML/.*' $GITHUB_WORKSPACE + + - name: Upload Coverage Report to Codecov + if: matrix.platform.name == 'Windows VS2019' || matrix.platform.name == 'Windows VS2022' || matrix.platform.name == 'Linux GCC' || matrix.platform.name == 'Linux Clang' || (matrix.platform.name == 'MacOS XCode' && (matrix.config.name == 'Shared' || matrix.config.name == 'Static')) + uses: codecov/codecov-action@v2 + with: + directory: ./build + files: ./build/coverage.out + fail_ci_if_error: true diff --git a/CMakeLists.txt b/CMakeLists.txt index fd528daa..cd63e47a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,9 @@ endif() # add an option for building the test suite sfml_set_option(SFML_BUILD_TEST_SUITE FALSE BOOL "TRUE to build the SFML test suite, FALSE to ignore it") +# add an option for enabling coverage reporting +sfml_set_option(SFML_ENABLE_COVERAGE FALSE BOOL "TRUE to enable coverage reporting, FALSE to ignore it") + # macOS specific options if(SFML_OS_MACOSX) # add an option to build frameworks instead of dylibs (release only) diff --git a/cmake/Macros.cmake b/cmake/Macros.cmake index b0d13684..7ce29c7b 100644 --- a/cmake/Macros.cmake +++ b/cmake/Macros.cmake @@ -73,6 +73,17 @@ macro(sfml_add_library module) # enable C++17 support target_compile_features(${target} PUBLIC cxx_std_17) + # Add required flags for GCC if coverage reporting is enabled + if (SFML_ENABLE_COVERAGE AND (SFML_COMPILER_GCC OR SFML_COMPILER_CLANG)) + target_compile_options(${target} PUBLIC $<$:-O0> $<$:-g> $<$:-fprofile-arcs> $<$:-ftest-coverage>) + + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) + target_link_options(${target} PUBLIC $<$:--coverage>) + else() + target_link_libraries(${target} PUBLIC $<$:--coverage>) + endif() + endif() + set_file_warnings(${THIS_SOURCES}) # define the export symbol of the module @@ -318,6 +329,20 @@ function(sfml_add_test target SOURCES DEPENDS) # link the target to its SFML dependencies target_link_libraries(${target} PRIVATE ${DEPENDS}) + # If coverage is enabled for MSVC and we are linking statically, use /WHOLEARCHIVE + # to make sure the linker doesn't discard unused code sections before coverage can be measured + if (SFML_ENABLE_COVERAGE AND SFML_COMPILER_MSVC AND NOT BUILD_SHARED_LIBS) + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) + foreach (DEPENDENCY ${DEPENDS}) + target_link_options(${target} PRIVATE $<$:/WHOLEARCHIVE:$>) + endforeach() + else() + foreach (DEPENDENCY ${DEPENDS}) + target_link_libraries(${target} PRIVATE $<$:/WHOLEARCHIVE:$>) + endforeach() + endif() + endif() + # Add the test add_test(${target} ${target}) diff --git a/test/Audio/Dummy.cpp b/test/Audio/Dummy.cpp new file mode 100644 index 00000000..e69de29b diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 16414336..d366dadf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -51,12 +51,75 @@ if(SFML_BUILD_NETWORK) target_link_libraries(test-sfml-network PRIVATE sfml-test-main) endif() +if(SFML_BUILD_AUDIO) + SET(AUDIO_SRC + "${SRCROOT}/Audio/Dummy.cpp" # TODO: Remove when there are real tests + ) + sfml_add_test(test-sfml-audio "${AUDIO_SRC}" SFML::Audio) + target_link_libraries(test-sfml-audio PRIVATE sfml-test-main) +endif() + # Automatically run the tests at the end of the build add_custom_target(runtests ALL - DEPENDS test-sfml-system test-sfml-window test-sfml-graphics test-sfml-network + DEPENDS test-sfml-system test-sfml-window test-sfml-graphics test-sfml-network test-sfml-audio ) -add_custom_command(TARGET runtests - COMMENT "Run tests" - POST_BUILD COMMAND "${CMAKE_CTEST_COMMAND}" --output-on-failure -C $ -) +if(SFML_OS_WINDOWS AND NOT SFML_USE_SYSTEM_DEPS) + # Copy the binaries of SFML dependencies + list(APPEND BINARIES + "openal32.dll" + ) + + foreach (BINARY ${BINARIES}) + if(ARCH_32BITS) + list(APPEND BINARY_PATHS "${PROJECT_SOURCE_DIR}/extlibs/bin/x86/${BINARY}") + elseif(ARCH_64BITS) + list(APPEND BINARY_PATHS "${PROJECT_SOURCE_DIR}/extlibs/bin/x64/${BINARY}") + endif() + endforeach() + + add_custom_command(TARGET runtests + COMMENT "Copy binaries" + POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy ${BINARY_PATHS} "$" + ) +endif() + +if(SFML_ENABLE_COVERAGE AND SFML_COMPILER_MSVC) + # Try to find and use OpenCppCoverage for coverage reporting when building with MSVC + find_program(OpenCppCoverage_BINARY "OpenCppCoverage.exe") + + if(OpenCppCoverage_BINARY) + execute_process(COMMAND "${OpenCppCoverage_BINARY}" --help ERROR_VARIABLE OpenCppCoverage_HELP_OUTPUT OUTPUT_QUIET) + + if(OpenCppCoverage_HELP_OUTPUT MATCHES "OpenCppCoverage Version: ([.0-9]+)") + set(OpenCppCoverage_VERSION "${CMAKE_MATCH_1}") + endif() + endif() + + include(FindPackageHandleStandardArgs) + + find_package_handle_standard_args(OpenCppCoverage + REQUIRED_VARS OpenCppCoverage_BINARY + VERSION_VAR OpenCppCoverage_VERSION + ) +endif() + +if(SFML_ENABLE_COVERAGE AND OpenCppCoverage_FOUND) + # Use OpenCppCoverage + message(STATUS "Using OpenCppCoverage to generate coverage report") + + string(REPLACE "/" "\\" COVERAGE_EXCLUDE "${CMAKE_CTEST_COMMAND}") + string(REPLACE "/" "\\" COVERAGE_SRC "${PROJECT_SOURCE_DIR}/src") + string(REPLACE "/" "\\" COVERAGE_INCLUDE "${PROJECT_SOURCE_DIR}/include") + + add_custom_command(TARGET runtests + COMMENT "Run tests" + POST_BUILD COMMAND "${OpenCppCoverage_BINARY}" ARGS --quiet --export_type cobertura:"${CMAKE_BINARY_DIR}/coverage.out" --cover_children --excluded_modules "${COVERAGE_EXCLUDE}" --sources "${COVERAGE_SRC}" --sources "${COVERAGE_INCLUDE}" -- "${CMAKE_CTEST_COMMAND}" --output-on-failure -C $ + ) +else() + # Run tests without a coverage runner + add_custom_command(TARGET runtests + COMMENT "Run tests" + POST_BUILD COMMAND "${CMAKE_CTEST_COMMAND}" --output-on-failure -C $ + ) +endif()