From 3baf483523174425b018ceb2772548fb64203877 Mon Sep 17 00:00:00 2001 From: binary1248 Date: Sat, 18 Jan 2025 17:55:37 +0100 Subject: [PATCH] Added support for mutation testing using Mull. Closes #3334 --- .github/workflows/mutation.yml | 95 ++++++++++++++++++++++++++++++++++ mull.yml | 50 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 .github/workflows/mutation.yml create mode 100644 mull.yml diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 000000000..713910ad4 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,95 @@ +name: Mutation + +# Run workflow when changing mutation configuration or manually triggered +on: + push: + paths: + - 'mull.yml' + - '.github/workflows/mutation.yml' + workflow_dispatch: + +concurrency: + group: environment-mutation-${{github.ref}} + cancel-in-progress: true + +env: + DISPLAY: ":99" # Display number to use for the X server + +defaults: + run: + shell: bash + +jobs: + build: + name: ${{ matrix.platform.name }} ${{ matrix.config.name }} ${{ matrix.type.name }} + runs-on: ${{ matrix.platform.os }} + + strategy: + fail-fast: false + matrix: + platform: + - { name: Linux Clang, os: ubuntu-24.04, flags: -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -GNinja -DSFML_BUILD_EXAMPLES=OFF } + config: + - { name: Static, flags: -DBUILD_SHARED_LIBS=OFF } + type: + - { name: Debug, flags: -DCMAKE_BUILD_TYPE=Debug } + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Get CMake and Ninja + uses: lukka/get-cmake@latest + with: + cmakeVersion: '3.22' + ninjaVersion: latest + + - name: Install Linux Dependencies and Tools + if: runner.os == 'Linux' + run: | + curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash + CLANG_VERSION=$(clang++ --version | sed -n 's/.*version \([0-9]\+\)\..*/\1/p') + echo "CLANG_VERSION=$CLANG_VERSION" >> $GITHUB_ENV + sudo apt-get update && sudo apt-get install xorg-dev libxrandr-dev libxcursor-dev libxi-dev libudev-dev libflac-dev libvorbis-dev libgl1-mesa-dev libegl1-mesa-dev libdrm-dev libgbm-dev x11-utils xvfb fluxbox eterm daemonize lib32stdc++6 llvm-$CLANG_VERSION mull-$CLANG_VERSION + mull-runner-$CLANG_VERSION --version + + - name: Remove ALSA Library + if: runner.os == 'Linux' + run: sudo apt-get remove -y libasound2 + + - name: Configure CMake + run: cmake --preset dev -DCMAKE_VERBOSE_MAKEFILE=ON ${{matrix.platform.flags}} ${{matrix.config.flags}} ${{matrix.type.flags}} -DCMAKE_CXX_FLAGS="-O0 -fpass-plugin=/usr/lib/mull-ir-frontend-$CLANG_VERSION -g -grecord-command-line -fprofile-instr-generate -fcoverage-mapping" + + - name: Build + run: cmake --build build --config ${{ matrix.type.name == 'Debug' && 'Debug' || 'Release' }} + + - name: Prepare Test + run: | + set -e + set -o pipefail + # Start up Xvfb and fluxbox to host display tests + daemonize /usr/bin/Xvfb $DISPLAY -screen 0 1920x1080x24 -ac +extension GLX +extension DPMS +render -noreset -s 1440 +bs -maxclients 256 +xinerama + sleep 5 + daemonize /usr/bin/fluxbox + sleep 5 + # Test if the X server is working properly by outputting display information + xdpyinfo -queryExtensions | awk "/^name of display/,/^ significant bits in color specification/" + # Make sure the build/bin directory exists so that the find command does not fail if no executables are built + mkdir -p build/bin + # Make use of a test to print OpenGL vendor/renderer/version info to the console + find build/bin -name test-sfml-window -or -name test-sfml-window.exe -exec sh -c "{} *sf::Context* --section=\"Version String\" --success | grep OpenGL" \; + + - name: Test + run: | + cd test + mull-runner-$CLANG_VERSION --allow-surviving --reporters SQLite --report-name SFML ../build/bin/test-sfml-system + mull-runner-$CLANG_VERSION --allow-surviving --reporters SQLite --report-name SFML ../build/bin/test-sfml-network + mull-runner-$CLANG_VERSION --allow-surviving --reporters SQLite --report-name SFML ../build/bin/test-sfml-audio + mull-runner-$CLANG_VERSION --allow-surviving --reporters SQLite --report-name SFML ../build/bin/test-sfml-window + mull-runner-$CLANG_VERSION --allow-surviving --reporters SQLite --report-name SFML ../build/bin/test-sfml-graphics + + - name: Report (IDE) + run: mull-reporter-$CLANG_VERSION --allow-surviving --reporters IDE test/SFML.sqlite + + - name: Report (Github Annotations) + run: mull-reporter-$CLANG_VERSION --allow-surviving --reporters GithubAnnotations test/SFML.sqlite diff --git a/mull.yml b/mull.yml new file mode 100644 index 000000000..07fc97c6b --- /dev/null +++ b/mull.yml @@ -0,0 +1,50 @@ +mutators: + - cxx_add_assign_to_sub_assign # Replaces += with -= + - cxx_add_to_sub # Replaces + with - + - cxx_and_assign_to_or_assign # Replaces &= with |= + - cxx_and_to_or # Replaces & with | + - cxx_assign_const # Replaces 'a = b' with 'a = 42' + - cxx_bitwise_not_to_noop # Replaces ~x with x + - cxx_div_assign_to_mul_assign # Replaces /= with *= + - cxx_div_to_mul # Replaces / with * + - cxx_eq_to_ne # Replaces == with != + - cxx_ge_to_gt # Replaces >= with > + - cxx_ge_to_lt # Replaces >= with < + - cxx_gt_to_ge # Replaces > with >= + - cxx_gt_to_le # Replaces > with <= + - cxx_init_const # Replaces 'T a = b' with 'T a = 42' + - cxx_le_to_gt # Replaces <= with > + - cxx_le_to_lt # Replaces <= with < + - cxx_lshift_assign_to_rshift_assign # Replaces <<= with >>= + - cxx_lshift_to_rshift # Replaces << with >> + - cxx_lt_to_ge # Replaces < with >= + - cxx_lt_to_le # Replaces < with <= + - cxx_minus_to_noop # Replaces -x with x + - cxx_mul_assign_to_div_assign # Replaces *= with /= + - cxx_mul_to_div # Replaces * with / + - cxx_ne_to_eq # Replaces != with == + - cxx_or_assign_to_and_assign # Replaces |= with &= + - cxx_or_to_and # Replaces | with & + - cxx_post_dec_to_post_inc # Replaces x-- with x++ + - cxx_post_inc_to_post_dec # Replaces x++ with x-- + - cxx_pre_dec_to_pre_inc # Replaces --x with ++x + - cxx_pre_inc_to_pre_dec # Replaces ++x with --x + - cxx_rem_assign_to_div_assign # Replaces %= with /= + - cxx_rem_to_div # Replaces % with / + - cxx_remove_void_call # Removes calls to a function returning void + - cxx_replace_scalar_call # Replaces call to a function with 42 + - cxx_rshift_assign_to_lshift_assign # Replaces >>= with <<= + - cxx_rshift_to_lshift # Replaces << with >> + - cxx_sub_assign_to_add_assign # Replaces -= with += + - cxx_sub_to_add # Replaces - with + + - cxx_xor_assign_to_or_assign # Replaces ^= with |= + - cxx_xor_to_or # Replaces ^ with | + # Experimental Mutators + # - negate_mutator # Negates conditionals !x to x and x to !x + # - scalar_value_mutator # Replaces zeros with 42, and non-zeros with 0 + # - cxx_logical_and_to_or # Replaces && with || + # - cxx_logical_or_to_and # Replaces || with && + # - cxx_remove_negation # Replaces !a with a +excludePaths: + - .*/catch2/.* + - .*/test/.*