From 7ff8720a39fa2a2eccffb4b43b63a3847ba6c30e Mon Sep 17 00:00:00 2001
From: Janek Bevendorff <janek@keepassxc.org>
Date: Mon, 27 Sep 2021 14:13:30 +0200
Subject: [PATCH] Clean up code coverage reporting.
CTest is now run directly and `make coverage` (like `make test`) now
expects you to run `make` beforehand, which is more flexible for the
user. This patch also reduces clutter by properly excluding unwanted
files and reduces the number of explicit exlusion regexes that are
required.
Gcov reports are still confusing and report very low branch coverage
(which is picked up by Codecov, unfortunately), but the llvm-cov reports
are nice and clean now.
---
CMakeLists.txt | 22 ++++++------------
cmake/CodeCoverage.cmake | 48 +++++++++++++++++++++++++++-------------
codecov.yaml | 2 ++
3 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1f89c958d..ae0fa655c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -385,20 +385,13 @@ endif(WITH_TESTS)
if(WITH_COVERAGE)
# Include code coverage, use with -DCMAKE_BUILD_TYPE=Debug
include(CodeCoverage)
- set(COVERAGE_EXCLUDES
- "\\(.+/\\)?tests/.\\*"
- "\\(.+/\\)?build/.\\*"
- "\\(.+/\\)?thirdparty/.\\*"
- "\\(.+/\\)?CMakeFiles/.\\*"
- "src/main.cpp"
- ".\\*/moc_\\[^/\\]+\\.cpp"
- ".\\*/ui_\\[^/\\]+\\.h"
- ".\\*/\\[^/\\]+_autogen/.\\*"
- "\\(.+/\\)?zxcvbn/.\\*"
- "/Applications/.\\*"
- "/opt/.\\*")
append_coverage_compiler_flags()
+ set(COVERAGE_EXCLUDES
+ "'^(.+/)?(thirdparty|zxcvbn)/.*'"
+ "'^(.+/)?main\\.cpp$$'"
+ "'^(.+/)?cli/keepassxc-cli\\.cpp$$'"
+ "'^(.+/)?proxy/keepassxc-proxy\\.cpp$$'")
if(WITH_COVERAGE AND CMAKE_COMPILER_IS_CLANGXX)
set(MAIN_BINARIES
"$<TARGET_FILE:${PROGNAME}>"
@@ -406,14 +399,13 @@ if(WITH_COVERAGE)
"$<TARGET_FILE:keepassxc-proxy>")
setup_target_for_coverage_llvm(
NAME coverage
- EXECUTABLE $(MAKE) test
BINARY ${MAIN_BINARIES}
- SOURCES ${CMAKE_SOURCE_DIR}/src
+ SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
)
else()
setup_target_for_coverage_gcovr(
NAME coverage
- EXECUTABLE $(MAKE) test
+ SOURCES_ROOT ${CMAKE_SOURCE_DIR}/src
)
endif()
endif()
diff --git a/cmake/CodeCoverage.cmake b/cmake/CodeCoverage.cmake
index 78633dee8..d1f2bfdda 100644
--- a/cmake/CodeCoverage.cmake
+++ b/cmake/CodeCoverage.cmake
@@ -45,7 +45,7 @@ elseif(CMAKE_COMPILER_IS_CLANGXX)
endif()
set(CMAKE_COVERAGE_FORMAT
- "html" "txt"
+ "html" "xml"
CACHE STRING "Coverage report output format.")
set_property(CACHE CMAKE_COVERAGE_FORMAT PROPERTY STRINGS "html" "txt")
@@ -93,7 +93,7 @@ endif()
function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
set(options NONE)
- set(oneValueArgs NAME)
+ set(oneValueArgs NAME SOURCES_ROOT)
set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
@@ -110,33 +110,48 @@ function(SETUP_TARGET_FOR_COVERAGE_GCOVR)
add_custom_target(${Coverage_NAME}
# Run tests
- COMMAND $(MAKE)
- COMMAND ${Coverage_EXECUTABLE}
+ COMMAND ctest -C $<CONFIG> $ENV{ARGS} $$ARGS
- # Create folder
- COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
DEPENDS ${Coverage_DEPENDENCIES}
)
if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+ # Create folder
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME}-html
+
# Running gcovr HTML
COMMAND ${GCOVR_PATH} --html --html-details
- -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+ -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
--object-directory=${PROJECT_BINARY_DIR}
+ --exclude-unreachable-branches --exclude-throw-branches
-o ${Coverage_NAME}-html/index.html
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMENT "Running gcovr to produce HTML code coverage report ${Coverage_NAME}-html."
)
endif()
+ if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+ add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+ # Running gcovr TXT
+ COMMAND ${GCOVR_PATH} --xml
+ -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
+ --object-directory=${PROJECT_BINARY_DIR}
+ --exclude-unreachable-branches --exclude-throw-branches
+ -o ${Coverage_NAME}.xml
+ WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+ COMMENT "Running gcovr to produce XML code coverage report ${Coverage_NAME}.xml."
+ )
+ endif()
+
if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
- # Running gcovr XML
+ # Running gcovr TXT
COMMAND ${GCOVR_PATH}
- -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES}
+ -r ${Coverage_SOURCES_ROOT} ${GCOVR_EXCLUDES}
--object-directory=${PROJECT_BINARY_DIR}
+ --exclude-unreachable-branches --exclude-throw-branches
-o ${Coverage_NAME}.txt
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMENT "Running gcovr to produce TXT code coverage report ${Coverage_NAME}.txt."
@@ -158,8 +173,8 @@ endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR
function(SETUP_TARGET_FOR_COVERAGE_LLVM)
set(options NONE)
- set(oneValueArgs NAME PROF_FILE)
- set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS SOURCES DEPENDENCIES)
+ set(oneValueArgs NAME SOURCES_ROOT PROF_FILE)
+ set(multiValueArgs EXECUTABLE BINARY EXECUTABLE_ARGS DEPENDENCIES)
cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
if(XCRUN_PATH)
@@ -191,8 +206,7 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
endif()
add_custom_target(${Coverage_NAME}
- COMMAND $(MAKE)
- COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ${Coverage_EXECUTABLE}
+ COMMAND ${CMAKE_COMMAND} -E env LLVM_PROFILE_FILE=${LLVM_PROFILE_DIR}/profile-%p.profraw ctest -C $<CONFIG> $$ARGS
COMMAND ${LLVM_PROFDATA_PATH} merge -sparse ${LLVM_PROFILE_DIR}/* -o coverage.profdata
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
@@ -201,15 +215,19 @@ function(SETUP_TARGET_FOR_COVERAGE_LLVM)
if("html" IN_LIST CMAKE_COVERAGE_FORMAT)
add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
- --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES}
+ --format=html --output-dir=${Coverage_NAME}-html ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMENT "Running llvm-cov to produce HTML code coverage report ${Coverage_NAME}-html")
endif()
+ if("xml" IN_LIST CMAKE_COVERAGE_FORMAT)
+ message(WARNING "XML coverage report format not supported for llvm-cov")
+ endif()
+
if("txt" IN_LIST CMAKE_COVERAGE_FORMAT)
add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
COMMAND ${LLVM_COV_PATH} show -instr-profile=coverage.profdata ${COV_BINARY}
- --format=text ${COV_EXCLUDES} ${Coverage_SOURCES} > ${Coverage_NAME}.txt
+ --format=text ${COV_EXCLUDES} ${Coverage_SOURCES_ROOT} > ${Coverage_NAME}.txt
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMENT "Running llvm-cov to produce TXT code coverage report ${Coverage_NAME}.txt.")
diff --git a/codecov.yaml b/codecov.yaml
index e671b003b..d92656b6f 100644
--- a/codecov.yaml
+++ b/codecov.yaml
@@ -2,5 +2,7 @@ coverage:
range: 60..80
round: nearest
precision: 2
+fixes:
+ - "*/src/::"
comment:
require_changes: true