From a79ac015c60532739495396dddbcb86edd4f8518 Mon Sep 17 00:00:00 2001 From: Joseph Snyder Date: Mon, 4 May 2026 12:05:08 -0400 Subject: [PATCH] ctest: Merge llvm-cov's .profraw files into a .profdata file for each test If the CTEST_TEST_COVERAGE_TOOL is set to LLVM-COV, use the `llvm-profdata --merge` functionality to collect the `.profraw` files generated by running each test into a combined `.profdata` file. This is the file that could be used to parse or display coverage information for each test. Issue: #26932 --- .../ci/configure_fedora44_common_clang.cmake | 2 + Modules/CTestTargets.cmake | 8 ++ Modules/DartConfiguration.tcl.in | 1 + Source/CTest/cmCTestRunTest.cxx | 99 +++++++++++++++++-- Source/CTest/cmCTestRunTest.h | 4 +- Tests/RunCMake/CMakeLists.txt | 3 + Tests/RunCMake/CTestCoverage/CMakeLists.txt | 3 + .../ClangCoverage-ctest-check.cmake | 3 + .../ClangCoverage-ctest-stdout.txt | 2 + Tests/RunCMake/CTestCoverage/ClangCoverage.c | 4 + .../CTestCoverage/ClangCoverage.cmake | 12 +++ .../RunCMake/CTestCoverage/RunCMakeTest.cmake | 14 +++ 12 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 Tests/RunCMake/CTestCoverage/CMakeLists.txt create mode 100644 Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-check.cmake create mode 100644 Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-stdout.txt create mode 100644 Tests/RunCMake/CTestCoverage/ClangCoverage.c create mode 100644 Tests/RunCMake/CTestCoverage/ClangCoverage.cmake create mode 100644 Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake diff --git a/.gitlab/ci/configure_fedora44_common_clang.cmake b/.gitlab/ci/configure_fedora44_common_clang.cmake index c7b38b6bcf..bd0290c19c 100644 --- a/.gitlab/ci/configure_fedora44_common_clang.cmake +++ b/.gitlab/ci/configure_fedora44_common_clang.cmake @@ -5,6 +5,8 @@ set(CMAKE_Fortran_COMPILER_SUPPORTS_F90 "1" CACHE BOOL "") set(CMake_TEST_C_STANDARDS "90;99;11;17;23" CACHE STRING "") set(CMake_TEST_CXX_STANDARDS "98;11;14;17;20;23;26" CACHE STRING "") +set(CMake_TEST_CLANG_COVERAGE "ON" CACHE BOOL "") + set(CMake_TEST_FindOpenACC "ON" CACHE BOOL "") set(CMake_TEST_FindOpenACC_C "ON" CACHE BOOL "") set(CMake_TEST_FindOpenACC_CXX "ON" CACHE BOOL "") diff --git a/Modules/CTestTargets.cmake b/Modules/CTestTargets.cmake index 28408754a9..72181ee0d2 100644 --- a/Modules/CTestTargets.cmake +++ b/Modules/CTestTargets.cmake @@ -35,6 +35,14 @@ block() set(CTEST_TLS_VERIFY "$ENV{CMAKE_TLS_VERIFY}") endif() endif() + + if(CTEST_TEST_COVERAGE_TOOL STREQUAL "LLVM-COV") + find_program(CTEST_TEST_COVERAGE_MERGE_EXECUTABLE llvm-profdata) + if(NOT CTEST_TEST_COVERAGE_MERGE_EXECUTABLE) + set(CTEST_TEST_COVERAGE_MERGE_EXECUTABLE "llvm-profdata") + endif() + endif() + if(CTEST_NEW_FORMAT) configure_file( ${CMAKE_ROOT}/Modules/DartConfiguration.tcl.in diff --git a/Modules/DartConfiguration.tcl.in b/Modules/DartConfiguration.tcl.in index 409a5c60ca..0c978184e3 100644 --- a/Modules/DartConfiguration.tcl.in +++ b/Modules/DartConfiguration.tcl.in @@ -110,3 +110,4 @@ CTestSubmitRetryCount: @CTEST_SUBMIT_RETRY_COUNT@ # Invoke each test with environment variables configuring tool's collection. CTestTestCoverageTool: @CTEST_TEST_COVERAGE_TOOL@ +CTestTestCoverageMergeExecutable: @CTEST_TEST_COVERAGE_MERGE_EXECUTABLE@ diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index eaf583175b..130a40d5f8 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -17,6 +17,7 @@ #include #include +#include "cmsys/FStream.hxx" #include "cmsys/Glob.hxx" #include "cmsys/RegularExpression.hxx" @@ -883,27 +884,31 @@ bool cmCTestRunTest::ForkProcess() "LLVM-COV"_s || this->TestHandler->TestOptions.CoverageTool == "LLVM-COV"_s) { + this->UseLLVMCov = true; // Isolate the test from any ambient LLVM_PROFILE_FILE env.UnPutEnv("LLVM_PROFILE_FILE"); // Value is /_.profraw - std::string profileEnv = - cmStrCat(this->TestProperties->CTestDirectory, "/", - this->TestProperties->Name, "_%p.profraw"_s); + + std::string profRawPath = this->GenerateLLVMPath("_%p.profraw"); cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Index << ": Using environment variable LLVM_PROFILE_FILE="_s - << profileEnv << " \n", + << profRawPath << " \n", this->TestHandler->GetQuiet()); - env.PutEnv(cmStrCat("LLVM_PROFILE_FILE="_s, profileEnv)); + env.PutEnv(cmStrCat("LLVM_PROFILE_FILE="_s, profRawPath)); // ProcessID -> * to allow for glob to find all // files generated by the test - cmSystemTools::ReplaceString(profileEnv, "%p", "*"); + std::string profRawGlobPath = this->GenerateLLVMPath("_*.profraw"); cmsys::Glob glob; - glob.FindFiles(profileEnv); + glob.FindFiles(profRawGlobPath); for (std::string const& file : glob.GetFiles()) { cmSystemTools::RemoveFile(file); } + // Remove merged coverage data + + std::string profDataPath = this->GenerateLLVMPath(".profdata"); + cmSystemTools::RemoveFile(profDataPath); }; if (this->UseAllocatedResources) { @@ -1030,8 +1035,88 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) "Testing " << this->TestProperties->Name << " ... "); } +std::string cmCTestRunTest::GenerateLLVMPath(std::string fileString) +{ + std::string dir = this->TestProperties->CTestDirectory; + std::string profRawRoot = cmStrCat(dir, "/", this->TestProperties->Name); + return cmStrCat(profRawRoot, fileString); +} + +void cmCTestRunTest::CollectLLVMCoverage() +{ + // find all *.profraw files + cmsys::Glob gl; + std::vector profRawFiles; + std::string profRawPath = this->GenerateLLVMPath("_*.profraw"); + cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + " looking for .profraw files in: " << profRawPath + << std::endl, + this->TestHandler->Quiet); + gl.FindFiles(profRawPath); + // Keep a list of all profraw files + profRawFiles = gl.GetFiles(); + if (profRawFiles.empty()) { + cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + " Cannot find any profraw coverage files." << std::endl, + this->TestHandler->Quiet); + // No coverage files is a valid thing, so the exit code is 0 + return; + } + + // Write filenames to input file + std::string profManifestPath = this->GenerateLLVMPath(".manifest"); + cmsys::ofstream manifestStream; + manifestStream.open(profManifestPath); + for (std::string const& f : profRawFiles) { + manifestStream << f << "\n"; + } + manifestStream.close(); + // execute merge command : + // (xcrun) llvm-profdata merge -sparse --input-files=.manifest -o + // .profdata + std::vector covargs; + std::string mergeExecutable = + this->CTest->GetCTestConfiguration("CTestTestCoverageMergeExecutable"); + if (mergeExecutable.empty()) { + mergeExecutable = "llvm-profdata"; + } + std::string profDataPath = this->GenerateLLVMPath(".profdata"); +#ifdef __APPLE__ + covargs.push_back("xcrun"); +#endif + covargs.push_back(mergeExecutable); + covargs.push_back("merge"); + covargs.push_back("-sparse"); + covargs.push_back(cmStrCat("--input-files=", profManifestPath)); + covargs.push_back("-o"); + covargs.push_back(profDataPath); + covargs.push_back("--failure-mode=all"); + + std::string output; + std::string errors; + int retVal = 0; + this->CTest->RunCommand(covargs, &output, &errors, &retVal, + this->TestProperties->CTestDirectory.c_str(), + cmDuration::zero() /*this->TimeOut*/); + + if (!cmSystemTools::FileExists(profDataPath)) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Something went wrong while merging .profraw data.\n"); + return; + } + for (std::string const& f : profRawFiles) { + cmSystemTools::RemoveFile(f); + } + cmSystemTools::RemoveFile(profManifestPath); +} + void cmCTestRunTest::FinalizeTest(bool started) { + // Collect known LLVM coverage files into binary form + if (this->UseLLVMCov) { + this->CollectLLVMCoverage(); + } + if (this->CTest->GetInstrumentation().HasQuery()) { std::string data_file = this->CTest->GetInstrumentation().InstrumentTest( this->TestProperties->Name, this->ActualCommand, this->Arguments, diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 37ec24ad66..3e82e58b2e 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -114,7 +114,8 @@ private: void WriteLogOutputTop(size_t completed, size_t total); // Run post processing of the process output for MemCheck void MemCheckPostProcess(); - + std::string GenerateLLVMPath(std::string fileString); + void CollectLLVMCoverage(); void SetupResourcesEnvironment(cmEnvironment& env); // Returns "completed/total Test #Index: " @@ -141,6 +142,7 @@ private: int NumberOfRunsLeft = 1; // default to 1 run of the test int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again + bool UseLLVMCov = false; size_t TotalNumberOfTests; }; diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 8bf0af3f82..f9cca759fa 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -1171,6 +1171,9 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL) endif() add_RunCMake_test(FetchContent) add_RunCMake_test(FetchContent_find_package) +if(CMake_TEST_CLANG_COVERAGE) + add_RunCMake_test(CTestCoverage -DCMake_TEST_CLANG_COVERAGE=${CMake_TEST_CLANG_COVERAGE}) +endif() set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE} -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA} diff --git a/Tests/RunCMake/CTestCoverage/CMakeLists.txt b/Tests/RunCMake/CTestCoverage/CMakeLists.txt new file mode 100644 index 0000000000..f54f74670c --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 4.3) +project(${RunCMake_TEST} NONE) +include(${RunCMake_TEST}.cmake) diff --git a/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-check.cmake b/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-check.cmake new file mode 100644 index 0000000000..ae1d326c21 --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-check.cmake @@ -0,0 +1,3 @@ +if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/ClangCoverage.profdata") + string(APPEND RunCMake_TEST_FAILED "ClangCoverage.profdata not found\n") +endif() diff --git a/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-stdout.txt b/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-stdout.txt new file mode 100644 index 0000000000..c8ab6ed865 --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-stdout.txt @@ -0,0 +1,2 @@ +Using environment variable LLVM_PROFILE_FILE=[^ +]*/Tests/RunCMake/CTestCoverage/ClangCoverage-build/ClangCoverage_%p\.profraw diff --git a/Tests/RunCMake/CTestCoverage/ClangCoverage.c b/Tests/RunCMake/CTestCoverage/ClangCoverage.c new file mode 100644 index 0000000000..8488f4e58f --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/ClangCoverage.c @@ -0,0 +1,4 @@ +int main(void) +{ + return 0; +} diff --git a/Tests/RunCMake/CTestCoverage/ClangCoverage.cmake b/Tests/RunCMake/CTestCoverage/ClangCoverage.cmake new file mode 100644 index 0000000000..26da358177 --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/ClangCoverage.cmake @@ -0,0 +1,12 @@ +enable_language(C) + +if(NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") + message(FATAL_ERROR "This test requires a Clang C compiler.") +endif() + +string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping") +set(CTEST_TEST_COVERAGE_TOOL "LLVM-COV") +include(CTest) + +add_executable(ClangCoverage ClangCoverage.c) +add_test(NAME ClangCoverage COMMAND ClangCoverage) diff --git a/Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake b/Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake new file mode 100644 index 0000000000..a703211207 --- /dev/null +++ b/Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake @@ -0,0 +1,14 @@ +include(RunCMake) + +if(CMake_TEST_CLANG_COVERAGE) + block() + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/ClangCoverage-build) + if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG) + list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug) + endif() + run_cmake(ClangCoverage) + set(RunCMake_TEST_NO_CLEAN 1) + run_cmake_command(ClangCoverage-build ${CMAKE_COMMAND} --build . --config Debug) + run_cmake_command(ClangCoverage-ctest ${CMAKE_CTEST_COMMAND} -C Debug -VV) + endblock() +endif()