ctest: Merge llvm-cov's .profraw files into a .profdata file for each test

If the CTEST_TEST_COVERAGE_TOOL is set to LLVM-COV, use the
`llvm-profdata --merge` functionality to collect the `.profraw` files
generated by running each test into a combined `.profdata` file.
This is the file that could be used to parse or display coverage
information for each test.

Issue: #26932

.gitlab/ci/configure_fedora44_common_clang.cmake

@@ -5,6 +5,8 @@ set(CMAKE_Fortran_COMPILER_SUPPORTS_F90 "1" CACHE BOOL "")
 set(CMake_TEST_C_STANDARDS "90;99;11;17;23" CACHE STRING "")
 set(CMake_TEST_CXX_STANDARDS "98;11;14;17;20;23;26" CACHE STRING "")
 
+set(CMake_TEST_CLANG_COVERAGE "ON" CACHE BOOL "")
+
 set(CMake_TEST_FindOpenACC "ON" CACHE BOOL "")
 set(CMake_TEST_FindOpenACC_C "ON" CACHE BOOL "")
 set(CMake_TEST_FindOpenACC_CXX "ON" CACHE BOOL "")

Modules/CTestTargets.cmake

@@ -35,6 +35,14 @@ block()
       set(CTEST_TLS_VERIFY "$ENV{CMAKE_TLS_VERIFY}")
     endif()
   endif()
+
+  if(CTEST_TEST_COVERAGE_TOOL STREQUAL "LLVM-COV")
+    find_program(CTEST_TEST_COVERAGE_MERGE_EXECUTABLE llvm-profdata)
+    if(NOT CTEST_TEST_COVERAGE_MERGE_EXECUTABLE)
+      set(CTEST_TEST_COVERAGE_MERGE_EXECUTABLE "llvm-profdata")
+    endif()
+  endif()
+
   if(CTEST_NEW_FORMAT)
     configure_file(
       ${CMAKE_ROOT}/Modules/DartConfiguration.tcl.in

Modules/DartConfiguration.tcl.in

@@ -110,3 +110,4 @@ CTestSubmitRetryCount: @CTEST_SUBMIT_RETRY_COUNT@
 
 # Invoke each test with environment variables configuring tool's collection.
 CTestTestCoverageTool: @CTEST_TEST_COVERAGE_TOOL@
+CTestTestCoverageMergeExecutable: @CTEST_TEST_COVERAGE_MERGE_EXECUTABLE@

Source/CTest/cmCTestRunTest.cxx

@@ -17,6 +17,7 @@
 #include <cm/string_view>
 #include <cmext/string_view>
 
+#include "cmsys/FStream.hxx"
 #include "cmsys/Glob.hxx"
 #include "cmsys/RegularExpression.hxx"
 
@@ -883,27 +884,31 @@ bool cmCTestRunTest::ForkProcess()
         "LLVM-COV"_s ||
       this->TestHandler->TestOptions.CoverageTool == "LLVM-COV"_s) {
 
+    this->UseLLVMCov = true;
     // Isolate the test from any ambient LLVM_PROFILE_FILE
     env.UnPutEnv("LLVM_PROFILE_FILE");
     // Value is <Test Dir>/<testName>_<processID>.profraw
-    std::string profileEnv =
-      cmStrCat(this->TestProperties->CTestDirectory, "/",
-               this->TestProperties->Name, "_%p.profraw"_s);
+
+    std::string profRawPath = this->GenerateLLVMPath("_%p.profraw");
     cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                        this->Index
                          << ": Using environment variable LLVM_PROFILE_FILE="_s
-                         << profileEnv << " \n",
+                         << profRawPath << " \n",
                        this->TestHandler->GetQuiet());
-    env.PutEnv(cmStrCat("LLVM_PROFILE_FILE="_s, profileEnv));
+    env.PutEnv(cmStrCat("LLVM_PROFILE_FILE="_s, profRawPath));
     // ProcessID -> * to allow for glob to find all
     // files generated by the test
-    cmSystemTools::ReplaceString(profileEnv, "%p", "*");
+    std::string profRawGlobPath = this->GenerateLLVMPath("_*.profraw");
 
     cmsys::Glob glob;
-    glob.FindFiles(profileEnv);
+    glob.FindFiles(profRawGlobPath);
     for (std::string const& file : glob.GetFiles()) {
       cmSystemTools::RemoveFile(file);
     }
+    // Remove merged coverage data
+
+    std::string profDataPath = this->GenerateLLVMPath(".profdata");
+    cmSystemTools::RemoveFile(profDataPath);
   };
 
   if (this->UseAllocatedResources) {
@@ -1030,8 +1035,88 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
              "Testing " << this->TestProperties->Name << " ... ");
 }
 
+std::string cmCTestRunTest::GenerateLLVMPath(std::string fileString)
+{
+  std::string dir = this->TestProperties->CTestDirectory;
+  std::string profRawRoot = cmStrCat(dir, "/", this->TestProperties->Name);
+  return cmStrCat(profRawRoot, fileString);
+}
+
+void cmCTestRunTest::CollectLLVMCoverage()
+{
+  // find all *.profraw files
+  cmsys::Glob gl;
+  std::vector<std::string> profRawFiles;
+  std::string profRawPath = this->GenerateLLVMPath("_*.profraw");
+  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                     "   looking for .profraw files in: " << profRawPath
+                                                          << std::endl,
+                     this->TestHandler->Quiet);
+  gl.FindFiles(profRawPath);
+  // Keep a list of all profraw files
+  profRawFiles = gl.GetFiles();
+  if (profRawFiles.empty()) {
+    cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                       " Cannot find any profraw coverage files." << std::endl,
+                       this->TestHandler->Quiet);
+    // No coverage files is a valid thing, so the exit code is 0
+    return;
+  }
+
+  // Write filenames to input file
+  std::string profManifestPath = this->GenerateLLVMPath(".manifest");
+  cmsys::ofstream manifestStream;
+  manifestStream.open(profManifestPath);
+  for (std::string const& f : profRawFiles) {
+    manifestStream << f << "\n";
+  }
+  manifestStream.close();
+  // execute merge command :
+  // (xcrun) llvm-profdata merge -sparse --input-files=<test_name>.manifest -o
+  // <test_name>.profdata
+  std::vector<std::string> covargs;
+  std::string mergeExecutable =
+    this->CTest->GetCTestConfiguration("CTestTestCoverageMergeExecutable");
+  if (mergeExecutable.empty()) {
+    mergeExecutable = "llvm-profdata";
+  }
+  std::string profDataPath = this->GenerateLLVMPath(".profdata");
+#ifdef __APPLE__
+  covargs.push_back("xcrun");
+#endif
+  covargs.push_back(mergeExecutable);
+  covargs.push_back("merge");
+  covargs.push_back("-sparse");
+  covargs.push_back(cmStrCat("--input-files=", profManifestPath));
+  covargs.push_back("-o");
+  covargs.push_back(profDataPath);
+  covargs.push_back("--failure-mode=all");
+
+  std::string output;
+  std::string errors;
+  int retVal = 0;
+  this->CTest->RunCommand(covargs, &output, &errors, &retVal,
+                          this->TestProperties->CTestDirectory.c_str(),
+                          cmDuration::zero() /*this->TimeOut*/);
+
+  if (!cmSystemTools::FileExists(profDataPath)) {
+    cmCTestLog(this->CTest, ERROR_MESSAGE,
+               "Something went wrong while merging .profraw data.\n");
+    return;
+  }
+  for (std::string const& f : profRawFiles) {
+    cmSystemTools::RemoveFile(f);
+  }
+  cmSystemTools::RemoveFile(profManifestPath);
+}
+
 void cmCTestRunTest::FinalizeTest(bool started)
 {
+  // Collect known LLVM coverage files into binary form
+  if (this->UseLLVMCov) {
+    this->CollectLLVMCoverage();
+  }
+
   if (this->CTest->GetInstrumentation().HasQuery()) {
     std::string data_file = this->CTest->GetInstrumentation().InstrumentTest(
       this->TestProperties->Name, this->ActualCommand, this->Arguments,

Source/CTest/cmCTestRunTest.h

@@ -114,7 +114,8 @@ private:
   void WriteLogOutputTop(size_t completed, size_t total);
   // Run post processing of the process output for MemCheck
   void MemCheckPostProcess();
-
+  std::string GenerateLLVMPath(std::string fileString);
+  void CollectLLVMCoverage();
   void SetupResourcesEnvironment(cmEnvironment& env);
 
   // Returns "completed/total Test #Index: "
@@ -141,6 +142,7 @@ private:
   int NumberOfRunsLeft = 1;  // default to 1 run of the test
   int NumberOfRunsTotal = 1; // default to 1 run of the test
   bool RunAgain = false;     // default to not having to run again
+  bool UseLLVMCov = false;
   size_t TotalNumberOfTests;
 };
 

Tests/RunCMake/CMakeLists.txt

@@ -1171,6 +1171,9 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL)
 endif()
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
+if(CMake_TEST_CLANG_COVERAGE)
+  add_RunCMake_test(CTestCoverage -DCMake_TEST_CLANG_COVERAGE=${CMake_TEST_CLANG_COVERAGE})
+endif()
 set(CTestCommandLine_ARGS
   -DPython_EXECUTABLE=${Python_EXECUTABLE}
   -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA}

Tests/RunCMake/CTestCoverage/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 4.3)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)

Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-check.cmake

@@ -0,0 +1,3 @@
+if(NOT EXISTS "${RunCMake_TEST_BINARY_DIR}/ClangCoverage.profdata")
+  string(APPEND RunCMake_TEST_FAILED "ClangCoverage.profdata not found\n")
+endif()

Tests/RunCMake/CTestCoverage/ClangCoverage-ctest-stdout.txt

@@ -0,0 +1,2 @@
+Using environment variable LLVM_PROFILE_FILE=[^
+]*/Tests/RunCMake/CTestCoverage/ClangCoverage-build/ClangCoverage_%p\.profraw

Tests/RunCMake/CTestCoverage/ClangCoverage.c

@@ -0,0 +1,4 @@
+int main(void)
+{
+  return 0;
+}

Tests/RunCMake/CTestCoverage/ClangCoverage.cmake

@@ -0,0 +1,12 @@
+enable_language(C)
+
+if(NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
+  message(FATAL_ERROR "This test requires a Clang C compiler.")
+endif()
+
+string(APPEND CMAKE_C_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
+set(CTEST_TEST_COVERAGE_TOOL "LLVM-COV")
+include(CTest)
+
+add_executable(ClangCoverage ClangCoverage.c)
+add_test(NAME ClangCoverage COMMAND ClangCoverage)

Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake

@@ -0,0 +1,14 @@
+include(RunCMake)
+
+if(CMake_TEST_CLANG_COVERAGE)
+  block()
+    set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/ClangCoverage-build)
+    if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG)
+      list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug)
+    endif()
+    run_cmake(ClangCoverage)
+    set(RunCMake_TEST_NO_CLEAN 1)
+    run_cmake_command(ClangCoverage-build ${CMAKE_COMMAND} --build . --config Debug)
+    run_cmake_command(ClangCoverage-ctest ${CMAKE_CTEST_COMMAND} -C Debug -VV)
+  endblock()
+endif()