ctest_coverage: Read CMake listfile coverage

Add a handler that reads the `<test-name>.cmake-coverage.info` files
that the `CMAKE` coverage tool writes for each test.  It searches the
build tree for them, without following symbolic links, and merges them
with the lcov tracefile code that ctest uses for the per-test merge.
The merge is needed because many tests run the same listfile, and the
lcov parser keeps only the first record of each file.  A file that
cannot be read is skipped with a warning.

The handler reports only files in the source tree.  It leaves out files
in the build tree, also when the build tree is inside the source tree,
unless the two trees are the same.  This keeps the modules of CMake
itself and generated files out of the report.  It also leaves out files
that no longer exist, because the report step would log an error for
them, and an error makes `ctest -S` fail.

Teach the lcov parser to count `FN` and `FNDA` records when asked, and
write `FunctionsTested` and `FunctionsUnTested` for CMake coverage.  The
Clang handler does not ask, so its output does not change.  The parser
now also ignores `DA` and `BRDA` records for a line past the end of the
source file, instead of writing out of range.  This protects the Clang
handler too.

Issue: #28192

Help/command/ctest_coverage.rst

@@ -15,6 +15,14 @@ Perform the :ref:`CTest Coverage Step` as a :ref:`Dashboard Client`.
 Collect coverage tool results and stores them in ``Coverage.xml``
 for submission with the :command:`ctest_submit` command.
 
+.. versionadded:: 4.6
+  The command reads the ``*.cmake-coverage.info`` files that the ``CMAKE``
+  :ref:`CoverageTool <ctest-CoverageTool>` writes for each test, and
+  reports coverage of CMake language code.  It sends the line counts and
+  the number of tested and untested functions of each file.  It reports
+  only files in the source directory, and leaves out files in the build
+  directory unless the two directories are the same.
+
 The options are:
 
 ``BUILD <build-dir>``

Help/release/dev/listfile-coverage.rst

@@ -7,3 +7,6 @@ listfile-coverage
 * :manual:`ctest(1)` gained a ``CMAKE`` value for its
   :ref:`CoverageTool <ctest-CoverageTool>` setting to record coverage of
   CMake language code for each test.
+
+* The :command:`ctest_coverage` command learned to read coverage of CMake
+  language code recorded by the ``CMAKE`` coverage tool.

Source/CTest/cmCTestCoverageHandler.cxx

@@ -6,6 +6,7 @@
 #include <chrono>
 #include <cstdio>
 #include <cstdlib>
+#include <functional>
 #include <iterator>
 #include <memory>
 #include <sstream>
@@ -20,6 +21,7 @@
 #include "cmCTest.h"
 #include "cmDuration.h"
 #include "cmGeneratedFileStream.h"
+#include "cmLcovTracefile.h"
 #include "cmParseBlanketJSCoverage.h"
 #include "cmParseCacheCoverage.h"
 #include "cmParseCoberturaCoverage.h"
@@ -254,6 +256,11 @@ int cmCTestCoverageHandler::ProcessHandler()
   if (file_count < 0) {
     return error;
   }
+  file_count += this->HandleCMakeListFileCoverage(&cont);
+  error = cont.Error;
+  if (file_count < 0) {
+    return error;
+  }
   file_count += this->HandleTracePyCoverage(&cont);
   error = cont.Error;
   if (file_count < 0) {
@@ -465,6 +472,12 @@ int cmCTestCoverageHandler::ProcessHandler()
     covSumXML.Element("LOCUnTested", untested);
     covSumXML.Element("BranchesTested", branchestested);
     covSumXML.Element("BranchesUnTested", branchesuntested);
+    auto const fFunctionCov = cont.TotalFunctionCoverage.find(file.first);
+    if (fFunctionCov != cont.TotalFunctionCoverage.end()) {
+      covSumXML.Element("FunctionsTested", fFunctionCov->second[0]);
+      covSumXML.Element("FunctionsUnTested",
+                        fFunctionCov->second[1] - fFunctionCov->second[0]);
+    }
     covSumXML.Element("PercentCoverage", cper);
     covSumXML.Element("CoverageMetric", cmet);
     this->WriteXMLLabels(covSumXML, shortFileName);
@@ -1306,7 +1319,8 @@ int cmCTestCoverageHandler::HandleClangSourceCodeCoverage(
 }
 
 void cmCTestCoverageHandler::HandleClangSourceCodeCoverageFile(
-  cmCTestCoverageHandlerContainer* cont, std::string output)
+  cmCTestCoverageHandlerContainer* cont, std::string output,
+  bool collectFunctions)
 {
   int coveredLineIndex = 0;
   int coveredLineHits = 0;
@@ -1338,6 +1352,9 @@ void cmCTestCoverageHandler::HandleClangSourceCodeCoverageFile(
             vec.push_back(-1);
             branchVec.push_back({ { 0, 0 } });
           }
+          if (collectFunctions) {
+            cont->TotalFunctionCoverage[coveredFile] = { { 0, 0 } };
+          }
         }
       } else {
         // Dont parse repeat copies of data, skip all lines of coverage info
@@ -1352,7 +1369,16 @@ void cmCTestCoverageHandler::HandleClangSourceCodeCoverageFile(
       coveredLineIndex =
         std::atoi(line.substr(line.find(':') + 1, line.find(',')).c_str()) - 1;
       coveredLineHits = std::atoi(line.substr(line.find(',') + 1).c_str());
-      cont->TotalCoverage[coveredFile][coveredLineIndex] = coveredLineHits;
+      cmCTestCoverageHandlerContainer::SingleFileCoverageVector& vec =
+        cont->TotalCoverage[coveredFile];
+      if (coveredLineIndex < 0 ||
+          static_cast<std::size_t>(coveredLineIndex) >= vec.size()) {
+        cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                           "Ignoring coverage of line ", coveredLineIndex + 1,
+                           " past the end of file: ", coveredFile, '\n');
+        continue;
+      }
+      vec[coveredLineIndex] = coveredLineHits;
     }
     // BRDA represents the line and number of Branch hits
     // BRDA:<LineNo>,<blockNo>,<BranchNo>,<NumHits>
@@ -1360,17 +1386,113 @@ void cmCTestCoverageHandler::HandleClangSourceCodeCoverageFile(
       // First increment count of branches for this line
       coveredLineIndex =
         std::atoi(line.substr(line.find(':') + 1, line.find(',')).c_str()) - 1;
-      cont->TotalBranchCoverage[coveredFile][coveredLineIndex][1]++;
+      cmCTestCoverageHandlerContainer::SingleFileBranchCoverageVector& vec =
+        cont->TotalBranchCoverage[coveredFile];
+      if (coveredLineIndex < 0 ||
+          static_cast<std::size_t>(coveredLineIndex) >= vec.size()) {
+        cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                           "Ignoring coverage of line ", coveredLineIndex + 1,
+                           " past the end of file: ", coveredFile, '\n');
+        continue;
+      }
+      vec[coveredLineIndex][1]++;
       // If lines hit > 0, increment the first number to indicate the branch
       // was hit
       coveredLineHits = std::atoi(line.substr(line.rfind(',') + 1).c_str());
       if (coveredLineHits > 0) {
-        cont->TotalBranchCoverage[coveredFile][coveredLineIndex][0]++;
+        vec[coveredLineIndex][0]++;
+      }
+    }
+    // FN represents a function FN:<LineNo>,<Name>
+    else if (lineType == "FN" && collectFunctions && !pauseCollecting) {
+      cont->TotalFunctionCoverage[coveredFile][1]++;
+    }
+    // FNDA represents the hits of a function FNDA:<NumHits>,<Name>
+    else if (lineType == "FNDA" && collectFunctions && !pauseCollecting) {
+      if (std::atoi(line.substr(line.find(':') + 1).c_str()) > 0) {
+        cont->TotalFunctionCoverage[coveredFile][0]++;
       }
     }
   }
 }
 
+int cmCTestCoverageHandler::HandleCMakeListFileCoverage(
+  cmCTestCoverageHandlerContainer* cont)
+{
+  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                     "Looking for CMake listfile coverage data: \n");
+  cmsys::Glob gl;
+  gl.RecurseOn();
+  gl.RecurseThroughSymlinksOff();
+  std::string const covGlob =
+    cmStrCat(this->CTest->GetBinaryDir(), "/*.cmake-coverage.info");
+  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                     "   looking for .cmake-coverage.info files in: ", covGlob,
+                     '\n');
+  gl.FindFiles(covGlob);
+  std::vector<std::string> files = gl.GetFiles();
+  if (files.empty()) {
+    cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                       " Cannot find any CMake listfile coverage files.\n");
+    return 0;
+  }
+  std::sort(files.begin(), files.end());
+
+  // Problems are warnings.  An error would make ctest -S fail.
+  cmLcovTracefile merged;
+  auto const warn = [this](std::string const& warning) {
+    cmCTestLog(this->CTest, WARNING, warning, '\n');
+  };
+  merged.MergeFiles(files, warn);
+
+  // Leave out files in the build tree, unless it is the source tree.  The
+  // per-test merge uses the same build tree.
+  std::string const& binaryDir =
+    cont->BinaryDir.empty() ? this->CTest->GetBinaryDir() : cont->BinaryDir;
+  auto const logRemoved = [this](std::string const& path) {
+    cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                       "Ignoring CMake listfile coverage of a file in "
+                       "the build tree: ",
+                       path, '\n');
+  };
+  merged.RemoveBuildTreeFiles(cont->SourceDir, binaryDir, logRemoved);
+
+  // Report only files in the source tree.  This keeps files of CMake itself,
+  // such as its modules, out of the report.
+  std::string sourceDir = cont->SourceDir;
+  if (!sourceDir.empty()) {
+    sourceDir = cmSystemTools::CollapseFullPath(sourceDir);
+  }
+  for (auto it = merged.Files.begin(); it != merged.Files.end();) {
+    std::string const path = cmSystemTools::CollapseFullPath(it->first);
+    if (!cmSystemTools::IsSubDirectory(path, sourceDir)) {
+      cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+                         "Ignoring CMake listfile coverage of a file outside "
+                         "the source tree: ",
+                         path, '\n');
+      it = merged.Files.erase(it);
+    } else if (!cmSystemTools::FileExists(path, true)) {
+      // The report step would log an error for a missing file.
+      cmCTestOptionalLog(
+        this->CTest, HANDLER_VERBOSE_OUTPUT, this->Quiet,
+        "Ignoring CMake listfile coverage of a missing file: ", path, '\n');
+      it = merged.Files.erase(it);
+    } else {
+      ++it;
+    }
+  }
+
+  std::ostringstream text;
+  merged.Write(text);
+  this->HandleClangSourceCodeCoverageFile(cont, text.str(), true);
+  // Write leaves out the files with no data, so do not count them.
+  return static_cast<int>(std::count_if(
+    merged.Files.begin(), merged.Files.end(),
+    [](std::pair<std::string const, cmLcovSourceFile> const& file) {
+      return file.second.HasData();
+    }));
+}
+
 int cmCTestCoverageHandler::HandleLCovCoverage(
   cmCTestCoverageHandlerContainer* cont)
 {

Source/CTest/cmCTestCoverageHandler.h

@@ -33,6 +33,8 @@ public:
     std::map<std::string, SingleFileBranchCoverageVector>;
   TotalCoverageMap TotalCoverage;
   TotalBranchCoverageMap TotalBranchCoverage;
+  // File -> {tested, total} functions.
+  std::map<std::string, std::array<int, 2>> TotalFunctionCoverage;
   std::ostream* OFS;
   bool Quiet;
 };
@@ -84,8 +86,14 @@ private:
   //! Handle coverage for Clang with llvm function
   int HandleClangSourceCodeCoverage(cmCTestCoverageHandlerContainer* cont);
 
+  //! Parse lcov text.  Collect FN and FNDA counts if collectFunctions.
   void HandleClangSourceCodeCoverageFile(cmCTestCoverageHandlerContainer* cont,
-                                         std::string output);
+                                         std::string output,
+                                         bool collectFunctions = false);
+
+  //! Handle coverage of CMake listfiles from the CMAKE coverage tool
+  int HandleCMakeListFileCoverage(cmCTestCoverageHandlerContainer* cont);
+
   //! Handle coverage for Python with coverage.py
   int HandleCoberturaCoverage(cmCTestCoverageHandlerContainer* cont);
 

Tests/RunCMake/CMakeLists.txt

@@ -1311,9 +1311,7 @@ endif()
 add_RunCMake_test(ExternalProject-CMP0225)
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
-if(CMake_TEST_CLANG_COVERAGE)
-  add_RunCMake_test(CTestCoverage -DCMake_TEST_CLANG_COVERAGE=${CMake_TEST_CLANG_COVERAGE})
-endif()
+add_RunCMake_test(CTestCoverage -DCMake_TEST_CLANG_COVERAGE=${CMake_TEST_CLANG_COVERAGE})
 set(CTestCommandLine_ARGS
   -DPython_EXECUTABLE=${Python_EXECUTABLE}
   -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA}

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage-coverage-check.cmake

@@ -0,0 +1,26 @@
+include(${CMAKE_CURRENT_LIST_DIR}/check-coverage.cmake)
+if(RunCMake_TEST_FAILED)
+  return()
+endif()
+
+get_file_element(shared "${coverage_xml}" "Shared.cmake")
+if(NOT shared)
+  string(APPEND RunCMake_TEST_FAILED "No File for Shared.cmake in Coverage.xml\n")
+else()
+  foreach(e IN ITEMS "<FunctionsTested>1</FunctionsTested>" "<FunctionsUnTested>1</FunctionsUnTested>")
+    string(FIND "${shared}" "${e}" pos)
+    if(pos EQUAL -1)
+      string(APPEND RunCMake_TEST_FAILED "Coverage.xml has no ${e} for Shared.cmake:\n${shared}\n")
+    endif()
+  endforeach()
+endif()
+
+# Number is zero-based, so this is line 2, which both tests ran.
+get_file_element(shared_log "${coverage_log_xml}" "Shared.cmake")
+string(FIND "${shared_log}" "<Line Number=\"1\" Count=\"2\"" pos)
+if(pos EQUAL -1)
+  string(APPEND RunCMake_TEST_FAILED "CoverageLog-0.xml has no merged count for line 2 of Shared.cmake:\n${shared_log}\n")
+endif()
+
+expect_no_file(CMakePrintHelpers.cmake)
+expect_no_file(Generated.cmake)

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage-empty-stderr.txt

@@ -0,0 +1 @@
+ Cannot find any coverage files\. Ignoring Coverage request\.$

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage-nested-coverage-check.cmake

@@ -0,0 +1,12 @@
+include(${CMAKE_CURRENT_LIST_DIR}/check-coverage.cmake)
+if(RunCMake_TEST_FAILED)
+  return()
+endif()
+
+get_file_element(shared "${coverage_xml}" "Shared.cmake")
+if(NOT shared)
+  string(APPEND RunCMake_TEST_FAILED "No File for Shared.cmake in Coverage.xml\n")
+endif()
+
+# The build tree is inside the source tree, but its files stay out.
+expect_no_file(Generated.cmake)

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage-range-stderr.txt

@@ -0,0 +1,2 @@
+Skipping unreadable CMake listfile coverage file: [^
+]*trunc\.cmake-coverage\.info

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage-range-stdout.txt

@@ -0,0 +1,2 @@
+Ignoring coverage of line 10 past the end of file: [^
+]*/three\.cmake

Tests/RunCMake/CTestCoverage/CMakeListFileCoverage.cmake

@@ -0,0 +1,10 @@
+set(CTEST_TEST_COVERAGE_TOOL "CMAKE")
+include(CTest)
+
+# Two tests run the same file, so its counts must be merged.
+add_test(NAME Shared1 COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Shared.cmake)
+add_test(NAME Shared2 COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Shared.cmake)
+
+# A file in the build tree must not be reported.
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/Generated.cmake "set(y 1)\n")
+add_test(NAME Generated COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/Generated.cmake)

Tests/RunCMake/CTestCoverage/RunCMakeTest.cmake

@@ -14,6 +14,73 @@ if(CMake_TEST_CLANG_COVERAGE)
   endblock()
 endif()
 
+# Isolate the coverage step from any surrounding coverage tool.  With
+# COVFILE set, the coverage handler uses Bullseye instead of CMAKE.
+set(ctest_coverage ${CMAKE_COMMAND} -E env COVNOSAVE=1
+  --unset=COVAPPDATADIR --unset=COVFILE ${CMAKE_CTEST_COMMAND})
+
+function(run_CMakeListFileCoverage case)
+  run_cmake(${case})
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(${case}-ctest ${CMAKE_CTEST_COMMAND} -C Debug -VV)
+  run_cmake_command(${case}-coverage ${ctest_coverage} -D ExperimentalCoverage -C Debug -VV)
+endfunction()
+
+block()
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CMakeListFileCoverage-build)
+  run_CMakeListFileCoverage(CMakeListFileCoverage)
+endblock()
+
+# A build tree inside the source tree.
+block()
+  set(src ${RunCMake_BINARY_DIR}/nested-src)
+  file(REMOVE_RECURSE "${src}")
+  file(MAKE_DIRECTORY "${src}")
+  file(COPY_FILE ${RunCMake_SOURCE_DIR}/CMakeLists.txt ${src}/CMakeLists.txt)
+  file(COPY_FILE ${RunCMake_SOURCE_DIR}/Shared.cmake ${src}/Shared.cmake)
+  file(COPY_FILE ${RunCMake_SOURCE_DIR}/CMakeListFileCoverage.cmake
+    ${src}/CMakeListFileCoverage-nested.cmake)
+  set(RunCMake_TEST_SOURCE_DIR ${src})
+  set(RunCMake_TEST_BINARY_DIR ${src}/build)
+  run_CMakeListFileCoverage(CMakeListFileCoverage-nested)
+endblock()
+
+# A line past the end of the file, and a truncated file.
+block()
+  set(tree ${RunCMake_BINARY_DIR}/CMakeListFileCoverage-range)
+  set(src ${RunCMake_BINARY_DIR}/CMakeListFileCoverage-range-src)
+  file(REMOVE_RECURSE "${tree}" "${src}")
+  file(WRITE "${src}/three.cmake" "set(a 1)\nset(b 2)\nset(c 3)\n")
+  file(WRITE "${tree}/DartConfiguration.tcl" "
+SourceDirectory: ${src}
+BuildDirectory: ${tree}
+")
+  file(WRITE "${tree}/bad.cmake-coverage.info"
+    "SF:${src}/three.cmake\nDA:10,1\nend_of_record\n")
+  file(WRITE "${tree}/trunc.cmake-coverage.info"
+    "SF:${src}/three.cmake\nDA:1,1\n")
+  set(RunCMake_TEST_BINARY_DIR ${tree})
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(CMakeListFileCoverage-range ${ctest_coverage} -D ExperimentalCoverage -VV)
+endblock()
+
+# Only a record with no lines and no functions: no coverage data.
+block()
+  set(tree ${RunCMake_BINARY_DIR}/CMakeListFileCoverage-empty)
+  set(src ${RunCMake_BINARY_DIR}/CMakeListFileCoverage-empty-src)
+  file(REMOVE_RECURSE "${tree}" "${src}")
+  file(WRITE "${src}/comments.cmake" "# Only a comment.\n")
+  file(WRITE "${tree}/DartConfiguration.tcl" "
+SourceDirectory: ${src}
+BuildDirectory: ${tree}
+")
+  file(WRITE "${tree}/empty.cmake-coverage.info"
+    "SF:${src}/comments.cmake\nend_of_record\n")
+  set(RunCMake_TEST_BINARY_DIR ${tree})
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(CMakeListFileCoverage-empty ${ctest_coverage} -D ExperimentalCoverage)
+endblock()
+
 # ctest records its own CMake code when the variable is set at startup.
 block()
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SelfCoverage)

Tests/RunCMake/CTestCoverage/Shared.cmake

@@ -0,0 +1,7 @@
+function(used)
+  set(x 1)
+endfunction()
+function(never)
+endfunction()
+used()
+include(CMakePrintHelpers)

Tests/RunCMake/CTestCoverage/check-coverage.cmake

@@ -0,0 +1,37 @@
+# Read Coverage.xml and CoverageLog-0.xml of the dashboard tag in the
+# test binary directory.
+file(STRINGS "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" tag LIMIT_COUNT 1)
+set(tag_dir "${RunCMake_TEST_BINARY_DIR}/Testing/${tag}")
+foreach(f IN ITEMS Coverage.xml CoverageLog-0.xml)
+  if(NOT EXISTS "${tag_dir}/${f}")
+    string(APPEND RunCMake_TEST_FAILED "${f} not found in\n  ${tag_dir}\n")
+  endif()
+endforeach()
+if(RunCMake_TEST_FAILED)
+  return()
+endif()
+file(READ "${tag_dir}/Coverage.xml" coverage_xml)
+file(READ "${tag_dir}/CoverageLog-0.xml" coverage_log_xml)
+
+# Set out to the File element named name in xml, or to "" if none.
+function(get_file_element out xml name)
+  set(elem "")
+  string(FIND "${xml}" "<File Name=\"${name}\"" start)
+  if(NOT start EQUAL -1)
+    string(SUBSTRING "${xml}" ${start} -1 elem)
+    string(FIND "${elem}" "</File>" end)
+    string(SUBSTRING "${elem}" 0 ${end} elem)
+  endif()
+  set(${out} "${elem}" PARENT_SCOPE)
+endfunction()
+
+# Fail if a File element named name exists in either file.
+function(expect_no_file name)
+  foreach(f IN ITEMS coverage_xml coverage_log_xml)
+    get_file_element(elem "${${f}}" "${name}")
+    if(elem)
+      string(APPEND RunCMake_TEST_FAILED "Unexpected File for ${name}:\n${elem}\n")
+    endif()
+  endforeach()
+  set(RunCMake_TEST_FAILED "${RunCMake_TEST_FAILED}" PARENT_SCOPE)
+endfunction()