instrumentation: Optionally Record Command Output

Add `stdout`  and `stderr` reporting to instrumentation for `compile`, `link`,
`custom`, `install` and `test` snippets when the `captureOutput` option is
enabled.

Fixes: #26704

Help/manual/cmake-instrumentation.7.rst

@@ -300,6 +300,13 @@ key is required, but all other fields are optional.
       generated by CMake, and includes information from immediately before and
       after the command is executed.
 
+    ``captureOutput``
+      .. versionadded:: 4.4
+
+      Enables collection of command output in generated `v1 Snippet Files`_.
+      When enabled, snippets for ``compile``, ``link``, ``custom``, ``test``, and
+      ``install`` commands include ``stdout`` and ``stderr`` fields.
+
     ``cdashSubmit``
       Enables including instrumentation data in CDash. This is
       equivalent to having the :envvar:`CTEST_USE_INSTRUMENTATION` environment
@@ -345,6 +352,7 @@ Example:
     "options": [
       "staticSystemInformation",
       "dynamicSystemInformation",
+      "captureOutput",
       "cdashSubmit",
       "trace"
     ]
@@ -358,13 +366,13 @@ The commands ``/usr/bin/python callback.py index-<timestamp>.json`` and
 ``/usr/bin/cmake -P callback.cmake arg index-<timestamp>.json`` will be
 executed in that order. The index file will contain the
 ``staticSystemInformation`` data and each snippet file listed in the index will
-contain the ``dynamicSystemInformation`` data. Additionally, the index file
-will contain the path to the generated `Google Trace File`_. Once both
-callbacks have completed, the index file and data files listed by it (including
-snippet files, but not the trace file) will be deleted from the project build
-tree. The instrumentation data will be present in the XML files submitted to
-CDash, but with truncated command strings because ``cdashVerbose`` was not
-enabled.
+contain the ``dynamicSystemInformation`` data and captured command output.
+Additionally, the index file will contain the path to the generated
+`Google Trace File`_. Once both callbacks have completed, the index file and
+data files listed by it (including snippet files, but not the trace file) will
+be deleted from the project build tree. The instrumentation data will be
+present in the XML files submitted to CDash, but with truncated command
+strings because ``cdashVerbose`` was not enabled.
 
 v1 Data Files
 -------------
@@ -426,6 +434,24 @@ Snippet files have a filename with the syntax
     The exit code of the command, an integer. This will be ``null`` when
     ``role`` is ``build``.
 
+  ``stdout``
+    .. versionadded:: 4.4
+
+    The standard output produced by the command. Only included when enabled by
+    the ``captureOutput`` `option <v1 Query Files_>`_ and when ``role`` is one
+    of: ``compile``, ``link``, ``custom``, ``install`` or ``test``. For
+    ``test`` snippets, this field contains the merged standard out and standard
+    error streams.
+
+  ``stderr``
+    .. versionadded:: 4.4
+
+    The standard error output produced by the command. Only included when
+    enabled by the ``captureOutput`` `option <v1 Query Files_>`_ and when
+    ``role`` is one of: ``compile``, ``link``, ``custom``, ``install`` or
+    ``test``. For ``test`` snippets, error output is merged with ``stdout``,
+    and the value of ``stderr`` is always empty.
+
   ``role``
     The type of command executed, which will be one of the following values:
 
@@ -525,6 +551,8 @@ Example:
     "command" : "\"/usr/bin/c++\" \"-MD\" \"-MT\" \"CMakeFiles/main.dir/main.cxx.o\" \"-MF\" \"CMakeFiles/main.dir/main.cxx.o.d\" \"-o\" \"CMakeFiles/main.dir/main.cxx.o\" \"-c\" \"<src>/main.cxx\"",
     "role" : "compile",
     "result" : 1,
+    "stdout" : "<compiler stdout>",
+    "stderr" : "<compiler stderr>",
     "target": "main",
     "language" : "C++",
     "outputs" : [ "CMakeFiles/main.dir/main.cxx.o" ],

Help/manual/instrumentation/query-v1-schema.json

@@ -64,7 +64,8 @@
           "dynamicSystemInformation",
           "cdashSubmit",
           "cdashVerbose",
-          "trace"
+          "trace",
+          "captureOutput"
         ],
         "type": "string"
       }

Source/CTest/cmCTestLaunch.cxx

@@ -21,6 +21,7 @@
 #include "cmCTestLaunchReporter.h"
 #include "cmGlobalGenerator.h"
 #include "cmInstrumentation.h"
+#include "cmInstrumentationQuery.h"
 #include "cmMakefile.h"
 #include "cmProcessOutput.h"
 #include "cmState.h"
@@ -206,6 +207,9 @@ void cmCTestLaunch::RunChild()
     return;
   }
 
+  this->CapturedStdOut.clear();
+  this->CapturedStdErr.clear();
+
   // Prepare to run the real command.
   cmUVProcessChainBuilder builder;
   builder.AddCommand(this->RealArgV);
@@ -213,18 +217,19 @@ void cmCTestLaunch::RunChild()
   // We always share the input pipe.
   builder.SetExternalStream(cmUVProcessChainBuilder::Stream_INPUT, stdin);
 
+  builder.SetBuiltinStream(cmUVProcessChainBuilder::Stream_OUTPUT)
+    .SetBuiltinStream(cmUVProcessChainBuilder::Stream_ERROR);
   cmsys::ofstream fout;
   cmsys::ofstream ferr;
-  if (this->Reporter.Passthru) {
-    // In passthru mode we just share the output pipes.
-    builder.SetExternalStream(cmUVProcessChainBuilder::Stream_OUTPUT, stdout)
-      .SetExternalStream(cmUVProcessChainBuilder::Stream_ERROR, stderr);
-  } else {
+  cmsys::ofstream* foutPtr = nullptr;
+  cmsys::ofstream* ferrPtr = nullptr;
+
+  if (!this->Reporter.Passthru) {
     // In full mode we record the child output pipes to log files.
-    builder.SetBuiltinStream(cmUVProcessChainBuilder::Stream_OUTPUT)
-      .SetBuiltinStream(cmUVProcessChainBuilder::Stream_ERROR);
     fout.open(this->Reporter.LogOut.c_str(), std::ios::out | std::ios::binary);
     ferr.open(this->Reporter.LogErr.c_str(), std::ios::out | std::ios::binary);
+    foutPtr = &fout;
+    ferrPtr = &ferr;
   }
 
 #ifdef _WIN32
@@ -243,36 +248,43 @@ void cmCTestLaunch::RunChild()
   cmProcessOutput processOutput;
   std::unique_ptr<cmUVStreamReadHandle> outputHandle;
   std::unique_ptr<cmUVStreamReadHandle> errorHandle;
-  if (!this->Reporter.Passthru) {
-    auto beginRead =
-      [&processOutput](uv_stream_t* stream, std::ostream& out,
-                       cmsys::ofstream& file, bool& haveData, bool& finished,
-                       int id) -> std::unique_ptr<cmUVStreamReadHandle> {
-      finished = false;
-      return cmUVStreamRead(
-        stream,
-        [&processOutput, &out, &file, id, &haveData](std::vector<char> data) {
-          std::string strdata;
-          processOutput.DecodeText(data.data(), data.size(), strdata, id);
-          file.write(strdata.c_str(), strdata.size());
-          out.write(strdata.c_str(), strdata.size());
-          haveData = true;
-        },
-        [&processOutput, &out, &file, &finished, id]() {
-          std::string strdata;
-          processOutput.DecodeText(std::string(), strdata, id);
-          if (!strdata.empty()) {
-            file.write(strdata.c_str(), strdata.size());
-            out.write(strdata.c_str(), strdata.size());
+  auto beginRead =
+    [&processOutput](
+      uv_stream_t* stream, std::ostream& out, cmsys::ofstream* file,
+      bool& haveData, bool& finished, int id,
+      std::string& capture) -> std::unique_ptr<cmUVStreamReadHandle> {
+    finished = false;
+    return cmUVStreamRead(
+      stream,
+      [&processOutput, &out, file, id, &haveData,
+       &capture](std::vector<char> data) {
+        std::string strdata;
+        processOutput.DecodeText(data.data(), data.size(), strdata, id);
+        if (file && file->is_open()) {
+          file->write(strdata.c_str(), strdata.size());
+        }
+        out.write(strdata.c_str(), strdata.size());
+        capture.append(strdata);
+        haveData = true;
+      },
+      [&processOutput, &out, file, &finished, id, &capture]() {
+        std::string strdata;
+        processOutput.DecodeText(std::string(), strdata, id);
+        if (!strdata.empty()) {
+          if (file && file->is_open()) {
+            file->write(strdata.c_str(), strdata.size());
           }
-          finished = true;
-        });
-    };
-    outputHandle = beginRead(chain.OutputStream(), std::cout, fout,
-                             this->HaveOut, outFinished, 1);
-    errorHandle = beginRead(chain.ErrorStream(), std::cerr, ferr,
-                            this->HaveErr, errFinished, 2);
-  }
+          out.write(strdata.c_str(), strdata.size());
+          capture.append(strdata);
+        }
+        finished = true;
+      });
+  };
+  outputHandle =
+    beginRead(chain.OutputStream(), std::cout, foutPtr, this->HaveOut,
+              outFinished, 1, this->CapturedStdOut);
+  errorHandle = beginRead(chain.ErrorStream(), std::cerr, ferrPtr,
+                          this->HaveErr, errFinished, 2, this->CapturedStdErr);
 
   // Wait for the real command to finish.
   while (!(chain.Finished() && outFinished && errFinished)) {
@@ -291,6 +303,8 @@ void cmCTestLaunch::RunChild()
 int cmCTestLaunch::Run()
 {
   auto instrumentation = cmInstrumentation(this->Reporter.OptionBuildDir);
+  bool const captureOutput =
+    instrumentation.HasOption(cmInstrumentationQuery::Option::CaptureOutput);
   std::map<std::string, std::string> options;
   if (this->Reporter.OptionTargetName != "TARGET_NAME") {
     options["target"] = this->Reporter.OptionTargetName;
@@ -305,9 +319,15 @@ int cmCTestLaunch::Run()
   arrayOptions["targetLabels"] = this->Reporter.OptionTargetLabels;
   instrumentation.InstrumentCommand(
     this->Reporter.OptionCommandType, this->RealArgV,
-    [this]() -> int {
+    [this, captureOutput]() -> cmInstrumentation::CommandResult {
       this->RunChild();
-      return this->Reporter.ExitCode;
+      cmInstrumentation::CommandResult result;
+      result.ExitCode = this->Reporter.ExitCode;
+      if (captureOutput) {
+        result.StdOut = this->CapturedStdOut;
+        result.StdErr = this->CapturedStdErr;
+      }
+      return result;
     },
     options, arrayOptions);
 

Source/CTest/cmCTestLaunch.h

@@ -59,6 +59,8 @@ private:
   // Whether or not any data have been written to stdout or stderr.
   bool HaveOut;
   bool HaveErr;
+  std::string CapturedStdOut;
+  std::string CapturedStdErr;
 
   // Load custom rules to match warnings and their exceptions.
   bool ScrapeRulesLoaded;

Source/CTest/cmCTestRunTest.cxx

@@ -1122,7 +1122,7 @@ void cmCTestRunTest::FinalizeTest(bool started)
       this->TestProperties->Name, this->ActualCommand, this->Arguments,
       this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(),
       this->TestProcess->GetSystemStartTime(),
-      this->GetCTest()->GetConfigType());
+      this->GetCTest()->GetConfigType(), this->ProcessOutput);
     this->TestResult.InstrumentationFile = data_file;
   }
   this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(),

Source/cmCTest.cxx

@@ -2713,8 +2713,12 @@ int cmCTest::ExecuteTests(std::vector<std::string> const& args)
   };
   std::map<std::string, std::string> data;
   data["showOnly"] = this->GetShowOnly() ? "1" : "0";
-  int ret =
-    instrumentation.InstrumentCommand("ctest", args, processHandler, data);
+  int ret = instrumentation.InstrumentCommand(
+    "ctest", args,
+    [processHandler]() -> cmInstrumentation::CommandResult {
+      return { processHandler(), cm::nullopt, cm::nullopt };
+    },
+    data);
   instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostCTest);
   if (ret == cmCTest::TEST_ERRORS) {
     cmCTestLog(this, ERROR_MESSAGE, "Errors while running CTest\n");

Source/cmInstrumentation.cxx

@@ -586,7 +586,8 @@ std::string cmInstrumentation::InstrumentTest(
   std::string const& name, std::string const& command,
   std::vector<std::string> const& args, int64_t result,
   std::chrono::steady_clock::time_point steadyStart,
-  std::chrono::system_clock::time_point systemStart, std::string config)
+  std::chrono::system_clock::time_point systemStart, std::string config,
+  cm::optional<std::string> output)
 {
   // Store command info
   Json::Value root(this->preTestStats);
@@ -597,6 +598,10 @@ std::string cmInstrumentation::InstrumentTest(
   root["result"] = static_cast<Json::Value::Int64>(result);
   root["config"] = config;
   root["workingDir"] = cmSystemTools::GetLogicalWorkingDirectory();
+  if (this->HasOption(cmInstrumentationQuery::Option::CaptureOutput)) {
+    root["stdout"] = output ? *output : "";
+    root["stderr"] = "";
+  }
 
   // Post-Command
   this->InsertTimingData(root, steadyStart, systemStart);
@@ -626,7 +631,7 @@ void cmInstrumentation::GetPreTestStats()
 
 int cmInstrumentation::InstrumentCommand(
   std::string command_type, std::vector<std::string> const& command,
-  std::function<int()> const& callback,
+  std::function<cmInstrumentation::CommandResult()> const& callback,
   cm::optional<std::map<std::string, std::string>> data,
   cm::optional<std::map<std::string, std::string>> arrayData,
   LoadQueriesAfter reloadQueriesAfterCommand)
@@ -635,7 +640,7 @@ int cmInstrumentation::InstrumentCommand(
   // Always begin gathering data for configure in case cmake_instrumentation
   // command creates a query
   if (!this->hasQuery && reloadQueriesAfterCommand == LoadQueriesAfter::No) {
-    return callback();
+    return callback().ExitCode;
   }
 
   // Store command info
@@ -660,7 +665,16 @@ int cmInstrumentation::InstrumentCommand(
   }
 
   // Execute Command
-  int ret = callback();
+  cmInstrumentation::CommandResult callbackResult = callback();
+  int ret = callbackResult.ExitCode;
+  if (this->HasOption(cmInstrumentationQuery::Option::CaptureOutput)) {
+    if (callbackResult.StdOut) {
+      root["stdout"] = *callbackResult.StdOut;
+    }
+    if (callbackResult.StdErr) {
+      root["stderr"] = *callbackResult.StdErr;
+    }
+  }
 
   // Exit early if configure didn't generate a query
   if (reloadQueriesAfterCommand == LoadQueriesAfter::Yes) {
@@ -916,8 +930,11 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid)
     return 0;
   };
   int ret = this->InstrumentCommand(
-    "build", {}, [waitForBuild]() { return waitForBuild(); }, cm::nullopt,
-    cm::nullopt, LoadQueriesAfter::Yes);
+    "build", {},
+    [waitForBuild]() -> cmInstrumentation::CommandResult {
+      return { waitForBuild(), cm::nullopt, cm::nullopt };
+    },
+    cm::nullopt, cm::nullopt, LoadQueriesAfter::Yes);
   this->buildLock.Release();
   this->CollectTimingData(cmInstrumentationQuery::Hook::PostBuild);
   return ret;

Source/cmInstrumentation.h

@@ -42,9 +42,15 @@ public:
                     LoadQueriesAfter loadQueries = LoadQueriesAfter::Yes);
   void LoadQueries();
   void CheckCDashVariable();
+  struct CommandResult
+  {
+    int ExitCode;
+    cm::optional<std::string> StdOut;
+    cm::optional<std::string> StdErr;
+  };
   int InstrumentCommand(
     std::string command_type, std::vector<std::string> const& command,
-    std::function<int()> const& callback,
+    std::function<CommandResult()> const& callback,
     cm::optional<std::map<std::string, std::string>> options = cm::nullopt,
     cm::optional<std::map<std::string, std::string>> arrayOptions =
       cm::nullopt,
@@ -55,7 +61,8 @@ public:
                              int64_t result,
                              std::chrono::steady_clock::time_point steadyStart,
                              std::chrono::system_clock::time_point systemStart,
-                             std::string config);
+                             std::string config,
+                             cm::optional<std::string> output = cm::nullopt);
   void GetPreTestStats();
   bool HasQuery() const;
   bool HasOption(cmInstrumentationQuery::Option option) const;

Source/cmInstrumentationQuery.cxx

@@ -15,8 +15,12 @@
 #include "cmStringAlgorithms.h"
 
 std::vector<std::string> const cmInstrumentationQuery::OptionString{
-  "staticSystemInformation", "dynamicSystemInformation", "cdashSubmit",
-  "cdashVerbose", "trace"
+  "staticSystemInformation",
+  "dynamicSystemInformation",
+  "captureOutput",
+  "cdashSubmit",
+  "cdashVerbose",
+  "trace"
 };
 std::vector<std::string> const cmInstrumentationQuery::HookString{
   "postGenerate",    "preBuild",  "postBuild",        "preCMakeBuild",

Source/cmInstrumentationQuery.h

@@ -16,6 +16,7 @@ public:
   {
     StaticSystemInformation,
     DynamicSystemInformation,
+    CaptureOutput,
     CDashSubmit,
     CDashVerbose,
     Trace

Source/cmake.cxx

@@ -2802,7 +2802,10 @@ int cmake::ActualConfigure()
     return 0;
   };
   int ret = this->Instrumentation->InstrumentCommand(
-    "configure", this->cmdArgs, [doConfigure]() { return doConfigure(); },
+    "configure", this->cmdArgs,
+    [doConfigure]() -> cmInstrumentation::CommandResult {
+      return { doConfigure(), cm::nullopt, cm::nullopt };
+    },
     cm::nullopt, cm::nullopt,
     this->GetIsInTryCompile() ? cmInstrumentation::LoadQueriesAfter::No
                               : cmInstrumentation::LoadQueriesAfter::Yes);
@@ -3254,7 +3257,10 @@ int cmake::Generate()
   };
 
   int ret = this->Instrumentation->InstrumentCommand(
-    "generate", this->cmdArgs, [doGenerate]() { return doGenerate(); });
+    "generate", this->cmdArgs,
+    [doGenerate]() -> cmInstrumentation::CommandResult {
+      return { doGenerate(), cm::nullopt, cm::nullopt };
+    });
   if (ret != 0) {
     return ret;
   }
@@ -4191,8 +4197,10 @@ int cmake::Build(cmBuildArgs buildArgs, std::vector<std::string> targets,
   // Block the instrumentation build daemon from spawning during this build.
   // This lock will be released when the process exits at the end of the build.
   instrumentation.LockBuildDaemon();
-  int buildresult =
-    instrumentation.InstrumentCommand("cmakeBuild", args, doBuild);
+  int buildresult = instrumentation.InstrumentCommand(
+    "cmakeBuild", args, [doBuild]() -> cmInstrumentation::CommandResult {
+      return { doBuild(), cm::nullopt, cm::nullopt };
+    });
   instrumentation.CollectTimingData(
     cmInstrumentationQuery::Hook::PostCMakeBuild);
 #else

Source/cmakemain.cxx

@@ -977,7 +977,9 @@ int do_install(int ac, char const* const* av)
   std::vector<std::string> cmd;
   cm::append(cmd, av, av + ac);
   ret = instrumentation.InstrumentCommand(
-    "cmakeInstall", cmd, [doInstall]() { return doInstall(); });
+    "cmakeInstall", cmd, [doInstall]() -> cmInstrumentation::CommandResult {
+      return { doInstall(), cm::nullopt, cm::nullopt };
+    });
   instrumentation.CollectTimingData(
     cmInstrumentationQuery::Hook::PostCMakeInstall);
   return ret;

Tests/RunCMake/Instrumentation/RunCMakeTest.cmake

@@ -16,6 +16,7 @@ function(instrument test)
     "COPY_QUERIES_GENERATED"
     "STATIC_QUERY"
     "DYNAMIC_QUERY"
+    "CAPTURE_OUTPUT_QUERY"
     "TRACE_QUERY"
     "MANUAL_HOOK"
     "PRESERVE_DATA"
@@ -82,7 +83,7 @@ function(instrument test)
         )
         set(cmake_file "${RunCMake_TEST_BINARY_DIR}/${cmake_filename}")
       else ()
-        set(cmake_file ${query_dir}/default.cmake)
+        set(cmake_file "${query_dir}/default.cmake")
       endif()
     endif()
     list(APPEND ARGS_CONFIGURE_ARG "-DINSTRUMENT_COMMAND_FILE=${cmake_file}")
@@ -133,9 +134,11 @@ function(instrument test)
         COPYONLY
       )
     endforeach()
+    set(RunCMake_QUIET_ERROR 1)
     set(v1 ${RunCMake_TEST_BINARY_DIR}/build/.cmake/instrumentation/v1)
     run_cmake_command(${test}-workflow ${CMAKE_COMMAND} --workflow default)
     set(ARGS_NO_CONFIGURE TRUE)
+    unset(RunCMake_QUIET_ERROR)
   endif()
   if (NOT ARGS_NO_CONFIGURE)
     run_cmake_with_options(${test} ${ARGS_CONFIGURE_ARG} ${maybe_CMAKE_BUILD_TYPE})
@@ -160,19 +163,23 @@ function(instrument test)
       # errors to different places.
       set(RunCMake_TEST_OUTPUT_MERGE 1)
     endif()
+    set(RunCMake_QUIET_ERROR 1)
     run_cmake_command(${test}-build
       ${CMAKE_COMMAND} --build . ${cmake_build_args} -- ${additional_build_args}
     )
+    unset(RunCMake_QUIET_ERROR)
     if (ARGS_FAIL)
       unset(RunCMake_TEST_OUTPUT_MERGE)
     endif()
   endif()
   if (ARGS_BUILD_MAKE_PROGRAM)
     set(RunCMake_TEST_OUTPUT_MERGE 1)
+    set(RunCMake_QUIET_ERROR 1)
     # Force reconfigure to test for double preBuild & postBuild hooks
     file(TOUCH ${RunCMake_TEST_BINARY_DIR}/CMakeCache.txt)
     run_cmake_command(${test}-make-program ${RunCMake_MAKE_PROGRAM})
     unset(RunCMake_TEST_OUTPUT_MERGE)
+    unset(RunCMake_QUIET_ERROR)
   endif()
   if (ARGS_INSTALL)
     run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . --prefix install --config Debug)
@@ -246,7 +253,7 @@ instrument(dynamic-query
   CHECK_SCRIPT check-data-dir.cmake
 )
 instrument(both-query
-  BUILD INSTALL TEST DYNAMIC_QUERY
+  BUILD INSTALL TEST STATIC_QUERY DYNAMIC_QUERY CAPTURE_OUTPUT_QUERY
   CHECK_SCRIPT check-data-dir.cmake
 )
 
@@ -339,6 +346,12 @@ instrument(cmake-command-trace
   CHECK_SCRIPT check-trace-removed.cmake
 )
 
+# Test capture output
+instrument(cmake-command-capture-output
+  BUILD CAPTURE_OUTPUT_QUERY
+  CHECK_SCRIPT check-data-dir.cmake
+)
+
 # Test make/ninja hooks
 if(RunCMake_GENERATOR STREQUAL "FASTBuild")
   # FIXME(#27184): This does not work for FASTBuild.

Tests/RunCMake/Instrumentation/check-data-dir.cmake

@@ -21,6 +21,17 @@ foreach(snippet IN LISTS snippets)
     list(APPEND FOUND_SNIPPETS ${role})
   endif()
 
+  # Ensure launcher-driven snippets record stdout/stderr and tests keep merged output.
+  if (filename MATCHES "^(compile|link|custom|install|test)-")
+    if (ARGS_CAPTURE_OUTPUT_QUERY)
+      json_has_key("${snippet}" "${contents}" stdout)
+      json_has_key("${snippet}" "${contents}" stderr)
+    else()
+      json_missing_key("${snippet}" "${contents}" stdout)
+      json_missing_key("${snippet}" "${contents}" stderr)
+    endif()
+  endif()
+
   # Verify target
   string(JSON target ERROR_VARIABLE noTarget GET "${contents}" target)
   if (target)
@@ -75,6 +86,16 @@ foreach(snippet IN LISTS snippets)
   # Verify contents of custom-* Snippets
   if (filename MATCHES "^custom-")
     string(JSON outputs GET "${contents}" outputs)
+    if (ARGS_CAPTURE_OUTPUT_QUERY AND outputs MATCHES "output3")
+      string(JSON stdout GET "${contents}" stdout)
+      string(JSON stderr GET "${contents}" stderr)
+      if (NOT stdout MATCHES "test stdout")
+        json_error("${snippet}" "Expected custom command stdout to be captured")
+      endif()
+      if (NOT stderr MATCHES "test stderr")
+        json_error("${snippet}" "Expected custom command stderr to be captured")
+      endif()
+    endif()
     # if "outputs" is CMakeFiles/customTarget, should not have a "target"
     if (outputs MATCHES "customTarget")
       json_missing_key("${snippet}" "${contents}" target)
@@ -88,7 +109,7 @@ foreach(snippet IN LISTS snippets)
       endif()
       json_missing_key("${snippet}" "${contents}" target)
     # unrecognized outputs
-    elseif (NOT outputs MATCHES "shell_redirect\\.out")
+    elseif (NOT outputs MATCHES "shell_redirect\\.out|output3")
       json_error("${snippet}" "Custom command has unexpected outputs\n${outputs}")
     endif()
   endif()
@@ -116,6 +137,15 @@ foreach(snippet IN LISTS snippets)
           "Expected zero exit code for test, got: ${result}"
         )
       endif()
+      if (ARGS_CAPTURE_OUTPUT_QUERY)
+        string(JSON output GET "${contents}" stdout)
+        if (NOT output MATCHES "test stdout")
+          json_error("${snippet}" "Expected test output to contain stdout")
+        endif()
+        if (NOT output MATCHES "test stderr")
+          json_error("${snippet}" "Expected test output to contain stderr")
+        endif()
+      endif()
     endif()
   endif()
 

Tests/RunCMake/Instrumentation/project/CMakeLists.txt

@@ -15,6 +15,11 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E true
   OUTPUT output1 output2
 )
+add_custom_command(
+  COMMAND $<TARGET_FILE:main>
+  OUTPUT output3
+  DEPENDS main
+)
 
 file(COPY_FILE shell_redirect.txt ${CMAKE_CURRENT_BINARY_DIR}/shell_redirect.in)
 if(CMAKE_GENERATOR STREQUAL "Watcom WMake")
@@ -35,10 +40,10 @@ else()
   )
 endif()
 
-set_property(SOURCE output1 output2 PROPERTY SYMBOLIC 1)
+set_property(SOURCE output1 output2 output3 PROPERTY SYMBOLIC 1)
 add_custom_target(customTarget ALL
   COMMAND ${CMAKE_COMMAND} -E true
-  DEPENDS output1 shell_redirect.out
+  DEPENDS output1 output3 shell_redirect.out
 )
 add_test(NAME test COMMAND $<TARGET_FILE:main>)
 if(DISABLE_TEST)

Tests/RunCMake/Instrumentation/project/main.c

@@ -1,5 +1,10 @@
+#include <stdio.h>
+
 #include "lib.h"
+
 int main(void)
 {
+  fprintf(stdout, "test stdout\n");
+  fprintf(stderr, "test stderr\n");
   return lib();
 }

Tests/RunCMake/Instrumentation/query/both-query.json.in

@@ -5,6 +5,7 @@
   },
   "options": [
     "staticSystemInformation",
-    "dynamicSystemInformation"
+    "dynamicSystemInformation",
+    "captureOutput"
   ]
 }

Tests/RunCMake/Instrumentation/query/cmake-command-capture-output.cmake

@@ -0,0 +1,5 @@
+cmake_instrumentation(
+  API_VERSION 1
+  DATA_VERSION 1
+  OPTIONS captureOutput
+)

Tests/RunCMake/Instrumentation/query/dynamic-query.json.in

@@ -1,7 +1,6 @@
 {
   "version": 1,
   "options": [
-    "staticSystemInformation",
     "dynamicSystemInformation"
   ]
 }