diff --git a/Help/command/cmake_instrumentation.rst b/Help/command/cmake_instrumentation.rst index 9f555035ae..3de4cb09f0 100644 --- a/Help/command/cmake_instrumentation.rst +++ b/Help/command/cmake_instrumentation.rst @@ -25,7 +25,7 @@ The ``API_VERSION`` and ``DATA_VERSION`` must always be given. See :ref:`cmake-instrumentation API v1` for details. ``DATA_VERSION`` is a version value of the form ``major`` or ``major.minor``. -Currently, the only supported version is ``1.0``. See +Currently, the maximum supported version is ``1.1``. See :ref:`cmake-instrumentation Data Version` for details. Each of the optional keywords ``HOOKS``, ``OPTIONS``, and ``CALLBACK`` diff --git a/Help/manual/cmake-instrumentation.7.rst b/Help/manual/cmake-instrumentation.7.rst index d0612c3875..ae8bba798e 100644 --- a/Help/manual/cmake-instrumentation.7.rst +++ b/Help/manual/cmake-instrumentation.7.rst @@ -17,8 +17,7 @@ information and system diagnostic information during the configure, generate, build, test and install steps for a CMake project. All interactions with the CMake instrumentation API must specify both an API -version and a Data version. At this time, there is only one version for each of -these: see the `API v1`_ and `Data Version`_. +version and a `Data Version`_. There is only one API version, see the `API v1`_. .. note:: @@ -229,11 +228,11 @@ request a specific Data Version, and `v1 Data Files`_ of the corresponding version will be generated and sent to the user `Callbacks`_ defined in that query. -Currently, the only supported version is ``1.0``. A new major version number -will be created whenever previously included data is removed or reformatted such -that scripts written to parse this data may become incompatible with the new -format. A new minor version number will be created whenever new data becomes -available. +Currently, the only supported major version is ``1``, and the maximum supported +minor version is also ``1``. A new major version number will be created whenever +previously included data is removed or reformatted such that scripts written to +parse this data may become incompatible with the new format. A new minor version +number will be created whenever new data becomes available. .. _`cmake-instrumentation v1 Query Files`: @@ -300,6 +299,15 @@ key is required, but all other fields are optional. generated by CMake, and includes information from immediately before and after the command is executed. + ``captureOutput`` + .. versionadded:: 4.4 + + Enables collection of command output in generated `v1 Snippet Files`_. + When enabled, snippets for ``compile``, ``link``, ``custom``, ``test``, and + ``install`` commands include ``stdout`` and ``stderr`` fields. + + Only available as of data version ``1.1``. + ``cdashSubmit`` Enables including instrumentation data in CDash. This is equivalent to having the :envvar:`CTEST_USE_INSTRUMENTATION` environment @@ -345,6 +353,7 @@ Example: "options": [ "staticSystemInformation", "dynamicSystemInformation", + "captureOutput", "cdashSubmit", "trace" ] @@ -358,13 +367,13 @@ The commands ``/usr/bin/python callback.py index-.json`` and ``/usr/bin/cmake -P callback.cmake arg index-.json`` will be executed in that order. The index file will contain the ``staticSystemInformation`` data and each snippet file listed in the index will -contain the ``dynamicSystemInformation`` data. Additionally, the index file -will contain the path to the generated `Google Trace File`_. Once both -callbacks have completed, the index file and data files listed by it (including -snippet files, but not the trace file) will be deleted from the project build -tree. The instrumentation data will be present in the XML files submitted to -CDash, but with truncated command strings because ``cdashVerbose`` was not -enabled. +contain the ``dynamicSystemInformation`` data and captured command output. +Additionally, the index file will contain the path to the generated +`Google Trace File`_. Once both callbacks have completed, the index file and +data files listed by it (including snippet files, but not the trace file) will +be deleted from the project build tree. The instrumentation data will be +present in the XML files submitted to CDash, but with truncated command +strings because ``cdashVerbose`` was not enabled. v1 Data Files ------------- @@ -426,6 +435,24 @@ Snippet files have a filename with the syntax The exit code of the command, an integer. This will be ``null`` when ``role`` is ``build``. + ``stdout`` + .. versionadded:: 4.4 + + The standard output produced by the command. Only included when enabled by + the ``captureOutput`` `option `_ and when ``role`` is one + of: ``compile``, ``link``, ``custom``, ``install`` or ``test``. For + ``test`` snippets, this field contains the merged standard out and standard + error streams. + + ``stderr`` + .. versionadded:: 4.4 + + The standard error output produced by the command. Only included when + enabled by the ``captureOutput`` `option `_ and when + ``role`` is one of: ``compile``, ``link``, ``custom``, ``install`` or + ``test``. For ``test`` snippets, error output is merged with ``stdout``, + and the value of ``stderr`` is always empty. + ``role`` The type of command executed, which will be one of the following values: @@ -520,11 +547,13 @@ Example: { "version": { "major": 1, - "minor": 0 + "minor": 1 }, "command" : "\"/usr/bin/c++\" \"-MD\" \"-MT\" \"CMakeFiles/main.dir/main.cxx.o\" \"-MF\" \"CMakeFiles/main.dir/main.cxx.o.d\" \"-o\" \"CMakeFiles/main.dir/main.cxx.o\" \"-c\" \"/main.cxx\"", "role" : "compile", "result" : 1, + "stdout" : "", + "stderr" : "", "target": "main", "language" : "C++", "outputs" : [ "CMakeFiles/main.dir/main.cxx.o" ], diff --git a/Help/manual/instrumentation/index-v1-schema.json b/Help/manual/instrumentation/index-v1-schema.json index 5f8ec57d08..540cb5bf3a 100644 --- a/Help/manual/instrumentation/index-v1-schema.json +++ b/Help/manual/instrumentation/index-v1-schema.json @@ -47,7 +47,10 @@ }, "minor": { "type": "integer", - "const": 0 + "enum": [ + 0, + 1 + ] } }, "additionalProperties": false diff --git a/Help/manual/instrumentation/query-v1-schema.json b/Help/manual/instrumentation/query-v1-schema.json index 9dba21599f..26bca55efe 100644 --- a/Help/manual/instrumentation/query-v1-schema.json +++ b/Help/manual/instrumentation/query-v1-schema.json @@ -22,7 +22,10 @@ }, "minor": { "type": "integer", - "const": 0 + "enum": [ + 0, + 1 + ] } }, "additionalProperties": false @@ -64,7 +67,8 @@ "dynamicSystemInformation", "cdashSubmit", "cdashVerbose", - "trace" + "trace", + "captureOutput" ], "type": "string" } diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 5d81676859..39206a041f 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -21,6 +21,7 @@ #include "cmCTestLaunchReporter.h" #include "cmGlobalGenerator.h" #include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmMakefile.h" #include "cmProcessOutput.h" #include "cmState.h" @@ -206,6 +207,9 @@ void cmCTestLaunch::RunChild() return; } + this->CapturedStdOut.clear(); + this->CapturedStdErr.clear(); + // Prepare to run the real command. cmUVProcessChainBuilder builder; builder.AddCommand(this->RealArgV); @@ -213,18 +217,19 @@ void cmCTestLaunch::RunChild() // We always share the input pipe. builder.SetExternalStream(cmUVProcessChainBuilder::Stream_INPUT, stdin); + builder.SetBuiltinStream(cmUVProcessChainBuilder::Stream_OUTPUT) + .SetBuiltinStream(cmUVProcessChainBuilder::Stream_ERROR); cmsys::ofstream fout; cmsys::ofstream ferr; - if (this->Reporter.Passthru) { - // In passthru mode we just share the output pipes. - builder.SetExternalStream(cmUVProcessChainBuilder::Stream_OUTPUT, stdout) - .SetExternalStream(cmUVProcessChainBuilder::Stream_ERROR, stderr); - } else { + cmsys::ofstream* foutPtr = nullptr; + cmsys::ofstream* ferrPtr = nullptr; + + if (!this->Reporter.Passthru) { // In full mode we record the child output pipes to log files. - builder.SetBuiltinStream(cmUVProcessChainBuilder::Stream_OUTPUT) - .SetBuiltinStream(cmUVProcessChainBuilder::Stream_ERROR); fout.open(this->Reporter.LogOut.c_str(), std::ios::out | std::ios::binary); ferr.open(this->Reporter.LogErr.c_str(), std::ios::out | std::ios::binary); + foutPtr = &fout; + ferrPtr = &ferr; } #ifdef _WIN32 @@ -243,36 +248,43 @@ void cmCTestLaunch::RunChild() cmProcessOutput processOutput; std::unique_ptr outputHandle; std::unique_ptr errorHandle; - if (!this->Reporter.Passthru) { - auto beginRead = - [&processOutput](uv_stream_t* stream, std::ostream& out, - cmsys::ofstream& file, bool& haveData, bool& finished, - int id) -> std::unique_ptr { - finished = false; - return cmUVStreamRead( - stream, - [&processOutput, &out, &file, id, &haveData](std::vector data) { - std::string strdata; - processOutput.DecodeText(data.data(), data.size(), strdata, id); - file.write(strdata.c_str(), strdata.size()); - out.write(strdata.c_str(), strdata.size()); - haveData = true; - }, - [&processOutput, &out, &file, &finished, id]() { - std::string strdata; - processOutput.DecodeText(std::string(), strdata, id); - if (!strdata.empty()) { - file.write(strdata.c_str(), strdata.size()); - out.write(strdata.c_str(), strdata.size()); + auto beginRead = + [&processOutput]( + uv_stream_t* stream, std::ostream& out, cmsys::ofstream* file, + bool& haveData, bool& finished, int id, + std::string& capture) -> std::unique_ptr { + finished = false; + return cmUVStreamRead( + stream, + [&processOutput, &out, file, id, &haveData, + &capture](std::vector data) { + std::string strdata; + processOutput.DecodeText(data.data(), data.size(), strdata, id); + if (file && file->is_open()) { + file->write(strdata.c_str(), strdata.size()); + } + out.write(strdata.c_str(), strdata.size()); + capture.append(strdata); + haveData = true; + }, + [&processOutput, &out, file, &finished, id, &capture]() { + std::string strdata; + processOutput.DecodeText(std::string(), strdata, id); + if (!strdata.empty()) { + if (file && file->is_open()) { + file->write(strdata.c_str(), strdata.size()); } - finished = true; - }); - }; - outputHandle = beginRead(chain.OutputStream(), std::cout, fout, - this->HaveOut, outFinished, 1); - errorHandle = beginRead(chain.ErrorStream(), std::cerr, ferr, - this->HaveErr, errFinished, 2); - } + out.write(strdata.c_str(), strdata.size()); + capture.append(strdata); + } + finished = true; + }); + }; + outputHandle = + beginRead(chain.OutputStream(), std::cout, foutPtr, this->HaveOut, + outFinished, 1, this->CapturedStdOut); + errorHandle = beginRead(chain.ErrorStream(), std::cerr, ferrPtr, + this->HaveErr, errFinished, 2, this->CapturedStdErr); // Wait for the real command to finish. while (!(chain.Finished() && outFinished && errFinished)) { @@ -291,6 +303,8 @@ void cmCTestLaunch::RunChild() int cmCTestLaunch::Run() { auto instrumentation = cmInstrumentation(this->Reporter.OptionBuildDir); + bool const captureOutput = + instrumentation.HasOption(cmInstrumentationQuery::Option::CaptureOutput); std::map options; if (this->Reporter.OptionTargetName != "TARGET_NAME") { options["target"] = this->Reporter.OptionTargetName; @@ -305,9 +319,15 @@ int cmCTestLaunch::Run() arrayOptions["targetLabels"] = this->Reporter.OptionTargetLabels; instrumentation.InstrumentCommand( this->Reporter.OptionCommandType, this->RealArgV, - [this]() -> int { + [this, captureOutput]() -> cmInstrumentation::CommandResult { this->RunChild(); - return this->Reporter.ExitCode; + cmInstrumentation::CommandResult result; + result.ExitCode = this->Reporter.ExitCode; + if (captureOutput) { + result.StdOut = this->CapturedStdOut; + result.StdErr = this->CapturedStdErr; + } + return result; }, options, arrayOptions); diff --git a/Source/CTest/cmCTestLaunch.h b/Source/CTest/cmCTestLaunch.h index 0aeb211b9e..cf2d1ed65c 100644 --- a/Source/CTest/cmCTestLaunch.h +++ b/Source/CTest/cmCTestLaunch.h @@ -59,6 +59,8 @@ private: // Whether or not any data have been written to stdout or stderr. bool HaveOut; bool HaveErr; + std::string CapturedStdOut; + std::string CapturedStdErr; // Load custom rules to match warnings and their exceptions. bool ScrapeRulesLoaded; diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index a3f638c384..ebbd45254f 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -1122,7 +1122,7 @@ void cmCTestRunTest::FinalizeTest(bool started) this->TestProperties->Name, this->ActualCommand, this->Arguments, this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(), this->TestProcess->GetSystemStartTime(), - this->GetCTest()->GetConfigType()); + this->GetCTest()->GetConfigType(), this->ProcessOutput); this->TestResult.InstrumentationFile = data_file; } this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(), diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index cc99177e06..76ee2b9bc8 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -2713,8 +2713,12 @@ int cmCTest::ExecuteTests(std::vector const& args) }; std::map data; data["showOnly"] = this->GetShowOnly() ? "1" : "0"; - int ret = - instrumentation.InstrumentCommand("ctest", args, processHandler, data); + int ret = instrumentation.InstrumentCommand( + "ctest", args, + [processHandler]() -> cmInstrumentation::CommandResult { + return { processHandler(), cm::nullopt, cm::nullopt }; + }, + data); instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostCTest); if (ret == cmCTest::TEST_ERRORS) { cmCTestLog(this, ERROR_MESSAGE, "Errors while running CTest\n"); diff --git a/Source/cmInstrumentation.cxx b/Source/cmInstrumentation.cxx index fc5e99d7ed..eb2a53ad53 100644 --- a/Source/cmInstrumentation.cxx +++ b/Source/cmInstrumentation.cxx @@ -586,7 +586,8 @@ std::string cmInstrumentation::InstrumentTest( std::string const& name, std::string const& command, std::vector const& args, int64_t result, std::chrono::steady_clock::time_point steadyStart, - std::chrono::system_clock::time_point systemStart, std::string config) + std::chrono::system_clock::time_point systemStart, std::string config, + cm::optional output) { // Store command info Json::Value root(this->preTestStats); @@ -597,6 +598,10 @@ std::string cmInstrumentation::InstrumentTest( root["result"] = static_cast(result); root["config"] = config; root["workingDir"] = cmSystemTools::GetLogicalWorkingDirectory(); + if (this->HasOption(cmInstrumentationQuery::Option::CaptureOutput)) { + root["stdout"] = output ? *output : ""; + root["stderr"] = ""; + } // Post-Command this->InsertTimingData(root, steadyStart, systemStart); @@ -626,7 +631,7 @@ void cmInstrumentation::GetPreTestStats() int cmInstrumentation::InstrumentCommand( std::string command_type, std::vector const& command, - std::function const& callback, + std::function const& callback, cm::optional> data, cm::optional> arrayData, LoadQueriesAfter reloadQueriesAfterCommand) @@ -635,7 +640,7 @@ int cmInstrumentation::InstrumentCommand( // Always begin gathering data for configure in case cmake_instrumentation // command creates a query if (!this->hasQuery && reloadQueriesAfterCommand == LoadQueriesAfter::No) { - return callback(); + return callback().ExitCode; } // Store command info @@ -660,7 +665,16 @@ int cmInstrumentation::InstrumentCommand( } // Execute Command - int ret = callback(); + cmInstrumentation::CommandResult callbackResult = callback(); + int ret = callbackResult.ExitCode; + if (this->HasOption(cmInstrumentationQuery::Option::CaptureOutput)) { + if (callbackResult.StdOut) { + root["stdout"] = *callbackResult.StdOut; + } + if (callbackResult.StdErr) { + root["stderr"] = *callbackResult.StdErr; + } + } // Exit early if configure didn't generate a query if (reloadQueriesAfterCommand == LoadQueriesAfter::Yes) { @@ -916,8 +930,11 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid) return 0; }; int ret = this->InstrumentCommand( - "build", {}, [waitForBuild]() { return waitForBuild(); }, cm::nullopt, - cm::nullopt, LoadQueriesAfter::Yes); + "build", {}, + [waitForBuild]() -> cmInstrumentation::CommandResult { + return { waitForBuild(), cm::nullopt, cm::nullopt }; + }, + cm::nullopt, cm::nullopt, LoadQueriesAfter::Yes); this->buildLock.Release(); this->CollectTimingData(cmInstrumentationQuery::Hook::PostBuild); return ret; diff --git a/Source/cmInstrumentation.h b/Source/cmInstrumentation.h index 6c67e20b9b..ec59a20d75 100644 --- a/Source/cmInstrumentation.h +++ b/Source/cmInstrumentation.h @@ -42,9 +42,15 @@ public: LoadQueriesAfter loadQueries = LoadQueriesAfter::Yes); void LoadQueries(); void CheckCDashVariable(); + struct CommandResult + { + int ExitCode; + cm::optional StdOut; + cm::optional StdErr; + }; int InstrumentCommand( std::string command_type, std::vector const& command, - std::function const& callback, + std::function const& callback, cm::optional> options = cm::nullopt, cm::optional> arrayOptions = cm::nullopt, @@ -55,7 +61,8 @@ public: int64_t result, std::chrono::steady_clock::time_point steadyStart, std::chrono::system_clock::time_point systemStart, - std::string config); + std::string config, + cm::optional output = cm::nullopt); void GetPreTestStats(); bool HasQuery() const; bool HasOption(cmInstrumentationQuery::Option option) const; diff --git a/Source/cmInstrumentationCommand.cxx b/Source/cmInstrumentationCommand.cxx index 5b0bb2d16b..636478e220 100644 --- a/Source/cmInstrumentationCommand.cxx +++ b/Source/cmInstrumentationCommand.cxx @@ -81,7 +81,7 @@ bool validateDataVersion(std::string const& versionString, Version& version, if (!cmInstrumentationQuery::ValidDataVersion(version)) { status.SetError( cmStrCat("given an unsupported DATA_VERSION \"", versionString, - "\" (the only currently supported version is 1.0).")); + "\" (the maximum currently supported version is 1.1).")); return false; } diff --git a/Source/cmInstrumentationQuery.cxx b/Source/cmInstrumentationQuery.cxx index fac5c2c0c9..7a6c74fe1d 100644 --- a/Source/cmInstrumentationQuery.cxx +++ b/Source/cmInstrumentationQuery.cxx @@ -15,8 +15,12 @@ #include "cmStringAlgorithms.h" std::vector const cmInstrumentationQuery::OptionString{ - "staticSystemInformation", "dynamicSystemInformation", "cdashSubmit", - "cdashVerbose", "trace" + "staticSystemInformation", + "dynamicSystemInformation", + "captureOutput", + "cdashSubmit", + "cdashVerbose", + "trace" }; std::vector const cmInstrumentationQuery::HookString{ "postGenerate", "preBuild", "postBuild", "preCMakeBuild", @@ -161,13 +165,13 @@ bool cmInstrumentationQuery::ReadJSON(std::string const& filename, bool cmInstrumentationQuery::ValidDataVersion(Version version) { auto const latest = LatestDataVersion(); - return version.Major == latest.Major && version.Minor == latest.Minor; + return version.Major == latest.Major && version.Minor <= latest.Minor; } Version cmInstrumentationQuery::LatestDataVersion() { Version latest; latest.Major = 1; - latest.Minor = 0; + latest.Minor = 1; return latest; } diff --git a/Source/cmInstrumentationQuery.h b/Source/cmInstrumentationQuery.h index 37bbd0ca40..4f64db7b32 100644 --- a/Source/cmInstrumentationQuery.h +++ b/Source/cmInstrumentationQuery.h @@ -16,6 +16,7 @@ public: { StaticSystemInformation, DynamicSystemInformation, + CaptureOutput, CDashSubmit, CDashVerbose, Trace diff --git a/Source/cmake.cxx b/Source/cmake.cxx index 5d85bfd664..362adf1cd3 100644 --- a/Source/cmake.cxx +++ b/Source/cmake.cxx @@ -2802,7 +2802,10 @@ int cmake::ActualConfigure() return 0; }; int ret = this->Instrumentation->InstrumentCommand( - "configure", this->cmdArgs, [doConfigure]() { return doConfigure(); }, + "configure", this->cmdArgs, + [doConfigure]() -> cmInstrumentation::CommandResult { + return { doConfigure(), cm::nullopt, cm::nullopt }; + }, cm::nullopt, cm::nullopt, this->GetIsInTryCompile() ? cmInstrumentation::LoadQueriesAfter::No : cmInstrumentation::LoadQueriesAfter::Yes); @@ -3254,7 +3257,10 @@ int cmake::Generate() }; int ret = this->Instrumentation->InstrumentCommand( - "generate", this->cmdArgs, [doGenerate]() { return doGenerate(); }); + "generate", this->cmdArgs, + [doGenerate]() -> cmInstrumentation::CommandResult { + return { doGenerate(), cm::nullopt, cm::nullopt }; + }); if (ret != 0) { return ret; } @@ -4191,8 +4197,10 @@ int cmake::Build(cmBuildArgs buildArgs, std::vector targets, // Block the instrumentation build daemon from spawning during this build. // This lock will be released when the process exits at the end of the build. instrumentation.LockBuildDaemon(); - int buildresult = - instrumentation.InstrumentCommand("cmakeBuild", args, doBuild); + int buildresult = instrumentation.InstrumentCommand( + "cmakeBuild", args, [doBuild]() -> cmInstrumentation::CommandResult { + return { doBuild(), cm::nullopt, cm::nullopt }; + }); instrumentation.CollectTimingData( cmInstrumentationQuery::Hook::PostCMakeBuild); #else diff --git a/Source/cmakemain.cxx b/Source/cmakemain.cxx index 842c4b87c2..b9545c9ba0 100644 --- a/Source/cmakemain.cxx +++ b/Source/cmakemain.cxx @@ -977,7 +977,9 @@ int do_install(int ac, char const* const* av) std::vector cmd; cm::append(cmd, av, av + ac); ret = instrumentation.InstrumentCommand( - "cmakeInstall", cmd, [doInstall]() { return doInstall(); }); + "cmakeInstall", cmd, [doInstall]() -> cmInstrumentation::CommandResult { + return { doInstall(), cm::nullopt, cm::nullopt }; + }); instrumentation.CollectTimingData( cmInstrumentationQuery::Hook::PostCMakeInstall); return ret; diff --git a/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake index ff32765a3d..69265ae073 100644 --- a/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake +++ b/Tests/RunCMake/Instrumentation/RunCMakeTest.cmake @@ -16,6 +16,7 @@ function(instrument test) "COPY_QUERIES_GENERATED" "STATIC_QUERY" "DYNAMIC_QUERY" + "CAPTURE_OUTPUT_QUERY" "TRACE_QUERY" "MANUAL_HOOK" "PRESERVE_DATA" @@ -82,7 +83,7 @@ function(instrument test) ) set(cmake_file "${RunCMake_TEST_BINARY_DIR}/${cmake_filename}") else () - set(cmake_file ${query_dir}/default.cmake) + set(cmake_file "${query_dir}/default.cmake") endif() endif() list(APPEND ARGS_CONFIGURE_ARG "-DINSTRUMENT_COMMAND_FILE=${cmake_file}") @@ -133,9 +134,11 @@ function(instrument test) COPYONLY ) endforeach() + set(RunCMake_QUIET_ERROR 1) set(v1 ${RunCMake_TEST_BINARY_DIR}/build/.cmake/instrumentation/v1) run_cmake_command(${test}-workflow ${CMAKE_COMMAND} --workflow default) set(ARGS_NO_CONFIGURE TRUE) + unset(RunCMake_QUIET_ERROR) endif() if (NOT ARGS_NO_CONFIGURE) run_cmake_with_options(${test} ${ARGS_CONFIGURE_ARG} ${maybe_CMAKE_BUILD_TYPE}) @@ -160,19 +163,23 @@ function(instrument test) # errors to different places. set(RunCMake_TEST_OUTPUT_MERGE 1) endif() + set(RunCMake_QUIET_ERROR 1) run_cmake_command(${test}-build ${CMAKE_COMMAND} --build . ${cmake_build_args} -- ${additional_build_args} ) + unset(RunCMake_QUIET_ERROR) if (ARGS_FAIL) unset(RunCMake_TEST_OUTPUT_MERGE) endif() endif() if (ARGS_BUILD_MAKE_PROGRAM) set(RunCMake_TEST_OUTPUT_MERGE 1) + set(RunCMake_QUIET_ERROR 1) # Force reconfigure to test for double preBuild & postBuild hooks file(TOUCH ${RunCMake_TEST_BINARY_DIR}/CMakeCache.txt) run_cmake_command(${test}-make-program ${RunCMake_MAKE_PROGRAM}) unset(RunCMake_TEST_OUTPUT_MERGE) + unset(RunCMake_QUIET_ERROR) endif() if (ARGS_INSTALL) run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . --prefix install --config Debug) @@ -246,7 +253,7 @@ instrument(dynamic-query CHECK_SCRIPT check-data-dir.cmake ) instrument(both-query - BUILD INSTALL TEST DYNAMIC_QUERY + BUILD INSTALL TEST STATIC_QUERY DYNAMIC_QUERY CAPTURE_OUTPUT_QUERY CHECK_SCRIPT check-data-dir.cmake ) @@ -339,6 +346,12 @@ instrument(cmake-command-trace CHECK_SCRIPT check-trace-removed.cmake ) +# Test capture output +instrument(cmake-command-capture-output + BUILD CAPTURE_OUTPUT_QUERY + CHECK_SCRIPT check-data-dir.cmake +) + # Test make/ninja hooks if(RunCMake_GENERATOR STREQUAL "FASTBuild") # FIXME(#27184): This does not work for FASTBuild. diff --git a/Tests/RunCMake/Instrumentation/check-custom-content.cmake b/Tests/RunCMake/Instrumentation/check-custom-content.cmake index ac358ef3c2..fd6a68ee19 100644 --- a/Tests/RunCMake/Instrumentation/check-custom-content.cmake +++ b/Tests/RunCMake/Instrumentation/check-custom-content.cmake @@ -18,7 +18,7 @@ foreach(content_file IN LISTS content_files) # Check version string(JSON version GET "${contents}" version) json_assert_key("${content_file}" "${version}" major "1") - json_assert_key("${content_file}" "${version}" minor "0") + json_assert_key("${content_file}" "${version}" minor "1") # Check project name json_assert_key("${content_file}" "${contents}" project "instrumentation") diff --git a/Tests/RunCMake/Instrumentation/check-data-dir.cmake b/Tests/RunCMake/Instrumentation/check-data-dir.cmake index 0658e63fcc..8f0b4f60a0 100644 --- a/Tests/RunCMake/Instrumentation/check-data-dir.cmake +++ b/Tests/RunCMake/Instrumentation/check-data-dir.cmake @@ -21,6 +21,17 @@ foreach(snippet IN LISTS snippets) list(APPEND FOUND_SNIPPETS ${role}) endif() + # Ensure launcher-driven snippets record stdout/stderr and tests keep merged output. + if (filename MATCHES "^(compile|link|custom|install|test)-") + if (ARGS_CAPTURE_OUTPUT_QUERY) + json_has_key("${snippet}" "${contents}" stdout) + json_has_key("${snippet}" "${contents}" stderr) + else() + json_missing_key("${snippet}" "${contents}" stdout) + json_missing_key("${snippet}" "${contents}" stderr) + endif() + endif() + # Verify target string(JSON target ERROR_VARIABLE noTarget GET "${contents}" target) if (target) @@ -75,6 +86,16 @@ foreach(snippet IN LISTS snippets) # Verify contents of custom-* Snippets if (filename MATCHES "^custom-") string(JSON outputs GET "${contents}" outputs) + if (ARGS_CAPTURE_OUTPUT_QUERY AND outputs MATCHES "output3") + string(JSON stdout GET "${contents}" stdout) + string(JSON stderr GET "${contents}" stderr) + if (NOT stdout MATCHES "test stdout") + json_error("${snippet}" "Expected custom command stdout to be captured") + endif() + if (NOT stderr MATCHES "test stderr") + json_error("${snippet}" "Expected custom command stderr to be captured") + endif() + endif() # if "outputs" is CMakeFiles/customTarget, should not have a "target" if (outputs MATCHES "customTarget") json_missing_key("${snippet}" "${contents}" target) @@ -88,7 +109,7 @@ foreach(snippet IN LISTS snippets) endif() json_missing_key("${snippet}" "${contents}" target) # unrecognized outputs - elseif (NOT outputs MATCHES "shell_redirect\\.out") + elseif (NOT outputs MATCHES "shell_redirect\\.out|output3") json_error("${snippet}" "Custom command has unexpected outputs\n${outputs}") endif() endif() @@ -116,6 +137,15 @@ foreach(snippet IN LISTS snippets) "Expected zero exit code for test, got: ${result}" ) endif() + if (ARGS_CAPTURE_OUTPUT_QUERY) + string(JSON output GET "${contents}" stdout) + if (NOT output MATCHES "test stdout") + json_error("${snippet}" "Expected test output to contain stdout") + endif() + if (NOT output MATCHES "test stderr") + json_error("${snippet}" "Expected test output to contain stderr") + endif() + endif() endif() endif() diff --git a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-data-version-stderr.txt b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-data-version-stderr.txt index d8c3ec7bcc..5864c2ba75 100644 --- a/Tests/RunCMake/Instrumentation/cmake-command-unsupported-data-version-stderr.txt +++ b/Tests/RunCMake/Instrumentation/cmake-command-unsupported-data-version-stderr.txt @@ -1,6 +1,6 @@ CMake Error at [^ ]*\(cmake_instrumentation\): - cmake_instrumentation given an unsupported DATA_VERSION "1\.1" \(the only - currently supported version is 1\.0\)\. + cmake_instrumentation given an unsupported DATA_VERSION "2\.1" \(the maximum + currently supported version is 1\.1\)\. Call Stack \(most recent call first\): CMakeLists\.txt:5 \(include\) diff --git a/Tests/RunCMake/Instrumentation/hook.cmake b/Tests/RunCMake/Instrumentation/hook.cmake index 21fe59a194..e1d29088e6 100644 --- a/Tests/RunCMake/Instrumentation/hook.cmake +++ b/Tests/RunCMake/Instrumentation/hook.cmake @@ -68,8 +68,8 @@ json_has_key("${index}" "${contents}" version) string(JSON version_major GET "${contents}" version major) string(JSON version_minor GET "${contents}" version minor) -if (NOT version_major EQUAL 1 OR NOT version_minor EQUAL 0) - add_error("Version must be 1.0, got: ${version_major}.${version_minor}") +if (NOT version_major EQUAL 1 OR NOT version_minor LESS_EQUAL 1) + add_error("Version must be <= 1.1, got: ${version_major}.${version_minor}") endif() json_has_key("${index}" "${contents}" buildDir) diff --git a/Tests/RunCMake/Instrumentation/project/CMakeLists.txt b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt index f646ec4752..42748e7f8b 100644 --- a/Tests/RunCMake/Instrumentation/project/CMakeLists.txt +++ b/Tests/RunCMake/Instrumentation/project/CMakeLists.txt @@ -15,6 +15,11 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E true OUTPUT output1 output2 ) +add_custom_command( + COMMAND $ + OUTPUT output3 + DEPENDS main +) file(COPY_FILE shell_redirect.txt ${CMAKE_CURRENT_BINARY_DIR}/shell_redirect.in) if(CMAKE_GENERATOR STREQUAL "Watcom WMake") @@ -35,10 +40,10 @@ else() ) endif() -set_property(SOURCE output1 output2 PROPERTY SYMBOLIC 1) +set_property(SOURCE output1 output2 output3 PROPERTY SYMBOLIC 1) add_custom_target(customTarget ALL COMMAND ${CMAKE_COMMAND} -E true - DEPENDS output1 shell_redirect.out + DEPENDS output1 output3 shell_redirect.out ) add_test(NAME test COMMAND $) if(DISABLE_TEST) diff --git a/Tests/RunCMake/Instrumentation/project/main.c b/Tests/RunCMake/Instrumentation/project/main.c index df19fd4498..78085f2268 100644 --- a/Tests/RunCMake/Instrumentation/project/main.c +++ b/Tests/RunCMake/Instrumentation/project/main.c @@ -1,5 +1,10 @@ +#include + #include "lib.h" + int main(void) { + fprintf(stdout, "test stdout\n"); + fprintf(stderr, "test stderr\n"); return lib(); } diff --git a/Tests/RunCMake/Instrumentation/query/bad-version-minor.json.in b/Tests/RunCMake/Instrumentation/query/bad-version-minor.json.in index de6bc6b3f8..afe9a55591 100644 --- a/Tests/RunCMake/Instrumentation/query/bad-version-minor.json.in +++ b/Tests/RunCMake/Instrumentation/query/bad-version-minor.json.in @@ -1,6 +1,6 @@ { "version": { "major": 1, - "minor": 1 + "minor": 9 } } diff --git a/Tests/RunCMake/Instrumentation/query/bad-version.json.in b/Tests/RunCMake/Instrumentation/query/bad-version.json.in index de6bc6b3f8..afe9a55591 100644 --- a/Tests/RunCMake/Instrumentation/query/bad-version.json.in +++ b/Tests/RunCMake/Instrumentation/query/bad-version.json.in @@ -1,6 +1,6 @@ { "version": { "major": 1, - "minor": 1 + "minor": 9 } } diff --git a/Tests/RunCMake/Instrumentation/query/both-query.json.in b/Tests/RunCMake/Instrumentation/query/both-query.json.in index 4f8ecb2084..0990d94483 100644 --- a/Tests/RunCMake/Instrumentation/query/both-query.json.in +++ b/Tests/RunCMake/Instrumentation/query/both-query.json.in @@ -5,6 +5,7 @@ }, "options": [ "staticSystemInformation", - "dynamicSystemInformation" + "dynamicSystemInformation", + "captureOutput" ] } diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-capture-output.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-capture-output.cmake new file mode 100644 index 0000000000..17c4362548 --- /dev/null +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-capture-output.cmake @@ -0,0 +1,5 @@ +cmake_instrumentation( + API_VERSION 1 + DATA_VERSION 1.1 + OPTIONS captureOutput +) diff --git a/Tests/RunCMake/Instrumentation/query/cmake-command-unsupported-data-version.cmake b/Tests/RunCMake/Instrumentation/query/cmake-command-unsupported-data-version.cmake index 94aa71fc2d..952b92eda9 100644 --- a/Tests/RunCMake/Instrumentation/query/cmake-command-unsupported-data-version.cmake +++ b/Tests/RunCMake/Instrumentation/query/cmake-command-unsupported-data-version.cmake @@ -1,4 +1,4 @@ cmake_instrumentation( API_VERSION 1 - DATA_VERSION 1.1 + DATA_VERSION 2.1 ) diff --git a/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in b/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in index 7a34ac8f85..487b507501 100644 --- a/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in +++ b/Tests/RunCMake/Instrumentation/query/dynamic-query.json.in @@ -1,7 +1,6 @@ { "version": 1, "options": [ - "staticSystemInformation", "dynamicSystemInformation" ] } diff --git a/Tests/RunCMake/Instrumentation/query/hooks-invalid-version-ignored.json.in b/Tests/RunCMake/Instrumentation/query/hooks-invalid-version-ignored.json.in index c99213d16d..0aeed54993 100644 --- a/Tests/RunCMake/Instrumentation/query/hooks-invalid-version-ignored.json.in +++ b/Tests/RunCMake/Instrumentation/query/hooks-invalid-version-ignored.json.in @@ -1,7 +1,7 @@ { "version": { "major": 1, - "minor": 1 + "minor": 9 }, "hooks": ["postCMakeBuild"], "callbacks": ["@GET_HOOK@"] diff --git a/Tests/RunCMake/Instrumentation/verify-snippet.cmake b/Tests/RunCMake/Instrumentation/verify-snippet.cmake index d5c87bc962..cf9acd07ce 100644 --- a/Tests/RunCMake/Instrumentation/verify-snippet.cmake +++ b/Tests/RunCMake/Instrumentation/verify-snippet.cmake @@ -77,8 +77,8 @@ function(verify_snippet_data snippet contents) snippet_valid_timing("${contents}") string(JSON version_major GET "${contents}" version major) string(JSON version_minor GET "${contents}" version minor) - if (NOT version_major EQUAL 1 OR NOT version_minor EQUAL 0) - json_error("${snippet}" "Version must be 1.0, got: ${version_major}.${version_minor}") + if (NOT version_major EQUAL 1 OR NOT version_minor LESS_EQUAL 1) + json_error("${snippet}" "Version must be <= 1.1, got: ${version_major}.${version_minor}") endif() get_filename_component(filename "${snippet}" NAME) string(JSON result GET "${contents}" result) diff --git a/Tests/RunCMake/RunCMake.cmake b/Tests/RunCMake/RunCMake.cmake index b849634efb..bf83507049 100644 --- a/Tests/RunCMake/RunCMake.cmake +++ b/Tests/RunCMake/RunCMake.cmake @@ -105,6 +105,9 @@ function(run_cmake test) else() set(maybe_input_file "") endif() + if (RunCMake_QUIET_ERROR) + set(maybe_quiet_error "ERROR_QUIET") + endif() if(NOT RunCMake_TEST_COMMAND) if(NOT DEFINED RunCMake_TEST_OPTIONS) set(RunCMake_TEST_OPTIONS "") @@ -157,6 +160,7 @@ function(run_cmake test) ENCODING UTF8 ${maybe_timeout} ${maybe_input_file} + ${maybe_quiet_error} )]]) else() set(expect_result "")