diff --git a/Help/command/ctest_memcheck.rst b/Help/command/ctest_memcheck.rst index 4ca7364f03..0011632f60 100644 --- a/Help/command/ctest_memcheck.rst +++ b/Help/command/ctest_memcheck.rst @@ -26,6 +26,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`. [CAPTURE_CMAKE_ERROR ] [REPEAT :] [OUTPUT_JUNIT ] + [PRESET ] [DEFECT_COUNT ] [QUIET] ) diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 47dfe237e4..ad34994fdf 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -28,6 +28,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [CAPTURE_CMAKE_ERROR ] [REPEAT :] [OUTPUT_JUNIT ] + [PRESET ] [QUIET] ) @@ -190,6 +191,15 @@ The options are: of the tests. See the CTest :ref:`CoverageTool ` setting for details. +``PRESET `` + .. versionadded:: 4.4 + + Specify a :manual:`preset ` to use when running tests. + Any value set in the CTest script will take priority over a corresponding + setting from the preset. For example, the ``INCLUDE`` argument will override + the :preset:`filter.include.name ` setting + from the chosen preset. + ``QUIET`` .. versionadded:: 3.3 diff --git a/Help/release/dev/ctest-script-preset-arg.rst b/Help/release/dev/ctest-script-preset-arg.rst new file mode 100644 index 0000000000..cfe2fe6cf1 --- /dev/null +++ b/Help/release/dev/ctest-script-preset-arg.rst @@ -0,0 +1,7 @@ +ctest-script-preset-arg +------------------------ + +* The :command:`ctest_configure`, :command:`ctest_build`, + :command:`ctest_test`, and :command:`ctest_memcheck` commands gained + ``PRESET`` arguments to support using :manual:`presets ` + for their :ref:`Dashboard Client` steps. diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 1c0d4e7811..0b022b124e 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -6,25 +6,60 @@ #include #include #include +#include #include #include #include +#include +#include "cmArgumentParserTypes.h" +#include "cmCMakePresetsGraph.h" #include "cmCTest.h" #include "cmCTestGenericHandler.h" #include "cmCTestTestHandler.h" #include "cmDuration.h" #include "cmExecutionStatus.h" +#include "cmJSONState.h" #include "cmMakefile.h" #include "cmStringAlgorithms.h" #include "cmValue.h" +using TestPreset = cmCMakePresetsGraph::TestPreset; + std::unique_ptr cmCTestTestCommand::InitializeHandler( HandlerArguments& arguments, cmExecutionStatus& status) const { cmMakefile& mf = status.GetMakefile(); auto& args = static_cast(arguments); + + std::unique_ptr presetsGraph; + TestPreset const* expandedPreset = nullptr; + if (!args.Preset.empty()) { + std::string const sourceDirectory = + mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY"); + + presetsGraph = cm::make_unique(); + if (!presetsGraph->ReadProjectPresets(sourceDirectory, "")) { + status.SetError( + cmStrCat("Could not read presets from \"", sourceDirectory, + "\": ", presetsGraph->parseState.GetErrorMessage())); + return nullptr; + } + + auto resolveResult = + presetsGraph->ResolvePreset(args.Preset, presetsGraph->TestPresets); + auto resolveError = cmCMakePresetsGraph::FormatPresetError( + resolveResult.StatusCode, resolveResult.ErrorPresetName, + sourceDirectory); + if (resolveError) { + status.SetError(*resolveError); + return nullptr; + } + + expandedPreset = resolveResult.Preset; + } + cmValue ctestTimeout = mf.GetDefinition("CTEST_TEST_TIMEOUT"); cmDuration timeout; @@ -39,12 +74,56 @@ std::unique_ptr cmCTestTestCommand::InitializeHandler( } this->CTest->SetTimeOut(timeout); - cmValue resourceSpecFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE"); - if (args.ResourceSpecFile.empty() && resourceSpecFile) { - args.ResourceSpecFile = *resourceSpecFile; + auto handler = this->InitializeActualHandler(args, status); + + // Load settings from the preset if one was specified. + if (expandedPreset) { + cmCTestApplyTestPresetToOptions(handler->TestOptions, *expandedPreset); + + if (!args.ParallelLevel && expandedPreset->Execution) { + if (auto const& jobs = expandedPreset->Execution->Jobs) { + ArgumentParser::Maybe level; + if (jobs->has_value()) { + level = std::to_string(**jobs); + } + handler->ParallelLevel = level; + } + } + + if (args.Repeat.empty() && expandedPreset->Execution && + expandedPreset->Execution->Repeat) { + auto const& rep = *expandedPreset->Execution->Repeat; + using RepeatMode = TestPreset::ExecutionOptions::RepeatOptions::ModeEnum; + std::string modeStr; + switch (rep.Mode) { + case RepeatMode::UntilFail: + modeStr = "UNTIL_FAIL"; + break; + case RepeatMode::UntilPass: + modeStr = "UNTIL_PASS"; + break; + case RepeatMode::AfterTimeout: + modeStr = "AFTER_TIMEOUT"; + break; + } + handler->Repeat = cmStrCat(modeStr, ':', rep.Count); + } + + if (args.TestLoad.empty() && expandedPreset->Execution && + expandedPreset->Execution->TestLoad) { + args.TestLoad = std::to_string(*expandedPreset->Execution->TestLoad); + } } - auto handler = this->InitializeActualHandler(args, status); + if (args.ResourceSpecFile.empty()) { + cmValue resourceSpecFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE"); + if (resourceSpecFile) { + args.ResourceSpecFile = *resourceSpecFile; + } + } + + // Apply explicitly specified ctest_test() options, + // overriding any conflicting preset values. if (!args.Start.empty() || !args.End.empty() || !args.Stride.empty()) { handler->TestOptions.TestsToRunInformation = cmStrCat(args.Start, ',', args.End, ',', args.Stride); @@ -56,10 +135,12 @@ std::unique_ptr cmCTestTestCommand::InitializeHandler( handler->TestOptions.IncludeRegularExpression = args.Include; } if (!args.ExcludeLabel.empty()) { + handler->TestOptions.ExcludeLabelRegularExpression.clear(); handler->TestOptions.ExcludeLabelRegularExpression.push_back( args.ExcludeLabel); } if (!args.IncludeLabel.empty()) { + handler->TestOptions.LabelRegularExpression.clear(); handler->TestOptions.LabelRegularExpression.push_back(args.IncludeLabel); } diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 00297f6414..ed054ca52b 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -48,6 +48,7 @@ protected: std::string OutputJUnit; std::string CoverageTool; bool StopOnFailure = false; + std::string Preset; }; template @@ -76,7 +77,8 @@ protected: .Bind("RESOURCE_SPEC_FILE"_s, &TestArguments::ResourceSpecFile) .Bind("STOP_ON_FAILURE"_s, &TestArguments::StopOnFailure) .Bind("COVERAGE_TOOL"_s, &TestArguments::CoverageTool) - .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit); + .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit) + .Bind("PRESET"_s, &TestArguments::Preset); } private: diff --git a/Tests/RunCMake/ctest_memcheck/CMakePresets.json.in b/Tests/RunCMake/ctest_memcheck/CMakePresets.json.in new file mode 100644 index 0000000000..c6b2174b94 --- /dev/null +++ b/Tests/RunCMake/ctest_memcheck/CMakePresets.json.in @@ -0,0 +1,26 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "my-configure-preset", + "generator": "@RunCMake_GENERATOR@", + "binaryDir": "${sourceDir}/build" + } + ], + "testPresets": [ + { + "name": "my-include-preset", + "configurePreset": "my-configure-preset", + "filter": { + "include": { + "name": "^test1$" + } + } + } + ] +} diff --git a/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake b/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake index 3f6501efaa..bae39a9cd1 100644 --- a/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake @@ -198,3 +198,19 @@ run_mc_test(DummyCudaSanitizer "${PSEUDO_CUDA_SANITIZER}") unset(CTEST_MEMCHECK_ARGS) unset(CTEST_SUFFIX_CODE) unset(CTEST_EXTRA_CODE) + +#----------------------------------------------------------------------------- +block() + set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/TestPresetInclude") + set(CMAKELISTS_EXTRA_CODE [[ +foreach(i RANGE 1 3) + add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) +endforeach() +]]) + configure_file( + "${RunCMake_SOURCE_DIR}/CMakePresets.json.in" + "${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json" + @ONLY) + set(CTEST_MEMCHECK_ARGS "PRESET my-include-preset") + run_mc_test(TestPresetInclude "${PSEUDO_VALGRIND}") +endblock() diff --git a/Tests/RunCMake/ctest_memcheck/TestPresetInclude-stdout.txt b/Tests/RunCMake/ctest_memcheck/TestPresetInclude-stdout.txt new file mode 100644 index 0000000000..7ec28f0648 --- /dev/null +++ b/Tests/RunCMake/ctest_memcheck/TestPresetInclude-stdout.txt @@ -0,0 +1,8 @@ +1/1 MemCheck #[0-9]+: test1 .+ Passed +[0-9]+\.[0-9]+ sec + +100% tests passed out of 1 +.* +-- Processing memory checking output: +MemCheck log files can be found here:.*corresponds to test number. +.*MemoryChecker.*log +Memory checking results: diff --git a/Tests/RunCMake/ctest_test/CMakePresets.json.in b/Tests/RunCMake/ctest_test/CMakePresets.json.in new file mode 100644 index 0000000000..3d0d3d0861 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CMakePresets.json.in @@ -0,0 +1,35 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "my-configure-preset", + "generator": "@RunCMake_GENERATOR@", + "binaryDir": "${sourceDir}/build" + } + ], + "testPresets": [ + { + "name": "my-include-preset", + "configurePreset": "my-configure-preset", + "filter": { + "include": { + "name": "^test1$" + } + } + }, + { + "name": "my-exclude-preset", + "configurePreset": "my-configure-preset", + "filter": { + "exclude": { + "name": "^test3$" + } + } + } + ] +} diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 9f014d0d62..32ee9cc123 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -13,6 +13,29 @@ endfunction() run_ctest_test(TestQuiet QUIET) +block() + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +foreach(i RANGE 1 3) + add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) +endforeach() +]]) + foreach(case IN ITEMS + TestPresetBadName + TestPresetExclude + TestPresetInclude + TestPresetOverride + ) + configure_file( + "${RunCMake_SOURCE_DIR}/CMakePresets.json.in" + "${RunCMake_BINARY_DIR}/${case}/CMakePresets.json" + @ONLY) + endforeach() + run_ctest_test(TestPresetBadName PRESET nonexistent-preset) + run_ctest_test(TestPresetExclude PRESET my-exclude-preset) + run_ctest_test(TestPresetInclude PRESET my-include-preset) + run_ctest_test(TestPresetOverride PRESET my-include-preset INCLUDE test2) +endblock() + set(CASE_CMAKELISTS_SUFFIX_CODE [[ foreach(i RANGE 1 4) add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) diff --git a/Tests/RunCMake/ctest_test/TestPresetBadName-result.txt b/Tests/RunCMake/ctest_test/TestPresetBadName-result.txt new file mode 100644 index 0000000000..b57e2deb77 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestPresetBadName-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestPresetBadName-stderr.txt b/Tests/RunCMake/ctest_test/TestPresetBadName-stderr.txt new file mode 100644 index 0000000000..a986c2817c --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestPresetBadName-stderr.txt @@ -0,0 +1,4 @@ +Cannot instantiate test handler ctest_test +CMake Error at [^ +]*/test\.cmake:[0-9]+ \(ctest_test\): + ctest_test No such test preset diff --git a/Tests/RunCMake/ctest_test/TestPresetExclude-stdout.txt b/Tests/RunCMake/ctest_test/TestPresetExclude-stdout.txt new file mode 100644 index 0000000000..271af6443d --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestPresetExclude-stdout.txt @@ -0,0 +1,6 @@ + Start [0-9]+: test1 +[0-9]+/3 Test #[0-9]+: test1 \.+ Passed +[0-9\.]+ sec + Start [0-9]+: test2 +[0-9]+/3 Test #[0-9]+: test2 \.+ Passed +[0-9\.]+ sec ++ +100% tests passed out of 3 diff --git a/Tests/RunCMake/ctest_test/TestPresetInclude-stdout.txt b/Tests/RunCMake/ctest_test/TestPresetInclude-stdout.txt new file mode 100644 index 0000000000..a4bf544cfd --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestPresetInclude-stdout.txt @@ -0,0 +1,4 @@ + Start [0-9]+: test1 +1/1 Test #[0-9]+: test1 \.+ Passed +[0-9\.]+ sec ++ +100% tests passed out of 1 diff --git a/Tests/RunCMake/ctest_test/TestPresetOverride-stdout.txt b/Tests/RunCMake/ctest_test/TestPresetOverride-stdout.txt new file mode 100644 index 0000000000..3529cf662b --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestPresetOverride-stdout.txt @@ -0,0 +1,4 @@ + Start [0-9]+: test2 +1/1 Test #[0-9]+: test2 \.+ Passed +[0-9\.]+ sec ++ +100% tests passed out of 1