ctest: add PRESET option for ctest_test() and ctest_memcheck()

Allow users to specify a test preset when using CTest dashboard scripts.

Fixes #27630

Help/command/ctest_memcheck.rst

@@ -26,6 +26,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`.
                  [CAPTURE_CMAKE_ERROR <result-var>]
                  [REPEAT <mode>:<n>]
                  [OUTPUT_JUNIT <file>]
+                 [PRESET <preset>]
                  [DEFECT_COUNT <defect-count-var>]
                  [QUIET]
                  )

Help/command/ctest_test.rst

@@ -28,6 +28,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [CAPTURE_CMAKE_ERROR <result-var>]
              [REPEAT <mode>:<n>]
              [OUTPUT_JUNIT <file>]
+             [PRESET <preset>]
              [QUIET]
              )
 
@@ -190,6 +191,15 @@ The options are:
   of the tests.  See the CTest :ref:`CoverageTool <ctest-CoverageTool>`
   setting for details.
 
+``PRESET <preset>``
+  .. versionadded:: 4.4
+
+  Specify a :manual:`preset <cmake-presets(7)>` to use when running tests.
+  Any value set in the CTest script will take priority over a corresponding
+  setting from the preset.  For example, the ``INCLUDE`` argument will override
+  the :preset:`filter.include.name <testPresets.filter.include.name>` setting
+  from the chosen preset.
+
 ``QUIET``
   .. versionadded:: 3.3
 

Help/release/dev/ctest-script-preset-arg.rst

@@ -0,0 +1,7 @@
+ctest-script-preset-arg
+------------------------
+
+* The :command:`ctest_configure`,  :command:`ctest_build`,
+  :command:`ctest_test`, and :command:`ctest_memcheck` commands gained
+  ``PRESET`` arguments to support using :manual:`presets <cmake-presets(7)>`
+  for their :ref:`Dashboard Client` steps.

Source/CTest/cmCTestTestCommand.cxx

@@ -6,25 +6,60 @@
 #include <cstdlib>
 #include <ratio>
 #include <sstream>
+#include <string>
 #include <utility>
 #include <vector>
 
 #include <cm/memory>
+#include <cm/optional>
 
+#include "cmArgumentParserTypes.h"
+#include "cmCMakePresetsGraph.h"
 #include "cmCTest.h"
 #include "cmCTestGenericHandler.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
 #include "cmExecutionStatus.h"
+#include "cmJSONState.h"
 #include "cmMakefile.h"
 #include "cmStringAlgorithms.h"
 #include "cmValue.h"
 
+using TestPreset = cmCMakePresetsGraph::TestPreset;
+
 std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
   HandlerArguments& arguments, cmExecutionStatus& status) const
 {
   cmMakefile& mf = status.GetMakefile();
   auto& args = static_cast<TestArguments&>(arguments);
+
+  std::unique_ptr<cmCMakePresetsGraph> presetsGraph;
+  TestPreset const* expandedPreset = nullptr;
+  if (!args.Preset.empty()) {
+    std::string const sourceDirectory =
+      mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY");
+
+    presetsGraph = cm::make_unique<cmCMakePresetsGraph>();
+    if (!presetsGraph->ReadProjectPresets(sourceDirectory, "")) {
+      status.SetError(
+        cmStrCat("Could not read presets from \"", sourceDirectory,
+                 "\": ", presetsGraph->parseState.GetErrorMessage()));
+      return nullptr;
+    }
+
+    auto resolveResult =
+      presetsGraph->ResolvePreset(args.Preset, presetsGraph->TestPresets);
+    auto resolveError = cmCMakePresetsGraph::FormatPresetError<TestPreset>(
+      resolveResult.StatusCode, resolveResult.ErrorPresetName,
+      sourceDirectory);
+    if (resolveError) {
+      status.SetError(*resolveError);
+      return nullptr;
+    }
+
+    expandedPreset = resolveResult.Preset;
+  }
+
   cmValue ctestTimeout = mf.GetDefinition("CTEST_TEST_TIMEOUT");
 
   cmDuration timeout;
@@ -39,12 +74,56 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
   }
   this->CTest->SetTimeOut(timeout);
 
-  cmValue resourceSpecFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE");
-  if (args.ResourceSpecFile.empty() && resourceSpecFile) {
-    args.ResourceSpecFile = *resourceSpecFile;
+  auto handler = this->InitializeActualHandler(args, status);
+
+  // Load settings from the preset if one was specified.
+  if (expandedPreset) {
+    cmCTestApplyTestPresetToOptions(handler->TestOptions, *expandedPreset);
+
+    if (!args.ParallelLevel && expandedPreset->Execution) {
+      if (auto const& jobs = expandedPreset->Execution->Jobs) {
+        ArgumentParser::Maybe<std::string> level;
+        if (jobs->has_value()) {
+          level = std::to_string(**jobs);
+        }
+        handler->ParallelLevel = level;
+      }
+    }
+
+    if (args.Repeat.empty() && expandedPreset->Execution &&
+        expandedPreset->Execution->Repeat) {
+      auto const& rep = *expandedPreset->Execution->Repeat;
+      using RepeatMode = TestPreset::ExecutionOptions::RepeatOptions::ModeEnum;
+      std::string modeStr;
+      switch (rep.Mode) {
+        case RepeatMode::UntilFail:
+          modeStr = "UNTIL_FAIL";
+          break;
+        case RepeatMode::UntilPass:
+          modeStr = "UNTIL_PASS";
+          break;
+        case RepeatMode::AfterTimeout:
+          modeStr = "AFTER_TIMEOUT";
+          break;
+      }
+      handler->Repeat = cmStrCat(modeStr, ':', rep.Count);
+    }
+
+    if (args.TestLoad.empty() && expandedPreset->Execution &&
+        expandedPreset->Execution->TestLoad) {
+      args.TestLoad = std::to_string(*expandedPreset->Execution->TestLoad);
+    }
   }
 
-  auto handler = this->InitializeActualHandler(args, status);
+  if (args.ResourceSpecFile.empty()) {
+    cmValue resourceSpecFile = mf.GetDefinition("CTEST_RESOURCE_SPEC_FILE");
+    if (resourceSpecFile) {
+      args.ResourceSpecFile = *resourceSpecFile;
+    }
+  }
+
+  // Apply explicitly specified ctest_test() options,
+  // overriding any conflicting preset values.
   if (!args.Start.empty() || !args.End.empty() || !args.Stride.empty()) {
     handler->TestOptions.TestsToRunInformation =
       cmStrCat(args.Start, ',', args.End, ',', args.Stride);
@@ -56,10 +135,12 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
     handler->TestOptions.IncludeRegularExpression = args.Include;
   }
   if (!args.ExcludeLabel.empty()) {
+    handler->TestOptions.ExcludeLabelRegularExpression.clear();
     handler->TestOptions.ExcludeLabelRegularExpression.push_back(
       args.ExcludeLabel);
   }
   if (!args.IncludeLabel.empty()) {
+    handler->TestOptions.LabelRegularExpression.clear();
     handler->TestOptions.LabelRegularExpression.push_back(args.IncludeLabel);
   }
 

Source/CTest/cmCTestTestCommand.h

@@ -48,6 +48,7 @@ protected:
     std::string OutputJUnit;
     std::string CoverageTool;
     bool StopOnFailure = false;
+    std::string Preset;
   };
 
   template <typename Args>
@@ -76,7 +77,8 @@ protected:
       .Bind("RESOURCE_SPEC_FILE"_s, &TestArguments::ResourceSpecFile)
       .Bind("STOP_ON_FAILURE"_s, &TestArguments::StopOnFailure)
       .Bind("COVERAGE_TOOL"_s, &TestArguments::CoverageTool)
-      .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit);
+      .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit)
+      .Bind("PRESET"_s, &TestArguments::Preset);
   }
 
 private:

Tests/RunCMake/ctest_memcheck/CMakePresets.json.in

@@ -0,0 +1,26 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "my-configure-preset",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "my-include-preset",
+      "configurePreset": "my-configure-preset",
+      "filter": {
+        "include": {
+          "name": "^test1$"
+        }
+      }
+    }
+  ]
+}

Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake

@@ -198,3 +198,19 @@ run_mc_test(DummyCudaSanitizer "${PSEUDO_CUDA_SANITIZER}")
 unset(CTEST_MEMCHECK_ARGS)
 unset(CTEST_SUFFIX_CODE)
 unset(CTEST_EXTRA_CODE)
+
+#-----------------------------------------------------------------------------
+block()
+  set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/TestPresetInclude")
+  set(CMAKELISTS_EXTRA_CODE [[
+foreach(i RANGE 1 3)
+  add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true)
+endforeach()
+]])
+  configure_file(
+    "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+    "${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json"
+    @ONLY)
+  set(CTEST_MEMCHECK_ARGS "PRESET my-include-preset")
+  run_mc_test(TestPresetInclude "${PSEUDO_VALGRIND}")
+endblock()

Tests/RunCMake/ctest_memcheck/TestPresetInclude-stdout.txt

@@ -0,0 +1,8 @@
+1/1 MemCheck #[0-9]+: test1 .+   Passed +[0-9]+\.[0-9]+ sec
+
+100% tests passed out of 1
+.*
+-- Processing memory checking output:
+MemCheck log files can be found here:.*corresponds to test number.
+.*MemoryChecker.*log
+Memory checking results:

Tests/RunCMake/ctest_test/CMakePresets.json.in

@@ -0,0 +1,35 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "my-configure-preset",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "my-include-preset",
+      "configurePreset": "my-configure-preset",
+      "filter": {
+        "include": {
+          "name": "^test1$"
+        }
+      }
+    },
+    {
+      "name": "my-exclude-preset",
+      "configurePreset": "my-configure-preset",
+      "filter": {
+        "exclude": {
+          "name": "^test3$"
+        }
+      }
+    }
+  ]
+}

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -13,6 +13,29 @@ endfunction()
 
 run_ctest_test(TestQuiet QUIET)
 
+block()
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+foreach(i RANGE 1 3)
+  add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true)
+endforeach()
+]])
+  foreach(case IN ITEMS
+    TestPresetBadName
+    TestPresetExclude
+    TestPresetInclude
+    TestPresetOverride
+  )
+    configure_file(
+      "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+      "${RunCMake_BINARY_DIR}/${case}/CMakePresets.json"
+      @ONLY)
+  endforeach()
+  run_ctest_test(TestPresetBadName PRESET nonexistent-preset)
+  run_ctest_test(TestPresetExclude PRESET my-exclude-preset)
+  run_ctest_test(TestPresetInclude PRESET my-include-preset)
+  run_ctest_test(TestPresetOverride PRESET my-include-preset INCLUDE test2)
+endblock()
+
 set(CASE_CMAKELISTS_SUFFIX_CODE [[
 foreach(i RANGE 1 4)
   add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true)

Tests/RunCMake/ctest_test/TestPresetBadName-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_test/TestPresetBadName-stderr.txt

@@ -0,0 +1,4 @@
+Cannot instantiate test handler ctest_test
+CMake Error at [^
+]*/test\.cmake:[0-9]+ \(ctest_test\):
+  ctest_test No such test preset

Tests/RunCMake/ctest_test/TestPresetExclude-stdout.txt

@@ -0,0 +1,6 @@
+    Start [0-9]+: test1
+[0-9]+/3 Test #[0-9]+: test1 \.+   Passed +[0-9\.]+ sec
+    Start [0-9]+: test2
+[0-9]+/3 Test #[0-9]+: test2 \.+   Passed +[0-9\.]+ sec
++
+100% tests passed out of 3

Tests/RunCMake/ctest_test/TestPresetInclude-stdout.txt

@@ -0,0 +1,4 @@
+    Start [0-9]+: test1
+1/1 Test #[0-9]+: test1 \.+   Passed +[0-9\.]+ sec
++
+100% tests passed out of 1

Tests/RunCMake/ctest_test/TestPresetOverride-stdout.txt

@@ -0,0 +1,4 @@
+    Start [0-9]+: test2
+1/1 Test #[0-9]+: test2 \.+   Passed +[0-9\.]+ sec
++
+100% tests passed out of 1