ctest: add PRESETS_FILE option to CTest commands

Allow CTest script authors to use presets from arbitrary files.

Help/command/ctest_build.rst

@@ -16,6 +16,7 @@ Perform the :ref:`CTest Build Step` as a :ref:`Dashboard Client`.
               [RETURN_VALUE <result-var>]
               [CAPTURE_CMAKE_ERROR <result-var>]
               [PRESET <preset>]
+              [PRESETS_FILE <file>]
               )
 
 Build the project and store results in ``Build.xml``
@@ -85,6 +86,14 @@ The options are:
   argument will override the :preset:`buildPresets.targets` setting from
   the chosen preset.
 
+``PRESETS_FILE <file>``
+  .. versionadded:: 4.4
+
+  Specify a :manual:`presets <cmake-presets(7)>` file to use instead of the
+  default ``CMakePresets.json`` in the source directory.
+  A relative path is interpreted relative to the source directory.
+  Has no effect unless ``PRESET`` is also specified.
+
 ``RETURN_VALUE <result-var>``
   Store the return value of the native build tool in the given variable.
 

Help/command/ctest_configure.rst

@@ -9,6 +9,7 @@ Perform the :ref:`CTest Configure Step` as a :ref:`Dashboard Client`.
                   [OPTIONS <options>] [RETURN_VALUE <result-var>] [QUIET]
                   [CAPTURE_CMAKE_ERROR <result-var>]
                   [PRESET <preset>]
+                  [PRESETS_FILE <file>]
                   )
 
 Configure the project build tree and record results in ``Configure.xml``
@@ -48,6 +49,14 @@ The options are:
   This option is ignored when :variable:`CTEST_CONFIGURE_COMMAND`
   is used.
 
+``PRESETS_FILE <file>``
+  .. versionadded:: 4.4
+
+  Specify a :manual:`presets <cmake-presets(7)>` file to use instead of the
+  default ``CMakePresets.json`` in the source directory.
+  A relative path is interpreted relative to the source directory.
+  Has no effect unless ``PRESET`` is also specified.
+
 ``RETURN_VALUE <result-var>``
   Store in the ``<result-var>`` variable the return value of the native
   configuration tool.

Help/command/ctest_memcheck.rst

@@ -27,6 +27,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`.
                  [REPEAT <mode>:<n>]
                  [OUTPUT_JUNIT <file>]
                  [PRESET <preset>]
+                 [PRESETS_FILE <path>]
                  [DEFECT_COUNT <defect-count-var>]
                  [QUIET]
                  )

Help/command/ctest_test.rst

@@ -29,6 +29,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [REPEAT <mode>:<n>]
              [OUTPUT_JUNIT <file>]
              [PRESET <preset>]
+             [PRESETS_FILE <file>]
              [QUIET]
              )
 
@@ -200,6 +201,14 @@ The options are:
   the :preset:`filter.include.name <testPresets.filter.include.name>` setting
   from the chosen preset.
 
+``PRESETS_FILE <file>``
+  .. versionadded:: 4.4
+
+  Specify a :manual:`presets <cmake-presets(7)>` file to use instead of the
+  default ``CMakePresets.json`` in the source directory.
+  A relative path is interpreted relative to the source directory.
+  Has no effect unless ``PRESET`` is also specified.
+
 ``QUIET``
   .. versionadded:: 3.3
 

Help/release/dev/ctest-script-preset-arg.rst

@@ -3,5 +3,5 @@ ctest-script-preset-arg
 
 * The :command:`ctest_configure`,  :command:`ctest_build`,
   :command:`ctest_test`, and :command:`ctest_memcheck` commands gained
-  ``PRESET`` arguments to support using :manual:`presets <cmake-presets(7)>`
+  ``PRESET`` and ``PRESETS_FILE`` arguments to support using
-  for their :ref:`Dashboard Client` steps.
+  :manual:`presets <cmake-presets(7)>` for their :ref:`Dashboard Client` steps.

Source/CTest/cmCTestBuildCommand.cxx

@@ -38,7 +38,8 @@ bool cmCTestBuildCommand::InitialPass(std::vector<std::string> const& args,
       .Bind("FLAGS"_s, &BuildArguments::Flags)
       .Bind("PROJECT_NAME"_s, &BuildArguments::ProjectName)
       .Bind("PARALLEL_LEVEL"_s, &BuildArguments::ParallelLevel)
-      .Bind("PRESET"_s, &BuildArguments::Preset);
+      .Bind("PRESET"_s, &BuildArguments::Preset)
+      .Bind("PRESETS_FILE"_s, &BuildArguments::PresetsFile);
 
   return this->Invoke(parser, args, status, [&](BuildArguments& a) {
     return this->ExecuteHandlerCommand(a, status);
@@ -80,11 +81,15 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestBuildCommand::InitializeHandler(
     std::string const sourceDirectory =
       mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY");
 
+    std::string const presetsFile = args.PresetsFile.empty()
+      ? ""
+      : cmSystemTools::CollapseFullPath(args.PresetsFile, sourceDirectory);
+
     cmCMakePresetsGraph presetsGraph;
-    if (!presetsGraph.ReadProjectPresets(sourceDirectory, "")) {
+    if (!presetsGraph.ReadProjectPresets(sourceDirectory, presetsFile)) {
-      status.SetError(
+      status.SetError(cmStrCat("\n Could not read presets from \"",
-        cmStrCat("Could not read presets from \"", sourceDirectory,
+                               sourceDirectory, "\":\n ",
-                 "\": ", presetsGraph.parseState.GetErrorMessage()));
+                               presetsGraph.parseState.GetErrorMessage()));
       return nullptr;
     }
 
@@ -104,6 +109,12 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestBuildCommand::InitializeHandler(
     buildCommand += args.Preset;
     buildCommand += "\"";
 
+    if (!presetsFile.empty()) {
+      buildCommand += " --presets-file \"";
+      buildCommand += presetsFile;
+      buildCommand += "\"";
+    }
+
     if (!cmakeBuildConfiguration.empty()) {
       buildCommand += " --config \"";
       buildCommand += cmakeBuildConfiguration;

Source/CTest/cmCTestBuildCommand.h

@@ -24,6 +24,7 @@ protected:
     std::string ProjectName;
     std::string ParallelLevel;
     std::string Preset;
+    std::string PresetsFile;
   };
 
 private:

Source/CTest/cmCTestConfigureCommand.cxx

@@ -39,6 +39,7 @@ bool ConstructConfigureCommand(cmExecutionStatus& status, cmMakefile& mf,
                                std::string const buildDirectory,
                                std::string const options,
                                std::string const presetName,
+                               std::string const presetsFile,
                                std::string& configureCommand)
 {
   configureCommand = cmStrCat('"', cmSystemTools::GetCMakeCommand(), '"');
@@ -64,10 +65,10 @@ bool ConstructConfigureCommand(cmExecutionStatus& status, cmMakefile& mf,
 
   if (!presetName.empty()) {
     cmCMakePresetsGraph presetsGraph;
-    if (!presetsGraph.ReadProjectPresets(sourceDirectory, "")) {
+    if (!presetsGraph.ReadProjectPresets(sourceDirectory, presetsFile)) {
-      status.SetError(
+      status.SetError(cmStrCat("\n Could not read presets from \"",
-        cmStrCat("Could not read presets from \"", sourceDirectory,
+                               sourceDirectory, "\":\n ",
-                 "\": ", presetsGraph.parseState.GetErrorMessage()));
+                               presetsGraph.parseState.GetErrorMessage()));
       return false;
     }
 
@@ -89,6 +90,13 @@ bool ConstructConfigureCommand(cmExecutionStatus& status, cmMakefile& mf,
     configureCommand += presetName;
     configureCommand += "\"";
 
+    if (!presetsFile.empty()) {
+      configureCommand += " \"--presets-file\"";
+      configureCommand += " \"";
+      configureCommand += presetsFile;
+      configureCommand += "\"";
+    }
+
     if (!expandedPreset->BinaryDir.empty()) {
       presetProvidesBuildDir = true;
     }
@@ -183,10 +191,14 @@ bool cmCTestConfigureCommand::ExecuteConfigure(ConfigureArguments const& args,
     return false;
   }
 
+  std::string const presetsFile = args.PresetsFile.empty()
+    ? ""
+    : cmSystemTools::CollapseFullPath(args.PresetsFile, sourceDirectory);
+
   std::string configureCommand = mf.GetDefinition("CTEST_CONFIGURE_COMMAND");
   if (configureCommand.empty() &&
       !ConstructConfigureCommand(status, mf, sourceDirectory, buildDirectory,
-                                 args.Options, args.Preset,
+                                 args.Options, args.Preset, presetsFile,
                                  configureCommand)) {
     return false;
   }
@@ -291,7 +303,8 @@ bool cmCTestConfigureCommand::InitialPass(std::vector<std::string> const& args,
   static auto const parser =
     cmArgumentParser<Args>{ MakeHandlerParser<Args>() } //
       .Bind("OPTIONS"_s, &ConfigureArguments::Options)
-      .Bind("PRESET"_s, &ConfigureArguments::Preset);
+      .Bind("PRESET"_s, &ConfigureArguments::Preset)
+      .Bind("PRESETS_FILE"_s, &ConfigureArguments::PresetsFile);
 
   return this->Invoke(parser, args, status, [&](ConfigureArguments& a) {
     return this->ExecuteConfigure(a, status);

Source/CTest/cmCTestConfigureCommand.h

@@ -20,6 +20,7 @@ protected:
   {
     std::string Options;
     std::string Preset;
+    std::string PresetsFile;
   };
 
 private:

Source/CTest/cmCTestTestCommand.cxx

@@ -23,6 +23,7 @@
 #include "cmJSONState.h"
 #include "cmMakefile.h"
 #include "cmStringAlgorithms.h"
+#include "cmSystemTools.h"
 #include "cmValue.h"
 
 using TestPreset = cmCMakePresetsGraph::TestPreset;
@@ -39,11 +40,15 @@ std::unique_ptr<cmCTestGenericHandler> cmCTestTestCommand::InitializeHandler(
     std::string const sourceDirectory =
       mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY");
 
+    std::string const presetsFile = args.PresetsFile.empty()
+      ? ""
+      : cmSystemTools::CollapseFullPath(args.PresetsFile, sourceDirectory);
+
     presetsGraph = cm::make_unique<cmCMakePresetsGraph>();
-    if (!presetsGraph->ReadProjectPresets(sourceDirectory, "")) {
+    if (!presetsGraph->ReadProjectPresets(sourceDirectory, presetsFile)) {
-      status.SetError(
+      status.SetError(cmStrCat("\n Could not read presets from \"",
-        cmStrCat("Could not read presets from \"", sourceDirectory,
+                               sourceDirectory, "\":\n ",
-                 "\": ", presetsGraph->parseState.GetErrorMessage()));
+                               presetsGraph->parseState.GetErrorMessage()));
       return nullptr;
     }
 

Source/CTest/cmCTestTestCommand.h

@@ -49,6 +49,7 @@ protected:
     std::string CoverageTool;
     bool StopOnFailure = false;
     std::string Preset;
+    std::string PresetsFile;
   };
 
   template <typename Args>
@@ -78,7 +79,8 @@ protected:
       .Bind("STOP_ON_FAILURE"_s, &TestArguments::StopOnFailure)
       .Bind("COVERAGE_TOOL"_s, &TestArguments::CoverageTool)
       .Bind("OUTPUT_JUNIT"_s, &TestArguments::OutputJUnit)
-      .Bind("PRESET"_s, &TestArguments::Preset);
+      .Bind("PRESET"_s, &TestArguments::Preset)
+      .Bind("PRESETS_FILE"_s, &TestArguments::PresetsFile);
   }
 
 private:

Tests/RunCMake/ctest_build/BuildPresetBadFile-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_build/BuildPresetBadFile-stderr.txt

@@ -0,0 +1 @@
+File not found: /nonexistent/path/presets.json$

Tests/RunCMake/ctest_build/BuildPresetFromFile-check.cmake

@@ -0,0 +1,14 @@
+file(GLOB build_xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/Build.xml")
+if(build_xml_file)
+  file(READ "${build_xml_file}" build_xml)
+  if(NOT build_xml MATCHES "--preset.*my-build-preset")
+    set(RunCMake_TEST_FAILED
+        "Build.xml does not contain the expected --preset argument")
+  endif()
+  if(NOT build_xml MATCHES "--presets-file")
+    set(RunCMake_TEST_FAILED
+        "Build.xml does not contain the expected --presets-file argument")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "Build.xml not found")
+endif()

Tests/RunCMake/ctest_build/RunCMakeTest.cmake

@@ -80,6 +80,23 @@ configure_file(
 run_ctest_build(BuildPreset PRESET my-build-preset)
 unset(RunCMake_TEST_SOURCE_DIR)
 
+set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/BuildPresetFromFile")
+set(custom_presets_file
+  "${RunCMake_BINARY_DIR}/BuildPresetFromFile/custom-presets.json")
+configure_file(
+  "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+  "${custom_presets_file}"
+  @ONLY)
+run_ctest_build(BuildPresetFromFile
+  PRESET my-build-preset
+  PRESETS_FILE "${custom_presets_file}")
+unset(RunCMake_TEST_SOURCE_DIR)
+unset(custom_presets_file)
+
+run_ctest_build(BuildPresetBadFile
+  PRESET my-build-preset
+  PRESETS_FILE /nonexistent/path/presets.json)
+
 set(RunCMake_USE_CUSTOM_BUILD_COMMAND FALSE)
 if(RunCMake_GENERATOR MATCHES "Ninja")
   function(run_NinjaLauncherSingleBuildFailure)

Tests/RunCMake/ctest_configure/ConfigurePresetFromFile-check.cmake

@@ -0,0 +1,29 @@
+if(IS_DIRECTORY "${RunCMake_TEST_SOURCE_DIR}/build")
+ set(RunCMake_TEST_FAILED
+     "CTEST_BINARY_DIRECTORY did not override buildDir from preset")
+endif()
+
+file(GLOB configure_xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/Configure.xml")
+if(configure_xml_file)
+  file(READ "${configure_xml_file}" configure_xml)
+  if(NOT configure_xml MATCHES "\"--preset\" \"my-preset\"")
+     set(RunCMake_TEST_FAILED
+         "Configure.xml does not contain the expected --preset argument")
+  endif()
+  if(NOT configure_xml MATCHES "\"--presets-file\"")
+     set(RunCMake_TEST_FAILED
+         "Configure.xml does not contain the expected --presets-file argument")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "Configure.xml not found")
+endif()
+
+set(cmakecache_file "${RunCMake_TEST_BINARY_DIR}/CMakeCache.txt")
+if(EXISTS "${cmakecache_file}")
+  file(READ "${cmakecache_file}" cmakecache_txt)
+  if(NOT cmakecache_txt MATCHES "MY_CUSTOM_VAR:STRING=this-gets-set")
+     set(RunCMake_TEST_FAILED "CMakeCache.txt does not contain MY_CUSTOM_VAR")
+  endif()
+else()
+  set(RunCMake_TEST_FAILED "CMakeCache.txt not found")
+endif()

Tests/RunCMake/ctest_configure/RunCMakeTest.cmake

@@ -15,3 +15,12 @@ configure_file(
   "${RunCMake_TEST_SOURCE_DIR}/CMakePresets.json"
   @ONLY)
 run_ctest_configure(ConfigurePreset PRESET my-preset)
+
+set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/ConfigurePresetFromFile")
+set(custom_presets_file "${RunCMake_BINARY_DIR}/ConfigurePresetFromFile/custom-presets.json")
+configure_file(
+  "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+  "${custom_presets_file}"
+  @ONLY)
+run_ctest_configure(ConfigurePresetFromFile PRESET my-preset PRESETS_FILE "${custom_presets_file}")
+unset(custom_presets_file)

Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake

@@ -214,3 +214,27 @@ endforeach()
   set(CTEST_MEMCHECK_ARGS "PRESET my-include-preset")
   run_mc_test(TestPresetInclude "${PSEUDO_VALGRIND}")
 endblock()
+
+block()
+  set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/TestPresetFromFile")
+  set(custom_presets_file
+    "${RunCMake_BINARY_DIR}/TestPresetFromFile/custom-presets.json")
+  set(CMAKELISTS_EXTRA_CODE [[
+foreach(i RANGE 1 3)
+  add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true)
+endforeach()
+]])
+  configure_file(
+    "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+    "${custom_presets_file}"
+    @ONLY)
+  set(CTEST_MEMCHECK_ARGS
+    "PRESET my-include-preset PRESETS_FILE \"${custom_presets_file}\"")
+  run_mc_test(TestPresetFromFile "${PSEUDO_VALGRIND}")
+endblock()
+
+block()
+  set(CTEST_MEMCHECK_ARGS
+    "PRESET my-include-preset PRESETS_FILE /nonexistent/path/presets.json")
+  run_mc_test(TestPresetBadFile "${PSEUDO_VALGRIND}")
+endblock()

Tests/RunCMake/ctest_memcheck/TestPresetBadFile-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_memcheck/TestPresetBadFile-stderr.txt

@@ -0,0 +1 @@
+File not found: /nonexistent/path/presets.json$

Tests/RunCMake/ctest_memcheck/TestPresetFromFile-stdout.txt

@@ -0,0 +1,8 @@
+1/1 MemCheck #[0-9]+: test1 .+   Passed +[0-9]+\.[0-9]+ sec
+
+100% tests passed out of 1
+.*
+-- Processing memory checking output:
+MemCheck log files can be found here:.*corresponds to test number.
+.*MemoryChecker.*log
+Memory checking results:

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -34,6 +34,23 @@ endforeach()
   run_ctest_test(TestPresetExclude PRESET my-exclude-preset)
   run_ctest_test(TestPresetInclude PRESET my-include-preset)
   run_ctest_test(TestPresetOverride PRESET my-include-preset INCLUDE test2)
+
+  set(custom_presets_file
+    "${RunCMake_BINARY_DIR}/TestPresetFileInclude/custom-presets.json")
+  configure_file(
+    "${RunCMake_SOURCE_DIR}/CMakePresets.json.in"
+    "${custom_presets_file}"
+    @ONLY)
+  set(CASE_CTEST_TEST_RAW_ARGS
+    "PRESET my-include-preset PRESETS_FILE \"${custom_presets_file}\"")
+  run_ctest(TestPresetFileInclude)
+  unset(CASE_CTEST_TEST_RAW_ARGS)
+  unset(custom_presets_file)
+
+  set(CASE_CTEST_TEST_RAW_ARGS
+    "PRESET my-include-preset PRESETS_FILE /nonexistent/path/presets.json")
+  run_ctest(TestPresetBadFile)
+  unset(CASE_CTEST_TEST_RAW_ARGS)
 endblock()
 
 set(CASE_CMAKELISTS_SUFFIX_CODE [[

Tests/RunCMake/ctest_test/TestPresetBadFile-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_test/TestPresetBadFile-stderr.txt

@@ -0,0 +1 @@
+File not found: /nonexistent/path/presets.json$

Tests/RunCMake/ctest_test/TestPresetFileInclude-stdout.txt

@@ -0,0 +1,4 @@
+    Start [0-9]+: test1
+1/1 Test #[0-9]+: test1 \.+   Passed +[0-9\.]+ sec
++
+100% tests passed out of 1