ctest: Add --out-of-date option for running out of date tests

Add an --out-of-date option to ctest which runs only those tests with
recorded build dependencies whose timestamps are newer than the timestamp
at which the test last finished.

Fixes: #27614

Help/command/add_test.rst

@@ -111,6 +111,12 @@ directory the test is created in.
   Generators`. The test name must be a valid target name in order to list
   build dependencies with this keyword.
 
+  * .. versionchanged:: 4.5
+
+    Build dependencies added by this argument, or referenced in the test
+    ``COMMAND`` also enable the :option:`--out-of-date <ctest --out-of-date>`
+    behavior of :manual:`ctest(1)`.
+
 ``COMMAND_EXPAND_LISTS``
   .. versionadded:: 3.16
 

Help/manual/ctest.1.rst

@@ -590,6 +590,25 @@ The options for running tests are:
  :preset:`testPresets.execution.testPassthroughArguments` are provided,
  the test preset arguments will appear first, followed by the ``<test-args>``.
 
+.. option:: --out-of-date
+
+ .. versionadded:: 4.5
+
+ Run only tests whose build dependencies are newer than the last time the
+ test ran.
+
+ This option tells CTest to skip tests that are already up to date with
+ respect to their recorded build dependencies. Build dependencies include
+ executables and targets in generator expressions as part of the test
+ ``COMMAND``, as well as the outputs of targets or files added as explicit
+ dependencies with the ``BUILD_DEPENDS`` argument of :command:`add_test`.
+
+ A test is selected to run when any of its recorded build dependencies are
+ newer than the test's last-run timestamp, or when the test has not been
+ run before. Tests without any known build dependencies, including any tests
+ not added by the :command:`add_test` command, are excluded when this argument
+ is provided.
+
 View Help
 =========
 

Help/release/dev/ctest-out-of-date.rst

@@ -0,0 +1,6 @@
+ctest-out-of-date
+-----------------
+
+* :manual:`ctest(1)` gained a :option:`--out-of-date <ctest --out-of-date>`
+  option to run only tests whose recorded build dependencies are newer than
+  the last time the tests ran.

Source/CTest/cmCTestRunTest.cxx

@@ -375,6 +375,12 @@ cmCTestRunTest::EndTestResult cmCTestRunTest::EndTest(size_t completed,
     this->TestResult.ExecutionTime = this->TestProcess->GetTotalTime();
     this->MemCheckPostProcess();
     this->ComputeWeightedCost();
+
+    std::string const stampDir = this->CTest->GetStampDir();
+    cmSystemTools::MakeDirectory(stampDir);
+    std::string const stampFile =
+      stampDir + "/" + this->TestProperties->GetStampFile();
+    cmSystemTools::Touch(stampFile, true);
   }
   // If the test does not need to rerun push the current TestResult onto the
   // TestHandler vector

Source/CTest/cmCTestTestHandler.cxx

@@ -41,6 +41,7 @@
 #include "cmCTestMultiProcessHandler.h"
 #include "cmCTestResourceGroupsLexerHelper.h"
 #include "cmCTestTestMeasurementXMLParser.h"
+#include "cmCryptoHash.h"
 #include "cmDuration.h"
 #include "cmExecutionStatus.h"
 #include "cmGeneratedFileStream.h"
@@ -967,6 +968,9 @@ bool cmCTestTestHandler::ComputeTestList()
   if (this->TestOptions.RerunFailed) {
     return this->ComputeTestListForRerunFailed();
   }
+  if (this->TestOptions.OutOfDateOnly) {
+    this->ComputeOutOfDateTests();
+  }
 
   cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
   // how many tests are in based on RegExp?
@@ -1069,6 +1073,49 @@ bool cmCTestTestHandler::ComputeTestListForRerunFailed()
   return true;
 }
 
+void cmCTestTestHandler::ComputeOutOfDateTests()
+{
+  ListOfTests finalList;
+  std::string const stampDir = this->CTest->GetStampDir();
+  cmSystemTools::MakeDirectory(stampDir);
+
+  for (cmCTestTestProperties& tp : this->TestList) {
+    if (tp.BuildDepends.empty()) {
+      continue;
+    }
+
+    std::string const stampFile = stampDir + "/" + tp.GetStampFile();
+
+    if (!cmSystemTools::FileExists(stampFile)) {
+      finalList.push_back(tp);
+      continue;
+    }
+
+    cmList deps{ tp.BuildDepends };
+    bool outOfDate = false;
+    for (std::string const& dep : deps) {
+      if (dep.empty()) {
+        continue;
+      }
+      if (!cmSystemTools::FileExists(dep)) {
+        // If any dependencies don't exist, skip the test
+        outOfDate = false;
+        break;
+      }
+      int result = 0;
+      cmSystemTools::FileTimeCompare(dep, stampFile, &result);
+      if (result >= 0) {
+        // At least one newer dependency, add to list
+        outOfDate = true;
+      }
+    }
+    if (outOfDate) {
+      finalList.push_back(tp);
+    }
+  }
+  this->TestList = finalList;
+}
+
 void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
 {
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
@@ -2298,6 +2345,11 @@ void cmCTestTestHandler::cmCTestTestProperties::AppendError(
   }
 }
 
+std::string cmCTestTestHandler::cmCTestTestProperties::GetStampFile()
+{
+  return cmCryptoHash(cmCryptoHash::AlgoMD5).HashString(this->Name) + ".stamp";
+}
+
 bool cmCTestTestHandler::SetTestsProperties(
   std::vector<std::string> const& args)
 {
@@ -2498,6 +2550,8 @@ bool cmCTestTestHandler::SetTestsProperties(
                 rt.TimeoutRegularExpressions.emplace_back(cr, cr);
               }
             }
+          } else if (key == "_CMAKE_TEST_BUILD_DEPENDS"_s) {
+            rt.BuildDepends = val;
           } else {
             rt.CustomProperties[key] = val;
           }

Source/CTest/cmCTestTestHandler.h

@@ -31,6 +31,7 @@ class cmXMLWriter;
 struct cmCTestTestOptions
 {
   bool RerunFailed = false;
+  bool OutOfDateOnly = false;
   bool ScheduleRandom = false;
   bool StopOnFailure = false;
   bool UseUnion = false;
@@ -144,6 +145,7 @@ public:
   struct cmCTestTestProperties
   {
     void AppendError(cm::string_view err);
+    std::string GetStampFile();
     cm::optional<std::string> Error;
     std::string Name;
     // working directory for test, overridden by WORKING_DIRECTORY property
@@ -192,6 +194,7 @@ public:
     std::set<std::string> RequireSuccessDepends;
     std::vector<std::vector<cmCTestTestResourceRequirement>> ResourceGroups;
     std::string GeneratedResourceSpecFile;
+    std::string BuildDepends;
     // Private test generator properties used to track backtraces
     cmListFileBacktrace Backtrace;
   };
@@ -342,6 +345,7 @@ private:
   // compute the lists of tests that will actually run
   // based on LastTestFailed.log
   bool ComputeTestListForRerunFailed();
+  void ComputeOutOfDateTests();
 
   // add required setup/cleanup tests not already in the
   // list of tests to be run and update dependencies between

Source/cmCTest.cxx

@@ -2522,6 +2522,11 @@ int cmCTest::Run(std::vector<std::string> const& args)
                        this->Impl->TestOptions.RerunFailed = true;
                        return true;
                      } },
+    CommandArgument{ "--out-of-date", CommandArgument::Values::Zero,
+                     [this](std::string const&) -> bool {
+                       this->Impl->TestOptions.OutOfDateOnly = true;
+                       return true;
+                     } },
   };
 
   // Process command line arguments for presets first, since other arguments
@@ -3045,6 +3050,11 @@ std::string cmCTest::GetBinaryDir()
   return this->Impl->BinaryDir;
 }
 
+std::string cmCTest::GetStampDir()
+{
+  return this->Impl->BinaryDir + "/Testing/Temporary/LastTestRun";
+}
+
 std::string const& cmCTest::GetConfigType()
 {
   return this->Impl->ConfigType;

Source/cmCTest.h

@@ -253,6 +253,9 @@ public:
   /** Get the path to the build tree */
   std::string GetBinaryDir();
 
+  /** Get the path to the test stamp file tree */
+  std::string GetStampDir();
+
   /**
    * Get the short path to the file.
    *

Source/cmGlobalGenerator.cxx

@@ -3010,6 +3010,34 @@ bool cmGlobalGenerator::NameResolvesToFramework(
   return false;
 }
 
+std::vector<std::string> cmGlobalGenerator::GetTestBuildDependencyPaths(
+  std::string const& config,
+  cmTestGenerator::BuildDependencies const& deps) const
+{
+  std::set<std::string> uniqueDeps;
+  for (auto const& file : deps.Files) {
+    uniqueDeps.insert(file.Path);
+  }
+  for (cmGeneratorTarget* target : deps.Targets) {
+    if (target->GetType() == cmStateEnums::TargetType::UTILITY ||
+        target->GetType() == cmStateEnums::TargetType::GLOBAL_TARGET ||
+        target->GetType() == cmStateEnums::TargetType::INTERFACE_LIBRARY) {
+      continue;
+    }
+    if (target->GetType() == cmStateEnums::TargetType::OBJECT_LIBRARY) {
+      std::vector<std::string> objects;
+      target->GetTargetObjectNames(config, objects);
+      for (auto const& object : objects) {
+        uniqueDeps.insert(
+          cmStrCat(target->GetObjectDirectory(config), object));
+      }
+      continue;
+    }
+    uniqueDeps.insert(target->GetFullPath(config));
+  }
+  return { uniqueDeps.begin(), uniqueDeps.end() };
+}
+
 // If the file has no extension it's either a raw executable or might
 // be a direct reference to a binary within a framework (bad practice!).
 // This is where we change the path to point to the framework directory.

Source/cmGlobalGenerator.h

@@ -34,6 +34,7 @@
 #include "cmSystemTools.h"
 #include "cmTarget.h"
 #include "cmTargetDepend.h"
+#include "cmTestGenerator.h"
 #include "cmValue.h"
 #include "cmXcFramework.h"
 
@@ -130,6 +131,10 @@ public:
   //! Get the name for this generator
   virtual std::string GetName() const { return "Generic"; }
 
+  virtual std::vector<std::string> GetTestBuildDependencyPaths(
+    std::string const& config,
+    cmTestGenerator::BuildDependencies const& deps) const;
+
   /** Check whether the given name matches the current generator.  */
   virtual bool MatchesGeneratorName(std::string const& name) const
   {

Source/cmTestGenerator.cxx

@@ -303,6 +303,17 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
        << cmScriptGenerator::Quote(
             ge.Parse(i.second)->Evaluate(this->LG, config));
   }
+  BuildDependencies deps;
+  if (this->GetBuildDependencies(this->LG, deps)) {
+    cmList depList;
+    for (std::string const& dep :
+         this->LG->GetGlobalGenerator()->GetTestBuildDependencyPaths(config,
+                                                                     deps)) {
+      depList.append(dep);
+    }
+    os << " _CMAKE_TEST_BUILD_DEPENDS "
+       << cmScriptGenerator::Quote(depList.to_string());
+  }
   os << ' ';
   this->GenerateBacktrace(os, this->Test->GetBacktrace());
   os << ")\n";

Source/ctest.cxx

@@ -117,6 +117,8 @@ cmDocumentationEntry const cmDocumentationOptions[] = {
     "Run a specific number of tests by number." },
   { "-U, --union", "Take the Union of -I and -R" },
   { "--rerun-failed", "Run only the tests that failed previously" },
+  { "--out-of-date",
+    "Run only tests whose build dependencies changed since they last ran" },
   { "--tests-from-file <file>", "Run the tests listed in the given file" },
   { "--exclude-from-file <file>",
     "Run tests except those listed in the given file" },

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -117,6 +117,52 @@ add_test(fails \"${CMAKE_COMMAND}\" -E false)
   run_cmake_command(rerun-failed ${CMAKE_CTEST_COMMAND} --rerun-failed)
 endblock()
 
+block()
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/out-of-date)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  set(out_of_date_build_dir "${RunCMake_TEST_BINARY_DIR}/build")
+  set(out_of_date_config "Debug")
+  if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG)
+    set(out_of_date_config "")
+  endif()
+  set(out_of_date_build_config_args)
+  set(out_of_date_ctest_config_args)
+  if(out_of_date_config)
+    set(out_of_date_build_config_args --config ${out_of_date_config})
+    set(out_of_date_ctest_config_args -C ${out_of_date_config})
+  endif()
+  file(COPY "${RunCMake_SOURCE_DIR}/out-of-date-src"
+       DESTINATION "${RunCMake_TEST_BINARY_DIR}")
+  set(RunCMake_TEST_SOURCE_DIR "${RunCMake_TEST_BINARY_DIR}/out-of-date-src")
+  set(RunCMake_TEST_BINARY_DIR "${out_of_date_build_dir}")
+  run_cmake(out-of-date-configure)
+  run_cmake_command(out-of-date-build "${CMAKE_COMMAND}" --build "${out_of_date_build_dir}" ${out_of_date_build_config_args})
+  run_cmake_command(out-of-date-initial-test
+    "${CMAKE_CTEST_COMMAND}" ${out_of_date_ctest_config_args} -V --test-dir "${out_of_date_build_dir}")
+
+  # Sleep for timestamp compare
+  execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 1.125)
+
+  execute_process(
+    COMMAND "${CMAKE_COMMAND}" -P
+    "${RunCMake_TEST_SOURCE_DIR}/update-deps.cmake"
+    COMMAND_ERROR_IS_FATAL ANY)
+  execute_process(COMMAND "${CMAKE_COMMAND}" -E touch
+    "${RunCMake_TEST_BINARY_DIR}/file_gen"
+    COMMAND_ERROR_IS_FATAL ANY)
+  run_cmake_command(out-of-date-rebuild
+    "${CMAKE_COMMAND}" --build "${out_of_date_build_dir}" ${out_of_date_build_config_args})
+  set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY "${out_of_date_build_dir}")
+
+  # Sleep for timestamp compare
+  execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 1.125)
+
+  run_cmake_command(out-of-date ${CMAKE_CTEST_COMMAND} ${out_of_date_ctest_config_args} -V --out-of-date)
+  run_cmake_command(up-to-date ${CMAKE_CTEST_COMMAND} ${out_of_date_ctest_config_args} -V --out-of-date)
+  unset(RunCMake_TEST_COMMAND_WORKING_DIRECTORY)
+endblock()
+
 function(run_BadCTestTestfile)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/BadCTestTestfile)
   set(RunCMake_TEST_NO_CLEAN 1)
@@ -768,6 +814,7 @@ function(run_ctest_configure_cli_preset CASE_NAME)
       -D "CTEST_SITE=cli-site")
   endif()
   set(RunCMake_TEST_SOURCE_DIR "${src}")
+  set(RunCMake_TEST_SOURCE_DIR "${src}")
   set(RunCMake_TEST_BINARY_DIR "${bin}")
   set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(${CASE_NAME}

Tests/RunCMake/CTestCommandLine/out-of-date-src/CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 4.4)
+project(OutOfDateDeps C)
+
+enable_testing()
+
+add_executable(app "${CMAKE_CURRENT_SOURCE_DIR}/main.c")
+add_library(staticlib STATIC "${CMAKE_CURRENT_SOURCE_DIR}/lib.c")
+add_library(sharedlib SHARED "${CMAKE_CURRENT_SOURCE_DIR}/lib.c")
+add_library(modulelib MODULE "${CMAKE_CURRENT_SOURCE_DIR}/lib.c")
+add_library(objlib OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/lib.c")
+
+set(file_gen "${CMAKE_CURRENT_BINARY_DIR}/file_gen")
+add_custom_target(custom COMMAND "${CMAKE_COMMAND}" -E touch ${file_gen})
+
+add_test(NAME exec-test COMMAND app)
+add_test(NAME static-test COMMAND "${CMAKE_COMMAND}" -E echo $<TARGET_FILE:staticlib>)
+add_test(NAME shared-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS sharedlib)
+add_test(NAME module-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS modulelib)
+if (CMAKE_GENERATOR STREQUAL "Xcode")
+  # Can't get the outputs of the object library, depend on the generated file
+  #   instead for parity with other environments.
+  add_test(NAME object-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS ${file_gen})
+else()
+  add_test(NAME object-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS objlib)
+endif()
+add_test(NAME custom-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS custom)
+add_test(NAME nodeps-test COMMAND "${CMAKE_COMMAND}" -E true)
+add_test(NAME file-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS ${file_gen})

Tests/RunCMake/CTestCommandLine/out-of-date-src/lib.c

@@ -0,0 +1,4 @@
+int lib(void)
+{
+  return 0;
+}

Tests/RunCMake/CTestCommandLine/out-of-date-src/main.c

@@ -0,0 +1,4 @@
+int main(void)
+{
+  return 0;
+}

Tests/RunCMake/CTestCommandLine/out-of-date-src/update-deps.cmake

@@ -0,0 +1,4 @@
+set(src_dir "${CMAKE_CURRENT_LIST_DIR}")
+
+file(APPEND "${src_dir}/main.c" "\n/* update main */\n")
+file(APPEND "${src_dir}/lib.c" "\n/* update lib */\n")

Tests/RunCMake/CTestCommandLine/out-of-date-stdout.txt

@@ -0,0 +1,7 @@
+Test project .*
+.*Start 1: exec-test.*
+.*Start 2: static-test.*
+.*Start 3: shared-test.*
+.*Start 4: module-test.*
+.*Start 5: object-test.*
+.*Start 6: file-test.*

Tests/RunCMake/CTestCommandLine/up-to-date-stderr.txt

@@ -0,0 +1 @@
+No tests were found!!!