diff --git a/Help/command/add_test.rst b/Help/command/add_test.rst index f5dc26f6b1..c630932e26 100644 --- a/Help/command/add_test.rst +++ b/Help/command/add_test.rst @@ -111,6 +111,12 @@ directory the test is created in. Generators`. The test name must be a valid target name in order to list build dependencies with this keyword. + * .. versionchanged:: 4.5 + + Build dependencies added by this argument, or referenced in the test + ``COMMAND`` also enable the :option:`--out-of-date ` + behavior of :manual:`ctest(1)`. + ``COMMAND_EXPAND_LISTS`` .. versionadded:: 3.16 diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index e807425489..9151aec460 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -590,6 +590,25 @@ The options for running tests are: :preset:`testPresets.execution.testPassthroughArguments` are provided, the test preset arguments will appear first, followed by the ````. +.. option:: --out-of-date + + .. versionadded:: 4.5 + + Run only tests whose build dependencies are newer than the last time the + test ran. + + This option tells CTest to skip tests that are already up to date with + respect to their recorded build dependencies. Build dependencies include + executables and targets in generator expressions as part of the test + ``COMMAND``, as well as the outputs of targets or files added as explicit + dependencies with the ``BUILD_DEPENDS`` argument of :command:`add_test`. + + A test is selected to run when any of its recorded build dependencies are + newer than the test's last-run timestamp, or when the test has not been + run before. Tests without any known build dependencies, including any tests + not added by the :command:`add_test` command, are excluded when this argument + is provided. + View Help ========= diff --git a/Help/release/dev/ctest-out-of-date.rst b/Help/release/dev/ctest-out-of-date.rst new file mode 100644 index 0000000000..823aa2348a --- /dev/null +++ b/Help/release/dev/ctest-out-of-date.rst @@ -0,0 +1,6 @@ +ctest-out-of-date +----------------- + +* :manual:`ctest(1)` gained a :option:`--out-of-date ` + option to run only tests whose recorded build dependencies are newer than + the last time the tests ran. diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index ebbd45254f..74ce022468 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -375,6 +375,12 @@ cmCTestRunTest::EndTestResult cmCTestRunTest::EndTest(size_t completed, this->TestResult.ExecutionTime = this->TestProcess->GetTotalTime(); this->MemCheckPostProcess(); this->ComputeWeightedCost(); + + std::string const stampDir = this->CTest->GetStampDir(); + cmSystemTools::MakeDirectory(stampDir); + std::string const stampFile = + stampDir + "/" + this->TestProperties->GetStampFile(); + cmSystemTools::Touch(stampFile, true); } // If the test does not need to rerun push the current TestResult onto the // TestHandler vector diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 1661b69a40..8baaebc7cc 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -41,6 +41,7 @@ #include "cmCTestMultiProcessHandler.h" #include "cmCTestResourceGroupsLexerHelper.h" #include "cmCTestTestMeasurementXMLParser.h" +#include "cmCryptoHash.h" #include "cmDuration.h" #include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" @@ -967,6 +968,9 @@ bool cmCTestTestHandler::ComputeTestList() if (this->TestOptions.RerunFailed) { return this->ComputeTestListForRerunFailed(); } + if (this->TestOptions.OutOfDateOnly) { + this->ComputeOutOfDateTests(); + } cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size(); // how many tests are in based on RegExp? @@ -1069,6 +1073,49 @@ bool cmCTestTestHandler::ComputeTestListForRerunFailed() return true; } +void cmCTestTestHandler::ComputeOutOfDateTests() +{ + ListOfTests finalList; + std::string const stampDir = this->CTest->GetStampDir(); + cmSystemTools::MakeDirectory(stampDir); + + for (cmCTestTestProperties& tp : this->TestList) { + if (tp.BuildDepends.empty()) { + continue; + } + + std::string const stampFile = stampDir + "/" + tp.GetStampFile(); + + if (!cmSystemTools::FileExists(stampFile)) { + finalList.push_back(tp); + continue; + } + + cmList deps{ tp.BuildDepends }; + bool outOfDate = false; + for (std::string const& dep : deps) { + if (dep.empty()) { + continue; + } + if (!cmSystemTools::FileExists(dep)) { + // If any dependencies don't exist, skip the test + outOfDate = false; + break; + } + int result = 0; + cmSystemTools::FileTimeCompare(dep, stampFile, &result); + if (result >= 0) { + // At least one newer dependency, add to list + outOfDate = true; + } + } + if (outOfDate) { + finalList.push_back(tp); + } + } + this->TestList = finalList; +} + void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const { cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, @@ -2298,6 +2345,11 @@ void cmCTestTestHandler::cmCTestTestProperties::AppendError( } } +std::string cmCTestTestHandler::cmCTestTestProperties::GetStampFile() +{ + return cmCryptoHash(cmCryptoHash::AlgoMD5).HashString(this->Name) + ".stamp"; +} + bool cmCTestTestHandler::SetTestsProperties( std::vector const& args) { @@ -2498,6 +2550,8 @@ bool cmCTestTestHandler::SetTestsProperties( rt.TimeoutRegularExpressions.emplace_back(cr, cr); } } + } else if (key == "_CMAKE_TEST_BUILD_DEPENDS"_s) { + rt.BuildDepends = val; } else { rt.CustomProperties[key] = val; } diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 276ad16045..926cad3132 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -31,6 +31,7 @@ class cmXMLWriter; struct cmCTestTestOptions { bool RerunFailed = false; + bool OutOfDateOnly = false; bool ScheduleRandom = false; bool StopOnFailure = false; bool UseUnion = false; @@ -144,6 +145,7 @@ public: struct cmCTestTestProperties { void AppendError(cm::string_view err); + std::string GetStampFile(); cm::optional Error; std::string Name; // working directory for test, overridden by WORKING_DIRECTORY property @@ -192,6 +194,7 @@ public: std::set RequireSuccessDepends; std::vector> ResourceGroups; std::string GeneratedResourceSpecFile; + std::string BuildDepends; // Private test generator properties used to track backtraces cmListFileBacktrace Backtrace; }; @@ -342,6 +345,7 @@ private: // compute the lists of tests that will actually run // based on LastTestFailed.log bool ComputeTestListForRerunFailed(); + void ComputeOutOfDateTests(); // add required setup/cleanup tests not already in the // list of tests to be run and update dependencies between diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7b15c9721c..3b707c508b 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -2522,6 +2522,11 @@ int cmCTest::Run(std::vector const& args) this->Impl->TestOptions.RerunFailed = true; return true; } }, + CommandArgument{ "--out-of-date", CommandArgument::Values::Zero, + [this](std::string const&) -> bool { + this->Impl->TestOptions.OutOfDateOnly = true; + return true; + } }, }; // Process command line arguments for presets first, since other arguments @@ -3045,6 +3050,11 @@ std::string cmCTest::GetBinaryDir() return this->Impl->BinaryDir; } +std::string cmCTest::GetStampDir() +{ + return this->Impl->BinaryDir + "/Testing/Temporary/LastTestRun"; +} + std::string const& cmCTest::GetConfigType() { return this->Impl->ConfigType; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 4227e0e2a0..72f49eb76f 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -253,6 +253,9 @@ public: /** Get the path to the build tree */ std::string GetBinaryDir(); + /** Get the path to the test stamp file tree */ + std::string GetStampDir(); + /** * Get the short path to the file. * diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index 3e6e47606f..c8612b9adb 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -3010,6 +3010,34 @@ bool cmGlobalGenerator::NameResolvesToFramework( return false; } +std::vector cmGlobalGenerator::GetTestBuildDependencyPaths( + std::string const& config, + cmTestGenerator::BuildDependencies const& deps) const +{ + std::set uniqueDeps; + for (auto const& file : deps.Files) { + uniqueDeps.insert(file.Path); + } + for (cmGeneratorTarget* target : deps.Targets) { + if (target->GetType() == cmStateEnums::TargetType::UTILITY || + target->GetType() == cmStateEnums::TargetType::GLOBAL_TARGET || + target->GetType() == cmStateEnums::TargetType::INTERFACE_LIBRARY) { + continue; + } + if (target->GetType() == cmStateEnums::TargetType::OBJECT_LIBRARY) { + std::vector objects; + target->GetTargetObjectNames(config, objects); + for (auto const& object : objects) { + uniqueDeps.insert( + cmStrCat(target->GetObjectDirectory(config), object)); + } + continue; + } + uniqueDeps.insert(target->GetFullPath(config)); + } + return { uniqueDeps.begin(), uniqueDeps.end() }; +} + // If the file has no extension it's either a raw executable or might // be a direct reference to a binary within a framework (bad practice!). // This is where we change the path to point to the framework directory. diff --git a/Source/cmGlobalGenerator.h b/Source/cmGlobalGenerator.h index 39aad5d618..930da2c4cb 100644 --- a/Source/cmGlobalGenerator.h +++ b/Source/cmGlobalGenerator.h @@ -34,6 +34,7 @@ #include "cmSystemTools.h" #include "cmTarget.h" #include "cmTargetDepend.h" +#include "cmTestGenerator.h" #include "cmValue.h" #include "cmXcFramework.h" @@ -130,6 +131,10 @@ public: //! Get the name for this generator virtual std::string GetName() const { return "Generic"; } + virtual std::vector GetTestBuildDependencyPaths( + std::string const& config, + cmTestGenerator::BuildDependencies const& deps) const; + /** Check whether the given name matches the current generator. */ virtual bool MatchesGeneratorName(std::string const& name) const { diff --git a/Source/cmTestGenerator.cxx b/Source/cmTestGenerator.cxx index d884cbf879..d8abc21189 100644 --- a/Source/cmTestGenerator.cxx +++ b/Source/cmTestGenerator.cxx @@ -303,6 +303,17 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os, << cmScriptGenerator::Quote( ge.Parse(i.second)->Evaluate(this->LG, config)); } + BuildDependencies deps; + if (this->GetBuildDependencies(this->LG, deps)) { + cmList depList; + for (std::string const& dep : + this->LG->GetGlobalGenerator()->GetTestBuildDependencyPaths(config, + deps)) { + depList.append(dep); + } + os << " _CMAKE_TEST_BUILD_DEPENDS " + << cmScriptGenerator::Quote(depList.to_string()); + } os << ' '; this->GenerateBacktrace(os, this->Test->GetBacktrace()); os << ")\n"; diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 8e600b003a..10c0d8a784 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -117,6 +117,8 @@ cmDocumentationEntry const cmDocumentationOptions[] = { "Run a specific number of tests by number." }, { "-U, --union", "Take the Union of -I and -R" }, { "--rerun-failed", "Run only the tests that failed previously" }, + { "--out-of-date", + "Run only tests whose build dependencies changed since they last ran" }, { "--tests-from-file ", "Run the tests listed in the given file" }, { "--exclude-from-file ", "Run tests except those listed in the given file" }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 08cacfcf38..49115e1dbf 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -117,6 +117,52 @@ add_test(fails \"${CMAKE_COMMAND}\" -E false) run_cmake_command(rerun-failed ${CMAKE_CTEST_COMMAND} --rerun-failed) endblock() +block() + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/out-of-date) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + set(out_of_date_build_dir "${RunCMake_TEST_BINARY_DIR}/build") + set(out_of_date_config "Debug") + if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG) + set(out_of_date_config "") + endif() + set(out_of_date_build_config_args) + set(out_of_date_ctest_config_args) + if(out_of_date_config) + set(out_of_date_build_config_args --config ${out_of_date_config}) + set(out_of_date_ctest_config_args -C ${out_of_date_config}) + endif() + file(COPY "${RunCMake_SOURCE_DIR}/out-of-date-src" + DESTINATION "${RunCMake_TEST_BINARY_DIR}") + set(RunCMake_TEST_SOURCE_DIR "${RunCMake_TEST_BINARY_DIR}/out-of-date-src") + set(RunCMake_TEST_BINARY_DIR "${out_of_date_build_dir}") + run_cmake(out-of-date-configure) + run_cmake_command(out-of-date-build "${CMAKE_COMMAND}" --build "${out_of_date_build_dir}" ${out_of_date_build_config_args}) + run_cmake_command(out-of-date-initial-test + "${CMAKE_CTEST_COMMAND}" ${out_of_date_ctest_config_args} -V --test-dir "${out_of_date_build_dir}") + + # Sleep for timestamp compare + execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 1.125) + + execute_process( + COMMAND "${CMAKE_COMMAND}" -P + "${RunCMake_TEST_SOURCE_DIR}/update-deps.cmake" + COMMAND_ERROR_IS_FATAL ANY) + execute_process(COMMAND "${CMAKE_COMMAND}" -E touch + "${RunCMake_TEST_BINARY_DIR}/file_gen" + COMMAND_ERROR_IS_FATAL ANY) + run_cmake_command(out-of-date-rebuild + "${CMAKE_COMMAND}" --build "${out_of_date_build_dir}" ${out_of_date_build_config_args}) + set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY "${out_of_date_build_dir}") + + # Sleep for timestamp compare + execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep 1.125) + + run_cmake_command(out-of-date ${CMAKE_CTEST_COMMAND} ${out_of_date_ctest_config_args} -V --out-of-date) + run_cmake_command(up-to-date ${CMAKE_CTEST_COMMAND} ${out_of_date_ctest_config_args} -V --out-of-date) + unset(RunCMake_TEST_COMMAND_WORKING_DIRECTORY) +endblock() + function(run_BadCTestTestfile) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/BadCTestTestfile) set(RunCMake_TEST_NO_CLEAN 1) @@ -768,6 +814,7 @@ function(run_ctest_configure_cli_preset CASE_NAME) -D "CTEST_SITE=cli-site") endif() set(RunCMake_TEST_SOURCE_DIR "${src}") + set(RunCMake_TEST_SOURCE_DIR "${src}") set(RunCMake_TEST_BINARY_DIR "${bin}") set(RunCMake_TEST_NO_CLEAN 1) run_cmake_command(${CASE_NAME} diff --git a/Tests/RunCMake/CTestCommandLine/out-of-date-src/CMakeLists.txt b/Tests/RunCMake/CTestCommandLine/out-of-date-src/CMakeLists.txt new file mode 100644 index 0000000000..7c4bb727ae --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/out-of-date-src/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 4.4) +project(OutOfDateDeps C) + +enable_testing() + +add_executable(app "${CMAKE_CURRENT_SOURCE_DIR}/main.c") +add_library(staticlib STATIC "${CMAKE_CURRENT_SOURCE_DIR}/lib.c") +add_library(sharedlib SHARED "${CMAKE_CURRENT_SOURCE_DIR}/lib.c") +add_library(modulelib MODULE "${CMAKE_CURRENT_SOURCE_DIR}/lib.c") +add_library(objlib OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/lib.c") + +set(file_gen "${CMAKE_CURRENT_BINARY_DIR}/file_gen") +add_custom_target(custom COMMAND "${CMAKE_COMMAND}" -E touch ${file_gen}) + +add_test(NAME exec-test COMMAND app) +add_test(NAME static-test COMMAND "${CMAKE_COMMAND}" -E echo $) +add_test(NAME shared-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS sharedlib) +add_test(NAME module-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS modulelib) +if (CMAKE_GENERATOR STREQUAL "Xcode") + # Can't get the outputs of the object library, depend on the generated file + # instead for parity with other environments. + add_test(NAME object-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS ${file_gen}) +else() + add_test(NAME object-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS objlib) +endif() +add_test(NAME custom-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS custom) +add_test(NAME nodeps-test COMMAND "${CMAKE_COMMAND}" -E true) +add_test(NAME file-test COMMAND "${CMAKE_COMMAND}" -E true BUILD_DEPENDS ${file_gen}) diff --git a/Tests/RunCMake/CTestCommandLine/out-of-date-src/lib.c b/Tests/RunCMake/CTestCommandLine/out-of-date-src/lib.c new file mode 100644 index 0000000000..abe3b787c6 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/out-of-date-src/lib.c @@ -0,0 +1,4 @@ +int lib(void) +{ + return 0; +} diff --git a/Tests/RunCMake/CTestCommandLine/out-of-date-src/main.c b/Tests/RunCMake/CTestCommandLine/out-of-date-src/main.c new file mode 100644 index 0000000000..8488f4e58f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/out-of-date-src/main.c @@ -0,0 +1,4 @@ +int main(void) +{ + return 0; +} diff --git a/Tests/RunCMake/CTestCommandLine/out-of-date-src/update-deps.cmake b/Tests/RunCMake/CTestCommandLine/out-of-date-src/update-deps.cmake new file mode 100644 index 0000000000..8960de3264 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/out-of-date-src/update-deps.cmake @@ -0,0 +1,4 @@ +set(src_dir "${CMAKE_CURRENT_LIST_DIR}") + +file(APPEND "${src_dir}/main.c" "\n/* update main */\n") +file(APPEND "${src_dir}/lib.c" "\n/* update lib */\n") diff --git a/Tests/RunCMake/CTestCommandLine/out-of-date-stdout.txt b/Tests/RunCMake/CTestCommandLine/out-of-date-stdout.txt new file mode 100644 index 0000000000..c1b25ce896 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/out-of-date-stdout.txt @@ -0,0 +1,7 @@ +Test project .* +.*Start 1: exec-test.* +.*Start 2: static-test.* +.*Start 3: shared-test.* +.*Start 4: module-test.* +.*Start 5: object-test.* +.*Start 6: file-test.* diff --git a/Tests/RunCMake/CTestCommandLine/up-to-date-stderr.txt b/Tests/RunCMake/CTestCommandLine/up-to-date-stderr.txt new file mode 100644 index 0000000000..eafba1c692 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/up-to-date-stderr.txt @@ -0,0 +1 @@ +No tests were found!!!