From a39d4139d07eee841016ddccde0e7c35d40c894c Mon Sep 17 00:00:00 2001 From: Stefan Dinkelacker Date: Fri, 31 Jan 2020 18:09:35 +0100 Subject: [PATCH] Add --no-tests=<[error|ignore]> option to CTest If no tests were found, the default behavior of CTest is to always log an error message but to return an error code in script mode only. This option unifies the behavior of CTest by either returning an error code if no tests were found or by ignoring it. Signed-off-by: Stefan Dinkelacker --- Help/manual/ctest.1.rst | 8 +++++ Help/release/dev/ctest-no-tests.rst | 6 ++++ Source/CTest/cmCTestTestHandler.cxx | 13 ++++++++- Source/cmCTest.cxx | 20 +++++++++++++ Source/cmCTest.h | 8 +++++ Source/ctest.cxx | 2 ++ .../CTestCommandLine/RunCMakeTest.cmake | 29 +++++++++++++++++++ .../no-tests-script_error-result.txt | 1 + .../no-tests-script_error-stderr.txt | 1 + .../no-tests-script_legacy-result.txt | 1 + .../no-tests-script_legacy-stderr.txt | 1 + .../CTestCommandLine/no-tests_bad-result.txt | 1 + .../CTestCommandLine/no-tests_bad-stderr.txt | 1 + .../no-tests_error-result.txt | 1 + .../no-tests_error-stderr.txt | 1 + .../no-tests_legacy-stderr.txt | 1 + 16 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 Help/release/dev/ctest-no-tests.rst create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 25cb639669..2bfaafeade 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -370,6 +370,14 @@ See `Build and Test Mode`_. This option will not run any tests, it will simply print the list of all labels associated with the test set. +``--no-tests=<[error|ignore]>`` + Regard no tests found either as error or ignore it. + + If no tests were found, the default behavior of CTest is to always log an + error message but to return an error code in script mode only. This option + unifies the behavior of CTest by either returning an error code if no tests + were found or by ignoring it. + .. include:: OPTIONS_HELP.txt .. _`Label and Subproject Summary`: diff --git a/Help/release/dev/ctest-no-tests.rst b/Help/release/dev/ctest-no-tests.rst new file mode 100644 index 0000000000..be0f28be8e --- /dev/null +++ b/Help/release/dev/ctest-no-tests.rst @@ -0,0 +1,6 @@ +ctest-no-tests +-------------- + +* The :manual:`ctest(1)` tool gained a ``--no-tests=<[error|ignore]>`` option + to explicitly set and unify the behavior between direct invocation and + script mode if no tests were found. diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index e70bc5a76d..78c68befdb 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -410,10 +410,15 @@ int cmCTestTestHandler::ProcessHandler() auto clock_finish = std::chrono::steady_clock::now(); + bool noTestsFoundError = false; if (passed.size() + failed.size() == 0) { - if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels()) { + if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() && + this->CTest->GetNoTestsMode() != cmCTest::NoTests::Ignore) { cmCTestLog(this->CTest, ERROR_MESSAGE, "No tests were found!!!" << std::endl); + if (this->CTest->GetNoTestsMode() == cmCTest::NoTests::Error) { + noTestsFoundError = true; + } } } else { if (this->HandlerVerbose && !passed.empty() && @@ -459,6 +464,12 @@ int cmCTestTestHandler::ProcessHandler() this->LogFile = nullptr; return -1; } + + if (noTestsFoundError) { + this->LogFile = nullptr; + return -1; + } + this->LogFile = nullptr; return 0; } diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 22a6e3869e..04f75bd506 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -208,6 +208,8 @@ struct cmCTest::Private bool OutputColorCode = cmCTest::ColoredOutputSupportedByConsole(); std::map Definitions; + + cmCTest::NoTests NoTestsMode = cmCTest::NoTests::Legacy; }; struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag) @@ -2059,6 +2061,19 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, this->SetNotesFiles(args[i].c_str()); } + const std::string noTestsPrefix = "--no-tests="; + if (cmHasPrefix(arg, noTestsPrefix)) { + const std::string noTestsMode = arg.substr(noTestsPrefix.length()); + if (noTestsMode == "error") { + this->Impl->NoTestsMode = cmCTest::NoTests::Error; + } else if (noTestsMode != "ignore") { + errormsg = "'--no-tests=' given unknown value '" + noTestsMode + "'"; + return false; + } else { + this->Impl->NoTestsMode = cmCTest::NoTests::Ignore; + } + } + // options that control what tests are run if (this->CheckArgument(arg, "-I", "--tests-information") && i < args.size() - 1) { @@ -2896,6 +2911,11 @@ cmCTest::Repeat cmCTest::GetRepeatMode() const return this->Impl->RepeatMode; } +cmCTest::NoTests cmCTest::GetNoTestsMode() const +{ + return this->Impl->NoTestsMode; +} + void cmCTest::SetBuildID(const std::string& id) { this->Impl->BuildID = id; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index e0ae1000b0..7f8f913a93 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -442,6 +442,14 @@ public: }; Repeat GetRepeatMode() const; + enum class NoTests + { + Legacy, + Error, + Ignore + }; + NoTests GetNoTestsMode() const; + void GenerateSubprojectsOutput(cmXMLWriter& xml); std::vector GetLabelsForSubprojects(); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 9b45bb05c0..fbdf75ada1 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -144,6 +144,8 @@ static const char* cmDocumentationOptions[][2] = { { "--http1.0", "Submit using HTTP 1.0." }, { "--no-compress-output", "Do not compress test output when submitting." }, { "--print-labels", "Print all available test labels." }, + { "--no-tests=<[error|ignore]>", + "Regard no tests found either as 'error' or 'ignore' it." }, { nullptr, nullptr } }; diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index b2de596d1a..9b9ae65b0b 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -314,5 +314,34 @@ function(run_ShowOnly) endfunction() run_ShowOnly() +function(run_NoTests) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/NoTests) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "") + run_cmake_command(no-tests_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore) + run_cmake_command(no-tests_error ${CMAKE_CTEST_COMMAND} --no-tests=error) + run_cmake_command(no-tests_bad ${CMAKE_CTEST_COMMAND} --no-tests=bad) + run_cmake_command(no-tests_legacy ${CMAKE_CTEST_COMMAND}) + file(WRITE "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake" " + set(CTEST_COMMAND \"${CMAKE_CTEST_COMMAND}\") + set(CTEST_SOURCE_DIRECTORY \"${RunCMake_SOURCE_DIR}\") + set(CTEST_BINARY_DIRECTORY \"${RunCMake_TEST_BINARY_DIR}\") + ctest_start(Experimental) + ctest_test() +") + run_cmake_command( + no-tests-script_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore + -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake") + run_cmake_command( + no-tests-script_error ${CMAKE_CTEST_COMMAND} --no-tests=error + -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake") + run_cmake_command( + no-tests-script_legacy ${CMAKE_CTEST_COMMAND} + -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake") +endfunction() +run_NoTests() + # Check the configuration type variable is passed run_ctest(check-configuration-type) diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt new file mode 100644 index 0000000000..b57e2deb77 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt new file mode 100644 index 0000000000..a7c4b11f76 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt new file mode 100644 index 0000000000..b57e2deb77 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt new file mode 100644 index 0000000000..a7c4b11f76 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt new file mode 100644 index 0000000000..170353982b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--no-tests=' given unknown value 'bad'$ diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt new file mode 100644 index 0000000000..45a4fb75db --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt @@ -0,0 +1 @@ +8 diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt new file mode 100644 index 0000000000..eafba1c692 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt new file mode 100644 index 0000000000..a7c4b11f76 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$