diff --git a/Help/command/ctest_memcheck.rst b/Help/command/ctest_memcheck.rst index 7e91f4d129..250eb46fa6 100644 --- a/Help/command/ctest_memcheck.rst +++ b/Help/command/ctest_memcheck.rst @@ -11,8 +11,8 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`. [STRIDE ] [EXCLUDE ] [INCLUDE ] - [EXCLUDE_LABEL ] - [INCLUDE_LABEL ] + [EXCLUDE_LABEL ...] + [INCLUDE_LABEL ...] [EXCLUDE_FIXTURE ] [EXCLUDE_FIXTURE_SETUP ] [EXCLUDE_FIXTURE_CLEANUP ] diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index e5a174cdbf..57fec0b1b9 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -11,8 +11,8 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [STRIDE ] [EXCLUDE ] [INCLUDE ] - [EXCLUDE_LABEL ] - [INCLUDE_LABEL ] + [EXCLUDE_LABEL ...] + [INCLUDE_LABEL ...] [EXCLUDE_FROM_FILE ] [INCLUDE_FROM_FILE ] [EXCLUDE_FIXTURE ] @@ -69,13 +69,31 @@ The options are: Specify a regular expression matching test names to include. Tests not matching this expression are excluded. -``EXCLUDE_LABEL `` +``EXCLUDE_LABEL ...`` Specify a regular expression matching test labels to exclude. -``INCLUDE_LABEL `` + .. versionchanged:: 4.5 + + More than one ```` may be given, in which case a + test is excluded only if each regular expression matches at least one + of the test's labels (i.e. the expressions form an ``AND`` relationship). + See the :ref:`Label Matching` section of the :manual:`ctest(1)` manual. + This mirrors the behavior of repeating the + :option:`ctest --label-exclude` command-line option. + +``INCLUDE_LABEL ...`` Specify a regular expression matching test labels to include. Tests not matching this expression are excluded. + .. versionchanged:: 4.5 + + More than one ```` may be given, in which case a + test is included only if each regular expression matches at least one + of the test's labels (i.e. the expressions form an ``AND`` relationship). + See the :ref:`Label Matching` section of the :manual:`ctest(1)` manual. + This mirrors the behavior of repeating the + :option:`ctest --label-regex` command-line option. + ``EXCLUDE_FROM_FILE `` .. versionadded:: 3.29 diff --git a/Help/release/dev/ctest-test-label-and.rst b/Help/release/dev/ctest-test-label-and.rst new file mode 100644 index 0000000000..32f82b88fd --- /dev/null +++ b/Help/release/dev/ctest-test-label-and.rst @@ -0,0 +1,10 @@ +ctest-test-label-and +--------------------- + +* The :command:`ctest_test` and :command:`ctest_memcheck` commands gained the + ability to accept more than one value for their ``INCLUDE_LABEL`` and + ``EXCLUDE_LABEL`` options. When multiple regular expressions are given, a + test is selected only if each expression matches at least one of the test's + labels (i.e. the expressions form an ``AND`` relationship), matching the + behavior of repeating the :option:`ctest --label-regex` and + :option:`ctest --label-exclude` command-line options. diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 3e8e319a38..1a31d7bfe9 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -176,13 +176,10 @@ std::unique_ptr cmCTestTestCommand::InitializeHandler( handler->TestOptions.IncludeRegularExpression = args.Include; } if (!args.ExcludeLabel.empty()) { - handler->TestOptions.ExcludeLabelRegularExpression.clear(); - handler->TestOptions.ExcludeLabelRegularExpression.push_back( - args.ExcludeLabel); + handler->TestOptions.ExcludeLabelRegularExpression = args.ExcludeLabel; } if (!args.IncludeLabel.empty()) { - handler->TestOptions.LabelRegularExpression.clear(); - handler->TestOptions.LabelRegularExpression.push_back(args.IncludeLabel); + handler->TestOptions.LabelRegularExpression = args.IncludeLabel; } if (!args.ExcludeTestsFromFile.empty()) { diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index d8782654cd..533df90a94 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -31,8 +32,8 @@ protected: std::string Stride; std::string Exclude; std::string Include; - std::string ExcludeLabel; - std::string IncludeLabel; + ArgumentParser::NonEmpty> ExcludeLabel; + ArgumentParser::NonEmpty> IncludeLabel; std::string IncludeTestsFromFile; std::string ExcludeTestsFromFile; std::string ExcludeFixture; diff --git a/Tests/RunCMake/ctest_memcheck/LabelAndExclude-stdout.txt b/Tests/RunCMake/ctest_memcheck/LabelAndExclude-stdout.txt new file mode 100644 index 0000000000..0b8a3d372e --- /dev/null +++ b/Tests/RunCMake/ctest_memcheck/LabelAndExclude-stdout.txt @@ -0,0 +1,10 @@ +Memory check project [^ +]*/LabelAndExclude-build + Start [0-9]+: RunCMake +[0-9]+/3 MemCheck #[0-9]+: RunCMake \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test1 +[0-9]+/3 MemCheck #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test2 +[0-9]+/3 MemCheck #[0-9]+: test2 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 3 diff --git a/Tests/RunCMake/ctest_memcheck/LabelAndInclude-stdout.txt b/Tests/RunCMake/ctest_memcheck/LabelAndInclude-stdout.txt new file mode 100644 index 0000000000..b5b710954c --- /dev/null +++ b/Tests/RunCMake/ctest_memcheck/LabelAndInclude-stdout.txt @@ -0,0 +1,6 @@ +Memory check project [^ +]*/LabelAndInclude-build + Start [0-9]+: test3 +1/1 MemCheck #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 1 diff --git a/Tests/RunCMake/ctest_memcheck/LabelAndIncludeSingle-stdout.txt b/Tests/RunCMake/ctest_memcheck/LabelAndIncludeSingle-stdout.txt new file mode 100644 index 0000000000..d93c62863a --- /dev/null +++ b/Tests/RunCMake/ctest_memcheck/LabelAndIncludeSingle-stdout.txt @@ -0,0 +1,8 @@ +Memory check project [^ +]*/LabelAndIncludeSingle-build + Start [0-9]+: test1 +[0-9]+/2 MemCheck #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test3 +[0-9]+/2 MemCheck #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 2 diff --git a/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake b/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake index 6e5efb94ce..d5d725cdb8 100644 --- a/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_memcheck/RunCMakeTest.cmake @@ -238,3 +238,26 @@ block() "PRESET my-include-preset PRESETS_FILE /nonexistent/path/presets.json") run_mc_test(TestPresetBadFile "${PSEUDO_VALGRIND}") endblock() + +#----------------------------------------------------------------------------- +# Verify that INCLUDE_LABEL / EXCLUDE_LABEL accept more than one value that +# form an AND relationship, and that the change propagates to ctest_memcheck(). +block() + set(CMAKELISTS_EXTRA_CODE [[ +add_test(NAME test1 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test1 PROPERTY LABELS foo) +add_test(NAME test2 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test2 PROPERTY LABELS bar) +add_test(NAME test3 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test3 PROPERTY LABELS foo bar) +]]) + set(CTEST_MEMCHECK_ARGS "INCLUDE_LABEL foo bar") + run_mc_test(LabelAndInclude "${PSEUDO_VALGRIND}") + + set(CTEST_MEMCHECK_ARGS "EXCLUDE_LABEL foo bar") + run_mc_test(LabelAndExclude "${PSEUDO_VALGRIND}") + + set(CTEST_MEMCHECK_ARGS "INCLUDE_LABEL foo") + run_mc_test(LabelAndIncludeSingle "${PSEUDO_VALGRIND}") + unset(CTEST_MEMCHECK_ARGS) +endblock() diff --git a/Tests/RunCMake/ctest_test/LabelAndDuplicate-result.txt b/Tests/RunCMake/ctest_test/LabelAndDuplicate-result.txt new file mode 100644 index 0000000000..b57e2deb77 --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndDuplicate-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/LabelAndDuplicate-stderr.txt b/Tests/RunCMake/ctest_test/LabelAndDuplicate-stderr.txt new file mode 100644 index 0000000000..d7b05af26d --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndDuplicate-stderr.txt @@ -0,0 +1,3 @@ +CMake Error at [^ +]*/test\.cmake:[0-9]+ \(ctest_test\): + ctest_test called with more than one value for INCLUDE_LABEL diff --git a/Tests/RunCMake/ctest_test/LabelAndExclude-stdout.txt b/Tests/RunCMake/ctest_test/LabelAndExclude-stdout.txt new file mode 100644 index 0000000000..8404b17bff --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndExclude-stdout.txt @@ -0,0 +1,8 @@ + Start [0-9]+: RunCMakeVersion +[0-9]+/3 Test #[0-9]+: RunCMakeVersion \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test1 +[0-9]+/3 Test #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test2 +[0-9]+/3 Test #[0-9]+: test2 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 3 diff --git a/Tests/RunCMake/ctest_test/LabelAndExcludeEmptyClears-stdout.txt b/Tests/RunCMake/ctest_test/LabelAndExcludeEmptyClears-stdout.txt new file mode 100644 index 0000000000..f8d989419d --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndExcludeEmptyClears-stdout.txt @@ -0,0 +1,10 @@ + Start [0-9]+: RunCMakeVersion +[0-9]+/4 Test #[0-9]+: RunCMakeVersion \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test1 +[0-9]+/4 Test #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test2 +[0-9]+/4 Test #[0-9]+: test2 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test3 +[0-9]+/4 Test #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 4 diff --git a/Tests/RunCMake/ctest_test/LabelAndInclude-stdout.txt b/Tests/RunCMake/ctest_test/LabelAndInclude-stdout.txt new file mode 100644 index 0000000000..4c2f5e940c --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndInclude-stdout.txt @@ -0,0 +1,4 @@ + Start [0-9]+: test3 +1/1 Test #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 1 diff --git a/Tests/RunCMake/ctest_test/LabelAndIncludeEmptyClears-stdout.txt b/Tests/RunCMake/ctest_test/LabelAndIncludeEmptyClears-stdout.txt new file mode 100644 index 0000000000..f8d989419d --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndIncludeEmptyClears-stdout.txt @@ -0,0 +1,10 @@ + Start [0-9]+: RunCMakeVersion +[0-9]+/4 Test #[0-9]+: RunCMakeVersion \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test1 +[0-9]+/4 Test #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test2 +[0-9]+/4 Test #[0-9]+: test2 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test3 +[0-9]+/4 Test #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 4 diff --git a/Tests/RunCMake/ctest_test/LabelAndIncludeSingle-stdout.txt b/Tests/RunCMake/ctest_test/LabelAndIncludeSingle-stdout.txt new file mode 100644 index 0000000000..c4d84b8d07 --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndIncludeSingle-stdout.txt @@ -0,0 +1,6 @@ + Start [0-9]+: test1 +[0-9]+/2 Test #[0-9]+: test1 \.+ +Passed +[0-9\.]+ sec + Start [0-9]+: test3 +[0-9]+/2 Test #[0-9]+: test3 \.+ +Passed +[0-9\.]+ sec ++ +100% tests passed out of 2 diff --git a/Tests/RunCMake/ctest_test/LabelAndMissingValue-result.txt b/Tests/RunCMake/ctest_test/LabelAndMissingValue-result.txt new file mode 100644 index 0000000000..b57e2deb77 --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndMissingValue-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/LabelAndMissingValue-stderr.txt b/Tests/RunCMake/ctest_test/LabelAndMissingValue-stderr.txt new file mode 100644 index 0000000000..d75e662fbc --- /dev/null +++ b/Tests/RunCMake/ctest_test/LabelAndMissingValue-stderr.txt @@ -0,0 +1,3 @@ +CMake Error at [^ +]*/test\.cmake:[0-9]+ \(ctest_test\): + Error after keyword "INCLUDE_LABEL": diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 59cbb801ad..b3f34fad2d 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -418,6 +418,34 @@ set_property(TEST b PROPERTY LABELS b) endfunction() run_changing_labels() +# Verify that INCLUDE_LABEL / EXCLUDE_LABEL accept more than one value that +# form an AND relationship, matching the `ctest -L` / `ctest -LE` behavior. +function(run_label_and) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME test1 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test1 PROPERTY LABELS foo) +add_test(NAME test2 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test2 PROPERTY LABELS bar) +add_test(NAME test3 COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST test3 PROPERTY LABELS foo bar) + ]]) + run_ctest_test(LabelAndInclude INCLUDE_LABEL foo bar) + run_ctest_test(LabelAndExclude EXCLUDE_LABEL foo bar) + run_ctest_test(LabelAndIncludeSingle INCLUDE_LABEL foo) + run_ctest_test(LabelAndMissingValue INCLUDE_LABEL) + run_ctest_test(LabelAndDuplicate INCLUDE_LABEL foo INCLUDE_LABEL bar) + + # An explicit empty-string label argument clears a filter pre-seeded on the + # `ctest` command line, and thus runs every test. + set(CASE_CTEST_TEST_RAW_ARGS "INCLUDE_LABEL \"\"") + run_ctest(LabelAndIncludeEmptyClears -L "^foo$") + unset(CASE_CTEST_TEST_RAW_ARGS) + set(CASE_CTEST_TEST_RAW_ARGS "EXCLUDE_LABEL \"\"") + run_ctest(LabelAndExcludeEmptyClears -LE "^foo$") + unset(CASE_CTEST_TEST_RAW_ARGS) +endfunction() +run_label_and() + # Verify that test output can add additional labels function(run_extra_labels) set(CASE_CMAKELISTS_SUFFIX_CODE [[