CTest: Throw error if resource spec is invalid

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -83,7 +83,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
 cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
 
 // Set the tests
-void cmCTestMultiProcessHandler::SetTests(TestMap tests,
+bool cmCTestMultiProcessHandler::SetTests(TestMap tests,
                                           PropertiesMap properties)
 {
   this->PendingTests = std::move(tests);
@@ -95,10 +95,11 @@ void cmCTestMultiProcessHandler::SetTests(TestMap tests,
     this->HasInvalidGeneratedResourceSpec =
       !this->CheckGeneratedResourceSpec();
     if (this->HasCycles || this->HasInvalidGeneratedResourceSpec) {
-      return;
+      return false;
     }
     this->CreateTestCostList();
   }
+  return true;
 }
 
 // Set the max number of tests that can be run at the same time.

Source/CTest/cmCTestMultiProcessHandler.h

@@ -61,7 +61,7 @@ public:
   cmCTestMultiProcessHandler();
   virtual ~cmCTestMultiProcessHandler();
   // Set the tests
-  void SetTests(TestMap tests, PropertiesMap properties);
+  bool SetTests(TestMap tests, PropertiesMap properties);
   // Set the max number of tests that can be run at the same time.
   void SetParallelLevel(size_t);
   void SetTestLoad(unsigned long load);

Source/CTest/cmCTestTestHandler.cxx

@@ -1410,7 +1410,9 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
     properties[p.Index] = &p;
   }
   parallel->SetResourceSpecFile(this->ResourceSpecFile);
-  parallel->SetTests(std::move(tests), std::move(properties));
+  if (!parallel->SetTests(std::move(tests), std::move(properties))) {
+    return false;
+  }
   parallel->SetPassFailVectors(&passed, &failed);
   this->TestResults.clear();
   parallel->SetTestResults(&this->TestResults);

Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake

@@ -167,6 +167,16 @@ function(verify_ctest_resources)
   endif()
 endfunction()
 
+set(RunCMake_TEST_BINARY_DIR "${RunCMake_BINARY_DIR}/invalid-build")
+set(RunCMake_TEST_NO_CLEAN 1)
+file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "add_test(true \"${CMAKE_COMMAND}\" -E true)\n")
+run_cmake_command(invalid-nofile-ctest ${CMAKE_CTEST_COMMAND} --resource-spec-file "${RunCMake_BINARY_DIR}/noexist.json")
+run_cmake_command(invalid-not-json-ctest ${CMAKE_CTEST_COMMAND} --resource-spec-file "${RunCMake_SOURCE_DIR}/invalid.json")
+unset(RunCMake_TEST_NO_CLEAN)
+unset(RunCMake_TEST_BINARY_DIR)
+
 run_ctest_resource(lotsoftests 10 1 0)
 run_ctest_resource(checkfree1 2 0 1)
 run_ctest_resource(checkfree2 1 0 0)

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s-stderr.txt

@@ -1,3 +1,5 @@
 ^Error: a cycle exists in the test dependency graph for the test "GenerateSpecFile"\.
-Please fix the cycle and run ctest again.
+Please fix the cycle and run ctest again\.
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-circular-no-required-fixtures-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^GENERATED_RESOURCE_SPEC_FILE test property cannot be used in conjunction with ResourceSpecFile option
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-conflicting-spec-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^Only one test may define the GENERATED_RESOURCE_SPEC_FILE property
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-generators-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-multiple-setup-fixtures-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^All tests that have RESOURCE_GROUPS must include the resource spec generator fixture in their FIXTURES_REQUIRED
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-required-fixture-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^Test that defines GENERATED_RESOURCE_SPEC_FILE must have exactly one FIXTURES_SETUP
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-no-setup-fixture-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s-stderr.txt

@@ -1,2 +1,4 @@
 ^GENERATED_RESOURCE_SPEC_FILE must be an absolute path
-No tests were found!!!$
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/dynamic-resource-relative-path-ctest-s/test\.cmake:[0-9]+ \(message\):
+  Tests did not pass$

Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CTestResourceAllocation/invalid-nofile-ctest-stderr.txt

@@ -0,0 +1,8 @@
+^Could not read/parse resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/noexist\.json:[ ]
+File not found: [^
+]*/Tests/RunCMake/CTestResourceAllocation/noexist.json
+Errors while running CTest
+Output from these tests are in: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid-build/Testing/Temporary/LastTest\.log
+Use "--rerun-failed --output-on-failure" to re-run the failed cases verbosely\.$

Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-result.txt

@@ -0,0 +1 @@
+8

Tests/RunCMake/CTestResourceAllocation/invalid-not-json-ctest-stderr.txt

@@ -0,0 +1,15 @@
+^Could not read/parse resource spec file [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid\.json:[ ]
+JSON Parse Error: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid\.json:
+\* Line 1, Column 1
+  Syntax error: value, object or array expected\.
+(\* Line 1, Column 2
+  Extra non-whitespace after JSON value\.
+|\* Line 1, Column 1
+  A valid JSON document must be either an array or an object value\.
+)
+Errors while running CTest
+Output from these tests are in: [^
+]*/Tests/RunCMake/CTestResourceAllocation/invalid-build/Testing/Temporary/LastTest\.log
+Use "--rerun-failed --output-on-failure" to re-run the failed cases verbosely\.$

Tests/RunCMake/CTestResourceAllocation/invalid.json

@@ -0,0 +1 @@
+This is not a valid JSON file!

Tests/RunCMake/ctest_fixtures/cyclicCleanup-stderr.txt

@@ -1,3 +1,2 @@
-Error: a cycle exists in the test dependency graph for the test "cyclicCleanup".
+^Error: a cycle exists in the test dependency graph for the test "cyclicCleanup"\.
-Please fix the cycle and run ctest again.
+Please fix the cycle and run ctest again\.$
-No tests were found!!!

Tests/RunCMake/ctest_fixtures/cyclicSetup-stderr.txt

@@ -1,3 +1,2 @@
-Error: a cycle exists in the test dependency graph for the test "cyclicSetup".
+^Error: a cycle exists in the test dependency graph for the test "cyclicSetup"\.
-Please fix the cycle and run ctest again.
+Please fix the cycle and run ctest again\.$
-No tests were found!!!$