Merge topic 'ctest-launch-signal-passthrough'

899e7231ab CTest: Report failure when a launched command is killed by a signal

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !12248

Help/release/dev/ctest-launch-signal-passthrough.rst

@@ -0,0 +1,8 @@
+ctest-launch-signal-passthrough
+-------------------------------
+
+* When CTest build launchers are enabled (see the :module:`CTestUseLaunchers`
+  module), a build rule that is terminated by a signal is now reported as a
+  build failure.  Previously such a rule was reported as success, so a crashed
+  compiler, linker, or custom command could be recorded as a passing build on
+  the dashboard.

Source/CTest/cmCTestLaunch.cxx

@@ -8,7 +8,6 @@
 #include <iostream>
 #include <map>
 #include <memory>
-#include <utility>
 
 #include <cm/optional>
 
@@ -291,8 +290,10 @@ void cmCTestLaunch::RunChild()
     uv_run(&chain.GetLoop(), UV_RUN_ONCE);
   }
   this->Reporter.Status = chain.GetStatus(0);
-  if (this->Reporter.Status.GetException().first ==
-      cmUVProcessChain::ExceptionCode::Spawn) {
+  if (this->Reporter.Status.SpawnResult != 0 ||
+      this->Reporter.Status.TermSignal != 0) {
+    // The child process could not be spawned or was terminated by a signal
+    // (POSIX).
     this->Reporter.ExitCode = 1;
   } else {
     this->Reporter.ExitCode =

Tests/RunCMake/CMakeLists.txt

@@ -279,6 +279,7 @@ if(CMAKE_GENERATOR MATCHES "Ninja")
 endif()
 add_RunCMake_test(InstallParallel)
 add_RunCMake_test(CTest)
+add_RunCMake_test(CTestLaunch)
 
 if(NOT CMake_TEST_EXTERNAL_CMAKE)
   add_RunCMake_test(ctest_memcheck

Tests/RunCMake/CTestLaunch/RunCMakeTest.cmake

@@ -0,0 +1,67 @@
+include(RunCMake)
+
+# cmCTestLaunch backs both `ctest --launch` and `ctest --instrument`.  It runs a
+# wrapped command and classifies how it terminated.  In the default "passthru"
+# mode (the CTEST_LAUNCH_LOGS environment variable is not set) a bare
+# `ctest --launch -- <command>` simply runs <command> and returns its
+# classified exit code, which makes these cases exercise
+# cmCTestLaunch::RunChild's status classification directly.
+
+# Merge stderr into stdout so incidental child output does not trip the default
+# empty-stderr expectation; none of these cases assert stdout content.
+set(RunCMake_TEST_OUTPUT_MERGE 1)
+
+# --- Regression guards (cross-platform): normal exits pass through unchanged.
+
+# A clean success stays 0.
+run_cmake_command(launch-success
+  ${CMAKE_CTEST_COMMAND} --launch -- ${CMAKE_COMMAND} -E true)
+
+# A normal non-zero exit is reported as failure.
+run_cmake_command(launch-fail
+  ${CMAKE_CTEST_COMMAND} --launch -- ${CMAKE_COMMAND} -E false)
+
+# The no-op make rule (":") short-circuits to success before spawning.
+run_cmake_command(launch-noop
+  ${CMAKE_CTEST_COMMAND} --launch -- :)
+
+# A command that cannot be spawned is reported as failure.
+run_cmake_command(launch-spawn-failure
+  ${CMAKE_CTEST_COMMAND} --launch -- command-that-does-not-exist-cmctl)
+
+# The signal cases require POSIX signal semantics.
+if(UNIX)
+  # A normal non-zero exit code is passed through exactly, not collapsed to 1.
+  run_cmake_command(launch-exit-code
+    ${CMAKE_CTEST_COMMAND} --launch -- /bin/sh -c "exit 3")
+
+  # A command terminated by a signal reports a non-zero exit code.
+  run_cmake_command(launch-signal
+    ${CMAKE_CTEST_COMMAND} --launch -- /bin/sh -c "kill -s TERM $$")
+
+  # Full launcher mode (CTEST_LAUNCH_LOGS set): a signal-killed rule is treated
+  # as an error and an error-*.xml fragment is written for the dashboard.
+  set(logdir ${RunCMake_BINARY_DIR}/launch-signal-logs-logs)
+  file(REMOVE_RECURSE "${logdir}")
+  file(MAKE_DIRECTORY "${logdir}")
+  set(ENV{CTEST_LAUNCH_LOGS} "${logdir}")
+  run_cmake_command(launch-signal-logs
+    ${CMAKE_CTEST_COMMAND} --launch
+      --target-name sig --build-dir ${RunCMake_BINARY_DIR}
+      -- /bin/sh -c "kill -s TERM $$")
+  unset(ENV{CTEST_LAUNCH_LOGS})
+
+  # `ctest --instrument`: with an instrumentation query present, the snippet's
+  # `result` field must carry the same non-zero exit code.  Use a dedicated
+  # directory name (not "instrument-signal-build") so the harness does not wipe
+  # the query we plant here when it cleans the default per-case binary dir.
+  set(instrbin ${RunCMake_BINARY_DIR}/instrument-signal-data)
+  set(v1 ${instrbin}/.cmake/instrumentation/v1)
+  file(REMOVE_RECURSE "${instrbin}")
+  file(MAKE_DIRECTORY "${v1}/query")
+  file(WRITE "${v1}/query/test.json" "{ \"version\": 1 }\n")
+  run_cmake_command(instrument-signal
+    ${CMAKE_CTEST_COMMAND} --instrument
+      --build-dir ${instrbin} --command-type test
+      -- /bin/sh -c "kill -s TERM $$")
+endif()

Tests/RunCMake/CTestLaunch/instrument-signal-check.cmake

@@ -0,0 +1,18 @@
+# ctest --instrument writes one snippet per wrapped command when an
+# instrumentation query is present.  The snippet's "result" field must carry
+# the wrapped command's exit code, which for a signal-killed command is
+# non-zero.
+set(datadir
+  "${RunCMake_BINARY_DIR}/instrument-signal-data/.cmake/instrumentation/v1/data")
+file(GLOB snippets "${datadir}/test-*.json")
+if(NOT snippets)
+  set(RunCMake_TEST_FAILED "No instrumentation snippet was written to ${datadir}")
+  return()
+endif()
+list(GET snippets 0 snippet)
+file(READ "${snippet}" contents)
+string(JSON result GET "${contents}" result)
+if(NOT result STREQUAL "1")
+  set(RunCMake_TEST_FAILED
+    "Instrumentation snippet result field is [${result}], expected [1]")
+endif()

Tests/RunCMake/CTestLaunch/instrument-signal-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestLaunch/launch-exit-code-result.txt

@@ -0,0 +1 @@
+3

Tests/RunCMake/CTestLaunch/launch-fail-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestLaunch/launch-signal-logs-check.cmake

@@ -0,0 +1,7 @@
+# Full launcher mode writes an error-*.xml fragment when the wrapped rule fails.
+# A signal-killed rule must produce one.
+file(GLOB error_files "${RunCMake_BINARY_DIR}/launch-signal-logs-logs/error-*.xml")
+if(NOT error_files)
+  set(RunCMake_TEST_FAILED
+    "No error-*.xml fragment was written for the signal-killed launched rule")
+endif()

Tests/RunCMake/CTestLaunch/launch-signal-logs-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestLaunch/launch-signal-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/CTestLaunch/launch-spawn-failure-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/ctest_build/NinjaLauncherSignalBuildFailure-check.cmake

@@ -0,0 +1,14 @@
+# A signal-killed build rule wrapped by ctest --launch must be recorded as a
+# build error in Build.xml.
+file(GLOB build_xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/Build.xml")
+if(NOT build_xml_file)
+  set(RunCMake_TEST_FAILED "Build.xml not found")
+  return()
+endif()
+file(READ "${build_xml_file}" build_xml)
+if(NOT build_xml MATCHES [[<Failure type="Error">]] OR
+   NOT build_xml MATCHES "Terminated abnormally")
+  string(REPLACE "\n" "\n  " build_xml "  ${build_xml}")
+  set(RunCMake_TEST_FAILED
+    "Build.xml does not report the signal-killed rule as an abnormal-termination error:\n${build_xml}")
+endif()

Tests/RunCMake/ctest_build/NinjaLauncherSignalBuildFailure-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_build/NinjaLauncherSignalBuildFailure-stderr.txt

@@ -0,0 +1 @@
+^Error\(s\) when building project

Tests/RunCMake/ctest_build/RunCMakeTest.cmake

@@ -191,4 +191,27 @@ if(RunCMake_GENERATOR MATCHES "Ninja")
     run_ctest(NinjaLauncherSingleBuildFailure)
   endfunction()
   run_NinjaLauncherSingleBuildFailure()
+
+  # A build rule that is terminated by a signal (rather than exiting non-zero)
+  # must still be reported as a build error.  POSIX-only: it relies on signal
+  # semantics.
+  if(UNIX)
+    function(run_NinjaLauncherSignalBuildFailure)
+      set(RunCMake_USE_LAUNCHERS TRUE)
+      set(RunCMake_TEST_SOURCE_DIR "${RunCMake_BINARY_DIR}/NinjaLauncherSignalBuildFailure")
+      # A tiny script that terminates its own shell with SIGTERM.  Kept in a
+      # file (rather than an inline "-c" command) so the "$$" is not mangled by
+      # build-tool command-line escaping.
+      file(WRITE "${RunCMake_TEST_SOURCE_DIR}/signal.sh" "kill -s TERM $$\n")
+      set(CASE_CMAKELISTS_SUFFIX_CODE [=[
+    add_custom_command(
+      OUTPUT signal.out
+      COMMAND /bin/sh "${CMAKE_CURRENT_SOURCE_DIR}/signal.sh"
+      VERBATIM)
+    add_custom_target(signal ALL DEPENDS signal.out)
+  ]=])
+      run_ctest(NinjaLauncherSignalBuildFailure)
+    endfunction()
+    run_NinjaLauncherSignalBuildFailure()
+  endif()
 endif()