instrumentation: Add environment variable to disable collection and indexing

Fixes: #28179

Help/envvar/CTEST_DISABLE_INSTRUMENTATION.rst

@@ -0,0 +1,22 @@
+CTEST_DISABLE_INSTRUMENTATION
+-----------------------------
+
+.. versionadded:: 4.6
+
+.. include:: include/ENV_VAR.rst
+
+When set to a true value, such as ``1``, ``True``, or ``ON``, this
+environment variable disables all :manual:`CMake instrumentation
+<cmake-instrumentation(7)>` data collection, indexing hooks, and callbacks.
+This includes manual indexing with :option:`ctest --collect-instrumentation`.
+
+This setting takes precedence over project-level and user-level queries,
+the :command:`cmake_instrumentation` command, and the
+:envvar:`CTEST_USE_INSTRUMENTATION` and
+:envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` environment variables.
+It also applies when building or testing an already-configured build tree.
+Queries are still loaded and generated normally, but no instrumentation data
+is collected or indexed.
+
+Unset the variable, or set it to a false value such as ``0``, ``OFF``, or an
+empty string, to allow instrumentation to operate normally.

Help/manual/cmake-env-variables.7.rst

@@ -125,6 +125,7 @@ Environment Variables for CTest
    :maxdepth: 1
 
    /envvar/CMAKE_CONFIG_TYPE
+   /envvar/CTEST_DISABLE_INSTRUMENTATION
    /envvar/CTEST_INTERACTIVE_DEBUG_MODE
    /envvar/CTEST_NO_TESTS_ACTION
    /envvar/CTEST_OUTPUT_ON_FAILURE

Help/manual/cmake-instrumentation.7.rst

@@ -108,6 +108,13 @@ for all CMake projects configured and built by a user. In all cases, a "query"
 represents a request for instrumentation behavior. See the `v1 Query Files`_
 for details on configuring this feature.
 
+.. versionadded:: 4.6
+
+  To disable all instrumentation data collection, indexing hooks, and callbacks,
+  set the :envvar:`CTEST_DISABLE_INSTRUMENTATION` environment variable to a
+  true value.  This overrides all methods of enabling instrumentation,
+  including in already-configured build trees.
+
 Enabling Instrumentation at the Project-Level
 ---------------------------------------------
 

Help/release/dev/instrumentation-disabled-env.rst

@@ -0,0 +1,5 @@
+instrumentation-disabled-env
+----------------------------
+
+* The :envvar:`CTEST_DISABLE_INSTRUMENTATION` environment variable was added
+  to disable all instrumentation data collection, indexing hooks, and callbacks.

Source/CTest/cmCTestLaunch.cxx

@@ -338,13 +338,15 @@ void cmCTestLaunch::RunChild()
 
 int cmCTestLaunch::Run()
 {
+  auto instrumentation = cmInstrumentation(this->Reporter.OptionBuildDir);
   if (this->Operation == Op::InstrumentTest) {
     this->RunChild();
-    this->WriteMetricsFile();
+    if (!instrumentation.IsDisabled()) {
+      this->WriteMetricsFile();
+    }
     return this->Reporter.ExitCode;
   }
 
-  auto instrumentation = cmInstrumentation(this->Reporter.OptionBuildDir);
   bool const captureOutput =
     instrumentation.HasOption(cmInstrumentationQuery::Option::CaptureOutput);
   bool const processMetrics =

Source/CTest/cmCTestRunTest.cxx

@@ -739,7 +739,8 @@ void cmCTestRunTest::ComputeArguments()
   this->InstrumentationCommand = this->ActualCommand;
   this->InstrumentationArguments = this->Arguments;
 
-  if (this->CTest->GetInstrumentation().HasOption(
+  if (!this->CTest->GetInstrumentation().IsDisabled() &&
+      this->CTest->GetInstrumentation().HasOption(
         cmInstrumentationQuery::Option::ProcessMetrics)) {
     std::string const realCommand = this->ActualCommand;
     std::vector<std::string> realArguments = this->Arguments;
@@ -1288,7 +1289,8 @@ void cmCTestRunTest::FinalizeTest(bool started)
     this->CollectCMakeCoverage();
   }
 
-  if (started && this->CTest->GetInstrumentation().HasQuery()) {
+  if (started && !this->CTest->GetInstrumentation().IsDisabled() &&
+      this->CTest->GetInstrumentation().HasQuery()) {
     if (this->CTest->GetInstrumentation().HasOption(
           cmInstrumentationQuery::Option::ProcessMetrics)) {
       std::string metricsFile = GetTestMetricsFile();

Source/cmGlobalFastbuildGenerator.cxx

@@ -285,10 +285,14 @@ void cmGlobalFastbuildGenerator::ProcessEnvironment()
     }
   }
 
-  // Empty strings are not allowed.
+  // Empty strings are not allowed. Instrumentation disabling is a build-time
+  // control, so do not freeze its configure-time value in the environment.
   LocalEnvironment.erase(
     std::remove_if(LocalEnvironment.begin(), LocalEnvironment.end(),
-                   [](std::string const& s) { return s.empty(); }),
+                   [](std::string const& s) {
+                     return s.empty() ||
+                       cmHasLiteralPrefix(s, "CTEST_DISABLE_INSTRUMENTATION=");
+                   }),
     LocalEnvironment.end());
 }
 
@@ -582,6 +586,12 @@ void cmGlobalFastbuildGenerator::WriteEnvironment()
 {
   if (!LocalEnvironment.empty()) {
     WriteArray(FASTBUILD_ENV_VAR_NAME, Wrap(LocalEnvironment), 0);
+    *this->BuildFileStream
+      << "#if exists(CTEST_DISABLE_INSTRUMENTATION)\n"
+         "  #import CTEST_DISABLE_INSTRUMENTATION\n"
+         "  ." FASTBUILD_ENV_VAR_NAME
+         " + 'CTEST_DISABLE_INSTRUMENTATION=$CTEST_DISABLE_INSTRUMENTATION$'\n"
+         "#endif\n";
   }
 }
 

Source/cmInstrumentation.cxx

@@ -103,6 +103,10 @@ std::map<std::string, std::string> cmInstrumentation::cdashSnippetsMap = {
 cmInstrumentation::cmInstrumentation(std::string const& binary_dir,
                                      LoadQueriesAfter loadQueries)
 {
+  std::string envVal;
+  this->disabled =
+    cmSystemTools::GetEnv("CTEST_DISABLE_INSTRUMENTATION", envVal) &&
+    !cmIsOff(envVal);
   this->binaryDir = binary_dir;
   this->timingDirv1 = cmStrCat(this->binaryDir, "/.cmake/instrumentation/v1");
   this->cdashDir = cmStrCat(this->timingDirv1, "/cdash");
@@ -239,6 +243,9 @@ void cmInstrumentation::AddCustomContent(std::string const& name,
 void cmInstrumentation::WriteCMakeContent(
   std::unique_ptr<cmGlobalGenerator> const& gg)
 {
+  if (this->disabled) {
+    return;
+  }
   Json::Value root;
   root["targets"] = this->DumpTargets(gg);
   root["custom"] = this->customContent;
@@ -352,8 +359,9 @@ bool cmInstrumentation::HasHook(cmInstrumentationQuery::Hook hook) const
 
 int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook)
 {
-  // Don't run collection if hook is disabled
-  if (hook != cmInstrumentationQuery::Hook::Manual && !this->HasHook(hook)) {
+  // Don't run collection if instrumentation or the hook is disabled.
+  if (this->disabled ||
+      (hook != cmInstrumentationQuery::Hook::Manual && !this->HasHook(hook))) {
     return 0;
   }
 
@@ -609,6 +617,9 @@ std::string cmInstrumentation::InstrumentTest(
   cm::optional<std::string> output,
   cm::optional<ProcessMetrics> processMetrics)
 {
+  if (this->disabled) {
+    return {};
+  }
   // Store command info
   Json::Value root(this->preTestStats);
   std::string command_str = cmStrCat(command, ' ', GetCommandStr(args));
@@ -648,6 +659,9 @@ std::string cmInstrumentation::InstrumentTest(
 
 void cmInstrumentation::GetPreTestStats()
 {
+  if (this->disabled) {
+    return;
+  }
   if (this->HasOption(
         cmInstrumentationQuery::Option::DynamicSystemInformation)) {
     this->InsertDynamicSystemInformation(this->preTestStats, "before");
@@ -661,9 +675,15 @@ int cmInstrumentation::InstrumentCommand(
   cm::optional<std::map<std::string, std::string>> arrayData,
   LoadQueriesAfter reloadQueriesAfterCommand)
 {
-
-  // Always begin gathering data for configure in case cmake_instrumentation
-  // command creates a query
+  if (this->disabled) {
+    int ret = callback().ExitCode;
+    if (reloadQueriesAfterCommand == LoadQueriesAfter::Yes) {
+      this->LoadQueries();
+    }
+    return ret;
+  }
+  // Unless instrumentation is disabled, begin gathering data for configure in
+  // case the cmake_instrumentation command creates a query.
   if (!this->hasQuery && reloadQueriesAfterCommand == LoadQueriesAfter::No) {
     return callback().ExitCode;
   }
@@ -928,6 +948,9 @@ bool cmInstrumentation::IsInstrumentableTargetType(cm::TargetType type)
  */
 int cmInstrumentation::SpawnBuildDaemon()
 {
+  if (this->disabled) {
+    return 0;
+  }
   // Do not inherit handles from the parent process, so that the daemon is
   // fully detached. This helps prevent deadlock between the two.
   uv_disable_stdio_inheritance();
@@ -972,6 +995,9 @@ bool cmInstrumentation::LockIndexing()
 bool cmInstrumentation::AcquireLock(std::string const& lock_file,
                                     cmFileLock& lock, unsigned long timeout)
 {
+  if (this->disabled) {
+    return false;
+  }
   std::string const lock_path = cmStrCat(this->timingDirv1, '/', lock_file);
   if (!cmSystemTools::FileExists(lock_path)) {
     cmSystemTools::Touch(lock_path, true);
@@ -987,6 +1013,9 @@ bool cmInstrumentation::AcquireLock(std::string const& lock_file,
  */
 int cmInstrumentation::CollectTimingAfterBuild(int ppid)
 {
+  if (this->disabled) {
+    return 0;
+  }
   // Check if another process is already instrumenting the build.
   // This lock will be released when the process exits at the end of the build.
   if (!this->LockBuildDaemon()) {

Source/cmInstrumentation.h

@@ -45,6 +45,7 @@ public:
                     LoadQueriesAfter loadQueries = LoadQueriesAfter::Yes);
   void LoadQueries();
   void CheckCDashVariable();
+  bool IsDisabled() const { return this->disabled; }
   struct TimeValue
   {
     long tv_sec;
@@ -180,6 +181,7 @@ private:
   static std::map<std::string, std::string> cdashSnippetsMap;
   Json::Value preTestStats;
   std::string errorMsg;
+  bool disabled = false;
   bool hasQuery = false;
   bool ranSystemChecks = false;
   bool ranOSCheck = false;

Tests/CMakeLists.txt

@@ -36,7 +36,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/CheckSwift.cmake)
 set(TEST_HOME "${CMake_BINARY_DIR}/Tests/CMakeFiles/TestHome")
 set(TEST_CONFIG_ENV_CODE "# Isolate tests from user-wide configuration.
 set(ENV{CMAKE_CONFIG_DIR} \"${TEST_HOME}/.config/cmake\")\n")
-set(TEST_INSTRUMENTATION_ENV_CODE "# Isolate tests from CTEST_USE_INSTRUMENTATION var
+set(TEST_INSTRUMENTATION_ENV_CODE "# Isolate tests from instrumentation environment variables.
+unset(ENV{CTEST_DISABLE_INSTRUMENTATION})
 unset(ENV{CTEST_USE_INSTRUMENTATION})\n")
 file(MAKE_DIRECTORY "${TEST_HOME}/.config/cmake")
 

Tests/RunCMake/Instrumentation/RunCMakeTest.cmake

@@ -33,6 +33,7 @@ function(instrument test)
     "PRESERVE_DATA"
     "NO_CONFIGURE"
     "DISABLE_TEST"
+    "DISABLE_AFTER_CONFIGURE"
     "FAIL"
     "BAD_QUERY"
   )
@@ -197,6 +198,17 @@ function(instrument test)
     run_cmake_with_options(${test} ${ARGS_CONFIGURE_ARGS} ${maybe_CMAKE_BUILD_TYPE})
   endif()
 
+  if (ARGS_DISABLE_AFTER_CONFIGURE)
+    if(NOT EXISTS ${RunCMake_TEST_BINARY_DIR}/instrumentation-hook)
+      message(SEND_ERROR "Instrumentation hook did not run despite CTEST_DISABLE_INSTRUMENTATION being OFF")
+    endif()
+    set(ENV{CTEST_DISABLE_INSTRUMENTATION} ON)
+    file(REMOVE_RECURSE ${v1}/data ${v1}/cdash)
+    file(REMOVE ${v1}/.index.lock ${RunCMake_TEST_BINARY_DIR}/instrumentation-hook)
+    file(WRITE ${v1}/data/do-not-remove.json "{}\n")
+    file(WRITE ${v1}/data/test/do-not-remove.json "{}\n")
+  endif()
+
   # Follow-up Commands
   if (ARGS_BUILD)
     set(cmake_build_args --config Debug)
@@ -461,8 +473,28 @@ function(instrument test)
     unset(RunCMake-check-file)
     unset(RunCMake_CHECK_ONLY)
   endif()
+  if (ARGS_DISABLE_AFTER_CONFIGURE)
+    unset(ENV{CTEST_DISABLE_INSTRUMENTATION})
+  endif()
 endfunction()
 
+# Disable collection and hooks while continuing to generate and load queries.
+set(ENV{CTEST_DISABLE_INSTRUMENTATION} ON)
+set(ENV{CTEST_USE_INSTRUMENTATION} ON)
+set(ENV{CTEST_USE_VERBOSE_INSTRUMENTATION} ON)
+instrument(instrumentation-disabled BUILD INSTALL TEST MANUAL_HOOK
+  CHECK_SCRIPT check-disabled.cmake)
+unset(ENV{CTEST_DISABLE_INSTRUMENTATION})
+unset(ENV{CTEST_USE_INSTRUMENTATION})
+unset(ENV{CTEST_USE_VERBOSE_INSTRUMENTATION})
+
+# Also disable instrumentation in an already-configured build tree.
+set(ENV{CTEST_DISABLE_INSTRUMENTATION} OFF)
+instrument(instrumentation-disabled BUILD INSTALL TEST MANUAL_HOOK
+  DISABLE_AFTER_CONFIGURE
+  CHECK_SCRIPT check-disabled.cmake)
+unset(ENV{CTEST_DISABLE_INSTRUMENTATION})
+
 # Bad Queries
 instrument(bad-option BAD_QUERY
   CHECK_SCRIPT check-query-dir.cmake

Tests/RunCMake/Instrumentation/check-disabled.cmake

@@ -0,0 +1,20 @@
+include(${CMAKE_CURRENT_LIST_DIR}/json.cmake)
+
+file(GLOB_RECURSE data_files LIST_DIRECTORIES false "${v1}/data/*")
+if(ARGS_DISABLE_AFTER_CONFIGURE)
+  set(expected_data_files "${v1}/data/do-not-remove.json;${v1}/data/test/do-not-remove.json")
+  if(NOT data_files STREQUAL expected_data_files)
+    add_error("Existing instrumentation data changed while disabled.")
+  endif()
+elseif(data_files)
+  add_error("Instrumentation data was collected while disabled.")
+endif()
+if(EXISTS "${v1}/cdash" OR
+   EXISTS "${v1}/.build.lock" OR EXISTS "${v1}/.index.lock" OR
+   EXISTS "${RunCMake_TEST_BINARY_DIR}/instrumentation-hook")
+  add_error("Instrumentation collected data or ran hooks while disabled.")
+endif()
+file(GLOB queries "${v1}/query/generated/*.json")
+if(NOT queries)
+  add_error("Instrumentation queries were not generated while disabled.")
+endif()

Tests/RunCMake/Instrumentation/query/instrumentation-disabled.cmake

@@ -0,0 +1,9 @@
+cmake_instrumentation(
+  API_VERSION 1
+  DATA_VERSION 1
+  OPTIONS staticSystemInformation dynamicSystemInformation captureOutput
+          processMetrics trace cdashSubmit cdashVerbose
+  HOOKS postGenerate preBuild postBuild preCMakeBuild postCMakeBuild
+        postCTest postCMakeInstall
+  CALLBACK ${CMAKE_COMMAND} -E touch "${CMAKE_BINARY_DIR}/instrumentation-hook"
+)