AutoRcc: Cache the rcc listing of qrc files

AutoGen ran "rcc --list" for every qrc file on every run.  Cache the
listing next to the AutoRcc info file and reuse it as long as neither
the rcc executable nor the content of the qrc file changed.  Cache only
listings in which rcc listed every entry of the qrc file as given.
Entries that refer to directories are not, because rcc lists the
content of the directories, which may change without the qrc file
changing.

Add a RunCMake.Autogen_5 RccListCache case checking that the cached rcc
listing is reused while the qrc file is unchanged, refreshed when it
changes, and not used for qrc files that refer to directories.

Source/cmQtAutoGen.cxx

@@ -4,8 +4,10 @@
 
 #include <algorithm>
 #include <array>
+#include <cstddef>
 #include <initializer_list>
 #include <sstream>
+#include <unordered_set>
 #include <utility>
 
 #include <cmext/algorithm>
@@ -13,7 +15,10 @@
 #include "cmsys/FStream.hxx"
 #include "cmsys/RegularExpression.hxx"
 
+#include "cmCryptoHash.h"
 #include "cmDuration.h"
+#include "cmFileTime.h"
+#include "cmGeneratedFileStream.h"
 #include "cmProcessOutput.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
@@ -243,6 +248,30 @@ static void RccListParseContent(std::string const& content,
   }
 }
 
+static bool RccListReadQrcContent(std::string const& qrcFile,
+                                  std::string& qrcContents)
+{
+  cmsys::ifstream ifs(qrcFile.c_str());
+  if (!ifs) {
+    return false;
+  }
+  std::ostringstream osst;
+  osst << ifs.rdbuf();
+  qrcContents = osst.str();
+  return true;
+}
+
+static bool RccListReadQrc(std::string const& qrcFile,
+                           std::vector<std::string>& files)
+{
+  std::string qrcContents;
+  if (!RccListReadQrcContent(qrcFile, qrcContents)) {
+    return false;
+  }
+  RccListParseContent(qrcContents, files);
+  return true;
+}
+
 static bool RccListParseOutput(std::string const& rccStdOut,
                                std::string const& rccStdErr,
                                std::vector<std::string>& files,
@@ -292,6 +321,97 @@ static bool RccListParseOutput(std::string const& rccStdOut,
   return true;
 }
 
+// The header of a cached rcc listing identifies the rcc executable, its
+// options, and the qrc file.  The qrc file is identified by a hash of its
+// content, because its modification time may not change if it is modified
+// again within the time resolution of the file system.
+static std::string RccListCacheHeader(std::string const& rccExecutable,
+                                      std::vector<std::string> const& options,
+                                      std::string const& qrcFile,
+                                      std::string const& qrcContent)
+{
+  cmFileTime rccTime;
+  if (!rccTime.Load(rccExecutable)) {
+    return std::string();
+  }
+  cmCryptoHash sha256(cmCryptoHash::AlgoSHA256);
+  return cmStrCat(
+    "rcc=", rccExecutable, "\nrcc_time=", std::to_string(rccTime.GetTime()),
+    "\nrcc_size=", std::to_string(cmSystemTools::FileLength(rccExecutable)),
+    "\noptions=", cmJoin(options, ";"), "\nqrc=", qrcFile,
+    "\nqrc_sha256=", sha256.HashString(qrcContent), '\n');
+}
+
+static bool RccListReadCache(std::string const& cacheFile,
+                             std::string const& header,
+                             std::vector<std::string>& files)
+{
+  cmsys::ifstream fin(cacheFile.c_str(), std::ios::in | std::ios::binary);
+  if (!fin) {
+    return false;
+  }
+  std::ostringstream content;
+  content << fin.rdbuf();
+  std::string const cache = content.str();
+  if (!cmHasPrefix(cache, header)) {
+    return false;
+  }
+  std::vector<std::string> cachedFiles;
+  std::istringstream lines(cache.substr(header.size()));
+  std::string line;
+  while (std::getline(lines, line)) {
+    // Tolerate line endings converted to CRLF.
+    if (!line.empty() && line.back() == '\r') {
+      line.pop_back();
+    }
+    if (line.empty()) {
+      continue;
+    }
+    // rcc lists the content of directories, so the listing is outdated
+    // if a listed path became a directory.
+    if (cmSystemTools::FileIsDirectory(line)) {
+      return false;
+    }
+    cachedFiles.push_back(std::move(line));
+  }
+  cm::append(files, cachedFiles);
+  return true;
+}
+
+static void RccListWriteCache(std::string const& cacheFile,
+                              std::string const& header,
+                              std::string const& qrcContent,
+                              std::string const& qrcDir,
+                              std::vector<std::string>::const_iterator first,
+                              std::vector<std::string>::const_iterator last)
+{
+  // rcc lists the content of directories referred to by the qrc file, which
+  // may change without the qrc file changing.  Do not cache such listings.
+  // Rather than interpreting the qrc file the way rcc does, cache only
+  // listings in which rcc listed each entry of the qrc file as given.
+  // Directories, and entries rcc resolves differently, e.g. names with
+  // XML character references, are not listed as given.
+  {
+    std::vector<std::string> entries;
+    RccListParseContent(qrcContent, entries);
+    std::unordered_set<std::string> const listed(first, last);
+    for (std::string const& entry : entries) {
+      if (!listed.count(cmSystemTools::CollapseFullPath(
+            cmTrimWhitespace(entry), qrcDir))) {
+        cmSystemTools::RemoveFile(cacheFile);
+        return;
+      }
+    }
+  }
+
+  std::string content = header;
+  for (; first != last; ++first) {
+    content += *first;
+    content += '\n';
+  }
+  cmGeneratedFileStream::WriteIfDifferent(cacheFile, content, true);
+}
+
 cmQtAutoGen::RccLister::RccLister() = default;
 
 cmQtAutoGen::RccLister::RccLister(std::string rccExecutable,
@@ -303,7 +423,8 @@ cmQtAutoGen::RccLister::RccLister(std::string rccExecutable,
 
 bool cmQtAutoGen::RccLister::list(std::string const& qrcFile,
                                   std::vector<std::string>& files,
-                                  std::string& error, bool verbose) const
+                                  std::string& error, bool verbose,
+                                  std::string const& cacheFile) const
 {
   error.clear();
 
@@ -319,10 +440,24 @@ bool cmQtAutoGen::RccLister::list(std::string const& qrcFile,
   // contains non-ASCII characters.
   std::string const fileDir = cmSystemTools::GetFilenamePath(qrcFile);
 
+  std::size_t const firstFile = files.size();
+  std::string cacheHeader;
+  std::string qrcContent;
   if (!this->RccExecutable_.empty() &&
       cmSystemTools::FileExists(this->RccExecutable_, true) &&
       !this->ListOptions_.empty()) {
 
+    // Running rcc is expensive.  Reuse the listing of a previous run if
+    // neither rcc nor the qrc file changed since.
+    if (!cacheFile.empty() && RccListReadQrcContent(qrcFile, qrcContent)) {
+      cacheHeader = RccListCacheHeader(
+        this->RccExecutable_, this->ListOptions_, qrcFile, qrcContent);
+      if (!cacheHeader.empty() &&
+          RccListReadCache(cacheFile, cacheHeader, files)) {
+        return true;
+      }
+    }
+
     bool result = false;
     int retVal = 0;
     std::string rccStdOut;
@@ -360,22 +495,10 @@ bool cmQtAutoGen::RccLister::list(std::string const& qrcFile,
   } else {
     // We can't use rcc for the file listing.
     // Read the qrc file content into string and parse it.
-    {
-      std::string qrcContents;
-      {
-        cmsys::ifstream ifs(qrcFile.c_str());
-        if (ifs) {
-          std::ostringstream osst;
-          osst << ifs.rdbuf();
-          qrcContents = osst.str();
-        } else {
-          error = cmStrCat("The resource file ", Quoted(qrcFile),
-                           " is not readable\n");
-          return false;
-        }
-      }
-      // Parse string content
-      RccListParseContent(qrcContents, files);
+    if (!RccListReadQrc(qrcFile, files)) {
+      error =
+        cmStrCat("The resource file ", Quoted(qrcFile), " is not readable\n");
+      return false;
     }
   }
 
@@ -383,5 +506,10 @@ bool cmQtAutoGen::RccLister::list(std::string const& qrcFile,
   for (std::string& entry : files) {
     entry = cmSystemTools::CollapseFullPath(entry, fileDir);
   }
+
+  if (!cacheHeader.empty()) {
+    RccListWriteCache(cacheFile, cacheHeader, qrcContent, fileDir,
+                      files.cbegin() + firstFile, files.cend());
+  }
   return true;
 }

Source/cmQtAutoGen.h

@@ -147,9 +147,12 @@ public:
      * @brief Lists a files in the qrcFile
      * @arg files The file names are appended to this list
      * @arg error contains the error message when the function fails
+     * @arg cacheFile if not empty, the listing is cached in this file and
+     *      reused as long as neither rcc nor the qrcFile change
      */
     bool list(std::string const& qrcFile, std::vector<std::string>& files,
-              std::string& error, bool verbose = false) const;
+              std::string& error, bool verbose = false,
+              std::string const& cacheFile = std::string()) const;
 
   private:
     std::string RccExecutable_;

Source/cmQtAutoGenInitializer.cxx

@@ -1332,6 +1332,7 @@ bool cmQtAutoGenInitializer::InitScanFiles()
                                         qrc.QrcName, '_', qrc.QrcPathChecksum);
       qrc.LockFile = cmStrCat(base, "_Lock.lock");
       qrc.InfoFile = cmStrCat(base, "_Info.json");
+      qrc.ListFileBase = cmStrCat(base, "_List");
       this->ConfigFileNames(qrc.SettingsFile, cmStrCat(base, "_Used"), ".txt");
     }
     // rcc options
@@ -1364,8 +1365,9 @@ bool cmQtAutoGenInitializer::InitScanFiles()
             RccLister const lister(
               this->Rcc.Executable.Config[config],
               this->Rcc.ExecutableFeatures.Config[config]->ListOptions);
-            if (!lister.list(qrc.QrcFile, qrc.Resources.Config[config],
-                             error)) {
+            if (!lister.list(
+                  qrc.QrcFile, qrc.Resources.Config[config], error, false,
+                  cmStrCat(qrc.ListFileBase, '_', config, ".txt"))) {
               cmSystemTools::Error(error);
               return false;
             }
@@ -1374,7 +1376,8 @@ bool cmQtAutoGenInitializer::InitScanFiles()
           RccLister const lister(
             this->Rcc.Executable.Default,
             this->Rcc.ExecutableFeatures.Default->ListOptions);
-          if (!lister.list(qrc.QrcFile, qrc.Resources.Default, error)) {
+          if (!lister.list(qrc.QrcFile, qrc.Resources.Default, error, false,
+                           cmStrCat(qrc.ListFileBase, ".txt"))) {
             cmSystemTools::Error(error);
             return false;
           }

Source/cmQtAutoGenInitializer.h

@@ -43,6 +43,8 @@ public:
     std::string QrcName;
     std::string QrcPathChecksum;
     std::string InfoFile;
+    // Base name of the files caching the rcc listing of the qrc file
+    std::string ListFileBase;
     ConfigString SettingsFile;
     ConfigString OutputFile;
     std::string OutputFileGenex;

Tests/RunCMake/Autogen_5/RccListCache-check.cmake

@@ -0,0 +1,30 @@
+file(GLOB_RECURSE info_files
+  "${RunCMake_TEST_BINARY_DIR}/AutoRcc_res_*_Info.json")
+if(NOT info_files)
+  set(RunCMake_TEST_FAILED "AutoRcc info file not found.")
+  return()
+endif()
+list(GET info_files 0 info_file)
+file(READ "${info_file}" info)
+
+foreach(name IN LISTS expect_inputs)
+  if(NOT info MATCHES "/${name}\"")
+    string(APPEND RunCMake_TEST_FAILED
+      "Resource \"${name}\" is missing from the inputs in\n  ${info_file}\n")
+  endif()
+endforeach()
+foreach(name IN LISTS expect_no_inputs)
+  if(info MATCHES "/${name}\"")
+    string(APPEND RunCMake_TEST_FAILED
+      "Resource \"${name}\" is unexpectedly in the inputs in\n  ${info_file}\n")
+  endif()
+endforeach()
+
+file(GLOB_RECURSE cache_files
+  "${RunCMake_TEST_BINARY_DIR}/AutoRcc_res_*_List*.txt")
+if(expect_cache AND NOT cache_files)
+  string(APPEND RunCMake_TEST_FAILED "The rcc listing was not cached.\n")
+elseif(NOT expect_cache AND cache_files)
+  string(APPEND RunCMake_TEST_FAILED
+    "The rcc listing was unexpectedly cached in\n  ${cache_files}\n")
+endif()

Tests/RunCMake/Autogen_5/RccListCache.cmake

@@ -0,0 +1,7 @@
+enable_language(CXX)
+
+find_package(Qt${with_qt_version} REQUIRED COMPONENTS Core)
+
+add_library(dummy STATIC ../Autogen_common/example.cpp "${RCC_DATA_DIR}/res.qrc")
+target_link_libraries(dummy Qt${with_qt_version}::Core)
+set_target_properties(dummy PROPERTIES AUTORCC ON)

Tests/RunCMake/Autogen_5/RunCMakeTest.cmake

@@ -28,4 +28,69 @@ if (DEFINED with_qt_version)
       endforeach()
     endif()
   endif()
+
+  # The rcc listing of a qrc file is cached between runs.  It must be
+  # refreshed when the qrc file changes, and must not be cached at all when
+  # the qrc file refers to a directory, whose content may change without the
+  # qrc file changing.
+  block()
+    set(data_dir "${RunCMake_BINARY_DIR}/RccListCache-data")
+    file(REMOVE_RECURSE "${data_dir}")
+    file(WRITE "${data_dir}/alpha.txt" "alpha\n")
+    file(WRITE "${data_dir}/beta.txt" "beta\n")
+    file(WRITE "${data_dir}/dir/gamma.txt" "gamma\n")
+    function(write_qrc entry)
+      file(WRITE "${data_dir}/res.qrc" "<RCC>
+  <qresource prefix=\"/\">
+    <file>${entry}</file>
+  </qresource>
+</RCC>
+")
+    endfunction()
+
+    set(RunCMake_TEST_BINARY_DIR "${RunCMake_BINARY_DIR}/RccListCache-build")
+    list(APPEND RunCMake_TEST_OPTIONS "-DRCC_DATA_DIR=${data_dir}")
+
+    write_qrc(alpha.txt)
+    set(expect_inputs alpha.txt)
+    set(expect_no_inputs beta.txt)
+    set(expect_cache 1)
+    set(RunCMake_TEST_VARIANT_DESCRIPTION "-1-initial")
+    run_cmake(RccListCache)
+    set(RunCMake_TEST_NO_CLEAN 1)
+
+    # A changed qrc file is listed again, even if its modification time
+    # did not change because it was modified within the time resolution of
+    # the file system.
+    write_qrc(beta.txt)
+    set(expect_inputs beta.txt)
+    set(expect_no_inputs alpha.txt)
+    set(RunCMake_TEST_VARIANT_DESCRIPTION "-2-qrc-changed")
+    run_cmake(RccListCache)
+
+    # An unchanged qrc file is not listed again.  Prove it by adding an
+    # entry to the cached listing.
+    file(GLOB_RECURSE cache_files
+      "${RunCMake_TEST_BINARY_DIR}/AutoRcc_res_*_List*.txt")
+    foreach(cache_file IN LISTS cache_files)
+      file(APPEND "${cache_file}" "${data_dir}/injected.txt\n")
+    endforeach()
+    set(expect_inputs beta.txt injected.txt)
+    set(RunCMake_TEST_VARIANT_DESCRIPTION "-3-qrc-unchanged")
+    run_cmake(RccListCache)
+
+    # A qrc file referring to a directory is not cached.
+    write_qrc(dir)
+    set(expect_inputs gamma.txt)
+    set(expect_no_inputs beta.txt injected.txt)
+    set(expect_cache 0)
+    set(RunCMake_TEST_VARIANT_DESCRIPTION "-4-qrc-directory")
+    run_cmake(RccListCache)
+
+    # New files in the directory are found without changing the qrc file.
+    file(WRITE "${data_dir}/dir/delta.txt" "delta\n")
+    set(expect_inputs gamma.txt delta.txt)
+    set(RunCMake_TEST_VARIANT_DESCRIPTION "-5-directory-changed")
+    run_cmake(RccListCache)
+  endblock()
 endif ()