From 4fdfa0db1a8d530767fe3d32171247627bd947c6 Mon Sep 17 00:00:00 2001 From: AJIOB Date: Wed, 3 Dec 2025 10:40:52 +0300 Subject: [PATCH] file(ARCHIVE_CREATE): support multithreading compression Relates: #27420 --- Help/command/file.rst | 9 ++ Help/release/dev/cli-tar-multithread.rst | 2 + Source/cmFileCommand.cxx | 32 +++++-- .../RunCMake/File_Archive/RunCMakeTest.cmake | 7 ++ ...-validation-compression-level-1-stderr.txt | 4 +- ...-validation-compression-level-2-stderr.txt | 4 +- .../argument-validation-threads-result.txt | 1 + .../argument-validation-threads-stderr.txt | 5 ++ .../argument-validation-threads.cmake | 8 ++ Tests/RunCMake/File_Archive/roundtrip.cmake | 17 ---- Tests/RunCMake/File_Archive/threads-bz2.cmake | 10 +++ Tests/RunCMake/File_Archive/threads-gz.cmake | 10 +++ Tests/RunCMake/File_Archive/threads-xz.cmake | 10 +++ .../RunCMake/File_Archive/threads-zstd.cmake | 10 +++ Tests/RunCMake/File_Archive/threads.cmake | 85 +++++++++++++++++++ 15 files changed, 188 insertions(+), 26 deletions(-) create mode 100644 Tests/RunCMake/File_Archive/argument-validation-threads-result.txt create mode 100644 Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt create mode 100644 Tests/RunCMake/File_Archive/argument-validation-threads.cmake create mode 100644 Tests/RunCMake/File_Archive/threads-bz2.cmake create mode 100644 Tests/RunCMake/File_Archive/threads-gz.cmake create mode 100644 Tests/RunCMake/File_Archive/threads-xz.cmake create mode 100644 Tests/RunCMake/File_Archive/threads-zstd.cmake create mode 100644 Tests/RunCMake/File_Archive/threads.cmake diff --git a/Help/command/file.rst b/Help/command/file.rst index 52725902c2..73af028761 100644 --- a/Help/command/file.rst +++ b/Help/command/file.rst @@ -915,6 +915,7 @@ Archiving [COMPRESSION [COMPRESSION_LEVEL ]] [MTIME ] + [THREADS ] [WORKING_DIRECTORY ] [VERBOSE]) :target: ARCHIVE_CREATE @@ -959,6 +960,14 @@ Archiving ``MTIME `` Specify the modification time recorded in tarball entries. + ``THREADS `` + .. versionadded:: 4.3 + + Use the ```` threads to operate on the archive. + + The number of available cores on the machine will be used if set to ``0``. + Note that not all compression modes support threading in all environments. + ``WORKING_DIRECTORY `` .. versionadded:: 3.31 diff --git a/Help/release/dev/cli-tar-multithread.rst b/Help/release/dev/cli-tar-multithread.rst index 7065d70206..af47b8d260 100644 --- a/Help/release/dev/cli-tar-multithread.rst +++ b/Help/release/dev/cli-tar-multithread.rst @@ -2,3 +2,5 @@ cli-tar-multithread ------------------- * The :manual:`cmake(1)` ``-E tar`` tool supports multithreading operations +* The :command:`file(ARCHIVE_CREATE)` command supports multithreading via the + ``THREADS`` option diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 1f4a6f8e47..06916dd1ac 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -3666,6 +3666,7 @@ bool HandleArchiveCreateCommand(std::vector const& args, // accepted without one and treated as if an empty value were given. // Fixing this would require a policy. ArgumentParser::Maybe MTime; + std::string Threads; std::string WorkingDirectory; bool Verbose = false; // "PATHS" requires at least one value, but use a custom check below. @@ -3679,6 +3680,7 @@ bool HandleArchiveCreateCommand(std::vector const& args, .Bind("COMPRESSION"_s, &Arguments::Compression) .Bind("COMPRESSION_LEVEL"_s, &Arguments::CompressionLevel) .Bind("MTIME"_s, &Arguments::MTime) + .Bind("THREADS"_s, &Arguments::Threads) .Bind("WORKING_DIRECTORY"_s, &Arguments::WorkingDirectory) .Bind("VERBOSE"_s, &Arguments::Verbose) .Bind("PATHS"_s, &Arguments::Paths); @@ -3738,7 +3740,7 @@ bool HandleArchiveCreateCommand(std::vector const& args, } int compressionLevel = 0; - int minCompressionLevel = 0; + constexpr int minCompressionLevel = 0; int maxCompressionLevel = 9; if (compress == cmSystemTools::TarCompressZstd) { maxCompressionLevel = 19; @@ -3773,16 +3775,36 @@ bool HandleArchiveCreateCommand(std::vector const& args, } } + // Use the single thread by default for backward compatibility + int threads = 1; + constexpr int minThreads = 0; + if (!parsedArgs.Threads.empty()) { + if (parsedArgs.Threads.size() != 1 && + !std::isdigit(parsedArgs.Threads[0])) { + status.SetError(cmStrCat("number of threads ", parsedArgs.Threads, + " should be at least ", minThreads)); + cmSystemTools::SetFatalErrorOccurred(); + return false; + } + threads = std::stoi(parsedArgs.Threads); + if (threads < minThreads) { + status.SetError(cmStrCat("number of threads ", parsedArgs.Threads, + " should be at least ", minThreads)); + cmSystemTools::SetFatalErrorOccurred(); + return false; + } + } + if (parsedArgs.Paths.empty()) { status.SetError("ARCHIVE_CREATE requires a non-empty list of PATHS"); cmSystemTools::SetFatalErrorOccurred(); return false; } - if (!cmSystemTools::CreateTar(parsedArgs.Output, parsedArgs.Paths, - parsedArgs.WorkingDirectory, compress, - parsedArgs.Verbose, parsedArgs.MTime, - parsedArgs.Format, compressionLevel)) { + if (!cmSystemTools::CreateTar( + parsedArgs.Output, parsedArgs.Paths, parsedArgs.WorkingDirectory, + compress, parsedArgs.Verbose, parsedArgs.MTime, parsedArgs.Format, + compressionLevel, threads)) { status.SetError(cmStrCat("failed to compress: ", parsedArgs.Output)); cmSystemTools::SetFatalErrorOccurred(); return false; diff --git a/Tests/RunCMake/File_Archive/RunCMakeTest.cmake b/Tests/RunCMake/File_Archive/RunCMakeTest.cmake index c50476c550..140995738c 100644 --- a/Tests/RunCMake/File_Archive/RunCMakeTest.cmake +++ b/Tests/RunCMake/File_Archive/RunCMakeTest.cmake @@ -12,6 +12,13 @@ run_cmake(zip) run_cmake(working-directory) +# Check the --cmake-tar-threads option +run_cmake(argument-validation-threads) +run_cmake(threads-bz2) +run_cmake(threads-gz) +run_cmake(threads-xz) +run_cmake(threads-zstd) + # Extracting only selected files or directories run_cmake(zip-filtered) diff --git a/Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt b/Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt index 90b57b8dcf..53cd7f59c4 100644 --- a/Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt +++ b/Tests/RunCMake/File_Archive/argument-validation-compression-level-1-stderr.txt @@ -1,5 +1,5 @@ -CMake Error at compression-level\.cmake:39 \(file\): +^CMake Error at compression-level\.cmake:39 \(file\): file compression level 100 for GZip should be in range 0 to 9 Call Stack \(most recent call first\): argument-validation-compression-level-1\.cmake:8 \(check_compression_level\) - CMakeLists\.txt:3 \(include\) + CMakeLists\.txt:3 \(include\)$ diff --git a/Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt b/Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt index 77c8bbfcbc..4a8bc0c9e8 100644 --- a/Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt +++ b/Tests/RunCMake/File_Archive/argument-validation-compression-level-2-stderr.txt @@ -1,5 +1,5 @@ -CMake Error at compression-level\.cmake:39 \(file\): +^CMake Error at compression-level\.cmake:39 \(file\): file compression level high for GZip should be in range 0 to 9 Call Stack \(most recent call first\): argument-validation-compression-level-2\.cmake:8 \(check_compression_level\) - CMakeLists\.txt:3 \(include\) + CMakeLists\.txt:3 \(include\)$ diff --git a/Tests/RunCMake/File_Archive/argument-validation-threads-result.txt b/Tests/RunCMake/File_Archive/argument-validation-threads-result.txt new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/Tests/RunCMake/File_Archive/argument-validation-threads-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt b/Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt new file mode 100644 index 0000000000..efbeb0c1a1 --- /dev/null +++ b/Tests/RunCMake/File_Archive/argument-validation-threads-stderr.txt @@ -0,0 +1,5 @@ +^CMake Error at threads\.cmake:39 \(file\): + file number of threads -1 should be at least 0 +Call Stack \(most recent call first\): + argument-validation-threads\.cmake:8 \(check_threads\) + CMakeLists\.txt:3 \(include\)$ diff --git a/Tests/RunCMake/File_Archive/argument-validation-threads.cmake b/Tests/RunCMake/File_Archive/argument-validation-threads.cmake new file mode 100644 index 0000000000..9af0cbdefb --- /dev/null +++ b/Tests/RunCMake/File_Archive/argument-validation-threads.cmake @@ -0,0 +1,8 @@ +set(OUTPUT_NAME "test.tar.gz") + +set(ARCHIVE_FORMAT gnutar) +set(COMPRESSION_TYPE GZip) + +include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake) + +check_threads("-1") diff --git a/Tests/RunCMake/File_Archive/roundtrip.cmake b/Tests/RunCMake/File_Archive/roundtrip.cmake index fa6c841dbd..99b87af5cb 100644 --- a/Tests/RunCMake/File_Archive/roundtrip.cmake +++ b/Tests/RunCMake/File_Archive/roundtrip.cmake @@ -95,20 +95,3 @@ function(check_magic EXPECTED) "Actual [${ACTUAL}] does not match expected [${EXPECTED}]") endif() endfunction() - - -function(check_compression_level COMPRESSION_LEVEL) - file(ARCHIVE_CREATE - OUTPUT "${FULL_OUTPUT_NAME}_compression_level" - FORMAT "${ARCHIVE_FORMAT}" - COMPRESSION_LEVEL ${COMPRESSION_LEVEL} - COMPRESSION "${COMPRESSION_TYPE}" - VERBOSE - PATHS ${COMPRESS_DIR}) - - file(ARCHIVE_EXTRACT - INPUT "${FULL_OUTPUT_NAME}_compression_level" - ${DECOMPRESSION_OPTIONS} - DESTINATION ${FULL_DECOMPRESS_DIR} - VERBOSE) -endfunction() diff --git a/Tests/RunCMake/File_Archive/threads-bz2.cmake b/Tests/RunCMake/File_Archive/threads-bz2.cmake new file mode 100644 index 0000000000..774a5d9d6a --- /dev/null +++ b/Tests/RunCMake/File_Archive/threads-bz2.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.bz2") + +set(ARCHIVE_FORMAT paxr) +set(COMPRESSION_TYPE BZip2) + +include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake) + +check_threads("0") +check_threads("1") +check_threads("4") diff --git a/Tests/RunCMake/File_Archive/threads-gz.cmake b/Tests/RunCMake/File_Archive/threads-gz.cmake new file mode 100644 index 0000000000..4b9d5058ae --- /dev/null +++ b/Tests/RunCMake/File_Archive/threads-gz.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.gz") + +set(ARCHIVE_FORMAT gnutar) +set(COMPRESSION_TYPE GZip) + +include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake) + +check_threads("0") +check_threads("1") +check_threads("4") diff --git a/Tests/RunCMake/File_Archive/threads-xz.cmake b/Tests/RunCMake/File_Archive/threads-xz.cmake new file mode 100644 index 0000000000..2a3ec70c19 --- /dev/null +++ b/Tests/RunCMake/File_Archive/threads-xz.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.xz") + +set(ARCHIVE_FORMAT pax) +set(COMPRESSION_TYPE XZ) + +include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake) + +check_threads("0") +check_threads("1") +check_threads("4") diff --git a/Tests/RunCMake/File_Archive/threads-zstd.cmake b/Tests/RunCMake/File_Archive/threads-zstd.cmake new file mode 100644 index 0000000000..d0588ba2d5 --- /dev/null +++ b/Tests/RunCMake/File_Archive/threads-zstd.cmake @@ -0,0 +1,10 @@ +set(OUTPUT_NAME "test.tar.zstd") + +set(ARCHIVE_FORMAT pax) +set(COMPRESSION_TYPE Zstd) + +include(${CMAKE_CURRENT_LIST_DIR}/threads.cmake) + +check_threads("0") +check_threads("1") +check_threads("4") diff --git a/Tests/RunCMake/File_Archive/threads.cmake b/Tests/RunCMake/File_Archive/threads.cmake new file mode 100644 index 0000000000..6a44ab77fa --- /dev/null +++ b/Tests/RunCMake/File_Archive/threads.cmake @@ -0,0 +1,85 @@ +foreach(parameter OUTPUT_NAME ARCHIVE_FORMAT) + if(NOT DEFINED ${parameter}) + message(FATAL_ERROR "missing required parameter ${parameter}") + endif() +endforeach() + +set(COMPRESS_DIR compress_dir) +set(FULL_COMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${COMPRESS_DIR}) + +set(DECOMPRESS_DIR decompress_dir) +set(FULL_DECOMPRESS_DIR ${CMAKE_CURRENT_BINARY_DIR}/${DECOMPRESS_DIR}) + +set(FULL_OUTPUT_NAME ${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME}) + +set(CHECK_FILES + "f1.txt" + "d1/f1.txt" + "d 2/f1.txt" + "d + 3/f1.txt" + "d_4/f1.txt" + "d-4/f1.txt" + "My Special Directory/f1.txt" +) + +function(check_threads THREADS) + foreach(file ${CHECK_FILES}) + configure_file(${CMAKE_CURRENT_LIST_FILE} ${FULL_COMPRESS_DIR}/${file} COPYONLY) + endforeach() + + if(UNIX) + execute_process(COMMAND ln -sf f1.txt ${FULL_COMPRESS_DIR}/d1/f2.txt) + list(APPEND CHECK_FILES "d1/f2.txt") + endif() + + file(REMOVE ${FULL_OUTPUT_NAME}) + file(REMOVE_RECURSE ${FULL_DECOMPRESS_DIR}) + file(MAKE_DIRECTORY ${FULL_DECOMPRESS_DIR}) + + file(ARCHIVE_CREATE + OUTPUT ${FULL_OUTPUT_NAME} + FORMAT "${ARCHIVE_FORMAT}" + COMPRESSION "${COMPRESSION_TYPE}" + THREADS ${THREADS} + VERBOSE + PATHS ${COMPRESS_DIR}) + + file(ARCHIVE_EXTRACT + INPUT ${FULL_OUTPUT_NAME} + ${DECOMPRESSION_OPTIONS} + DESTINATION ${FULL_DECOMPRESS_DIR} + VERBOSE) + + if(CUSTOM_CHECK_FILES) + set(CHECK_FILES ${CUSTOM_CHECK_FILES}) + endif() + + foreach(file ${CHECK_FILES}) + set(input ${FULL_COMPRESS_DIR}/${file}) + set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file}) + + if(NOT EXISTS ${input}) + message(SEND_ERROR "Cannot find input file ${output}") + endif() + + if(NOT EXISTS ${output}) + message(SEND_ERROR "Cannot find output file ${output}") + endif() + + file(MD5 ${input} input_md5) + file(MD5 ${output} output_md5) + + if(NOT input_md5 STREQUAL output_md5) + message(SEND_ERROR "Files \"${input}\" and \"${output}\" are different") + endif() + endforeach() + + foreach(file ${NOT_EXISTING_FILES_CHECK}) + set(output ${FULL_DECOMPRESS_DIR}/${COMPRESS_DIR}/${file}) + + if(EXISTS ${output}) + message(SEND_ERROR "File ${output} exists but it shouldn't") + endif() + endforeach() + +endfunction()