Merge branch 'cl/regexec-macos-leak' into jch

A compatibility workaround has been introduced for macOS to address
a memory leak in the system regex engine when it encounters invalid
multibyte sequences.  The workaround segments the input buffer at
invalid byte boundaries and searches each valid segment separately
using regexec(), avoiding the leaking path.

* cl/regexec-macos-leak:
  SQUASH???
  regexec: work around macOS TRE leak on invalid UTF-8

Makefile

@@ -110,6 +110,9 @@ include shared.mak
 # Define USE_HOMEBREW_LIBICONV to link against libiconv installed by
 # Homebrew, if present.
 #
+# Define DARWIN_REGEXEC if regexec() in your platform regex library
+# leaks when fed an invalid UTF-8 sequence.
+#
 # Define NO_APPLE_COMMON_CRYPTO if you are building on Darwin/Mac OS X
 # and do not want to use Apple's CommonCrypto library.  This allows you
 # to provide your own OpenSSL library, for example from MacPorts.
@@ -2282,6 +2285,10 @@ ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS
 	COMPAT_CFLAGS += -DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS
 	COMPAT_OBJS += compat/regcomp_enhanced.o
 endif
+ifdef DARWIN_REGEXEC
+	COMPAT_OBJS += compat/darwin/regexec.o
+	BASIC_CFLAGS += -DDARWIN_REGEXEC
+endif
 endif
 ifdef NATIVE_CRLF
 	BASIC_CFLAGS += -DNATIVE_CRLF

compat/darwin.h

@@ -0,0 +1,8 @@
+#ifndef COMPAT_DARWIN_H
+#define COMPAT_DARWIN_H
+
+int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size,
+		       size_t nmatch, regmatch_t pmatch[], int eflags);
+#define regexec_buf darwin_regexec_buf
+
+#endif

compat/darwin/regexec.c

@@ -0,0 +1,91 @@
+#include "git-compat-util.h"
+
+#include <wchar.h>
+
+/*
+ * Darwin's TRE regex engine leaks an internal buffer when it encounters an
+ * invalid multibyte sequence.  Since the leak has already happened when
+ * regexec() reports REG_ILLSEQ, keep invalid bytes out of regexec() by
+ * searching each valid segment separately.
+ */
+
+/*
+ * Search buf[start, end), where size is the full size of buf.  REG_STARTEND
+ * keeps match offsets relative to buf.  Do not let an internal segment create
+ * a false beginning or end of line.
+ */
+static int regexec_segment(const regex_t *preg, const char *buf,
+			   size_t size, size_t start, size_t end,
+			   size_t nmatch, regmatch_t pmatch[], int eflags)
+{
+	eflags |= REG_STARTEND;
+	if (start > 0)
+		eflags |= REG_NOTBOL;
+	if (end < size)
+		eflags |= REG_NOTEOL;
+	pmatch[0].rm_so = start;
+	pmatch[0].rm_eo = end;
+	return regexec(preg, buf, nmatch, pmatch, eflags);
+}
+
+int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size,
+		       size_t nmatch, regmatch_t pmatch[], int eflags)
+{
+	size_t seg_start = 0, i = 0;
+	mbstate_t mbs;
+
+	assert(nmatch > 0 && pmatch);
+
+	/*
+	 * A single-byte locale cannot contain an invalid multibyte sequence,
+	 * so use regexec() directly.
+	 */
+	if (MB_CUR_MAX == 1) {
+		pmatch[0].rm_so = 0;
+		pmatch[0].rm_eo = size;
+		return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND);
+	}
+
+	memset(&mbs, 0, sizeof(mbs));
+	while (i < size) {
+		unsigned char c = (unsigned char)buf[i];
+		size_t n;
+
+		if (c < 0x80) {
+			i++;
+			continue;
+		}
+
+		n = mbrtowc(NULL, buf + i, size - i, &mbs);
+		if (!n)
+			n = 1;
+		if (n != (size_t)-1 && n != (size_t)-2) {
+			i += n;
+			continue;
+		}
+
+		/*
+		 * -1 denotes an encoding error; -2 denotes an incomplete
+		 * trailing sequence.  In either case, buf[i] cannot begin a
+		 * complete valid character within this buffer.  Search an
+		 * empty initial segment to preserve zero-width matches at the
+		 * true beginning.
+		 */
+		if (i > seg_start || i == 0) {
+			int ret = regexec_segment(preg, buf, size, seg_start, i,
+						  nmatch, pmatch, eflags);
+			if (ret != REG_NOMATCH)
+				return ret;
+		}
+		i++;
+		seg_start = i;
+		memset(&mbs, 0, sizeof(mbs));
+	}
+
+	/*
+	 * Search the final segment even when it is empty, so an empty buffer
+	 * or a buffer ending in invalid bytes still has its true end.
+	 */
+	return regexec_segment(preg, buf, size, seg_start, size,
+			       nmatch, pmatch, eflags);
+}

config.mak.uname

@@ -154,6 +154,7 @@ ifeq ($(uname_S),Darwin)
 	HAVE_DEV_TTY = YesPlease
 	COMPAT_OBJS += compat/precompose_utf8.o
 	BASIC_CFLAGS += -DPRECOMPOSE_UNICODE
+	DARWIN_REGEXEC = YesPlease
 	BASIC_CFLAGS += -DPROTECT_HFS_DEFAULT=1
 	HAVE_BSD_SYSCTL = YesPlease
 	FREAD_READS_DIRECTORIES = UnfortunatelyYes

contrib/buildsystems/CMakeLists.txt

@@ -523,6 +523,9 @@ if(NOT HAVE_REGEX)
 	include_directories(${CMAKE_SOURCE_DIR}/compat/regex)
 	list(APPEND compat_SOURCES compat/regex/regex.c )
 	add_compile_definitions(NO_REGEX NO_MBSUPPORT GAWK)
+elseif(APPLE)
+	list(APPEND compat_SOURCES compat/darwin/regexec.c)
+	add_compile_definitions(DARWIN_REGEXEC)
 endif()
 
 

git-compat-util.h

@@ -162,6 +162,9 @@ static inline int is_xplatform_dir_sep(int c)
 #include "compat/win32/path-utils.h"
 #include "compat/msvc.h"
 #endif
+#ifdef DARWIN_REGEXEC
+#include "compat/darwin.h"
+#endif
 
 /* used on Mac OS X */
 #ifdef PRECOMPOSE_UNICODE
@@ -996,6 +999,7 @@ static inline int strtol_i(char const *s, int base, int *result)
 #error "Git requires REG_STARTEND support. Compile with NO_REGEX=NeedsStartEnd"
 #endif
 
+#ifndef regexec_buf
 static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size,
 			      size_t nmatch, regmatch_t pmatch[], int eflags)
 {
@@ -1004,6 +1008,7 @@ static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size,
 	pmatch[0].rm_eo = size;
 	return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND);
 }
+#endif
 
 #ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS
 int git_regcomp(regex_t *preg, const char *pattern, int cflags);

meson.build

@@ -1407,6 +1407,11 @@ if not get_option('b_sanitize').contains('address') and get_option('regex').allo
     libgit_c_args += '-DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS'
     compat_sources += 'compat/regcomp_enhanced.c'
   endif
+
+  if host_machine.system() == 'darwin'
+    libgit_c_args += '-DDARWIN_REGEXEC'
+    compat_sources += 'compat/darwin/regexec.c'
+  endif
 elif not get_option('regex').enabled()
   libgit_c_args += [
     '-DNO_REGEX',

t/t7810-grep.sh

@@ -89,6 +89,10 @@ test_expect_success setup '
 	function dummy() {}
 	EOF
 	printf "\200\nASCII\n" >invalid-utf8 &&
+	printf "before\346world\n" >invalid-utf8-embedded &&
+	printf "a\346b\347c\n" >invalid-utf8-multi &&
+	printf "\346world\n" >invalid-utf8-leading &&
+	printf "before\346\n" >invalid-utf8-trailing &&
 	if test_have_prereq FUNNYNAMES
 	then
 		echo unusual >"\"unusual\" pathname" &&
@@ -595,6 +599,39 @@ test_expect_success MB_REGEX 'grep two chars in single-char multibyte file' '
 	LC_ALL=en_US.UTF-8 test_expect_code 1 git grep ".." reverse-question-mark
 '
 
+test_expect_success MACOS,MB_REGEX 'grep matches valid text on both sides of invalid UTF-8' '
+	LC_ALL=en_US.UTF-8 git grep -h "befo[r]e" invalid-utf8-embedded >actual &&
+	test_cmp invalid-utf8-embedded actual &&
+	LC_ALL=en_US.UTF-8 git grep -h "worl[d]" invalid-utf8-embedded >actual &&
+	test_cmp invalid-utf8-embedded actual &&
+	LC_ALL=en_US.UTF-8 git grep -h -o "worl[d]" invalid-utf8-embedded >actual &&
+	echo world >expected &&
+	test_cmp expected actual
+'
+
+test_expect_success MACOS,MB_REGEX 'grep matches a run between two invalid sequences' '
+	LC_ALL=en_US.UTF-8 git grep -h "[b]" invalid-utf8-multi >actual &&
+	test_cmp invalid-utf8-multi actual
+'
+
+test_expect_success MB_REGEX 'grep does not anchor ^ or $ inside an invalid-byte line' '
+	test_expect_code 1 env LC_ALL=en_US.UTF-8 \
+		git grep -h "^world" invalid-utf8-embedded &&
+	test_expect_code 1 env LC_ALL=en_US.UTF-8 \
+		git grep -h "before\$" invalid-utf8-embedded
+'
+
+test_expect_success MACOS,MB_REGEX 'grep anchors ^ and $ at true line ends past invalid UTF-8' '
+	LC_ALL=en_US.UTF-8 git grep -h "^before" invalid-utf8-embedded >actual &&
+	test_cmp invalid-utf8-embedded actual &&
+	LC_ALL=en_US.UTF-8 git grep -h "world\$" invalid-utf8-embedded >actual &&
+	test_cmp invalid-utf8-embedded actual &&
+	LC_ALL=en_US.UTF-8 git grep -h "^" invalid-utf8-leading >actual &&
+	test_cmp invalid-utf8-leading actual &&
+	LC_ALL=en_US.UTF-8 git grep -h "\$" invalid-utf8-trailing >actual &&
+	test_cmp invalid-utf8-trailing actual
+'
+
 cat >expected <<EOF
 file
 EOF