fix(telegram): preserve astral chunk boundaries

This commit is contained in:
Vincent Koc
2026-06-17 18:28:51 +08:00
parent 42992f5502
commit 489afc41db
5 changed files with 26 additions and 4 deletions

View File

@@ -435,6 +435,10 @@ describe("markdownToTelegramHtml", () => {
expect(containsLoneSurrogate(chunk)).toBe(false);
}
});
it("keeps an astral char whole when a positive limit starts on its pair", () => {
expect(splitTelegramHtmlChunks("A😀B", 1)).toEqual(["A", "😀", "B"]);
});
});
function containsLoneSurrogate(text: string): boolean {

View File

@@ -1071,14 +1071,17 @@ function findTelegramHtmlEntityEnd(text: string, start: number): number {
}
// Never return a split index that lands between a UTF-16 surrogate pair, or
// both chunks would carry a lone surrogate that re-encodes to U+FFFD. Mirrors
// the guard in bot/native-quote.ts `truncateUtf16Safe`.
// both chunks would carry a lone surrogate that re-encodes to U+FFFD. If the
// pair starts the segment, keep it whole so chunking still advances.
function clampToSurrogateBoundary(text: string, index: number): number {
const high = text.charCodeAt(index - 1);
const low = text.charCodeAt(index);
const splitsPair =
index > 0 && high >= 0xd800 && high <= 0xdbff && low >= 0xdc00 && low <= 0xdfff;
return splitsPair ? index - 1 : index;
if (!splitsPair) {
return index;
}
return index > 1 ? index - 1 : index + 1;
}
function findTelegramHtmlSafeSplitIndex(text: string, maxLength: number): number {

View File

@@ -43,6 +43,17 @@ describe("telegramPlugin outbound", () => {
expect(telegramOutbound.chunker?.(text, 4000)).toEqual([text]);
});
it("keeps astral characters whole at positive configured chunk limits", () => {
clearTelegramRuntime();
expect(telegramOutbound.chunker?.("A😀B", 1)).toEqual(["A", "😀", "B"]);
expect(telegramOutbound.chunker?.("A😀B", 1, { formatting: { parseMode: "HTML" } })).toEqual([
"A",
"😀",
"B",
]);
});
it("preserves markdown tables for the configured delivery renderer", () => {
clearTelegramRuntime();
const text = ["| Name | Value |", "|------|-------|", "| A | 1 |"].join("\n");

View File

@@ -604,6 +604,10 @@ describe("chunkMarkdownTextWithMode", () => {
expect(chunks.every((chunk) => !/[\uD800-\uDBFF]$/u.test(chunk))).toBe(true);
expect(chunks.every((chunk) => !/^[\uDC00-\uDFFF]/u.test(chunk))).toBe(true);
});
it("keeps an astral character whole when a positive hard limit starts on its pair", () => {
expect(chunkMarkdownTextWithMode("A😀B", 1, "length")).toEqual(["A", "😀", "B"]);
});
});
describe("resolveChunkMode", () => {

View File

@@ -16,7 +16,7 @@ export function avoidTrailingHighSurrogateBreak(text: string, start: number, end
return end;
}
const adjusted = end - 1;
return adjusted > start ? adjusted : end;
return adjusted > start ? adjusted : end + 1;
}
export function chunkTextByBreakResolver(