mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-09 03:22:40 +00:00
fix(telegram): preserve astral chunk boundaries
This commit is contained in:
@@ -435,6 +435,10 @@ describe("markdownToTelegramHtml", () => {
|
||||
expect(containsLoneSurrogate(chunk)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an astral char whole when a positive limit starts on its pair", () => {
|
||||
expect(splitTelegramHtmlChunks("A😀B", 1)).toEqual(["A", "😀", "B"]);
|
||||
});
|
||||
});
|
||||
|
||||
function containsLoneSurrogate(text: string): boolean {
|
||||
|
||||
@@ -1071,14 +1071,17 @@ function findTelegramHtmlEntityEnd(text: string, start: number): number {
|
||||
}
|
||||
|
||||
// Never return a split index that lands between a UTF-16 surrogate pair, or
|
||||
// both chunks would carry a lone surrogate that re-encodes to U+FFFD. Mirrors
|
||||
// the guard in bot/native-quote.ts `truncateUtf16Safe`.
|
||||
// both chunks would carry a lone surrogate that re-encodes to U+FFFD. If the
|
||||
// pair starts the segment, keep it whole so chunking still advances.
|
||||
function clampToSurrogateBoundary(text: string, index: number): number {
|
||||
const high = text.charCodeAt(index - 1);
|
||||
const low = text.charCodeAt(index);
|
||||
const splitsPair =
|
||||
index > 0 && high >= 0xd800 && high <= 0xdbff && low >= 0xdc00 && low <= 0xdfff;
|
||||
return splitsPair ? index - 1 : index;
|
||||
if (!splitsPair) {
|
||||
return index;
|
||||
}
|
||||
return index > 1 ? index - 1 : index + 1;
|
||||
}
|
||||
|
||||
function findTelegramHtmlSafeSplitIndex(text: string, maxLength: number): number {
|
||||
|
||||
@@ -43,6 +43,17 @@ describe("telegramPlugin outbound", () => {
|
||||
expect(telegramOutbound.chunker?.(text, 4000)).toEqual([text]);
|
||||
});
|
||||
|
||||
it("keeps astral characters whole at positive configured chunk limits", () => {
|
||||
clearTelegramRuntime();
|
||||
|
||||
expect(telegramOutbound.chunker?.("A😀B", 1)).toEqual(["A", "😀", "B"]);
|
||||
expect(telegramOutbound.chunker?.("A😀B", 1, { formatting: { parseMode: "HTML" } })).toEqual([
|
||||
"A",
|
||||
"😀",
|
||||
"B",
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves markdown tables for the configured delivery renderer", () => {
|
||||
clearTelegramRuntime();
|
||||
const text = ["| Name | Value |", "|------|-------|", "| A | 1 |"].join("\n");
|
||||
|
||||
@@ -604,6 +604,10 @@ describe("chunkMarkdownTextWithMode", () => {
|
||||
expect(chunks.every((chunk) => !/[\uD800-\uDBFF]$/u.test(chunk))).toBe(true);
|
||||
expect(chunks.every((chunk) => !/^[\uDC00-\uDFFF]/u.test(chunk))).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps an astral character whole when a positive hard limit starts on its pair", () => {
|
||||
expect(chunkMarkdownTextWithMode("A😀B", 1, "length")).toEqual(["A", "😀", "B"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveChunkMode", () => {
|
||||
|
||||
@@ -16,7 +16,7 @@ export function avoidTrailingHighSurrogateBreak(text: string, start: number, end
|
||||
return end;
|
||||
}
|
||||
const adjusted = end - 1;
|
||||
return adjusted > start ? adjusted : end;
|
||||
return adjusted > start ? adjusted : end + 1;
|
||||
}
|
||||
|
||||
export function chunkTextByBreakResolver(
|
||||
|
||||
Reference in New Issue
Block a user