diff --git a/extensions/memory-core/src/concept-vocabulary.test.ts b/extensions/memory-core/src/concept-vocabulary.test.ts index 952051270881..23ca7b261b02 100644 --- a/extensions/memory-core/src/concept-vocabulary.test.ts +++ b/extensions/memory-core/src/concept-vocabulary.test.ts @@ -29,6 +29,30 @@ describe("concept vocabulary", () => { expect(tags).not.toContain("2026-04-04.md"); }); + it("preserves short protected-glossary terms past the latin minimum-length gate", () => { + const tags = deriveConceptTags({ + path: "memory/2026-04-04.md", + snippet: "Store the session in kv and back up to s3 nightly.", + }); + + // "kv" and "s3" are 2-char latin glossary entries that the generic min-length-3 gate would drop. + expect(tags).toContain("kv"); + expect(tags).toContain("s3"); + }); + + it("does not surface short glossary terms that only appear inside longer words", () => { + const tags = deriveConceptTags({ + path: "memory/2026-04-04.md", + snippet: "Played the mkv recording and tuned the css3 layout.", + }); + + // "kv"/"s3" are substrings of "mkv"/"css3"; whole-word matching must not emit them as tags. + expect(tags).not.toContain("kv"); + expect(tags).not.toContain("s3"); + expect(tags).toContain("mkv"); + expect(tags).toContain("css3"); + }); + it("extracts protected and segmented CJK concept tags", () => { const tags = deriveConceptTags({ path: "memory/2026-04-04.md", diff --git a/extensions/memory-core/src/concept-vocabulary.ts b/extensions/memory-core/src/concept-vocabulary.ts index 6876fa0ec608..78093d9c1c8c 100644 --- a/extensions/memory-core/src/concept-vocabulary.ts +++ b/extensions/memory-core/src/concept-vocabulary.ts @@ -330,7 +330,7 @@ function isKanaOnlyToken(value: string): boolean { ); } -function normalizeConceptToken(rawToken: string): string | null { +function normalizeConceptToken(rawToken: string, fromGlossary = false): string | null { const normalized = normalizeLowercaseStringOrEmpty( rawToken .normalize("NFKC") @@ -348,7 +348,9 @@ function normalizeConceptToken(rawToken: string): string | null { return null; } const script = classifyConceptTagScript(normalized); - if (normalized.length < minimumTokenLengthForScript(script)) { + // Glossary entries are an explicit allowlist of short technical terms (e.g. "kv", "s3"); they + // bypass the per-script minimum length that would otherwise discard them. + if (!fromGlossary && normalized.length < minimumTokenLengthForScript(script)) { return null; } if (isKanaOnlyToken(normalized) && normalized.length < 3) { @@ -360,14 +362,43 @@ function normalizeConceptToken(rawToken: string): string | null { return normalized; } +// Only entries shorter than their script's minimum token length rely on the glossary bypass, and +// only those need whole-word matching so they don't fire inside longer words ("kv" in "mkv"). Longer +// entries keep substring containment (the shipped behavior, e.g. "backup" tagging inside "backups"). +// Precomputed so derive() does not reclassify on every call. +const GLOSSARY_ENTRIES = PROTECTED_GLOSSARY.map((entry) => ({ + entry, + wholeWord: entry.length < minimumTokenLengthForScript(classifyConceptTagScript(entry)), +})); + +function isAlphanumericAt(source: string, index: number): boolean { + const ch = source[index]; + return ch !== undefined && LETTER_OR_NUMBER_RE.test(ch); +} + +// True when `entry` occurs as a delimiter-bounded token, not inside a longer word. Keeps short +// glossary entries like "kv"/"s3" from firing inside "mkv"/"css3" once they bypass the length gate. +function includesStandaloneTerm(source: string, entry: string): boolean { + let from = source.indexOf(entry); + while (from !== -1) { + if (!isAlphanumericAt(source, from - 1) && !isAlphanumericAt(source, from + entry.length)) { + return true; + } + from = source.indexOf(entry, from + 1); + } + return false; +} + function collectGlossaryMatches(source: string): string[] { const normalizedSource = normalizeLowercaseStringOrEmpty(source.normalize("NFKC")); const matches: string[] = []; - for (const entry of PROTECTED_GLOSSARY) { - if (!normalizedSource.includes(entry)) { - continue; + for (const { entry, wholeWord } of GLOSSARY_ENTRIES) { + const present = wholeWord + ? includesStandaloneTerm(normalizedSource, entry) + : normalizedSource.includes(entry); + if (present) { + matches.push(entry); } - matches.push(entry); } return matches; } @@ -385,8 +416,13 @@ function collectSegmentTokens(source: string): string[] { return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean); } -function pushNormalizedTag(tags: string[], rawToken: string, limit: number): void { - const normalized = normalizeConceptToken(rawToken); +function pushNormalizedTag( + tags: string[], + rawToken: string, + limit: number, + fromGlossary = false, +): void { + const normalized = normalizeConceptToken(rawToken, fromGlossary); if (!normalized || tags.includes(normalized)) { return; } @@ -410,14 +446,17 @@ export function deriveConceptTags(params: { } const tags: string[] = []; - for (const rawToken of [ - ...collectGlossaryMatches(source), - ...collectCompoundTokens(source), - ...collectSegmentTokens(source), - ]) { - pushNormalizedTag(tags, rawToken, limit); - if (tags.length >= limit) { - break; + const tokenSources: Array<{ tokens: string[]; fromGlossary: boolean }> = [ + { tokens: collectGlossaryMatches(source), fromGlossary: true }, + { tokens: collectCompoundTokens(source), fromGlossary: false }, + { tokens: collectSegmentTokens(source), fromGlossary: false }, + ]; + for (const { tokens, fromGlossary } of tokenSources) { + for (const rawToken of tokens) { + pushNormalizedTag(tags, rawToken, limit, fromGlossary); + if (tags.length >= limit) { + return tags; + } } } return tags; diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 3422425ee042..8d4f868fa701 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -3189,7 +3189,9 @@ describe("short-term promotion", () => { path: "memory/2026-04-03.md", snippet: "Move backups to S3 Glacier and sync QMD router notes.", }), - ).toStrictEqual(["backup", "backups", "glacier", "qmd", "router", "sync"]); + // "s3" is a protected-glossary term; it now surfaces as a standalone token past the + // per-script min-length gate (the longer terms still match as substrings). + ).toStrictEqual(["backup", "backups", "glacier", "qmd", "router", "s3", "sync"]); }); it("extracts multilingual concept tags across latin and cjk snippets", () => {