fix(memory-core): keep short protected-glossary terms past the min-length gate (#96304)

PROTECTED_GLOSSARY exists to preserve short technical terms that generic
filtering would discard, but every glossary match still flowed through
normalizeConceptToken's per-script minimum-length gate. The 2-char latin
entries "kv" and "s3" were therefore never emitted as concept tags despite
being on the protect-list. Thread a fromGlossary flag so glossary matches
bypass only that length check; all other gates still apply.

Because that bypass lets short entries through, a bare substring match would
also surface them from inside longer words ("kv" in "mkv", "s3" in "css3").
Match ONLY the short entries (those below their script's min length) as
delimiter-bounded whole tokens; longer entries keep substring containment, so
the shipped behavior of "backup" tagging inside "backups" is preserved. CJK
entries (no word delimiters) always use substring matching. Positive
(standalone kv/s3) and negative (mkv/css3 substrings) regression tests cover
both directions, and the short-term-promotion stable-tags assertion gains "s3".

Co-authored-by: ly-wang19 <ly-wang19@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

extensions/memory-core/src/concept-vocabulary.test.ts

@@ -29,6 +29,30 @@ describe("concept vocabulary", () => {
     expect(tags).not.toContain("2026-04-04.md");
   });
 
+  it("preserves short protected-glossary terms past the latin minimum-length gate", () => {
+    const tags = deriveConceptTags({
+      path: "memory/2026-04-04.md",
+      snippet: "Store the session in kv and back up to s3 nightly.",
+    });
+
+    // "kv" and "s3" are 2-char latin glossary entries that the generic min-length-3 gate would drop.
+    expect(tags).toContain("kv");
+    expect(tags).toContain("s3");
+  });
+
+  it("does not surface short glossary terms that only appear inside longer words", () => {
+    const tags = deriveConceptTags({
+      path: "memory/2026-04-04.md",
+      snippet: "Played the mkv recording and tuned the css3 layout.",
+    });
+
+    // "kv"/"s3" are substrings of "mkv"/"css3"; whole-word matching must not emit them as tags.
+    expect(tags).not.toContain("kv");
+    expect(tags).not.toContain("s3");
+    expect(tags).toContain("mkv");
+    expect(tags).toContain("css3");
+  });
+
   it("extracts protected and segmented CJK concept tags", () => {
     const tags = deriveConceptTags({
       path: "memory/2026-04-04.md",

extensions/memory-core/src/concept-vocabulary.ts

@@ -330,7 +330,7 @@ function isKanaOnlyToken(value: string): boolean {
   );
 }
 
-function normalizeConceptToken(rawToken: string): string | null {
+function normalizeConceptToken(rawToken: string, fromGlossary = false): string | null {
   const normalized = normalizeLowercaseStringOrEmpty(
     rawToken
       .normalize("NFKC")
@@ -348,7 +348,9 @@ function normalizeConceptToken(rawToken: string): string | null {
     return null;
   }
   const script = classifyConceptTagScript(normalized);
-  if (normalized.length < minimumTokenLengthForScript(script)) {
+  // Glossary entries are an explicit allowlist of short technical terms (e.g. "kv", "s3"); they
+  // bypass the per-script minimum length that would otherwise discard them.
+  if (!fromGlossary && normalized.length < minimumTokenLengthForScript(script)) {
     return null;
   }
   if (isKanaOnlyToken(normalized) && normalized.length < 3) {
@@ -360,14 +362,43 @@ function normalizeConceptToken(rawToken: string): string | null {
   return normalized;
 }
 
+// Only entries shorter than their script's minimum token length rely on the glossary bypass, and
+// only those need whole-word matching so they don't fire inside longer words ("kv" in "mkv"). Longer
+// entries keep substring containment (the shipped behavior, e.g. "backup" tagging inside "backups").
+// Precomputed so derive() does not reclassify on every call.
+const GLOSSARY_ENTRIES = PROTECTED_GLOSSARY.map((entry) => ({
+  entry,
+  wholeWord: entry.length < minimumTokenLengthForScript(classifyConceptTagScript(entry)),
+}));
+
+function isAlphanumericAt(source: string, index: number): boolean {
+  const ch = source[index];
+  return ch !== undefined && LETTER_OR_NUMBER_RE.test(ch);
+}
+
+// True when `entry` occurs as a delimiter-bounded token, not inside a longer word. Keeps short
+// glossary entries like "kv"/"s3" from firing inside "mkv"/"css3" once they bypass the length gate.
+function includesStandaloneTerm(source: string, entry: string): boolean {
+  let from = source.indexOf(entry);
+  while (from !== -1) {
+    if (!isAlphanumericAt(source, from - 1) && !isAlphanumericAt(source, from + entry.length)) {
+      return true;
+    }
+    from = source.indexOf(entry, from + 1);
+  }
+  return false;
+}
+
 function collectGlossaryMatches(source: string): string[] {
   const normalizedSource = normalizeLowercaseStringOrEmpty(source.normalize("NFKC"));
   const matches: string[] = [];
-  for (const entry of PROTECTED_GLOSSARY) {
-    if (!normalizedSource.includes(entry)) {
-      continue;
+  for (const { entry, wholeWord } of GLOSSARY_ENTRIES) {
+    const present = wholeWord
+      ? includesStandaloneTerm(normalizedSource, entry)
+      : normalizedSource.includes(entry);
+    if (present) {
+      matches.push(entry);
     }
-    matches.push(entry);
   }
   return matches;
 }
@@ -385,8 +416,13 @@ function collectSegmentTokens(source: string): string[] {
   return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
 }
 
-function pushNormalizedTag(tags: string[], rawToken: string, limit: number): void {
-  const normalized = normalizeConceptToken(rawToken);
+function pushNormalizedTag(
+  tags: string[],
+  rawToken: string,
+  limit: number,
+  fromGlossary = false,
+): void {
+  const normalized = normalizeConceptToken(rawToken, fromGlossary);
   if (!normalized || tags.includes(normalized)) {
     return;
   }
@@ -410,14 +446,17 @@ export function deriveConceptTags(params: {
   }
 
   const tags: string[] = [];
-  for (const rawToken of [
-    ...collectGlossaryMatches(source),
-    ...collectCompoundTokens(source),
-    ...collectSegmentTokens(source),
-  ]) {
-    pushNormalizedTag(tags, rawToken, limit);
-    if (tags.length >= limit) {
-      break;
+  const tokenSources: Array<{ tokens: string[]; fromGlossary: boolean }> = [
+    { tokens: collectGlossaryMatches(source), fromGlossary: true },
+    { tokens: collectCompoundTokens(source), fromGlossary: false },
+    { tokens: collectSegmentTokens(source), fromGlossary: false },
+  ];
+  for (const { tokens, fromGlossary } of tokenSources) {
+    for (const rawToken of tokens) {
+      pushNormalizedTag(tags, rawToken, limit, fromGlossary);
+      if (tags.length >= limit) {
+        return tags;
+      }
     }
   }
   return tags;

extensions/memory-core/src/short-term-promotion.test.ts

@@ -3189,7 +3189,9 @@ describe("short-term promotion", () => {
         path: "memory/2026-04-03.md",
         snippet: "Move backups to S3 Glacier and sync QMD router notes.",
       }),
-    ).toStrictEqual(["backup", "backups", "glacier", "qmd", "router", "sync"]);
+      // "s3" is a protected-glossary term; it now surfaces as a standalone token past the
+      // per-script min-length gate (the longer terms still match as substrings).
+    ).toStrictEqual(["backup", "backups", "glacier", "qmd", "router", "s3", "sync"]);
   });
 
   it("extracts multilingual concept tags across latin and cjk snippets", () => {