diff --git a/agent/i18n.py b/agent/i18n.py index 05f875cb55..1edb2a6a7b 100644 --- a/agent/i18n.py +++ b/agent/i18n.py @@ -106,6 +106,29 @@ def supported_languages(home: str | None = None) -> tuple[str, ...]: return result +@lru_cache(maxsize=4) +def _bundled_surfaces(locales_dir: str) -> frozenset[tuple[str, str]]: + """``(lang, surface)`` for every bundled ``locales/[.].yaml``. Keyed by directory so a + ``HERMES_BUNDLED_LOCALES`` switch re-scans; cleared with the other layer views by + :func:`reset_language_cache`.""" + return frozenset((lang, surface) for lang, surface, _path in i18n_layers.scan_locale_dir(Path(locales_dir))) + + +def language_surfaces(home: str | None = None) -> dict[str, tuple[str, ...]]: + """Per language, the surfaces some layer has strings for (bundled ∪ user overlay ∪ plugin packs), in + ``SURFACES`` order. An overlay file or pack that loaded no strings (empty, comment-only, unparseable) + does not count. A renderer that bundles its own catalog for a language does not need this; for any + other language a missing surface means that renderer would show English.""" + home = home or _current_home() + layered = {(lang, surface) for lang, surface in i18n_layers.overlay_surfaces(home) | i18n_layers.pack_surfaces() + if i18n_layers.overlay_layer(home, lang, surface) or i18n_layers.pack_layer(lang, surface)} + pairs = _bundled_surfaces(str(_locales_dir())) | layered + found: dict[str, set[str]] = {} + for lang, surface in pairs: + found.setdefault(lang, set()).add(surface) + return {lang: tuple(s for s in i18n_layers.SURFACES if s in surfaces) for lang, surfaces in found.items()} + + def resolve_language_id(value: Any, home: str | None = None) -> str | None: """Canonical supported id for a user-supplied value (code, alias, regional tag), or ``None`` when no layer supplies it — the validation ``hermes config set display.language`` runs.""" @@ -198,6 +221,7 @@ def reset_language_cache() -> None: ``save_config`` changes ``display.language``, after a pack registers/unregisters, or after editing an overlay file).""" _config_language_cached.cache_clear() + _bundled_surfaces.cache_clear() with _catalog_lock: _catalog_cache.clear() _supported_cache.clear() @@ -248,6 +272,7 @@ __all__ = [ "SUPPORTED_LANGUAGES", "get_language", "language_options", + "language_surfaces", "reset_language_cache", "resolve_language_id", "supported_languages", diff --git a/agent/i18n_languages.py b/agent/i18n_languages.py index 519f986af4..b160bb9d2d 100644 --- a/agent/i18n_languages.py +++ b/agent/i18n_languages.py @@ -8,7 +8,7 @@ and fall back to the bare id. from __future__ import annotations -from typing import TypedDict +from typing import Iterable, TypedDict class LanguageOption(TypedDict): @@ -16,6 +16,7 @@ class LanguageOption(TypedDict): endonym: str rtl: bool source: str + surfaces: list[str] # id -> (endonym, rtl). Keep sorted by id; ``en`` is listed first by ``language_options`` regardless. @@ -43,10 +44,12 @@ BUNDLED_SOURCE = "bundled" OVERLAY_SOURCE = "overlay" -def describe_language(lang: str, *, pack: dict | None, overlay: bool) -> LanguageOption: +def describe_language(lang: str, *, pack: dict | None, overlay: bool, surfaces: Iterable[str]) -> LanguageOption: """One picker row. Precedence for endonym/rtl: bundled table → pack metadata → bare id / LTR. ``source`` names the highest layer that supplies the language (``plugin:`` > ``overlay`` > - ``bundled``) so a picker can say where a language came from.""" + ``bundled``) so a picker can say where a language came from. ``surfaces`` are the surfaces some + layer has strings for (``agent.i18n.language_surfaces``), so a renderer can leave out a language it + would only show in English.""" bundled = BUNDLED_LANGUAGE_INFO.get(lang) endonym, rtl = bundled if bundled else ((pack or {}).get("endonym") or lang, bool((pack or {}).get("rtl"))) if pack is not None: @@ -55,18 +58,21 @@ def describe_language(lang: str, *, pack: dict | None, overlay: bool) -> Languag source = OVERLAY_SOURCE else: source = BUNDLED_SOURCE - return {"id": lang, "endonym": endonym, "rtl": rtl, "source": source} + return {"id": lang, "endonym": endonym, "rtl": rtl, "source": source, "surfaces": list(surfaces)} def language_options() -> list[LanguageOption]: - """``[{"id", "endonym", "rtl", "source"}, ...]`` for every supported language, ``en`` first then sorted + """``[{"id", "endonym", "rtl", "source", "surfaces"}, ...]`` for every supported language, ``en`` first then sorted by id — the list ``i18n.languages`` serves and every switcher renders (endonym only, no flags).""" from agent import i18n, i18n_layers from hermes_constants import get_hermes_home - overlay_langs = i18n_layers.overlay_languages(get_hermes_home()) + home = get_hermes_home() + overlay_langs = i18n_layers.overlay_languages(home) + surfaces = i18n.language_surfaces(str(home)) return [ - describe_language(lang, pack=i18n_layers.pack_info(lang), overlay=lang in overlay_langs) + describe_language(lang, pack=i18n_layers.pack_info(lang), overlay=lang in overlay_langs, + surfaces=surfaces.get(lang, ())) for lang in i18n.supported_languages() ] diff --git a/agent/i18n_layers.py b/agent/i18n_layers.py index d6d9a80107..8f0b800967 100644 --- a/agent/i18n_layers.py +++ b/agent/i18n_layers.py @@ -134,7 +134,7 @@ _tokens = count(1) _packs: list[PackEntry] = [] _pack_cache: dict[tuple[str, str], dict[str, str]] = {} _overlay_cache: dict[tuple[str, str, str], dict[str, str]] = {} -_overlay_languages_cache: dict[str, frozenset[str]] = {} +_overlay_files_cache: dict[str, frozenset[tuple[str, str]]] = {} def _reset_facade() -> None: @@ -184,9 +184,14 @@ def pack_layer(lang: str, surface: str = CORE_SURFACE) -> dict[str, str]: return merged -def pack_languages() -> frozenset[str]: +def pack_surfaces() -> frozenset[tuple[str, str]]: + """``(lang, surface)`` for every registered pack layer.""" with _lock: - return frozenset(entry.lang for entry in _packs) + return frozenset((entry.lang, entry.surface) for entry in _packs) + + +def pack_languages() -> frozenset[str]: + return frozenset(lang for lang, _surface in pack_surfaces()) def pack_info(lang: str) -> dict[str, Any] | None: @@ -230,17 +235,22 @@ def overlay_layer(home: Path | str, lang: str, surface: str = CORE_SURFACE) -> d return flat -def overlay_languages(home: Path | str) -> frozenset[str]: - """Language ids that have at least one overlay file (any surface) under *home*.""" +def overlay_surfaces(home: Path | str) -> frozenset[tuple[str, str]]: + """``(lang, surface)`` for every overlay file under *home*.""" key = str(home) with _lock: - cached = _overlay_languages_cache.get(key) + cached = _overlay_files_cache.get(key) if cached is not None: return cached - langs = frozenset(lang for lang, _surface, _path in scan_locale_dir(overlay_dir(home))) + found = frozenset((lang, surface) for lang, surface, _path in scan_locale_dir(overlay_dir(home))) with _lock: - _overlay_languages_cache[key] = langs - return langs + _overlay_files_cache[key] = found + return found + + +def overlay_languages(home: Path | str) -> frozenset[str]: + """Language ids that have at least one overlay file (any surface) under *home*.""" + return frozenset(lang for lang, _surface in overlay_surfaces(home)) def surface_catalog(home: Path | str, lang: str, surface: str = CORE_SURFACE) -> dict[str, str]: @@ -261,7 +271,7 @@ def clear_cache() -> None: with _lock: _pack_cache.clear() _overlay_cache.clear() - _overlay_languages_cache.clear() + _overlay_files_cache.clear() def _reset_registry_for_tests() -> None: @@ -289,9 +299,11 @@ __all__ = [ "overlay_dir", "overlay_languages", "overlay_layer", + "overlay_surfaces", "pack_info", "pack_languages", "pack_layer", + "pack_surfaces", "parse_locale_file", "register_pack", "registered_packs", diff --git a/apps/desktop/src/i18n/backend-packs.test.ts b/apps/desktop/src/i18n/backend-packs.test.ts index 294d34238c..36c1853acb 100644 --- a/apps/desktop/src/i18n/backend-packs.test.ts +++ b/apps/desktop/src/i18n/backend-packs.test.ts @@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { type BackendLocaleRequest, syncBackendLocalePacks } from './backend-packs' import { TRANSLATIONS } from './catalog' -import { languageOptions } from './languages' +import { isSupportedLocaleValue, languageOptions } from './languages' import { isRegisteredLocale, registerAppLocale, resetAppLocaleRegistry, resolveTranslations } from './registry' afterEach(() => { @@ -65,6 +65,53 @@ describe('syncBackendLocalePacks', () => { expect(isRegisteredLocale('uk')).toBe(true) }) + it('leaves out a language the backend has no Desktop strings for, and keeps rows from older backends', async () => { + const request = backend( + { + languages: [ + { id: 'ko', endonym: '한국어', rtl: false, source: 'bundled', surfaces: ['core', 'tui'] }, + { id: 'it', endonym: 'Italiano', rtl: false, source: 'plugin:hermes-lang-it', surfaces: ['core', 'desktop'] }, + { id: 'ja', endonym: '日本語', rtl: false, source: 'bundled', surfaces: ['core', 'tui'] }, + { id: 'uk', endonym: 'Українська' } + ] + }, + {} + ) + + await syncBackendLocalePacks(request, 'ko') + + // Picking ko here would only paint English: it is not offered and not renderable. + expect(isRegisteredLocale('ko')).toBe(false) + expect(isSupportedLocaleValue('ko')).toBe(false) + expect(languageOptions().map(option => option.id)).not.toContain('ko') + // A pack with Desktop strings, a Desktop-bundled language, and a row without `surfaces` stay. + expect(isRegisteredLocale('it')).toBe(true) + expect(isRegisteredLocale('ja')).toBe(true) + expect(isRegisteredLocale('uk')).toBe(true) + }) + + it('offers a language once its Desktop pack arrives', async () => { + const request = backend( + { + languages: [ + { + id: 'ko', + endonym: '한국어', + rtl: false, + source: 'plugin:hermes-lang-ko', + surfaces: ['core', 'tui', 'desktop'] + } + ] + }, + { ko: { 'settings.appearance.modelPricingTitle': '모델 가격' } } + ) + + await syncBackendLocalePacks(request, 'ko') + + expect(languageOptions().find(option => option.id === 'ko')).toMatchObject({ endonym: '한국어', source: 'backend' }) + expect(resolveTranslations('ko').settings.appearance.modelPricingTitle).toBe('모델 가격') + }) + it('is silent on a backend that predates the i18n methods and leaves the registry alone', async () => { const debug = vi.spyOn(console, 'debug').mockImplementation(() => {}) registerAppLocale('pl', { endonym: 'Polski' }, 'backend') diff --git a/apps/desktop/src/i18n/backend-packs.ts b/apps/desktop/src/i18n/backend-packs.ts index cbaa7a3485..ed48f077f4 100644 --- a/apps/desktop/src/i18n/backend-packs.ts +++ b/apps/desktop/src/i18n/backend-packs.ts @@ -11,10 +11,12 @@ * leaves whatever the registry already holds (never blank the UI over i18n). */ +import type { LocaleSurface } from '@hermes/shared' import { isRecord } from '@hermes/shared/i18n' import { isMissingRpcMethod } from '@/lib/gateway-rpc' +import { isBundledLocale } from './catalog' import { type AppLocaleRegistration, normalizeLocaleId, replaceAppLocaleSource } from './registry' export type BackendLocaleRequest = (method: string, params: Record) => Promise @@ -25,6 +27,16 @@ export interface BackendLanguageOption { endonym?: string rtl?: boolean source?: string + /** Surfaces some backend layer has strings for. Absent on older backends. */ + surfaces?: LocaleSurface[] +} + +/** The backend lists every language it translates for any surface (core, + * TUI). A language the Desktop does not bundle renders in English unless the + * backend has Desktop strings for it, so only those enter the picker. Rows + * without `surfaces` (older backends) are kept. */ +function hasDesktopStrings(id: string, row: BackendLanguageOption): boolean { + return isBundledLocale(id) || !Array.isArray(row.surfaces) || row.surfaces.includes(DESKTOP_SURFACE) } /** `i18n.catalog` reply: the pack + overlay layer for one surface, flat. */ @@ -104,7 +116,7 @@ export async function syncBackendLocalePacks( for (const row of languageRows(languages)) { const id = normalizeLocaleId(row.id) - if (id) { + if (id && hasDesktopStrings(id, row)) { registrations.set(id, { id, endonym: typeof row.endonym === 'string' ? row.endonym : undefined, diff --git a/apps/shared/src/gateway-contract.generated.ts b/apps/shared/src/gateway-contract.generated.ts index e3aff5727e..9ff2603605 100644 --- a/apps/shared/src/gateway-contract.generated.ts +++ b/apps/shared/src/gateway-contract.generated.ts @@ -1905,19 +1905,20 @@ export interface BotRelayReplyParams { export interface I18nLanguagesResult { languages: LanguageOption[] } -/** ``agent.i18n_languages.language_options`` row. ``source`` is ``bundled``, ``overlay`` or ``plugin:`` — the highest layer that supplies the language. */ +/** ``agent.i18n_languages.language_options`` row. ``source`` is ``bundled``, ``overlay`` or ``plugin:`` — the highest layer that supplies the language. ``surfaces`` are the surfaces some layer has strings for; a renderer leaves out a language it does not bundle and that lacks its surface, since picking it would only show English. */ export interface LanguageOption { id: string endonym: string rtl: boolean source: string + surfaces: LocaleSurface[] } +export type LocaleSurface = 'core' | 'tui' | 'desktop' export interface I18nCatalogParams { profile?: string | null lang: string surface?: LocaleSurface } -export type LocaleSurface = 'core' | 'tui' | 'desktop' /** ``messages`` is ONLY the pack + user-overlay layer for that surface (flat dotted keys); the client merges it over its bundled ``en``/````. ``lang`` is the canonical id the request resolved to (``pt-BR`` → ``pt-br``; an unknown id resolves to ``en`` with an empty layer). */ export interface I18nCatalogResult { lang: string diff --git a/apps/shared/src/gateway-contract.openrpc.json b/apps/shared/src/gateway-contract.openrpc.json index 61953cad0b..a279eded1c 100644 --- a/apps/shared/src/gateway-contract.openrpc.json +++ b/apps/shared/src/gateway-contract.openrpc.json @@ -21292,7 +21292,7 @@ }, "LanguageOption": { "additionalProperties": false, - "description": "``agent.i18n_languages.language_options`` row. ``source`` is ``bundled``, ``overlay`` or\n``plugin:`` \u2014 the highest layer that supplies the language.", + "description": "``agent.i18n_languages.language_options`` row. ``source`` is ``bundled``, ``overlay`` or\n``plugin:`` \u2014 the highest layer that supplies the language. ``surfaces`` are the surfaces some\nlayer has strings for; a renderer leaves out a language it does not bundle and that lacks its surface,\nsince picking it would only show English.", "properties": { "id": { "title": "Id", @@ -21309,13 +21309,21 @@ "source": { "title": "Source", "type": "string" + }, + "surfaces": { + "items": { + "$ref": "#/components/schemas/LocaleSurface" + }, + "title": "Surfaces", + "type": "array" } }, "required": [ "id", "endonym", "rtl", - "source" + "source", + "surfaces" ], "title": "LanguageOption", "type": "object" diff --git a/tests/agent/test_i18n_languages.py b/tests/agent/test_i18n_languages.py index bc1c1602ea..7d4fa53e8d 100644 --- a/tests/agent/test_i18n_languages.py +++ b/tests/agent/test_i18n_languages.py @@ -25,16 +25,61 @@ def test_language_options_are_en_first_sorted_and_sourced(monkeypatch, tmp_path) ids = [o["id"] for o in options] assert ids == ["en", *sorted(ids[1:])] and set(ids) == set(i18n.SUPPORTED_LANGUAGES) assert all(o["source"] == "bundled" for o in options) - assert set(options[0]) == {"id", "endonym", "rtl", "source"} + assert set(options[0]) == {"id", "endonym", "rtl", "source", "surfaces"} def test_describe_language_precedence(): # Pack metadata names a pack-only language; the bundled table wins for a bundled id even if a pack # re-declares it; an overlay-only language is reported as such and falls back to the bare id. pack = {"endonym": "Polski", "rtl": False, "source": "plugin:hermes-lang-pl"} - assert describe_language("pl", pack=pack, overlay=False) == { - "id": "pl", "endonym": "Polski", "rtl": False, "source": "plugin:hermes-lang-pl"} - assert describe_language("ar", pack={**pack, "endonym": "Other", "rtl": False}, overlay=False)["endonym"] == "العربية" - assert describe_language("ar", pack={**pack, "rtl": False}, overlay=False)["rtl"] is True - assert describe_language("eo", pack=None, overlay=True) == {"id": "eo", "endonym": "eo", "rtl": False, "source": "overlay"} - assert describe_language("de", pack=None, overlay=True)["source"] == "bundled" + assert describe_language("pl", pack=pack, overlay=False, surfaces=("core", "desktop")) == { + "id": "pl", "endonym": "Polski", "rtl": False, "source": "plugin:hermes-lang-pl", "surfaces": ["core", "desktop"]} + assert describe_language("ar", pack={**pack, "endonym": "Other", "rtl": False}, overlay=False, surfaces=())["endonym"] == "العربية" + assert describe_language("ar", pack={**pack, "rtl": False}, overlay=False, surfaces=())["rtl"] is True + assert describe_language("eo", pack=None, overlay=True, surfaces=()) == { + "id": "eo", "endonym": "eo", "rtl": False, "source": "overlay", "surfaces": []} + assert describe_language("de", pack=None, overlay=True, surfaces=())["source"] == "bundled" + + +def test_language_surfaces_report_each_layer(monkeypatch, tmp_path): + """Bundled Korean has core + TUI strings and no Desktop catalog; a user overlay or a pack adds the + surface it ships. ``i18n.languages`` carries this so the Desktop can leave out a language it would + only show in English.""" + import hermes_yaml as yaml + + from agent import i18n_layers + + home = tmp_path / "home" + (home / "locales").mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(home)) + i18n_layers._reset_registry_for_tests() + i18n.reset_language_cache() + try: + surfaces = i18n.language_surfaces() + assert surfaces["ko"] == ("core", "tui") + assert surfaces["en"] == ("core",) + + (home / "locales" / "ko.desktop.yaml").write_text(yaml.safe_dump({"settings": {"title": "설정"}}), + encoding="utf-8") + i18n_layers.register_pack("pl", "desktop", {"settings.title": "Ustawienia"}, source="plugin:hermes-lang-pl") + i18n.reset_language_cache() + surfaces = i18n.language_surfaces() + assert surfaces["ko"] == ("core", "tui", "desktop") + assert surfaces["pl"] == ("desktop",) + rows = {row["id"]: row for row in i18n.language_options()} + assert rows["ko"]["surfaces"] == ["core", "tui", "desktop"] + assert rows["pl"]["surfaces"] == ["desktop"] + assert rows["ja"]["surfaces"] == ["core", "tui"] + + # A layer that loaded no strings advertises nothing: an empty or unparseable overlay, an empty pack. + (home / "locales" / "ja.desktop.yaml").write_text("# nothing translated yet\n", encoding="utf-8") + (home / "locales" / "de.desktop.yaml").write_text("settings: [unclosed\n", encoding="utf-8") + i18n_layers.register_pack("uk", "desktop", {}, source="plugin:hermes-lang-uk") + i18n.reset_language_cache() + surfaces = i18n.language_surfaces() + assert "desktop" not in surfaces["ja"] + assert "desktop" not in surfaces["de"] + assert "desktop" not in surfaces["uk"] + finally: + i18n_layers._reset_registry_for_tests() + i18n.reset_language_cache() diff --git a/tests/agent/test_i18n_layers.py b/tests/agent/test_i18n_layers.py index ee3307be8e..6aa61737e9 100644 --- a/tests/agent/test_i18n_layers.py +++ b/tests/agent/test_i18n_layers.py @@ -143,7 +143,8 @@ def test_manifest_only_pack_registers_language_and_display_language_resolves(hom assert i18n.get_language() == "pl" assert i18n.t(_KEY) == "Zatwierdzenie" option = next(o for o in i18n.language_options() if o["id"] == "pl") - assert option == {"id": "pl", "endonym": "pl", "rtl": False, "source": "plugin:hermes-lang-pl"} + assert option == {"id": "pl", "endonym": "pl", "rtl": False, "source": "plugin:hermes-lang-pl", + "surfaces": ["core", "tui", "desktop"]} def test_pack_with_register_function_and_metadata(home): diff --git a/tests/tui_gateway/test_i18n_methods.py b/tests/tui_gateway/test_i18n_methods.py index 3b22c013e8..65b8f6a1fb 100644 --- a/tests/tui_gateway/test_i18n_methods.py +++ b/tests/tui_gateway/test_i18n_methods.py @@ -42,8 +42,9 @@ def test_languages_and_catalog_serve_pack_layers_per_surface(tmp_path, monkeypat languages = _call(server, "i18n.languages", {})["result"]["languages"] ids = [row["id"] for row in languages] assert ids == ["en", *sorted(ids[1:])] - assert {"id": "pl", "endonym": "Polski", "rtl": False, "source": "plugin:hermes-lang-pl"} in languages - assert {"id": "ar", "endonym": "العربية", "rtl": True, "source": "bundled"} in languages + assert {"id": "pl", "endonym": "Polski", "rtl": False, "source": "plugin:hermes-lang-pl", + "surfaces": ["core", "tui", "desktop"]} in languages + assert {"id": "ar", "endonym": "العربية", "rtl": True, "source": "bundled", "surfaces": ["core", "tui"]} in languages core = _call(server, "i18n.catalog", {"lang": "pl"})["result"] assert core == {"lang": "pl", "surface": "core", "messages": {"approval.denied": "Odrzucono"}} @@ -105,7 +106,7 @@ def test_cold_gateway_serves_an_installed_pack_before_any_agent_exists(tmp_path, assert "ko" not in i18n_layers.pack_languages() # nothing has loaded plugins in this home yet try: ko = next(row for row in _call(server, "i18n.languages", {})["result"]["languages"] if row["id"] == "ko") - assert ko["source"] == "plugin:hermes-lang-ko" + assert ko["source"] == "plugin:hermes-lang-ko" and "desktop" in ko["surfaces"] desktop = _call(server, "i18n.catalog", {"lang": "ko", "surface": "desktop"})["result"] assert desktop["messages"] == {"settings.appearance.modelPricingTitle": "모델 가격"} finally: @@ -125,7 +126,7 @@ def test_failed_plugin_discovery_still_answers_with_bundled_languages(tmp_path, i18n.reset_language_cache() languages = _call(server, "i18n.languages", {})["result"]["languages"] - assert {"id": "ko", "endonym": "한국어", "rtl": False, "source": "bundled"} in languages + assert {"id": "ko", "endonym": "한국어", "rtl": False, "source": "bundled", "surfaces": ["core", "tui"]} in languages assert _call(server, "i18n.catalog", {"lang": "ko", "surface": "desktop"})["result"]["messages"] == {} diff --git a/tui_gateway/contracts/i18n.py b/tui_gateway/contracts/i18n.py index cf388e36e2..86fec78af5 100644 --- a/tui_gateway/contracts/i18n.py +++ b/tui_gateway/contracts/i18n.py @@ -20,12 +20,15 @@ class LocaleSurface(WireEnum): class LanguageOption(Result): """``agent.i18n_languages.language_options`` row. ``source`` is ``bundled``, ``overlay`` or - ``plugin:`` — the highest layer that supplies the language.""" + ``plugin:`` — the highest layer that supplies the language. ``surfaces`` are the surfaces some + layer has strings for; a renderer leaves out a language it does not bundle and that lacks its surface, + since picking it would only show English.""" id: str endonym: str rtl: bool source: str + surfaces: list[LocaleSurface] class I18nLanguagesResult(Result):