mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
fix(context): read the fallback label from the resolver, and label /status and /context
The fallback marker re-derived the resolver's answer from a few local sources, so a window a custom endpoint reported through /models (or a local probe, provider profile, live catalog) read as "fallback" whenever it equalled the default. Its 60s memo was also keyed without the profile home, so one multiplexed profile's verdict labelled another's window. get_model_context_length now records when it ended on a static default (the generic 256K or Bedrock's 128K) per (profile home, model, base_url), and the marker reads that record. The memo is gone. Gateway /model drops the configured pin for a changed route before both the number and its label, through the same helper resolve_display_context_length uses, so a discarded 128K pin no longer shows "(pinned)" over a fallback. /status and /context now carry the same label as /usage for resident agents and non-resident sessions. Addresses Greptile review on #148801.
This commit is contained in:
1 parent
2111815fab
commit
0b388277f9
10 files changed
+313
-98
No files matched your search
@@ -1616,6 +1616,8 @@ def get_bedrock_context_length(model_id: str, region: str = "", probe: bool = Tr
|
||||
"Bedrock model %s has no known context window; using the static %s-token fallback "
|
||||
"(not a detected limit). %s", model_id, f"{BEDROCK_DEFAULT_CONTEXT_LENGTH:,}", override,
|
||||
)
|
||||
from agent.context_default_record import mark_context_defaulted
|
||||
mark_context_defaulted()
|
||||
return BEDROCK_DEFAULT_CONTEXT_LENGTH
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Which context-window resolutions ended on a static default, recorded by the resolver itself.
|
||||
|
||||
``get_model_context_length`` has many sources (config, endpoint metadata, /models, provider catalogs,
|
||||
local probes, Bedrock's table and probe). Only the resolver knows whether its answer came from one of
|
||||
them or from a default (the generic 256K, Bedrock's 128K), so the "fallback" label reads this record
|
||||
instead of re-deriving the answer. Keyed by profile home: a multiplexed gateway resolves for several
|
||||
profiles in one process.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import functools
|
||||
import inspect
|
||||
from typing import Callable, Optional
|
||||
|
||||
# Set while a recorded resolution runs; the default branches flip it via mark_context_defaulted().
|
||||
_DEFAULTED: contextvars.ContextVar[Optional[list]] = contextvars.ContextVar("context_defaulted", default=None)
|
||||
_RECORDS: dict[tuple[str, str, str], int] = {} # (profile home, model, base_url) -> the defaulted window
|
||||
_MAX_RECORDS = 512
|
||||
|
||||
|
||||
def _key(model, base_url) -> tuple[str, str, str]:
|
||||
from hermes_constants import hermes_home_key
|
||||
return hermes_home_key(), str(model or "").strip(), str(base_url or "").strip().rstrip("/")
|
||||
|
||||
|
||||
def mark_context_defaulted() -> None:
|
||||
"""Called by a resolver branch that returns a static default instead of a known window."""
|
||||
flag = _DEFAULTED.get()
|
||||
if flag is not None:
|
||||
flag[0] = True
|
||||
|
||||
|
||||
def records_context_defaults(resolve: Callable[..., int]) -> Callable[..., int]:
|
||||
"""Wrap the resolver: remember a defaulted answer for its (model, base_url), forget it otherwise."""
|
||||
signature = inspect.signature(resolve)
|
||||
|
||||
@functools.wraps(resolve)
|
||||
def wrapper(*args, **kwargs):
|
||||
flag = [False]
|
||||
token = _DEFAULTED.set(flag)
|
||||
try:
|
||||
context_length = resolve(*args, **kwargs)
|
||||
finally:
|
||||
_DEFAULTED.reset(token)
|
||||
call = signature.bind_partial(*args, **kwargs).arguments
|
||||
key = _key(call.get("model"), call.get("base_url"))
|
||||
if flag[0]:
|
||||
if len(_RECORDS) >= _MAX_RECORDS:
|
||||
_RECORDS.clear()
|
||||
_RECORDS[key] = context_length
|
||||
else:
|
||||
_RECORDS.pop(key, None)
|
||||
return context_length
|
||||
return wrapper
|
||||
|
||||
|
||||
def was_context_defaulted(context_length, model, base_url="") -> bool:
|
||||
"""True when the last resolution of ``model`` on ``base_url`` in this profile defaulted to ``context_length``."""
|
||||
return _RECORDS.get(_key(model, base_url)) == context_length
|
||||
+9
-45
@@ -8,16 +8,11 @@ window the resolver only defaulted to (``fallback``), so it is not read as a det
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_warned_pins: set = set()
|
||||
# (window, model, provider, base_url) -> (monotonic ts, is_fallback). Status-bar repaints read the marker, and the
|
||||
# check reads local caches; a short TTL lets a later probe or provider-confirmed write clear the label.
|
||||
_FALLBACK_MEMO: dict[tuple, tuple[float, bool]] = {}
|
||||
_FALLBACK_MEMO_TTL_SECONDS = 60.0
|
||||
_MARKER_SUFFIX = {"pinned": " (pinned)", "fallback": " (fallback)"}
|
||||
|
||||
|
||||
@@ -36,50 +31,19 @@ def context_pin_suffix(context_length, config_context_length) -> str:
|
||||
return " (pinned)" if is_context_pinned(context_length, config_context_length) else ""
|
||||
|
||||
|
||||
def _resolved_by_fallback(context_length: int, model: str, provider: str, base_url: str) -> bool:
|
||||
"""True when no source knows ``model``'s window and ``context_length`` is the resolver's default. Local
|
||||
reads only (config overrides, static tables, persistent cache, models.dev/OpenRouter disk caches)."""
|
||||
from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT, _is_bedrock_context, get_cached_context_length
|
||||
bedrock = _is_bedrock_context(base_url, provider)
|
||||
try:
|
||||
from agent.bedrock_adapter import BEDROCK_DEFAULT_CONTEXT_LENGTH, bedrock_static_context_length
|
||||
except ImportError: # no boto3: the resolver skipped the Bedrock step and fell through to the generic default
|
||||
bedrock, BEDROCK_DEFAULT_CONTEXT_LENGTH, bedrock_static_context_length = False, 0, None
|
||||
if context_length != (BEDROCK_DEFAULT_CONTEXT_LENGTH if bedrock else DEFAULT_FALLBACK_CONTEXT):
|
||||
return False # every other value came from a source; the reads below run only for the default
|
||||
from agent.models_dev import _override_context_window, lookup_models_dev_context
|
||||
from hermes_cli.config import get_custom_provider_context_length
|
||||
if provider and _override_context_window(provider, model):
|
||||
return False # model_overrides.<provider>.<model>.context_window: the operator's own number
|
||||
if base_url and get_custom_provider_context_length(model=model, base_url=base_url):
|
||||
return False
|
||||
if bedrock and bedrock_static_context_length is not None:
|
||||
confirmed = get_cached_context_length(model, base_url or "bedrock://", bedrock_confirmed=True)
|
||||
return confirmed != context_length and not bedrock_static_context_length(model)
|
||||
if advertised_context_length(model, base_url):
|
||||
return False
|
||||
return not (provider and lookup_models_dev_context(provider, model))
|
||||
|
||||
|
||||
def is_fallback_context_length(context_length, *, model: str, provider: str = "", base_url: str = "") -> bool:
|
||||
"""True when ``context_length`` is a default the resolver fell back to for ``model`` on this route (Bedrock's
|
||||
static 128K, or the generic 256K), not a known or detected limit. Fail-open: False on any error."""
|
||||
static 128K, or the generic 256K), not a known or detected limit. The resolver records that itself
|
||||
(agent.context_default_record), so this agrees with every source it consults; ``provider`` is accepted for
|
||||
call-site symmetry. Fail-open: False on any error."""
|
||||
if type(context_length) is not int or context_length <= 0 or not isinstance(model, str) or not model:
|
||||
return False
|
||||
key = (context_length, model, str(provider or ""), str(base_url or ""))
|
||||
now = time.monotonic()
|
||||
hit = _FALLBACK_MEMO.get(key)
|
||||
if hit is not None and now - hit[0] < _FALLBACK_MEMO_TTL_SECONDS:
|
||||
return hit[1]
|
||||
try:
|
||||
verdict = _resolved_by_fallback(*key)
|
||||
from agent.context_default_record import was_context_defaulted
|
||||
return was_context_defaulted(context_length, model, base_url)
|
||||
except Exception:
|
||||
logger.debug("context fallback check failed for %s", model, exc_info=True)
|
||||
verdict = False
|
||||
if len(_FALLBACK_MEMO) > 256:
|
||||
_FALLBACK_MEMO.clear()
|
||||
_FALLBACK_MEMO[key] = (now, verdict)
|
||||
return verdict
|
||||
return False
|
||||
|
||||
|
||||
def context_window_marker(context_length, config_context_length, *, model: str, provider: str = "",
|
||||
@@ -100,9 +64,9 @@ def agent_context_marker(agent) -> str:
|
||||
""":func:`context_window_marker` for an agent's live compressor window."""
|
||||
compressor = getattr(agent, "context_compressor", None)
|
||||
pin = getattr(compressor, "_config_context_length", getattr(agent, "_config_context_length", None))
|
||||
return context_window_marker(
|
||||
getattr(compressor, "context_length", None), pin, model=getattr(agent, "model", ""),
|
||||
provider=getattr(agent, "provider", "") or "", base_url=getattr(agent, "base_url", "") or "")
|
||||
# The compressor's route is what resolved the window; the agent's is the fallback for doubles without one.
|
||||
route = {name: getattr(compressor, name, None) or getattr(agent, name, "") or "" for name in ("model", "provider", "base_url")}
|
||||
return context_window_marker(getattr(compressor, "context_length", None), pin, **route)
|
||||
|
||||
|
||||
def agent_context_suffix(agent) -> str:
|
||||
|
||||
@@ -19,6 +19,7 @@ import hermes_yaml as yaml
|
||||
|
||||
from agent import model_metadata_http
|
||||
from agent.model_metadata_cache import _write_context_cache, locked_context_cache
|
||||
from agent.context_default_record import mark_context_defaulted, records_context_defaults
|
||||
from agent.model_metadata_llamacpp import llamacpp_context
|
||||
from agent.model_metadata_memo_bounds import MEMO_LOCK, bound_codex_caches, remember_endpoint_models as _remember_endpoint_models
|
||||
|
||||
@@ -264,14 +265,12 @@ _FALLBACK_WARNED: set = set() # the fallback is never cached, so dedupe its war
|
||||
def _warn_context_length_fallback(model: str, base_url: str) -> None:
|
||||
"""Warn once per model+endpoint that detection failed and the hard default is used,
|
||||
so small-context models (8K, 32K) don't silently get 256K and fail at the API."""
|
||||
mark_context_defaulted() # the "fallback" label reads this, not a re-derivation (agent.context_default_record)
|
||||
key = (model, base_url or "")
|
||||
if key not in _FALLBACK_WARNED:
|
||||
_FALLBACK_WARNED.add(key)
|
||||
logger.warning(
|
||||
"Could not determine context length for model %r (base_url=%s) — falling back to %s tokens. "
|
||||
"Set model.context_length in config.yaml to override.",
|
||||
model, base_url or "default", f"{DEFAULT_FALLBACK_CONTEXT:,}",
|
||||
)
|
||||
logger.warning("Could not determine context length for model %r (base_url=%s) — falling back to %s tokens. "
|
||||
"Set model.context_length in config.yaml to override.", model, base_url or "default", f"{DEFAULT_FALLBACK_CONTEXT:,}")
|
||||
|
||||
|
||||
# Sessions, model switches and cron jobs reject models below this (too little working memory).
|
||||
@@ -2208,6 +2207,7 @@ def _resolve_provider_aware_context_length(model: str, base_url: str, api_key: s
|
||||
return None
|
||||
|
||||
|
||||
@records_context_defaults
|
||||
def get_model_context_length(
|
||||
model: str, base_url: str = "", api_key: str = "", config_context_length: int | None = None,
|
||||
provider: str = "", custom_providers: list | None = None,
|
||||
|
||||
@@ -305,7 +305,8 @@ class GatewayModelCommandsMixin:
|
||||
) -> str:
|
||||
"""Confirmation text with full metadata (display form shortens opaque Palantir IDs)."""
|
||||
from gateway.run import _load_gateway_config
|
||||
from hermes_cli.model_switch import format_model_for_display, resolve_display_context_length_async
|
||||
from hermes_cli.model_switch import (
|
||||
effective_context_pin, format_model_for_display, resolve_display_context_length_async)
|
||||
|
||||
lines = [
|
||||
t("gateway.model.switched", model=format_model_for_display(result.new_model)),
|
||||
@@ -321,20 +322,20 @@ class GatewayModelCommandsMixin:
|
||||
config_ctx = int(model_cfg["context_length"])
|
||||
if not isinstance(model_cfg, dict):
|
||||
model_cfg = {}
|
||||
base_url = result.base_url or ctx.current_base_url or ""
|
||||
# The pin applies only while the route matches the config; the number and its label use the same answer.
|
||||
config_ctx = effective_context_pin(
|
||||
config_ctx, result.new_model, result.target_provider, base_url,
|
||||
model_cfg.get("default") or model_cfg.get("model"), model_cfg.get("provider"), model_cfg.get("base_url"))
|
||||
ctx_len = await resolve_display_context_length_async(
|
||||
result.new_model, result.target_provider,
|
||||
base_url=result.base_url or ctx.current_base_url or "",
|
||||
result.new_model, result.target_provider, base_url=base_url,
|
||||
api_key=result.api_key or ctx.current_api_key or "", model_info=mi,
|
||||
custom_providers=ctx.custom_provs, config_context_length=config_ctx,
|
||||
configured_model=model_cfg.get("default") or model_cfg.get("model"),
|
||||
configured_provider=model_cfg.get("provider"),
|
||||
configured_base_url=model_cfg.get("base_url"),
|
||||
)
|
||||
if ctx_len:
|
||||
from agent.context_pin import context_window_suffix
|
||||
lines.append(t("gateway.model.context_label", tokens=f"{ctx_len:,}") + context_window_suffix(
|
||||
ctx_len, config_ctx, model=result.new_model, provider=result.target_provider,
|
||||
base_url=result.base_url or ctx.current_base_url or ""))
|
||||
ctx_len, config_ctx, model=result.new_model, provider=result.target_provider, base_url=base_url))
|
||||
if mi and mi.max_output:
|
||||
lines.append(t("gateway.model.max_output_label", tokens=f"{mi.max_output:,}"))
|
||||
if mi:
|
||||
|
||||
@@ -292,8 +292,11 @@ class GatewayStatusCommandsMixin:
|
||||
mark = "~" if context_display_source(getattr(status_agent, "context_compressor", None)) != "provider_usage" else ""
|
||||
if context_total:
|
||||
pct = min(100, round((context_used / context_total) * 100))
|
||||
from agent.context_pin import agent_context_suffix # (pinned) / (fallback) for the agent's own window
|
||||
compressor_total = _n(getattr(status_agent, "context_compressor", None), "context_length")
|
||||
suffix = agent_context_suffix(status_agent) if compressor_total and compressor_total == context_total else ""
|
||||
lines.append(t("gateway.status.context", used=mark + _fmt(context_used), total=_fmt(context_total),
|
||||
pct=f"{mark}{pct}"))
|
||||
pct=f"{mark}{pct}") + suffix)
|
||||
elif context_used:
|
||||
lines.append(t("gateway.status.context_used", used=mark + _fmt(context_used)))
|
||||
state = t("gateway.status.state_yes") if fields["agent_running"] else t("gateway.status.state_no")
|
||||
@@ -355,7 +358,7 @@ class GatewayStatusCommandsMixin:
|
||||
# Running agent first (mid-turn), then cached agent (between turns).
|
||||
agent = self._resident_agent_for(self._session_key_for_source(source)) or None
|
||||
ctx = getattr(agent, "context_compressor", None) if agent else None
|
||||
used, context_length, model_name = await self._resolve_context_figures(
|
||||
used, context_length, model_name, window_suffix = await self._resolve_context_figures(
|
||||
agent, ctx, session_entry, source
|
||||
)
|
||||
from agent.context_breakdown import context_display_source
|
||||
@@ -367,7 +370,7 @@ class GatewayStatusCommandsMixin:
|
||||
lines = [
|
||||
t("gateway.context.header"), "",
|
||||
t("gateway.context.model", model=model_name or "?"),
|
||||
t("gateway.context.window", total=_fmt(context_length)),
|
||||
t("gateway.context.window", total=_fmt(context_length)) + window_suffix,
|
||||
t("gateway.context.in_use", used=mark + _fmt(used), total=_fmt(context_length), pct=f"{mark}{pct:.0f}"),
|
||||
t("gateway.context.bar", bar="█" * max(0, filled) + "░" * max(0, 24 - filled)),
|
||||
t("gateway.context.headroom", headroom=mark + _fmt(max(0, context_length - used))),
|
||||
@@ -396,10 +399,13 @@ class GatewayStatusCommandsMixin:
|
||||
])
|
||||
|
||||
async def _resolve_context_figures(self, agent, ctx, session_entry, source):
|
||||
"""``(used, context_length, model_name)`` for /context: used = compressor -> SessionStore;
|
||||
model = agent -> SessionDB row; window = compressor -> gateway model route -> model metadata."""
|
||||
"""``(used, context_length, model_name, window_suffix)`` for /context: used = compressor -> SessionStore;
|
||||
model = agent -> SessionDB row; window = compressor -> gateway model route -> model metadata. The suffix
|
||||
is `` (pinned)`` / `` (fallback)`` from whichever source supplied the window."""
|
||||
from agent.context_pin import agent_context_suffix, context_window_suffix
|
||||
used = max(0, _n(ctx, "last_prompt_tokens")) or max(0, _int_value(getattr(session_entry, "last_prompt_tokens", 0)))
|
||||
context_length = _n(ctx, "context_length")
|
||||
suffix = agent_context_suffix(agent) if context_length else ""
|
||||
model_name = _clean_str(getattr(agent, "model", "")) if agent is not None else ""
|
||||
if not model_name and self._session_db:
|
||||
row = await _quiet(lambda: self._session_db.get_session(session_entry.session_id))
|
||||
@@ -409,12 +415,14 @@ class GatewayStatusCommandsMixin:
|
||||
if resolved is not None:
|
||||
model_name = model_name or resolved.model
|
||||
context_length = _int_value(resolved.context_length)
|
||||
suffix = {"config": " (pinned)", "default": " (fallback)"}.get(resolved.context_source, "")
|
||||
if not context_length and model_name:
|
||||
from agent.model_metadata import get_model_context_length
|
||||
context_length = _int_value(
|
||||
await _quiet(lambda: asyncio.to_thread(get_model_context_length, model_name))
|
||||
)
|
||||
return used, context_length, model_name
|
||||
suffix = context_window_suffix(context_length, None, model=model_name)
|
||||
return used, context_length, model_name, suffix
|
||||
|
||||
async def _resolve_route_context(self, source, model_name: str, route: dict | None = None):
|
||||
"""``_resolve_gateway_model_context`` for a non-resident session, run off the event loop inside
|
||||
|
||||
@@ -891,6 +891,24 @@ def _resolve_alias_fallback(
|
||||
return next((r for r in results if r is not None), None)
|
||||
|
||||
|
||||
def effective_context_pin(
|
||||
config_context_length: int | None, model: str, provider: str, base_url: str = "",
|
||||
configured_model: str | None = None, configured_provider: str | None = None,
|
||||
configured_base_url: str | None = None) -> int | None:
|
||||
"""The ``model.context_length`` pin that still applies to ``model`` on this route, or None when the
|
||||
route changed from the configured one (the pin belongs to that model). Fail-closed: None on error."""
|
||||
if config_context_length is None or not (configured_model or configured_provider or configured_base_url):
|
||||
return config_context_length
|
||||
try:
|
||||
from hermes_cli.route_identity import should_clear_context_pin
|
||||
clear = should_clear_context_pin(
|
||||
configured_model, model, configured_base_url, base_url, configured_provider, provider)
|
||||
except Exception: # any route-identity failure drops the pin, as resolve_display_context_length always did
|
||||
logger.debug("context pin route check failed for %s", model, exc_info=True)
|
||||
return None
|
||||
return None if clear else config_context_length
|
||||
|
||||
|
||||
def resolve_display_context_length(
|
||||
model: str, provider: str, base_url: str = "", api_key: str = "",
|
||||
model_info: Optional[ModelInfo] = None, custom_providers: list | None = None,
|
||||
@@ -908,15 +926,8 @@ def resolve_display_context_length(
|
||||
``custom_providers[].models.<id>.context_length`` are honored — this closes #15779 where ``/model``
|
||||
switch ignored user-set overrides.
|
||||
"""
|
||||
if config_context_length is not None and (configured_model or configured_provider or configured_base_url):
|
||||
try:
|
||||
from hermes_cli.route_identity import should_clear_context_pin
|
||||
if should_clear_context_pin(
|
||||
configured_model, model, configured_base_url, base_url, configured_provider, provider):
|
||||
config_context_length = None
|
||||
except Exception:
|
||||
config_context_length = None
|
||||
|
||||
config_context_length = effective_context_pin(
|
||||
config_context_length, model, provider, base_url, configured_model, configured_provider, configured_base_url)
|
||||
try:
|
||||
from agent.model_metadata import get_model_context_length
|
||||
ctx = get_model_context_length(
|
||||
|
||||
@@ -11,47 +11,101 @@ TERRA_CARD_WINDOW = 1_050_000
|
||||
RUNTIME_URL = "https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
|
||||
|
||||
UNKNOWN = "acme.unknown-model-v1"
|
||||
LOCAL_URL = "http://127.0.0.1:8000/v1"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_state(monkeypatch):
|
||||
from agent import bedrock_adapter, context_pin
|
||||
from agent import bedrock_adapter, context_default_record
|
||||
bedrock_adapter.reset_client_cache()
|
||||
monkeypatch.setattr(context_pin, "_FALLBACK_MEMO", {}, raising=False)
|
||||
monkeypatch.setattr(context_default_record, "_RECORDS", {})
|
||||
monkeypatch.setattr("agent.bedrock_adapter.probe_bedrock_context_length", lambda *a, **k: None)
|
||||
yield
|
||||
bedrock_adapter.reset_client_cache()
|
||||
|
||||
|
||||
def _bedrock_fallback(model="acme.unknown-model-v1", **kw):
|
||||
def _resolve(model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL, config_context_length=None) -> int:
|
||||
"""The real resolver, as the compressor / gateway / /model call it (it records a defaulted answer)."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
return get_model_context_length(model, base_url=base_url, provider=provider,
|
||||
config_context_length=config_context_length)
|
||||
|
||||
|
||||
def _resolve_and_mark(model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL, config_context_length=None):
|
||||
from agent.context_pin import context_window_marker
|
||||
from agent.bedrock_adapter import BEDROCK_DEFAULT_CONTEXT_LENGTH
|
||||
kw.setdefault("provider", "bedrock")
|
||||
kw.setdefault("base_url", RUNTIME_URL)
|
||||
return context_window_marker(kw.pop("context_length", BEDROCK_DEFAULT_CONTEXT_LENGTH),
|
||||
kw.pop("config_context_length", None), model=model, **kw)
|
||||
ctx = _resolve(model, provider, base_url, config_context_length)
|
||||
return ctx, context_window_marker(ctx, config_context_length, model=model, provider=provider, base_url=base_url)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _quiet_custom_endpoint(monkeypatch):
|
||||
"""A custom endpoint whose live probes report nothing (tests opt in to a /models answer)."""
|
||||
for name in ("_resolve_endpoint_context_length", "_probe_local_context_length", "_query_ollama_api_show"):
|
||||
monkeypatch.setattr(f"agent.model_metadata.{name}", lambda *a, **k: None)
|
||||
return monkeypatch
|
||||
|
||||
|
||||
class TestFallbackMarker:
|
||||
def test_unknown_bedrock_model_window_is_marked_fallback(self):
|
||||
assert _bedrock_fallback() == "fallback"
|
||||
assert _resolve_and_mark() == (128_000, "fallback")
|
||||
|
||||
def test_card_sized_and_pinned_windows_are_not_marked_fallback(self):
|
||||
assert _bedrock_fallback("us.openai.gpt-6.1-terra", context_length=TERRA_CARD_WINDOW) == ""
|
||||
assert _bedrock_fallback("amazon.nova-micro-v1:0") == "" # table row of exactly 128K
|
||||
assert _bedrock_fallback(config_context_length=128_000) == "pinned"
|
||||
assert _resolve_and_mark("us.openai.gpt-6.1-terra") == (TERRA_CARD_WINDOW, "")
|
||||
assert _resolve_and_mark("amazon.nova-micro-v1:0") == (128_000, "") # table row of exactly 128K
|
||||
assert _resolve_and_mark(config_context_length=128_000) == (128_000, "pinned")
|
||||
|
||||
def test_provider_confirmed_window_equal_to_the_default_is_not_marked_fallback(self):
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=128_000):
|
||||
assert _bedrock_fallback() == ""
|
||||
assert _resolve_and_mark() == (128_000, "")
|
||||
|
||||
def test_non_bedrock_route_with_the_bedrock_default_is_not_marked_fallback(self):
|
||||
assert _bedrock_fallback(provider="openrouter", base_url="https://openrouter.ai/api/v1") == ""
|
||||
|
||||
def test_generic_default_for_an_uncatalogued_model_is_marked_fallback(self):
|
||||
def test_a_window_no_resolution_defaulted_is_not_marked_fallback(self):
|
||||
from agent.context_pin import context_window_marker
|
||||
_resolve() # the Bedrock route defaulted; another route showing the same number did not
|
||||
assert context_window_marker(128_000, None, model=UNKNOWN, provider="openrouter",
|
||||
base_url="https://openrouter.ai/api/v1") == ""
|
||||
|
||||
def test_generic_default_for_an_uncatalogued_model_is_marked_fallback(self, _quiet_custom_endpoint):
|
||||
from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT
|
||||
route = dict(provider="custom", base_url="http://127.0.0.1:8000/v1")
|
||||
assert context_window_marker(DEFAULT_FALLBACK_CONTEXT, None, model="mystery-model", **route) == "fallback"
|
||||
route = dict(provider="custom", base_url=LOCAL_URL)
|
||||
assert _resolve_and_mark("mystery-model", **route) == (DEFAULT_FALLBACK_CONTEXT, "fallback")
|
||||
# Same number from the catalog (a listed 256K model) is a known limit.
|
||||
assert context_window_marker(DEFAULT_FALLBACK_CONTEXT, None, model="gemma-4-31b", **route) == ""
|
||||
assert _resolve_and_mark("gemma-4-31b", **route) == (DEFAULT_FALLBACK_CONTEXT, "")
|
||||
|
||||
def test_a_window_the_endpoint_reported_is_not_marked_fallback(self, _quiet_custom_endpoint):
|
||||
"""Greptile: an uncatalogued model whose /models reports 256000 is detected, not defaulted."""
|
||||
from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT
|
||||
route = dict(provider="custom", base_url=LOCAL_URL)
|
||||
assert _resolve_and_mark("mystery-model", **route)[1] == "fallback" # endpoint down: defaulted
|
||||
_quiet_custom_endpoint.setattr("agent.model_metadata._resolve_endpoint_context_length",
|
||||
lambda *a, **k: DEFAULT_FALLBACK_CONTEXT)
|
||||
assert _resolve_and_mark("mystery-model", **route) == (DEFAULT_FALLBACK_CONTEXT, "") # reported: detected
|
||||
|
||||
@pytest.mark.parametrize("defaulted_first", [True, False])
|
||||
def test_profiles_keep_their_own_label(self, tmp_path, defaulted_first):
|
||||
"""Greptile: a multiplexed gateway resolves for several profiles; one profile's answer is not another's."""
|
||||
from agent.context_pin import context_window_marker
|
||||
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
|
||||
homes = {"defaulted": tmp_path / "a", "confirmed": tmp_path / "b"}
|
||||
for home in homes.values():
|
||||
home.mkdir()
|
||||
|
||||
def in_home(name, fn):
|
||||
token = set_hermes_home_override(homes[name])
|
||||
try:
|
||||
return fn()
|
||||
finally:
|
||||
reset_hermes_home_override(token)
|
||||
|
||||
def confirmed():
|
||||
with patch("agent.model_metadata.get_cached_context_length", return_value=128_000):
|
||||
return _resolve()
|
||||
order = [("defaulted", _resolve), ("confirmed", confirmed)]
|
||||
for name, fn in order if defaulted_first else order[::-1]:
|
||||
assert in_home(name, fn) == 128_000
|
||||
mark = lambda: context_window_marker(128_000, None, model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL)
|
||||
assert in_home("defaulted", mark) == "fallback"
|
||||
assert in_home("confirmed", mark) == ""
|
||||
|
||||
|
||||
class TestFallbackIsLabelledOnSurfaces:
|
||||
@@ -64,8 +118,7 @@ class TestFallbackIsLabelledOnSurfaces:
|
||||
runtime = {"provider": "bedrock", "base_url": RUNTIME_URL, "api_key": "aws-sdk"}
|
||||
with patch("gateway.run._hermes_home", tmp_path), \
|
||||
patch("gateway.run._resolve_gateway_model", return_value="acme.unknown-model-v1"), \
|
||||
patch("gateway.run._resolve_runtime_agent_kwargs", return_value=runtime), \
|
||||
patch("agent.bedrock_adapter.probe_bedrock_context_length", return_value=None):
|
||||
patch("gateway.run._resolve_runtime_agent_kwargs", return_value=runtime):
|
||||
info = runner._format_session_info()
|
||||
line = next(x for x in info.splitlines() if "128K" in x)
|
||||
assert "(fallback" in line and "(detected)" not in line
|
||||
@@ -92,6 +145,7 @@ class TestFallbackIsLabelledOnSurfaces:
|
||||
return cli_obj
|
||||
|
||||
def test_cli_usage_marks_the_fallback_window(self, capsys):
|
||||
_resolve()
|
||||
self._cli_with_bedrock_agent()._show_usage()
|
||||
line = next(x for x in capsys.readouterr().out.splitlines() if "128,000" in x)
|
||||
assert "(fallback)" in line
|
||||
@@ -101,6 +155,7 @@ class TestFallbackIsLabelledOnSurfaces:
|
||||
assert "(fallback)" not in capsys.readouterr().out
|
||||
|
||||
def test_cli_status_bar_marks_the_fallback_window(self):
|
||||
_resolve()
|
||||
text = self._cli_with_bedrock_agent()._build_status_bar_text(width=120)
|
||||
assert "/128K fallback" in text
|
||||
|
||||
@@ -112,6 +167,7 @@ class TestFallbackIsLabelledOnSurfaces:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_usage_marks_the_fallback_window(self):
|
||||
_resolve()
|
||||
agent = MagicMock(model="acme.unknown-model-v1", provider="bedrock", base_url=RUNTIME_URL,
|
||||
session_input_tokens=9_000, session_output_tokens=1_000, session_total_tokens=10_000,
|
||||
session_api_calls=2, session_cache_read_tokens=0, session_cache_write_tokens=0)
|
||||
@@ -135,7 +191,6 @@ class TestFallbackIsLabelledOnSurfaces:
|
||||
is_global=False,
|
||||
)
|
||||
from hermes_cli.cli_model_switch_mixin import _print_switch_summary
|
||||
with patch("agent.bedrock_adapter.probe_bedrock_context_length", return_value=None):
|
||||
_print_switch_summary(stub, result, "us.openai.gpt-6-sol", one_turn=False, strict_context=True)
|
||||
_print_switch_summary(stub, result, "us.openai.gpt-6-sol", one_turn=False, strict_context=True)
|
||||
line = next(x for x in captured if "128,000" in x)
|
||||
assert "(fallback)" in line
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Gateway views label a defaulted context window "fallback" like /usage does, and /model never shows a pin
|
||||
the route change discarded (Greptile on #148801)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.config import Platform
|
||||
from gateway.session import SessionEntry, build_session_key
|
||||
from tests.gateway.test_status_command import _make_event, _make_runner, _make_source
|
||||
|
||||
UNKNOWN = "acme.unknown-model-v1"
|
||||
RUNTIME_URL = "https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_state(monkeypatch):
|
||||
from agent import context_default_record
|
||||
monkeypatch.setattr(context_default_record, "_RECORDS", {})
|
||||
monkeypatch.setattr("agent.bedrock_adapter.probe_bedrock_context_length", lambda *a, **k: None)
|
||||
|
||||
|
||||
def _resolve_unknown_bedrock() -> int:
|
||||
"""What the agent's compressor does when it builds the window."""
|
||||
from agent.model_metadata import get_model_context_length
|
||||
return get_model_context_length(UNKNOWN, base_url=RUNTIME_URL, provider="bedrock")
|
||||
|
||||
|
||||
def _bedrock_agent(context_length: int) -> SimpleNamespace:
|
||||
compressor = SimpleNamespace(
|
||||
model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL, last_prompt_tokens=9_000,
|
||||
context_length=context_length, _config_context_length=None, threshold_tokens=96_000,
|
||||
threshold_percent=0.75, compression_count=0, _last_compression_savings_pct=None)
|
||||
return SimpleNamespace(
|
||||
model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL, context_compressor=compressor,
|
||||
interrupt=lambda *a, **k: None, session_api_calls=1, session_input_tokens=9_000,
|
||||
session_output_tokens=100, session_reasoning_tokens=0, session_total_tokens=9_100,
|
||||
session_cache_read_tokens=0, session_cache_write_tokens=0)
|
||||
|
||||
|
||||
def _runner():
|
||||
entry = SessionEntry(session_key=build_session_key(_make_source()), session_id="sess-fallback",
|
||||
created_at=datetime.now(), updated_at=datetime.now(), platform=Platform.TELEGRAM,
|
||||
chat_type="dm")
|
||||
entry.last_prompt_tokens = 9_000
|
||||
return _make_runner(entry)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_marks_a_live_agents_fallback_window():
|
||||
runner = _runner()
|
||||
runner._running_agents[build_session_key(_make_source())] = _bedrock_agent(_resolve_unknown_bedrock())
|
||||
result = await runner._handle_message(_make_event("/status"))
|
||||
line = next(x for x in result.splitlines() if "**Context:**" in x)
|
||||
assert "128,000" in line and line.endswith("(fallback)")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_leaves_a_detected_live_window_unmarked():
|
||||
runner = _runner()
|
||||
runner._running_agents[build_session_key(_make_source())] = _bedrock_agent(1_050_000)
|
||||
result = await runner._handle_message(_make_event("/status"))
|
||||
assert "(fallback)" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_marks_a_resident_agents_fallback_window():
|
||||
runner = _runner()
|
||||
agent = _bedrock_agent(_resolve_unknown_bedrock())
|
||||
with patch.object(type(runner), "_resident_agent_for", lambda self, key: agent, create=True), \
|
||||
patch.object(type(runner), "_context_breakdown_block", lambda self, *a: [], create=True):
|
||||
result = await runner._handle_context_command(_make_event("/context"))
|
||||
assert "Window: 128,000 tokens (fallback)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_marks_a_non_resident_sessions_defaulted_route():
|
||||
from gateway.run import _GatewayModelContext
|
||||
runner = _runner()
|
||||
resolved = _GatewayModelContext(model=UNKNOWN, provider="bedrock", base_url=RUNTIME_URL,
|
||||
context_length=128_000, context_source="default")
|
||||
|
||||
async def _route(self, source, model_name, route=None):
|
||||
return resolved
|
||||
with patch.object(type(runner), "_resident_agent_for", lambda self, key: None, create=True), \
|
||||
patch.object(type(runner), "_resolve_route_context", _route):
|
||||
result = await runner._handle_context_command(_make_event("/context"))
|
||||
assert "Window: 128,000 tokens (fallback)" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_switch_does_not_show_a_discarded_pin(tmp_path, monkeypatch):
|
||||
"""The configured model's 128K pin is dropped for the new route, so it must not label the new window."""
|
||||
import hermes_yaml as yaml
|
||||
from hermes_cli.model_switch import ModelSwitchResult
|
||||
from tests.gateway.test_model_command_context_offload import _event, _runner_with_store
|
||||
runner = _runner_with_store(tmp_path, monkeypatch)
|
||||
(tmp_path / ".hermes" / "config.yaml").write_text(yaml.safe_dump(
|
||||
{"model": {"default": "old-model", "provider": "openrouter", "context_length": 128_000}}), encoding="utf-8")
|
||||
monkeypatch.setattr("hermes_cli.model_switch.switch_model", lambda **kw: ModelSwitchResult(
|
||||
success=True, new_model=UNKNOWN, target_provider="bedrock", provider_changed=True, api_key="aws-sdk",
|
||||
base_url=RUNTIME_URL, api_mode="bedrock_converse", provider_label="AWS Bedrock"))
|
||||
result = await runner._handle_model_command(_event(f"/model {UNKNOWN}"))
|
||||
line = next(x for x in result.splitlines() if "128,000" in x)
|
||||
assert "(pinned)" not in line and line.endswith("(fallback)")
|
||||
@@ -293,14 +293,20 @@ async def test_status_command_resolves_window_of_session_only_model_switch():
|
||||
async def test_status_command_keeps_occupancy_only_for_unknown_model_window():
|
||||
"""A model the resolver cannot size falls to DEFAULT_FALLBACK_CONTEXT; /status must not present
|
||||
that invented number as the window (a catalog-listed model of the same size still counts)."""
|
||||
from agent.context_default_record import mark_context_defaulted, records_context_defaults
|
||||
from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT
|
||||
|
||||
@records_context_defaults # the resolver records that it defaulted; a catalog 256K below does not
|
||||
def _unsized(model, base_url="", **_kwargs):
|
||||
mark_context_defaulted()
|
||||
return DEFAULT_FALLBACK_CONTEXT
|
||||
|
||||
runner = _runner_with_session_override(
|
||||
{"model": "proxy-mystery-model", "provider": "custom:proxy", "base_url": "http://127.0.0.1:1/v1"},
|
||||
last_prompt_tokens=4_321,
|
||||
)
|
||||
with patch("gateway.run._load_gateway_config", return_value={}), patch(
|
||||
"agent.model_metadata.get_model_context_length", return_value=DEFAULT_FALLBACK_CONTEXT
|
||||
"agent.model_metadata.get_model_context_length", _unsized
|
||||
):
|
||||
result = await runner._handle_message(_make_event("/status"))
|
||||
assert "**Context:** ~4,321 tokens" in result
|
||||
|
||||
Reference in new issue
Block a user