fix(cli): treat credential-resolution 429 as quota, not auth

A Codex quota wall at startup was labeled "Primary auth failed" and
exited 1, so Kanban counted a failure and operators went looking for
a bad token. Use is_rate_limited_auth_error() for the fallback notice
and EX_TEMPFAIL (75) when HERMES_KANBAN_TASK is set.

cli.py

@@ -4140,7 +4140,7 @@ _TERMINAL_PROVIDER_REASONS = frozenset({
 })
 
 
-def _single_query_exit_code(result) -> int:
+def _single_query_exit_code(result, *, credentials_rate_limited: bool = False) -> int:
     """Map a one-shot turn result onto a process exit code, for both `-q` and `-Q`.
 
     0 only when the turn completed; 130 when it was interrupted; 1 when it failed, stopped
@@ -4149,10 +4149,15 @@ def _single_query_exit_code(result) -> int:
     failed purely on a provider rate-limit / billing wall exits ``KANBAN_RATE_LIMIT_EXIT_CODE``
     (EX_TEMPFAIL): the dispatcher books that run ``rate_limited`` and requeues the task
     WITHOUT counting a failure, so a quota window or a provider outage cannot trip the breaker.
-    One that failed on a terminal provider error (credential revoked, model gone) exits
-    ``KANBAN_TERMINAL_PROVIDER_EXIT_CODE`` (EX_CONFIG): the dispatcher blocks the card at once.
+    The same sentinel applies when credential resolution itself is a quota/rate-limit
+    AuthError (no turn result object is produced). One that failed on a terminal provider
+    error (credential revoked, model gone) exits ``KANBAN_TERMINAL_PROVIDER_EXIT_CODE``
+    (EX_CONFIG): the dispatcher blocks the card at once.
     """
     if not isinstance(result, dict):
+        if credentials_rate_limited and os.environ.get("HERMES_KANBAN_TASK"):
+            from hermes_cli.kanban_db import KANBAN_RATE_LIMIT_EXIT_CODE
+            return KANBAN_RATE_LIMIT_EXIT_CODE
         return 1
     if result.get("interrupted"):
         return 130
@@ -4608,10 +4613,12 @@ def _run_single_query_mode(cli, query, image, quiet, oneshot, stream_json: bool
                         emitter.attach(cli.agent)
                     _run_quiet_single_query(cli, effective_query, emitter=emitter)
 
+            fail_code = _single_query_exit_code(
+                None, credentials_rate_limited=getattr(cli, "_credentials_rate_limited", False))
             if emitter is not None:
                 emitter.emit_result({"failed": True, "error": "credentials or agent init failed"},
-                                    session_id=cli.session_id or "", exit_code=1)
-            exit_single_query(1)  # credentials or agent init failed
+                                    session_id=cli.session_id or "", exit_code=fail_code)
+            exit_single_query(fail_code)  # credentials or agent init failed
         # No welcome banner (~420 ms cold); session id / resume hint come from _print_exit_summary().
         _query_label = query or ("[image attached]" if single_query_images else "")
         if _query_label:
@@ -4629,7 +4636,9 @@ def _run_single_query_mode(cli, query, image, quiet, oneshot, stream_json: bool
         cli._print_exit_summary(clear_screen=False)
         # Same exit contract as `-Q`: scripts and the Kanban dispatcher read the outcome from
         # the exit code. This path used to fall through to an implicit 0 for every outcome.
-        exit_single_query(_single_query_exit_code(cli._last_turn_result))
+        exit_single_query(_single_query_exit_code(
+            cli._last_turn_result,
+            credentials_rate_limited=getattr(cli, "_credentials_rate_limited", False)))
     finally:
         _finalize_single_query(cli)
 

hermes_cli/cli_agent_setup_mixin.py

@@ -233,6 +233,7 @@ class CLIAgentSetupMixin:
         _primary_exc = None
         runtime = None
         _model_at_entry = self.model
+        self._credentials_rate_limited = False
         try:
             # target_model: the ladder's model-keyed rungs (Zen/Go api_mode, Copilot/Nous
             # api_mode) must see the model this CLI will actually send, not config's `default`,
@@ -248,6 +249,8 @@ class CLIAgentSetupMixin:
             if runtime is not None:
                 _primary_exc = None
         if runtime is None:
+            from hermes_cli.auth import is_rate_limited_auth_error
+            self._credentials_rate_limited = bool(_primary_exc) and is_rate_limited_auth_error(_primary_exc)
             message = format_runtime_provider_error(_primary_exc) if _primary_exc else "Provider resolution failed."
             if getattr(self, "tool_progress_mode", "full") == "off":
                 print(message, file=sys.stderr)  # quiet/stream-json: stdout is machine-readable
@@ -357,7 +360,7 @@ class CLIAgentSetupMixin:
         order and switch the CLI's requested_provider/model to the first that resolves.
         None when the error is not auth-related or no fallback resolves."""
         from cli import _cprint, logger
-        from hermes_cli.auth import AuthError
+        from hermes_cli.auth import AuthError, is_rate_limited_auth_error
         from hermes_cli.runtime_provider import resolve_runtime_provider
         if not isinstance(primary_exc, AuthError):
             return None
@@ -378,13 +381,23 @@ class CLIAgentSetupMixin:
                 if _fb_api_key:
                     _fb_kwargs["explicit_api_key"] = _fb_api_key
                 runtime = resolve_runtime_provider(**_fb_kwargs)
-                logger.warning(
-                    "Primary provider auth failed (%s). Falling through to fallback: %s/%s",
-                    primary_exc, _fb_provider, _fb_model)
-                from gateway.warning_notifications import render_notification
-                render_notification(
-                    lambda: _cprint(f"⚠️  Primary auth failed — switching to fallback: {_fb_provider} / {_fb_model}"),
-                    platform="cli")
+                if is_rate_limited_auth_error(primary_exc):
+                    logger.warning(
+                        "Primary provider rate-limited (429): %s. Falling through to fallback: %s/%s",
+                        primary_exc, _fb_provider, _fb_model)
+                    from gateway.warning_notifications import render_notification
+                    render_notification(
+                        lambda: _cprint(
+                            f"⚠️  Primary provider quota exhausted — switching to fallback: {_fb_provider} / {_fb_model}"),
+                        platform="cli")
+                else:
+                    logger.warning(
+                        "Primary provider auth failed (%s). Falling through to fallback: %s/%s",
+                        primary_exc, _fb_provider, _fb_model)
+                    from gateway.warning_notifications import render_notification
+                    render_notification(
+                        lambda: _cprint(f"⚠️  Primary auth failed — switching to fallback: {_fb_provider} / {_fb_model}"),
+                        platform="cli")
                 self.requested_provider = _fb_provider
                 self.model = _fb_model
                 # reasoning_config follows the swap in _ensure_runtime_credentials (the only caller).

tests/agent/test_grid_sweep_cli_tui_stream_diagnostics.py

@@ -170,7 +170,7 @@ def test_cli_provider_fallback_notice_honors_policy(tmp_path, monkeypatch, setti
     from hermes_cli import cli_agent_setup_mixin as m
     out = []
     monkeypatch.setattr(climod, "_cprint", lambda s: out.append(s))
-    _cli_block(m, "from gateway.warning_notifications import render_notification\n                render_notification(\n                    lambda: _cprint(f\"⚠️  Primary auth failed",
+    _cli_block(m, "from gateway.warning_notifications import render_notification\n                    render_notification(\n                        lambda: _cprint(f\"⚠️  Primary auth failed",
                "self.requested_provider = _fb_provider", {"_fb_provider": "p", "_fb_model": "m"})
     assert (len(out) == 1 and "Primary auth failed" in out[0]) is _visible(setting)
 

tests/hermes_cli/test_cli_provider_resolution.py

@@ -363,6 +363,107 @@ def test_fallback_runtime_resolves_the_fallback_entry_model(monkeypatch, tmp_pat
     assert runtime["base_url"] == "https://opencode.ai/zen/go/v1"
 
 
+def _quota_auth_error():
+    from hermes_cli.auth import CODEX_RATE_LIMITED_CODE, AuthError
+    return AuthError(
+        "Codex provider quota exhausted (429); retry after 1839s. Credentials are still valid.",
+        provider="openai-codex",
+        code=CODEX_RATE_LIMITED_CODE,
+        relogin_required=False,
+    )
+
+
+def test_fallback_runtime_quota_outage_is_not_labeled_auth_failure(monkeypatch, tmp_path):
+    """A 429 at credential resolution is quota, not bad credentials (#117482)."""
+    from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin
+
+    home = tmp_path / "hermes"
+    home.mkdir()
+    monkeypatch.setenv("HERMES_HOME", str(home))
+    printed = []
+    monkeypatch.setattr("cli._cprint", printed.append, raising=False)
+    monkeypatch.setattr(
+        "hermes_cli.runtime_provider.resolve_runtime_provider",
+        lambda **kw: {"provider": "custom", "base_url": "http://x/v1", "api_key": "k"},
+    )
+    monkeypatch.setattr("hermes_cli.fallback_config.resolve_entry_api_key", lambda entry: "k")
+
+    shell = CLIAgentSetupMixin.__new__(CLIAgentSetupMixin)
+    shell._fallback_model = [{"provider": "custom", "model": "local-model"}]
+    runtime = shell._resolve_fallback_runtime(_quota_auth_error())
+
+    assert runtime is not None
+    assert printed
+    assert "quota exhausted" in printed[-1]
+    assert "auth failed" not in printed[-1]
+
+
+def test_fallback_runtime_keeps_auth_failed_wording_for_bad_credentials(monkeypatch, tmp_path):
+    from hermes_cli.auth import AuthError
+    from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin
+
+    home = tmp_path / "hermes"
+    home.mkdir()
+    monkeypatch.setenv("HERMES_HOME", str(home))
+    printed = []
+    monkeypatch.setattr("cli._cprint", printed.append, raising=False)
+    monkeypatch.setattr(
+        "hermes_cli.runtime_provider.resolve_runtime_provider",
+        lambda **kw: {"provider": "custom", "base_url": "http://x/v1", "api_key": "k"},
+    )
+    monkeypatch.setattr("hermes_cli.fallback_config.resolve_entry_api_key", lambda entry: "k")
+
+    shell = CLIAgentSetupMixin.__new__(CLIAgentSetupMixin)
+    shell._fallback_model = [{"provider": "custom", "model": "local-model"}]
+    runtime = shell._resolve_fallback_runtime(
+        AuthError("no key", provider="openai-codex", code="missing_api_key"))
+
+    assert runtime is not None
+    assert printed
+    assert "Primary auth failed" in printed[-1]
+    assert "quota exhausted" not in printed[-1]
+
+
+def test_ensure_runtime_credentials_records_quota_vs_bad_key(monkeypatch, tmp_path):
+    """Kanban workers need this flag: a quota wall at startup is not a worker failure (#117482)."""
+    from hermes_cli.auth import AuthError
+    from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin
+
+    home = tmp_path / "hermes"
+    home.mkdir()
+    monkeypatch.setenv("HERMES_HOME", str(home))
+    monkeypatch.setattr("cli._cprint", lambda *a, **k: None, raising=False)
+
+    def _raise_quota(**kw):
+        raise _quota_auth_error()
+
+    monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise_quota)
+
+    quota_shell = CLIAgentSetupMixin.__new__(CLIAgentSetupMixin)
+    quota_shell.model = "gpt-x"
+    quota_shell.requested_provider = "openai-codex"
+    quota_shell._explicit_api_key = None
+    quota_shell._explicit_base_url = None
+    quota_shell._fallback_model = []
+    quota_shell.tool_progress_mode = "off"
+    assert quota_shell._ensure_runtime_credentials() is False
+    assert quota_shell._credentials_rate_limited is True
+
+    def _raise_missing(**kw):
+        raise AuthError("no key", provider="openai-codex", code="missing_api_key")
+
+    monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise_missing)
+    bad_shell = CLIAgentSetupMixin.__new__(CLIAgentSetupMixin)
+    bad_shell.model = "gpt-x"
+    bad_shell.requested_provider = "openai-codex"
+    bad_shell._explicit_api_key = None
+    bad_shell._explicit_base_url = None
+    bad_shell._fallback_model = []
+    bad_shell.tool_progress_mode = "off"
+    assert bad_shell._ensure_runtime_credentials() is False
+    assert bad_shell._credentials_rate_limited is False
+
+
 def test_cli_turn_routing_uses_primary_when_disabled(monkeypatch):
     cli = _import_cli()
     shell = cli.HermesCLI(model="gpt-5", compact=True, max_turns=1)

tests/hermes_cli/test_single_query_exit_contract.py

@@ -86,3 +86,49 @@ def test_dispatcher_spawned_worker_keeps_a_plain_failure_at_one(monkeypatch):
 )
 def test_a_plain_one_shot_run_reports_its_outcome(monkeypatch, turn_result, expected):
     assert _run_non_quiet(monkeypatch, turn_result) == expected
+
+
+def test_kanban_worker_quota_during_credential_resolution_is_tempfail_not_auth(monkeypatch):
+    """A 429 at startup never produces a turn result; the worker must still exit 75 (#117482)."""
+    monkeypatch.setenv("HERMES_KANBAN_TASK", "t_abc123")
+    assert cli._single_query_exit_code(None, credentials_rate_limited=True) == KANBAN_RATE_LIMIT_EXIT_CODE
+    monkeypatch.delenv("HERMES_KANBAN_TASK")
+    assert cli._single_query_exit_code(None, credentials_rate_limited=True) == 1
+    monkeypatch.setenv("HERMES_KANBAN_TASK", "t_abc123")
+    assert cli._single_query_exit_code(None, credentials_rate_limited=False) == 1
+
+
+def test_quiet_kanban_worker_exits_tempfail_when_credentials_are_rate_limited(monkeypatch):
+    monkeypatch.setenv("HERMES_KANBAN_TASK", "t_abc123")
+    monkeypatch.setattr(cli, "_should_seed_interactive", lambda *a, **k: False)
+    monkeypatch.setattr(cli, "_collect_query_images", lambda q, i: (q, []))
+    monkeypatch.setattr(cli, "_collect_kanban_task_images", lambda imgs: [])
+    monkeypatch.setattr(cli, "_finalize_single_query", lambda c: None)
+    stub = SimpleNamespace(
+        _claim_active_session=lambda *a, **k: True,
+        _ensure_runtime_credentials=lambda: False,
+        _credentials_rate_limited=True,
+        session_id="s1",
+        model="gpt-x",
+    )
+    with pytest.raises(SystemExit) as exc:
+        cli._run_single_query_mode(stub, "do the thing", None, True, True)
+    assert exc.value.code == KANBAN_RATE_LIMIT_EXIT_CODE
+
+
+def test_quiet_kanban_worker_still_exits_one_on_real_credential_failure(monkeypatch):
+    monkeypatch.setenv("HERMES_KANBAN_TASK", "t_abc123")
+    monkeypatch.setattr(cli, "_should_seed_interactive", lambda *a, **k: False)
+    monkeypatch.setattr(cli, "_collect_query_images", lambda q, i: (q, []))
+    monkeypatch.setattr(cli, "_collect_kanban_task_images", lambda imgs: [])
+    monkeypatch.setattr(cli, "_finalize_single_query", lambda c: None)
+    stub = SimpleNamespace(
+        _claim_active_session=lambda *a, **k: True,
+        _ensure_runtime_credentials=lambda: False,
+        _credentials_rate_limited=False,
+        session_id="s1",
+        model="gpt-x",
+    )
+    with pytest.raises(SystemExit) as exc:
+        cli._run_single_query_mode(stub, "do the thing", None, True, True)
+    assert exc.value.code == 1