diff --git a/docker-compose.yaml b/docker-compose.yaml index 09ab8f10..ce5ecb93 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,6 +33,27 @@ services: redis: condition: service_started + # The OpenAI-compatible chat front (horde/oai). It runs from the same image as the horde and reaches it over the + # compose network. See docs/how-to/run_openai_chat_front.md. + aihorde_oai: + image: aihorde:latest + container_name: aihorde_oai + command: ["python", "-m", "horde.oai"] + ports: + - "7002:7002" + environment: + - HORDE_OAI_HORDE_URLS=http://aihorde:7001/api + - HORDE_OAI_REDIS_URL=redis://redis:6379/7 + volumes: + - ./horde:/app/horde + networks: + - aihorde_network + depends_on: + aihorde: + condition: service_started + redis: + condition: service_started + postgres: image: ghcr.io/haidra-org/ai-horde-postgres:latest container_name: postgres diff --git a/horde/oai/__init__.py b/horde/oai/__init__.py new file mode 100644 index 00000000..b8d1a8aa --- /dev/null +++ b/horde/oai/__init__.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""An OpenAI-compatible chat completions front for the AI Horde. + +The front is a separate ASGI process (``python -m horde.oai``). It accepts ``POST /v1/chat/completions`` and +``GET /v1/models``, submits each chat request to the Horde's native ``/v2/generate/chat`` routes over HTTP, waits for +the Horde's Redis completion signal (with a status poll as fallback), and answers in the OpenAI response shape. It +imports nothing from the Flask application, so it runs beside the Horde without its database or Flask stack. + +Modules: + ``settings``: The environment-driven [`FrontSettings`][horde.oai.settings.FrontSettings]. + ``schema``: The OpenAI request, response and error models, the default no-op table and the refusal screen. + ``mapping``: The native request body, the completion built from a Horde status, and the error envelope. + ``sse``: The streamed chunks, keepalive comments and the error event. + ``horde_client``: The asynchronous HTTP client for the Horde API, with a separate cancel pool. + ``waiter``: The Redis completion signal listener, the per-request wait loop and the deadline rule. + +The ``horde`` extension object is validated against ``horde.chat_contract``, the model the native chat route shares. + ``app``: The FastAPI application factory, [`create_app`][horde.oai.app.create_app]. + +The package namespace imports nothing itself, so importing a submodule does not start any client or connection. +""" diff --git a/horde/oai/__main__.py b/horde/oai/__main__.py new file mode 100644 index 00000000..9b67d8b2 --- /dev/null +++ b/horde/oai/__main__.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Run the chat front with uvicorn: ``python -m horde.oai``. + +Settings come from the ``HORDE_OAI_*`` environment variables (see ``horde.oai.settings``). +``HORDE_OAI_FORWARDED_ALLOW_IPS`` is passed to uvicorn, which then takes the client address from the trusted +listener's ``X-Forwarded-For``; that address is the per-source admission key and the ``Proxied-For`` the Horde sees. +``HORDE_OAI_SHUTDOWN_DRAIN_SECONDS`` bounds how long a clean stop waits for open requests before cancelling them. +""" + +from __future__ import annotations + +import math + +import uvicorn + +from horde.oai.app import create_app +from horde.oai.settings import load_settings + + +def main() -> None: + """Serve the chat front until interrupted.""" + settings = load_settings() + uvicorn.run( + create_app(settings), + host=settings.listen, + port=settings.port, + proxy_headers=True, + forwarded_allow_ips=settings.forwarded_allow_ips, + timeout_graceful_shutdown=math.ceil(settings.shutdown_drain_seconds), + ) + + +if __name__ == "__main__": + main() diff --git a/horde/oai/app.py b/horde/oai/app.py new file mode 100644 index 00000000..f727fe60 --- /dev/null +++ b/horde/oai/app.py @@ -0,0 +1,1122 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""The FastAPI application serving the OpenAI-compatible chat completions front. + +[`create_app`][horde.oai.app.create_app] builds the app with these routes: + +- ``POST /v1/chat/completions``: reads the body under a byte ceiling and a read timeout, screens and validates it, + takes an inflight slot (per-source cap first, then the global cap), submits to the Horde's + ``/v2/generate/chat/async``, waits through [`ChatRequestWatcher`][horde.oai.waiter.ChatRequestWatcher] and answers + with a ``chat.completion`` body, or with server-sent events when ``stream`` is true. +- ``GET /v1/models`` and ``GET /v1/models/{id}``: the Horde's active text models in the OpenAI list shape; an id may + contain slashes. +- ``GET /healthz``: liveness, admission readiness and whether the completion signal subscription is up. + +Every error before headers is an OpenAI error envelope with a ``metadata`` object; after stream headers it is one SSE +error event. Once the Horde holds a request, or may hold one after a submit whose answer was lost, every error carries +``x-should-retry: false`` so a client SDK does not create a second charged job; a successful stream carries it too. +Cancels on disconnect, deadline, fail-fast and shutdown are bounded, shielded from the client's disconnect, sent at +most once per request and use the client's separate cancel pool. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import math +import time +from collections.abc import AsyncIterator, Mapping +from http import HTTPStatus + +import anyio +import httpx +import redis.asyncio +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse +from loguru import logger +from pydantic import ValidationError +from sse_starlette import EventSourceResponse, ServerSentEvent +from starlette.background import BackgroundTask + +from horde.chat_contract import CHAT_REQUEST_CONFLICT_ERROR_TYPE +from horde.consts import ANONYMOUS_API_KEY, HORDE_VERSION +from horde.oai.horde_client import ( + HordeActiveModel, + HordeAPIError, + HordeAsyncSubmission, + HordeChatStatus, + HordeClient, + HordeCredentials, + HordeUnavailableError, + create_http_client, +) +from horde.oai.mapping import build_completion, build_error_envelope, build_native_chat_body +from horde.oai.schema import ( + BAD_REQUEST_CODE, + CHAT_REQUEST_CONFLICT_CODE, + UNSUPPORTED_PARAMETER_CODE, + UNSUPPORTED_PARAMETER_ERROR_TYPE, + ChatCompletionRequest, + ErrorMetadata, + HordeModelDetails, + ModelCard, + ModelList, + OpenAIErrorEnvelope, + SubmitOutcome, + drop_default_no_ops, + find_refused_parameter, +) +from horde.oai.settings import FrontSettings +from horde.oai.sse import build_completion_events, build_error_events, build_keepalive_event +from horde.oai.waiter import ( + ChatRequestWatcher, + ChatWaitTimeoutError, + CompletionSignalListener, + CompletionSubscription, + NoEligibleWorkerError, + WaitBudget, +) + +__all__ = [ + "ChatFrontError", + "FrontState", + "InflightLimiter", + "InflightSlot", + "OpenRequest", + "create_app", +] + +CLIENT_AGENT_NAME: str = "horde-oai-front" +"""The name the front gives itself in the ``Client-Agent`` it sends the Horde.""" + +MAX_FORWARDED_USER_AGENT_CHARACTERS: int = 200 +"""The most characters of the client's ``User-Agent`` appended to the ``Client-Agent``; it is stored per request.""" + +BEARER_SCHEME: str = "bearer" +"""The lower-cased ``Authorization`` scheme carrying an API key.""" + +UNKNOWN_SOURCE: str = "unknown" +"""The source key of a request whose ASGI client address is not known; such requests share one per-source cap.""" + +MODEL_VARIANT_SEPARATOR: str = ":" +"""The character reserved for model id variants; no variant is served, so an id containing it is unknown.""" + +SHOULD_RETRY_HEADER: str = "x-should-retry" +"""The response header OpenAI SDKs honour to skip their automatic retries.""" + +SHOULD_NOT_RETRY: str = "false" +"""The ``x-should-retry`` value sent once a retry could create a second charged job.""" + +RETRY_AFTER_HEADER: str = "Retry-After" +"""The response header telling a client how many seconds to wait before retrying.""" + +STREAM_HEADERS: Mapping[str, str] = { + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + SHOULD_RETRY_HEADER: SHOULD_NOT_RETRY, +} +"""Headers of every streamed response: no caching or proxy buffering, and no automatic retry once a job exists.""" + +INFLIGHT_RETRY_AFTER_SECONDS: int = 5 +"""The ``Retry-After`` of a 503 for a full front; waiting requests usually finish within seconds of each other.""" + +NO_ELIGIBLE_WORKER_RETRY_AFTER_SECONDS: int = 30 +"""The ``Retry-After`` of a 503 for no eligible worker; long enough for a worker restart or model load to show.""" + +HORDE_UNAVAILABLE_RETRY_AFTER_SECONDS: int = 30 +"""The ``Retry-After`` added to a Horde 503 that carries none, such as chat admission switched off for maintenance.""" + +MODEL_LIST_CACHE_SECONDS: float = 5.0 +"""How long one process reuses the Horde's model list, so clients listing models often do not each cost a native call.""" + +CLIENT_CLOSED_REQUEST_STATUS: int = 499 +"""The status recorded for a non-streamed request whose client disconnected; no client receives it.""" + +TIMEOUT_CODE: str = "ChatRequestTimeout" +"""The error code of a request still waiting after its deadline and the final status read.""" + +FAULTED_CODE: str = "ChatRequestFaulted" +"""The error code of a request the Horde reports as faulted.""" + +FRONT_FULL_CODE: str = "FrontAtCapacity" +"""The error code of a request refused because the process or the request's source holds its maximum of requests.""" + +HORDE_UNAVAILABLE_CODE: str = "HordeUnavailable" +"""The error code of a request no Horde root answered.""" + +INVALID_AUTHORIZATION_CODE: str = "InvalidAuthorizationHeader" +"""The error code of a request whose ``Authorization`` header is not ``Bearer ``.""" + +REQUEST_TOO_LARGE_CODE: str = "RequestTooLarge" +"""The error code of a body over the byte ceiling.""" + +REQUEST_BODY_TIMEOUT_CODE: str = "RequestBodyTimeout" +"""The error code of a body not received within the body-read timeout; a slow sender is not an oversized body.""" + +MODEL_NOT_FOUND_CODE: str = "model_not_found" +"""The OpenAI error code of an unknown model id.""" + +NO_ELIGIBLE_WORKER_CODE: str = "no_eligible_worker" +"""The error code of a request no online worker could serve within the fail-fast grace.""" + + +class ChatFrontError(Exception): + """An error the front answers with an OpenAI error envelope, or an SSE error event after headers.""" + + def __init__( + self, + *, + status_code: int, + message: str, + code: str | None, + param: str | None = None, + metadata: ErrorMetadata | None = None, + headers: Mapping[str, str] | None = None, + job_may_exist: bool = False, + ) -> None: + """Create the error. + + Args: + status_code: The HTTP status. + message: The explanation for the client. + code: The Horde return code, or the front's own code. + param: The request path at fault, when one is. + metadata: The request id, return code and retry details that are known. + headers: Extra response headers, such as ``Retry-After``. + job_may_exist: Whether the Horde holds, or may hold, a request for this call; adds ``x-should-retry: false``. + """ + super().__init__(message) + self.status_code: int = status_code + """The HTTP status.""" + self.message: str = message + """The explanation for the client.""" + self.code: str | None = code + """The Horde return code, or the front's own code.""" + self.param: str | None = param + """The request path at fault.""" + self.metadata: ErrorMetadata = metadata if metadata is not None else ErrorMetadata() + """The request id, return code and retry details that are known.""" + self.headers: dict[str, str] = dict(headers or {}) + """Extra response headers.""" + self.job_may_exist: bool = job_may_exist + """Whether a retry could create a second charged job.""" + + def to_response(self) -> JSONResponse: + """Create the JSON error response, with ``x-should-retry: false`` when a job may exist.""" + headers = dict(self.headers) + if self.job_may_exist: + headers[SHOULD_RETRY_HEADER] = SHOULD_NOT_RETRY + return JSONResponse(status_code=self.status_code, content=self._envelope_payload(), headers=headers) + + def to_events(self) -> list[ServerSentEvent]: + """Create the SSE error event and ``[DONE]`` for an error after stream headers.""" + return build_error_events(self._envelope()) + + def _envelope(self) -> OpenAIErrorEnvelope: + """Create the OpenAI error envelope.""" + return build_error_envelope( + status_code=self.status_code, + message=self.message, + code=self.code, + param=self.param, + metadata=self.metadata, + ) + + def _envelope_payload(self) -> dict[str, object]: + """Return the envelope as JSON-ready values, null fields included.""" + payload: dict[str, object] = self._envelope().model_dump(mode="json") + return payload + + +class InflightLimiter: + """Count the chat requests the process holds open, globally and per source, and refuse new ones past the caps. + + Concurrency: + Acquire and release run on the event loop without awaiting, so the counts need no lock. + """ + + def __init__(self, *, max_inflight: int, max_inflight_per_source: int) -> None: + """Create the limiter. + + Args: + max_inflight: The most requests held open at once. + max_inflight_per_source: The most requests held open at once for one source address. + """ + self._max_inflight = max_inflight + self._max_inflight_per_source = max_inflight_per_source + self._inflight = 0 + self._inflight_by_source: dict[str, int] = {} + + @property + def inflight(self) -> int: + """Return how many requests are held open.""" + return self._inflight + + @property + def has_capacity(self) -> bool: + """Return whether the global cap admits one more request.""" + return self._inflight < self._max_inflight + + def inflight_for(self, source: str) -> int: + """Return how many requests are held open for ``source``.""" + return self._inflight_by_source.get(source, 0) + + def try_acquire(self, source: str) -> InflightSlot | None: + """Return a slot for one more request from ``source``, or None when the source's cap or the global cap is reached. + + The source's cap is checked first, so one client filling its share is told so even when the process is full. + """ + if self.inflight_for(source) >= self._max_inflight_per_source: + return None + if not self.has_capacity: + return None + self._inflight += 1 + self._inflight_by_source[source] = self.inflight_for(source) + 1 + return InflightSlot(self, source) + + def _release(self, source: str) -> None: + """Mutate the counts by returning one slot of ``source``.""" + self._inflight -= 1 + remaining_for_source = self.inflight_for(source) - 1 + if remaining_for_source > 0: + self._inflight_by_source[source] = remaining_for_source + else: + self._inflight_by_source.pop(source, None) + + +class InflightSlot: + """One request's place in the [`InflightLimiter`][horde.oai.app.InflightLimiter]; only the first release counts.""" + + def __init__(self, limiter: InflightLimiter, source: str) -> None: + """Create the slot. + + Args: + limiter: The limiter the slot belongs to. + source: The source address the slot counts against. + """ + self._limiter = limiter + self._source = source + self._released = False + + def release(self) -> None: + """Return the slot to the limiter, once.""" + if self._released: + return + self._released = True + self._limiter._release(self._source) + + +class OpenRequest: + """A submitted Horde request the process may still have to cancel; it is cancelled at most once.""" + + def __init__(self, *, request_id: str, credentials: HordeCredentials) -> None: + """Create the record. + + Args: + request_id: The Horde request id. + credentials: Who the cancel is made for. + """ + self.request_id: str = request_id + """The Horde request id.""" + self.credentials: HordeCredentials = credentials + """Who the cancel is made for.""" + self.cancel_started: bool = False + """Whether a cancel was sent or is being sent.""" + + +class ModelListCache: + """Hold the Horde's model list for [`MODEL_LIST_CACHE_SECONDS`][horde.oai.app.MODEL_LIST_CACHE_SECONDS].""" + + def __init__(self) -> None: + """Create an empty cache.""" + self._models: list[HordeActiveModel] | None = None + self._fetched_at: float = -math.inf + + async def get(self, horde_client: HordeClient, credentials: HordeCredentials) -> list[HordeActiveModel]: + """Return the cached model list, fetching it when it is older than the cache period. + + Raises: + HordeAPIError: The Horde answered with an error. + HordeUnavailableError: No Horde root accepted the connection. + """ + now = time.monotonic() + if self._models is None or now - self._fetched_at >= MODEL_LIST_CACHE_SECONDS: + self._models = await horde_client.list_text_models(credentials=credentials) + self._fetched_at = now + return self._models + + +class FrontState: + """Hold the process-wide objects the routes share.""" + + def __init__( + self, + *, + settings: FrontSettings, + horde_client: HordeClient, + listener: CompletionSignalListener, + ) -> None: + """Create the state. + + Args: + settings: The front's settings. + horde_client: The Horde client. + listener: The completion signal listener. + """ + self.settings: FrontSettings = settings + """The front's settings.""" + self.horde_client: HordeClient = horde_client + """The Horde client.""" + self.listener: CompletionSignalListener = listener + """The completion signal listener.""" + self.inflight_limiter: InflightLimiter = InflightLimiter( + max_inflight=settings.max_inflight, + max_inflight_per_source=settings.max_inflight_per_source, + ) + """The open request counts.""" + self.open_requests: dict[str, OpenRequest] = {} + """The submitted requests not yet finished, by Horde request id; a clean stop cancels them.""" + self.model_list_cache: ModelListCache = ModelListCache() + """The cached Horde model list.""" + self.accepting: bool = True + """False once the process is stopping; ``/healthz`` then reports it not ready.""" + + def open_request(self, *, request_id: str, credentials: HordeCredentials) -> OpenRequest: + """Create and register the record of a submitted request.""" + open_request = OpenRequest(request_id=request_id, credentials=credentials) + self.open_requests[request_id] = open_request + return open_request + + def close_request(self, open_request: OpenRequest) -> None: + """Mutate the registry by removing a finished request; removing it twice is harmless.""" + if self.open_requests.get(open_request.request_id) is open_request: + del self.open_requests[open_request.request_id] + + +def create_app( + settings: FrontSettings, + *, + http_client: httpx.AsyncClient | None = None, + cancel_http_client: httpx.AsyncClient | None = None, + redis_client: redis.asyncio.Redis | None = None, +) -> FastAPI: + """Create the chat front application. + + Args: + settings: The front's settings. + http_client: The HTTP client for submits, status reads and model lists; None creates one sized by + ``settings.native_max_connections`` that the app closes on shutdown. + cancel_http_client: The HTTP client for cancels; None creates a separate small pool the app closes on shutdown. + redis_client: The client for the Horde's chat signal database; None creates one from ``settings.redis_url`` + when it is set, and otherwise requests wait on the fallback poll alone. + + Returns: + The application; its lifespan starts the signal listener and, on stop, cancels every open request within + ``settings.shutdown_cancel_seconds`` before closing the subscription. + """ + owned_http_clients: list[httpx.AsyncClient] = [] + if http_client is None: + http_client = create_http_client(settings.native_max_connections) + owned_http_clients.append(http_client) + if cancel_http_client is None: + cancel_http_client = create_http_client() + owned_http_clients.append(cancel_http_client) + owns_redis_client = redis_client is None and settings.redis_url is not None + if redis_client is None and settings.redis_url is not None: + # The listener's own liveness ping detects a black-holed subscription. redis-py's health_check_interval is left + # off: its PINGs would interleave with the listener's and it drops the PONGs the listener waits for. + redis_client = redis.asyncio.Redis.from_url(settings.redis_url, socket_keepalive=True) + if redis_client is None: + logger.warning("HORDE_OAI_REDIS_URL is not set; chat requests wait on the fallback poll alone.") + front_state = FrontState( + settings=settings, + horde_client=HordeClient( + horde_urls=settings.horde_urls, + http_client=http_client, + cancel_http_client=cancel_http_client, + proxy_passkey=settings.proxy_passkey, + max_concurrent_calls=settings.native_max_connections, + ), + listener=CompletionSignalListener(redis_client, liveness_seconds=settings.redis_liveness_seconds), + ) + + @contextlib.asynccontextmanager + async def lifespan(_app: FastAPI) -> AsyncIterator[None]: + await front_state.listener.start() + try: + yield + finally: + front_state.accepting = False + await _cancel_open_requests(front_state) + await front_state.listener.stop() + if owns_redis_client and redis_client is not None: + await redis_client.close() + for owned_http_client in owned_http_clients: + await owned_http_client.aclose() + + app = FastAPI(title="AI Horde OpenAI-compatible front", version=HORDE_VERSION, lifespan=lifespan) + app.state.front = front_state + app.add_api_route("/v1/chat/completions", _chat_completions, methods=["POST"], response_model=None) + app.add_api_route("/v1/models", _list_models, methods=["GET"], response_model=None) + app.add_api_route("/v1/models/{model_id:path}", _get_model, methods=["GET"], response_model=None) + app.add_api_route("/healthz", _health, methods=["GET"], response_model=None) + return app + + +def _front_state(request: Request) -> FrontState: + """Return the shared state of the app serving ``request``.""" + front_state: FrontState = request.app.state.front + return front_state + + +def _request_source(request: Request) -> str: + """Return the end-user address of ``request``: the ASGI client address after uvicorn's forwarded handling.""" + return request.client.host if request.client is not None else UNKNOWN_SOURCE + + +def _horde_error(error: HordeAPIError, *, request_id: str | None, job_may_exist: bool) -> ChatFrontError: + """Create the front error for a Horde error, keeping its status, ``rc`` and forwarded headers.""" + retry_after_header = next( + ( + header_value + for header_name, header_value in error.forwarded_headers.items() + if header_name.lower() == RETRY_AFTER_HEADER.lower() + ), + None, + ) + return ChatFrontError( + status_code=error.status_code, + message=error.message, + code=error.return_code, + metadata=ErrorMetadata( + request_id=request_id, + rc=error.return_code, + retry_after=int(retry_after_header) if retry_after_header is not None and retry_after_header.isdigit() else None, + ), + headers=error.forwarded_headers, + job_may_exist=job_may_exist, + ) + + +def _build_credentials(request: Request) -> HordeCredentials: + """Return the Horde identity for ``request``: its bearer API key (or the anonymous key) and its address. + + Raises: + ChatFrontError: 401 when an ``Authorization`` header is present but is not ``Bearer ``. + """ + authorization = request.headers.get("authorization") + apikey = ANONYMOUS_API_KEY + if authorization is not None: + scheme, _, token = authorization.strip().partition(" ") + token = token.strip() + if scheme.lower() != BEARER_SCHEME or not token or any(not 33 <= ord(character) <= 126 for character in token): + raise ChatFrontError( + status_code=HTTPStatus.UNAUTHORIZED, + message="The Authorization header must be 'Bearer '; omit it to use the anonymous key.", + code=INVALID_AUTHORIZATION_CODE, + ) + apikey = token + user_agent = request.headers.get("user-agent", "unknown")[:MAX_FORWARDED_USER_AGENT_CHARACTERS] + return HordeCredentials( + apikey=apikey, + client_agent=f"{CLIENT_AGENT_NAME}:{HORDE_VERSION}:{user_agent}", + client_address=request.client.host if request.client is not None else None, + ) + + +def _request_too_large(message: str) -> ChatFrontError: + """Create the 413 for a body over the ceiling.""" + return ChatFrontError(status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, message=message, code=REQUEST_TOO_LARGE_CODE) + + +async def _read_body(request: Request, settings: FrontSettings) -> bytes: + """Return the request body, counted against the byte ceiling while it is received. + + Raises: + ChatFrontError: 413 when the declared or received size passes the ceiling, and 408 when the body is not + received within the body-read timeout. + """ + too_large_message = f"The request body is larger than {settings.max_body_bytes} bytes." + declared_length = request.headers.get("content-length") + if declared_length is not None and declared_length.isdigit() and int(declared_length) > settings.max_body_bytes: + raise _request_too_large(too_large_message) + body_chunks: list[bytes] = [] + received_bytes = 0 + try: + async with asyncio.timeout(settings.body_read_timeout_seconds): + async for body_chunk in request.stream(): + received_bytes += len(body_chunk) + if received_bytes > settings.max_body_bytes: + raise _request_too_large(too_large_message) + body_chunks.append(body_chunk) + except TimeoutError as error: + raise ChatFrontError( + status_code=HTTPStatus.REQUEST_TIMEOUT, + message=f"The request body was not received within {settings.body_read_timeout_seconds:g} seconds.", + code=REQUEST_BODY_TIMEOUT_CODE, + ) from error + return b"".join(body_chunks) + + +def _format_location(location: tuple[int | str, ...], request_body: object) -> str: + """Return a pydantic error location as a request path, such as ``messages[1].content``. + + A location also carries the union member or discriminator tag pydantic tried, such as ``str-enum[...]`` under + ``tool_choice`` or ``json_schema`` under ``response_format``. Those are not keys the client sent, so a part is + kept only where the body has it: an index of the array at that point, or a key of the object there. The last part + is kept under any object, because a required key the body left out is reported by that key. + + Args: + location: The pydantic error location. + request_body: The decoded body the model was validated against. + """ + request_path = "" + body_at_path: object = request_body + last_index = len(location) - 1 + for index, location_part in enumerate(location): + if isinstance(location_part, int): + if not isinstance(body_at_path, list) or not 0 <= location_part < len(body_at_path): + continue + request_path += f"[{location_part}]" + body_at_path = body_at_path[location_part] + else: + if not isinstance(body_at_path, dict) or (location_part not in body_at_path and index != last_index): + continue + request_path += f".{location_part}" if request_path else location_part + body_at_path = body_at_path.get(location_part) + return request_path + + +def _validation_error(error: ValidationError, request_body: object) -> ChatFrontError: + """Create the 400 for the first validation failure of a request body. + + Args: + error: The pydantic failure. + request_body: The decoded body the model was validated against, which anchors the error path. + """ + first_error = error.errors()[0] + error_location = _format_location(tuple(first_error["loc"]), request_body) + code = BAD_REQUEST_CODE + if first_error["type"] == UNSUPPORTED_PARAMETER_ERROR_TYPE: + code = UNSUPPORTED_PARAMETER_CODE + elif first_error["type"] == CHAT_REQUEST_CONFLICT_ERROR_TYPE: + code = CHAT_REQUEST_CONFLICT_CODE + return ChatFrontError( + status_code=HTTPStatus.BAD_REQUEST, + message=f"{error_location}: {first_error['msg']}" if error_location else first_error["msg"], + code=code, + param=error_location or None, + ) + + +def _message_text_bytes(chat_request: ChatCompletionRequest) -> int: + """Return the UTF-8 size of every message text of ``chat_request``.""" + text_bytes = 0 + for message in chat_request.messages: + if isinstance(message.content, str): + text_bytes += len(message.content.encode()) + else: + text_bytes += sum(len(content_part.text.encode()) for content_part in message.content) + return text_bytes + + +async def _read_chat_request(request: Request, settings: FrontSettings) -> ChatCompletionRequest: + """Return the validated chat request of ``request``. + + Raises: + ChatFrontError: 413 for an oversized body, 408 for a slow body, 400 for invalid JSON, a refused part, a value outside the + service contract or a size limit, and 404 for a model id with a variant. + """ + request_body = await _read_body(request, settings) + try: + decoded_body = json.loads(request_body) + except ValueError as error: + raise ChatFrontError( + status_code=HTTPStatus.BAD_REQUEST, message="The request body is not valid JSON.", code=BAD_REQUEST_CODE + ) from error + if not isinstance(decoded_body, dict): + raise ChatFrontError(status_code=HTTPStatus.BAD_REQUEST, message="The request body must be a JSON object.", code=BAD_REQUEST_CODE) + refused_parameter = find_refused_parameter(decoded_body) + if refused_parameter is not None: + raise ChatFrontError( + status_code=HTTPStatus.BAD_REQUEST, + message=refused_parameter.message, + code=UNSUPPORTED_PARAMETER_CODE, + param=refused_parameter.parameter, + ) + validated_body = drop_default_no_ops(decoded_body) + try: + chat_request = ChatCompletionRequest.model_validate(validated_body) + except ValidationError as error: + raise _validation_error(error, validated_body) from error + if len(chat_request.messages) > settings.max_message_count: + raise ChatFrontError( + status_code=HTTPStatus.BAD_REQUEST, + message=f"A chat request may carry at most {settings.max_message_count} messages.", + code=UNSUPPORTED_PARAMETER_CODE, + param="messages", + ) + if _message_text_bytes(chat_request) > settings.max_text_bytes: + raise ChatFrontError( + status_code=HTTPStatus.BAD_REQUEST, + message=f"The message text of a chat request may be at most {settings.max_text_bytes} bytes.", + code=UNSUPPORTED_PARAMETER_CODE, + param="messages", + ) + if MODEL_VARIANT_SEPARATOR in chat_request.model: + raise _model_not_found(chat_request.model) + return chat_request + + +def _model_not_found(model_id: str) -> ChatFrontError: + """Create the 404 for an unknown model id.""" + return ChatFrontError( + status_code=HTTPStatus.NOT_FOUND, + message=f"The model '{model_id}' does not exist on the AI Horde.", + code=MODEL_NOT_FOUND_CODE, + param="model", + ) + + +async def _submit( + front_state: FrontState, + *, + native_body: Mapping[str, object], + credentials: HordeCredentials, + deadline: float, +) -> HordeAsyncSubmission: + """Submit a chat request within the time left in its budget and return the Horde's acknowledgement. + + Raises: + ChatFrontError: The Horde refused the request, no root accepted the connection, or the outcome is unknown. + """ + try: + async with asyncio.timeout(deadline - time.monotonic()): + return await front_state.horde_client.submit_chat(request_body=native_body, credentials=credentials) + except HordeAPIError as error: + front_error = _horde_error(error, request_id=None, job_may_exist=False) + if error.status_code == HTTPStatus.SERVICE_UNAVAILABLE and front_error.metadata.retry_after is None: + front_error.headers[RETRY_AFTER_HEADER] = str(HORDE_UNAVAILABLE_RETRY_AFTER_SECONDS) + front_error.metadata = front_error.metadata.model_copy(update={"retry_after": HORDE_UNAVAILABLE_RETRY_AFTER_SECONDS}) + raise front_error from error + except HordeUnavailableError as error: + if error.request_sent: + raise _submit_outcome_unknown(str(error)) from error + raise ChatFrontError(status_code=HTTPStatus.BAD_GATEWAY, message=str(error), code=HORDE_UNAVAILABLE_CODE) from error + except TimeoutError as error: + raise _submit_outcome_unknown("The submit did not finish within the request's budget.") from error + + +def _submit_outcome_unknown(detail: str) -> ChatFrontError: + """Create the 502 for a submit that was sent and never answered, so a Horde request may exist. + + Recovery by operation key belongs here: re-sending the identical body with the same key returns the existing + request id, after which the wait continues as for any submit. Without it, the client is told not to retry. + """ + return ChatFrontError( + status_code=HTTPStatus.BAD_GATEWAY, + message=f"The AI Horde did not answer the submit, so the request may have been queued and charged. {detail}", + code=HORDE_UNAVAILABLE_CODE, + metadata=ErrorMetadata(submit_outcome=SubmitOutcome.UNKNOWN), + job_may_exist=True, + ) + + +async def _cancel_within_budget(front_state: FrontState, open_request: OpenRequest) -> None: + """Cancel a Horde chat request once, within the cancel budget, logging rather than raising when it fails. + + Every cancel logs exactly one outcome: cancelled, failed, or not completed within the budget. A request already + cancelled or being cancelled is left alone, so each request receives at most one cancel. + """ + if open_request.cancel_started: + return + open_request.cancel_started = True + cancel_timeout_seconds = front_state.settings.cancel_timeout_seconds + with anyio.move_on_after(cancel_timeout_seconds) as cancel_deadline: + try: + await front_state.horde_client.cancel_chat(request_id=open_request.request_id, credentials=open_request.credentials) + except (HordeAPIError, HordeUnavailableError, httpx.HTTPError) as error: + logger.warning(f"Cancelling chat request {open_request.request_id} failed: {error}") + return + logger.info(f"Cancelled chat request {open_request.request_id}") + if cancel_deadline.cancelled_caught: + logger.warning(f"Cancelling chat request {open_request.request_id} did not complete within {cancel_timeout_seconds:g}s") + + +async def _cancel_shielded(front_state: FrontState, open_request: OpenRequest) -> None: + """Cancel a Horde chat request once, shielded from the caller's cancellation. + + The cancel is usually issued because the caller is being torn down after its client disconnected; the shield lets + it finish within its own budget. + """ + with anyio.CancelScope(shield=True): + await _cancel_within_budget(front_state, open_request) + + +async def _cancel_open_requests(front_state: FrontState) -> None: + """Cancel every open request within the process-wide shutdown bound.""" + pending_requests = [open_request for open_request in front_state.open_requests.values() if not open_request.cancel_started] + if not pending_requests: + return + shutdown_cancel_seconds = front_state.settings.shutdown_cancel_seconds + logger.info(f"Cancelling {len(pending_requests)} open chat requests before stopping") + with anyio.move_on_after(shutdown_cancel_seconds) as shutdown_deadline: + await asyncio.gather(*(_cancel_within_budget(front_state, open_request) for open_request in pending_requests)) + if shutdown_deadline.cancelled_caught: + logger.warning(f"Cancelling open chat requests did not complete within {shutdown_cancel_seconds:g}s") + + +def _log_finished(watcher: ChatRequestWatcher, *, submitted_at: float, streamed: bool) -> None: + """Log one line per finished request with its submit and wake times, for latency analysis.""" + woke_at = watcher.woke_at if watcher.woke_at is not None else time.time() + logger.info( + f"Chat request {watcher.request_id} finished: streamed={streamed} submitted_at={submitted_at:.3f} " + f"woke_at={woke_at:.3f} woke_by={watcher.woke_by} waited={woke_at - submitted_at:.3f}s", + ) + + +async def _wait_for_disconnect(request: Request) -> None: + """Return once the client of ``request`` disconnects; the request body must already be read.""" + while True: + message = await request.receive() + if message["type"] == "http.disconnect": + return + + +def _create_watcher( + front_state: FrontState, + *, + open_request: OpenRequest, + subscription: CompletionSubscription, + deadline: float, +) -> ChatRequestWatcher: + """Create the watcher of a submitted request.""" + settings = front_state.settings + return ChatRequestWatcher( + horde_client=front_state.horde_client, + credentials=open_request.credentials, + subscription=subscription, + budget=WaitBudget( + deadline=deadline, + final_status_timeout_seconds=settings.final_status_timeout_seconds, + fallback_poll_seconds=settings.fallback_poll_seconds, + signal_recheck_seconds=settings.signal_recheck_seconds, + fail_fast_grace_seconds=settings.fail_fast_grace_seconds, + ), + ) + + +def _wait_error(error: ChatWaitTimeoutError | NoEligibleWorkerError | HordeAPIError, *, request_id: str) -> ChatFrontError: + """Create the front error for a wait that ended without a terminal outcome.""" + if isinstance(error, ChatWaitTimeoutError): + return ChatFrontError( + status_code=HTTPStatus.REQUEST_TIMEOUT, + message="The request was still waiting for a worker at its deadline and was cancelled.", + code=TIMEOUT_CODE, + metadata=ErrorMetadata(request_id=request_id), + job_may_exist=True, + ) + if isinstance(error, NoEligibleWorkerError): + return ChatFrontError( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + message="No online worker can serve this request at the moment; it was cancelled. Retry later.", + code=NO_ELIGIBLE_WORKER_CODE, + metadata=ErrorMetadata(request_id=request_id, retry_after=NO_ELIGIBLE_WORKER_RETRY_AFTER_SECONDS), + headers={RETRY_AFTER_HEADER: str(NO_ELIGIBLE_WORKER_RETRY_AFTER_SECONDS)}, + job_may_exist=True, + ) + return _horde_error(error, request_id=request_id, job_may_exist=True) + + +def _faulted_error(request_id: str) -> ChatFrontError: + """Create the front error for a request the Horde reports as faulted.""" + return ChatFrontError( + status_code=HTTPStatus.BAD_GATEWAY, + message=f"The AI Horde reports request {request_id} as faulted: the worker failed to generate it.", + code=FAULTED_CODE, + metadata=ErrorMetadata(request_id=request_id), + job_may_exist=True, + ) + + +async def _chat_completions(request: Request) -> Response: + """Serve ``POST /v1/chat/completions``.""" + front_state = _front_state(request) + settings = front_state.settings + try: + chat_request = await _read_chat_request(request, settings) + credentials = _build_credentials(request) + except ChatFrontError as error: + return error.to_response() + inflight_slot = front_state.inflight_limiter.try_acquire(_request_source(request)) + if inflight_slot is None: + return ChatFrontError( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + message="This front holds its maximum of open requests for this client or in total; retry shortly.", + code=FRONT_FULL_CODE, + metadata=ErrorMetadata(retry_after=INFLIGHT_RETRY_AFTER_SECONDS), + headers={RETRY_AFTER_HEADER: str(INFLIGHT_RETRY_AFTER_SECONDS)}, + ).to_response() + budget_seconds = settings.stream_timeout_seconds if chat_request.stream else settings.nonstream_timeout_seconds + deadline = time.monotonic() + budget_seconds + try: + submission = await _submit( + front_state, native_body=build_native_chat_body(chat_request), credentials=credentials, deadline=deadline + ) + except ChatFrontError as error: + inflight_slot.release() + return error.to_response() + except BaseException: + inflight_slot.release() + raise + submitted_at = time.time() + open_request = front_state.open_request(request_id=submission.request_id, credentials=credentials) + if chat_request.stream: + try: + return _stream_completion( + request, + chat_request=chat_request, + open_request=open_request, + submitted_at=submitted_at, + deadline=deadline, + inflight_slot=inflight_slot, + ) + except BaseException: + inflight_slot.release() + await _cancel_shielded(front_state, open_request) + front_state.close_request(open_request) + raise + try: + return await _wait_for_completion( + request, + chat_request=chat_request, + open_request=open_request, + submitted_at=submitted_at, + deadline=deadline, + inflight_slot=inflight_slot, + ) + except asyncio.CancelledError: + inflight_slot.release() + # A separate task survives repeated Task.cancel() calls; the AnyIO shield also protects against cancellation + # scopes during ASGI teardown. The cancel has its own bounded budget and finishes before cleanup returns. + cancel_task = asyncio.create_task(_cancel_within_budget(front_state, open_request)) + with anyio.CancelScope(shield=True): + while not cancel_task.done(): + try: + await asyncio.shield(cancel_task) + except asyncio.CancelledError: + continue + cancel_task.result() + raise + finally: + inflight_slot.release() + front_state.close_request(open_request) + + +async def _wait_for_completion( + request: Request, + *, + chat_request: ChatCompletionRequest, + open_request: OpenRequest, + submitted_at: float, + deadline: float, + inflight_slot: InflightSlot, +) -> Response: + """Wait for a submitted request and return its completion, or the error that ended the wait. + + The in-flight slot is released before an abandoned request's cancel, so a slow cancel does not count against new + requests; the cancel's own duration is bounded by the cancel budget. + """ + front_state = _front_state(request) + request_id = open_request.request_id + with front_state.listener.register(request_id) as subscription: + watcher = _create_watcher( + front_state, + open_request=open_request, + subscription=subscription, + deadline=deadline, + ) + wait_task = asyncio.create_task(watcher.wait_until_finished()) + disconnect_task = asyncio.create_task(_wait_for_disconnect(request)) + try: + await asyncio.wait({wait_task, disconnect_task}, return_when=asyncio.FIRST_COMPLETED) + finally: + disconnect_task.cancel() + if not wait_task.done(): + wait_task.cancel() + with anyio.CancelScope(shield=True): + await asyncio.gather(wait_task, disconnect_task, return_exceptions=True) + if wait_task.cancelled(): + logger.info(f"Client of chat request {request_id} disconnected while waiting") + inflight_slot.release() + await _cancel_shielded(front_state, open_request) + return Response(status_code=CLIENT_CLOSED_REQUEST_STATUS) + try: + status = wait_task.result() + except (ChatWaitTimeoutError, NoEligibleWorkerError) as error: + inflight_slot.release() + await _cancel_shielded(front_state, open_request) + return _wait_error(error, request_id=request_id).to_response() + except HordeAPIError as error: + return _wait_error(error, request_id=request_id).to_response() + _log_finished(watcher, submitted_at=submitted_at, streamed=False) + if status.faulted: + return _faulted_error(request_id).to_response() + completion = build_completion( + request_id=request_id, + created=int(submitted_at), + chat_request=chat_request, + status=status, + ) + return JSONResponse(content=completion.model_dump(mode="json")) + + +def _stream_completion( + request: Request, + *, + chat_request: ChatCompletionRequest, + open_request: OpenRequest, + submitted_at: float, + deadline: float, + inflight_slot: InflightSlot, +) -> EventSourceResponse: + """Return the server-sent event response that waits for a submitted request and then delivers it. + + The in-flight slot is released before an abandoned request's cancel. When the client disconnects before the event + generator first runs, sse-starlette never starts it, so the response's background task releases the slot and + cancels the request instead. + """ + front_state = _front_state(request) + settings = front_state.settings + request_id = open_request.request_id + latest_watcher: ChatRequestWatcher | None = None + stream_started = False + + def build_heartbeat() -> ServerSentEvent: + latest_status: HordeChatStatus | None = latest_watcher.latest_status if latest_watcher is not None else None + return build_keepalive_event(latest_status) + + async def stream_events() -> AsyncIterator[ServerSentEvent]: + nonlocal latest_watcher, stream_started + stream_started = True + outcome_settled = False + try: + with front_state.listener.register(request_id) as subscription: + latest_watcher = _create_watcher( + front_state, + open_request=open_request, + subscription=subscription, + deadline=deadline, + ) + try: + status = await latest_watcher.wait_until_finished() + except (ChatWaitTimeoutError, NoEligibleWorkerError) as error: + inflight_slot.release() + await _cancel_shielded(front_state, open_request) + for error_event in _wait_error(error, request_id=request_id).to_events(): + yield error_event + return + except HordeAPIError as error: + outcome_settled = True + for error_event in _wait_error(error, request_id=request_id).to_events(): + yield error_event + return + outcome_settled = True + _log_finished(latest_watcher, submitted_at=submitted_at, streamed=True) + if status.faulted: + for error_event in _faulted_error(request_id).to_events(): + yield error_event + return + completion = build_completion( + request_id=request_id, + created=int(submitted_at), + chat_request=chat_request, + status=status, + ) + include_usage = bool(chat_request.stream_options and chat_request.stream_options.include_usage) + for completion_event in build_completion_events(completion, include_usage=include_usage): + yield completion_event + finally: + inflight_slot.release() + if not outcome_settled: + if not open_request.cancel_started: + logger.info(f"Client of streamed chat request {request_id} disconnected while waiting") + await _cancel_shielded(front_state, open_request) + front_state.close_request(open_request) + + async def finish_unstarted_stream() -> None: + inflight_slot.release() + if not stream_started: + logger.info(f"Client of streamed chat request {request_id} disconnected before the stream started") + await _cancel_shielded(front_state, open_request) + front_state.close_request(open_request) + + return EventSourceResponse( + stream_events(), + headers=STREAM_HEADERS, + ping=settings.heartbeat_seconds, + ping_message_factory=build_heartbeat, + background=BackgroundTask(finish_unstarted_stream), + ) + + +def _build_model_card(active_model: HordeActiveModel) -> ModelCard: + """Create the OpenAI model list entry for one of the Horde's active text models.""" + return ModelCard( + model_id=active_model.name, + x_horde=HordeModelDetails(workers=active_model.count, queued=active_model.queued), + ) + + +async def _fetch_models(request: Request) -> list[HordeActiveModel]: + """Return the Horde's active text models through the process cache. + + Raises: + ChatFrontError: The Authorization header is malformed, or the Horde could not be asked. + """ + front_state = _front_state(request) + credentials = _build_credentials(request) + try: + return await front_state.model_list_cache.get(front_state.horde_client, credentials) + except HordeAPIError as error: + raise _horde_error(error, request_id=None, job_may_exist=False) from error + except HordeUnavailableError as error: + raise ChatFrontError(status_code=HTTPStatus.BAD_GATEWAY, message=str(error), code=HORDE_UNAVAILABLE_CODE) from error + + +async def _list_models(request: Request) -> Response: + """Serve ``GET /v1/models``.""" + try: + active_models = await _fetch_models(request) + except ChatFrontError as error: + return error.to_response() + model_list = ModelList(data=[_build_model_card(active_model) for active_model in active_models]) + return JSONResponse(content=model_list.model_dump(mode="json")) + + +async def _get_model(request: Request, model_id: str) -> Response: + """Serve ``GET /v1/models/{model_id}``; the id may contain slashes.""" + try: + active_models = await _fetch_models(request) + except ChatFrontError as error: + return error.to_response() + for active_model in active_models: + if active_model.name == model_id: + return JSONResponse(content=_build_model_card(active_model).model_dump(mode="json")) + return _model_not_found(model_id).to_response() + + +async def _health(request: Request) -> Response: + """Serve ``GET /healthz``: liveness, admission readiness and the signal subscription, as separate fields.""" + front_state = _front_state(request) + return JSONResponse( + content={ + "live": True, + "ready": front_state.accepting and front_state.inflight_limiter.has_capacity, + "signal_subscribed": front_state.listener.subscribed, + "inflight": front_state.inflight_limiter.inflight, + }, + ) diff --git a/horde/oai/horde_client.py b/horde/oai/horde_client.py new file mode 100644 index 00000000..154a2ae6 --- /dev/null +++ b/horde/oai/horde_client.py @@ -0,0 +1,574 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Call the Horde's native chat and model routes over HTTP. + +[`HordeClient`][horde.oai.horde_client.HordeClient] wraps shared ``httpx.AsyncClient`` instances, normally made by +[`create_http_client`][horde.oai.horde_client.create_http_client]. Calls rotate across the configured Horde API roots. +A root that refuses the connection is skipped for that call, which is safe because the request never reached it, and +any root that refuses or drops a connection is left out of the rotation for +[`UNHEALTHY_URL_BACKOFF_SECONDS`][horde.oai.horde_client.UNHEALTHY_URL_BACKOFF_SECONDS]. A Horde error response raises +[`HordeAPIError`][horde.oai.horde_client.HordeAPIError] carrying the status, the Horde ``rc`` and the rate-limit +headers; no root answering raises [`HordeUnavailableError`][horde.oai.horde_client.HordeUnavailableError], whose +``request_sent`` says whether the call may have reached a Horde. + +Submits, status reads and model lists share one pool whose size the caller sets (``HORDE_OAI_NATIVE_MAX_CONNECTIONS``); +the rest wait on a semaphore in arrival order before they reach the pool. Cancels use a second, smaller pool and +semaphore ([`CANCEL_MAX_CONNECTIONS`][horde.oai.horde_client.CANCEL_MAX_CONNECTIONS]), so a burst of slow cancels cannot +starve submits and a queue of status reads cannot delay a cancel. + +The Horde response models here are parsed leniently (unknown fields ignored), so a Horde that adds status fields does +not break the front. +""" + +from __future__ import annotations + +import asyncio +import itertools +import time +from collections.abc import Callable, Mapping, Sequence +from http import HTTPStatus + +import httpx +from loguru import logger +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +__all__ = [ + "CANCEL_MAX_CONNECTIONS", + "HTTP_MAX_CONNECTIONS", + "HordeAPIError", + "HordeActiveModel", + "HordeAsyncSubmission", + "HordeChatGeneration", + "HordeChatMessage", + "HordeChatStatus", + "HordeClient", + "HordeCredentials", + "HordeUnavailableError", + "create_http_client", +] + +CHAT_SUBMIT_PATH: str = "/v2/generate/chat/async" +"""The Horde route that queues a chat request.""" + +CHAT_STATUS_PATH_TEMPLATE: str = "/v2/generate/chat/status/{request_id}" +"""The Horde route that reports (GET) or cancels (DELETE) a chat request.""" + +MODELS_PATH: str = "/v2/status/models" +"""The Horde route that lists active models.""" + +APIKEY_HEADER: str = "apikey" +"""The header the Horde reads the API key from.""" + +CLIENT_AGENT_HEADER: str = "Client-Agent" +"""The header the Horde reads the client's name and version from.""" + +PROXY_AUTHORIZATION_HEADER: str = "Proxy-Authorization" +"""The header carrying a proxy passkey; the Horde trusts ``Proxied-For`` only when it holds a known passkey.""" + +PROXIED_FOR_HEADER: str = "Proxied-For" +"""The header carrying the end client's address for a trusted proxy.""" + +FORWARDED_RESPONSE_HEADERS: tuple[str, ...] = ("retry-after",) +"""Horde response headers an error passes back to the client, besides every ``x-ratelimit-*`` header.""" + +RATE_LIMIT_HEADER_PREFIX: str = "x-ratelimit-" +"""The prefix of the rate-limit headers an error passes back to the client.""" + +MAX_ERROR_BODY_CHARACTERS: int = 500 +"""The most characters of a non-JSON Horde error body kept in the message, such as an HTML gateway page.""" + +UNEXPECTED_RESPONSE_CODE: str = "HordeUnexpectedResponse" +"""The error code for a successful Horde response whose body could not be parsed.""" + +HTTP_MAX_CONNECTIONS: int = 200 +"""The default for the most simultaneous calls to the Horde, and the size of the connection pool. + +[`HordeClient`][horde.oai.horde_client.HordeClient] admits this many calls at once through a semaphore of the same +size, so the pool never queues a request. httpcore rescans every in-flight request against every pooled connection +on each pool event, probing each idle connection's socket, so the cost of one event grows with the square of this +number. The number also bounds how many slow calls can be in flight at once: with too few slots, submits queue behind +slow status reads until clients give up. +""" + +CANCEL_MAX_CONNECTIONS: int = 16 +"""The most simultaneous cancels, and the size of their separate connection pool. + +Cancels are rare next to status reads, but each can take its whole budget when the Horde is slow; a pool of their own +keeps them from holding the slots submits need, and keeps them moving when status reads fill the main pool. +""" + +HTTP_KEEPALIVE_EXPIRY_SECONDS: float = 20.0 +"""How long an idle pooled connection is kept before the front closes it. + +Well under waitress's 120 second ``channel_timeout``, so the front closes idle connections itself rather than having +the server drop them under a request about to be sent. +""" + +HTTP_CONNECT_TIMEOUT_SECONDS: float = 5.0 +"""How long a Horde call waits for a connection before the root counts as unreachable.""" + +HTTP_READ_TIMEOUT_SECONDS: float = 30.0 +"""How long a Horde call waits for its response; a submit runs moderation and kudos checks, so it is not instant.""" + +UNHEALTHY_URL_BACKOFF_SECONDS: float = 3.0 +"""How long a Horde root that refused or dropped a connection is left out of the rotation. + +Without it, every call that rotates onto a down or overloaded root opens a fresh connection to it at once, and the +connect-and-drop churn exhausts the server's descriptors faster than it can recover. +""" + + +def create_http_client(max_connections: int = HTTP_MAX_CONNECTIONS) -> httpx.AsyncClient: + """Create an HTTP client for Horde calls. + + Args: + max_connections: The pool size; keep it equal to the semaphore the calls through this client wait on. + """ + return httpx.AsyncClient( + timeout=httpx.Timeout(HTTP_READ_TIMEOUT_SECONDS, connect=HTTP_CONNECT_TIMEOUT_SECONDS), + limits=httpx.Limits( + max_connections=max_connections, + max_keepalive_connections=max_connections, + keepalive_expiry=HTTP_KEEPALIVE_EXPIRY_SECONDS, + ), + ) + + +class HordeCredentials(BaseModel): + """Represents the identity a Horde call is made with on behalf of one client request.""" + + model_config = ConfigDict(frozen=True) + + apikey: str = Field(repr=False) + """The client's Horde API key, or the anonymous key; left out of the representation so it is never logged.""" + + client_agent: str + """The ``Client-Agent`` value: the front's name and version plus the client's user agent.""" + + client_address: str | None = None + """The client's address, sent as ``Proxied-For`` when the front has a proxy passkey.""" + + +class HordeAPIError(Exception): + """The Horde answered a call with an error status.""" + + def __init__( + self, + *, + status_code: int, + return_code: str | None, + message: str, + forwarded_headers: Mapping[str, str], + ) -> None: + """Create the error. + + Args: + status_code: The Horde's HTTP status. + return_code: The Horde ``rc``, or None when the body had none. + message: The Horde's message, or the start of the body when it was not JSON. + forwarded_headers: The ``Retry-After`` and rate-limit headers to pass back to the client. + """ + super().__init__(f"{status_code} {return_code}: {message}") + self.status_code: int = status_code + """The Horde's HTTP status.""" + self.return_code: str | None = return_code + """The Horde ``rc``.""" + self.message: str = message + """The Horde's message.""" + self.forwarded_headers: dict[str, str] = dict(forwarded_headers) + """The headers to pass back to the client.""" + + +class HordeUnavailableError(Exception): + """No configured Horde API root answered the call.""" + + def __init__(self, message: str, *, request_sent: bool) -> None: + """Create the error. + + Args: + message: What failed. + request_sent: Whether the call may have reached a Horde: true after a transport failure once a connection + was made, false when no root accepted a connection. + """ + super().__init__(message) + self.request_sent: bool = request_sent + """Whether the call may have reached a Horde, which leaves a submit's outcome unknown.""" + + +class HordeAsyncSubmission(BaseModel): + """Represents the Horde's answer to a queued chat request.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + request_id: str = Field(validation_alias="id") + """The Horde request id.""" + + kudos: float | None = None + """The kudos the Horde expects the request to cost.""" + + message: str | None = None + """An informational message, if any.""" + + +class HordeChatMessage(BaseModel): + """Represents the assistant message a chat generation carries.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + role: str | None = None + """The role of the message, ``assistant`` for a reply.""" + + content: str | None = None + """The reply text; an empty string is a completed, empty reply.""" + + +class HordeChatGeneration(BaseModel): + """Represents one generation in a Horde chat status.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + text: str | None = None + """The generated text as a plain string; the fallback when ``message`` is absent.""" + + message: HordeChatMessage | None = None + """The assistant message the worker submitted.""" + + model: str | None = None + """The model that generated it.""" + + worker_id: str | None = None + """The worker that generated it.""" + + worker_name: str | None = None + """The name of that worker.""" + + state: str | None = None + """The generation state, such as ``ok`` or ``faulted``.""" + + seed: int | str | None = None + """The seed the worker used.""" + + finish_reason: str | None = None + """Why the generation stopped, normalised by the worker.""" + + native_finish_reason: str | None = None + """The backend's own reason, when the worker reported it.""" + + prompt_tokens: int | None = None + """The prompt token count, when the worker reported it.""" + + completion_tokens: int | None = None + """The generated token count, when the worker reported it.""" + + charge_basis: str | None = None + """Which count the generation was charged and rewarded on.""" + + queue_wait_seconds: float | None = None + """Seconds from the request's creation to this generation's assignment, when the Horde has an assignment time.""" + + +class HordeChatStatus(BaseModel): + """Represents the Horde's status of a chat request.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + done: bool = False + """True when every generation of the request is finished.""" + + faulted: bool = False + """True when the request failed and will not finish.""" + + finished: int = 0 + """How many generations are finished.""" + + processing: int = 0 + """How many generations a worker is working on.""" + + waiting: int = 0 + """How many generations wait for a worker.""" + + queue_position: int | None = None + """The request's position in the Horde-wide queue.""" + + wait_time: int | None = None + """The Horde-wide estimate of seconds until the request finishes.""" + + kudos: float = 0 + """The kudos the request consumed so far.""" + + is_possible: bool = True + """False when no current worker can serve the request.""" + + generations: list[HordeChatGeneration] = Field(default_factory=list) + """The finished generations, in the Horde's order.""" + + context_adjustments: list[object] | None = None + """The context changes the Horde applied to fit the messages, when it reports any.""" + + +class HordeActiveModel(BaseModel): + """Represents one entry of the Horde's active model list.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + name: str + """The model name.""" + + count: int = 0 + """How many worker threads serve it.""" + + queued: float | None = None + """The queued work for the model.""" + + +class HordeClient: + """Call the Horde's chat and model routes, rotating across the configured API roots. + + Concurrency: + Safe to share between concurrent requests on one event loop; the rotation counter and the unhealthy marks are + updated without awaiting. At most ``max_concurrent_calls`` calls and, separately, ``max_concurrent_cancels`` + cancels are in flight; the rest wait in arrival order. + """ + + def __init__( + self, + *, + horde_urls: Sequence[str], + http_client: httpx.AsyncClient, + proxy_passkey: str | None, + cancel_http_client: httpx.AsyncClient | None = None, + max_concurrent_calls: int = HTTP_MAX_CONNECTIONS, + max_concurrent_cancels: int = CANCEL_MAX_CONNECTIONS, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Create the client. + + Args: + horde_urls: The Horde API roots, each ending in ``/api``. + http_client: The HTTP client for submits, status reads and model lists; the caller owns and closes it. + proxy_passkey: The front operator's proxy passkey, or None to send no ``Proxied-For``. + cancel_http_client: The HTTP client for cancels; None sends them through ``http_client`` under their own + semaphore. The caller owns and closes it. + max_concurrent_calls: The most non-cancel calls in flight at once; keep it equal to the main pool's + ``max_connections`` so the pool never queues. + max_concurrent_cancels: The most cancels in flight at once. + clock: The monotonic clock the unhealthy marks are measured on. + """ + if not horde_urls: + raise ValueError("At least one Horde URL is required.") + self._horde_urls: tuple[str, ...] = tuple(horde_urls) + self._http_client = http_client + self._cancel_http_client = cancel_http_client if cancel_http_client is not None else http_client + self._proxy_passkey = proxy_passkey + self._rotation = itertools.count() + self._call_slots = asyncio.Semaphore(max_concurrent_calls) + self._cancel_slots = asyncio.Semaphore(max_concurrent_cancels) + self._clock = clock + self._unhealthy_until: dict[str, float] = {} + + async def submit_chat(self, *, request_body: Mapping[str, object], credentials: HordeCredentials) -> HordeAsyncSubmission: + """Queue a chat request and return the Horde's acknowledgement. + + Args: + request_body: The native chat request body, as JSON-ready values. + credentials: Who the call is made for. + + Raises: + HordeAPIError: The Horde rejected the request. + HordeUnavailableError: No Horde root accepted the connection, or the connection failed after the call was + sent (``request_sent`` true), so a request may exist. + """ + response = await self._send("POST", CHAT_SUBMIT_PATH, credentials=credentials, json_body=request_body) + return _parse_response(response, HordeAsyncSubmission) + + async def get_chat_status(self, *, request_id: str, credentials: HordeCredentials) -> HordeChatStatus: + """Return the Horde's status of a chat request. + + Args: + request_id: The Horde request id. + credentials: Who the call is made for. + + Raises: + HordeAPIError: The Horde answered with an error, such as 404 once the request expired. + HordeUnavailableError: No Horde root accepted the connection. + """ + status_path = CHAT_STATUS_PATH_TEMPLATE.format(request_id=request_id) + response = await self._send("GET", status_path, credentials=credentials) + return _parse_response(response, HordeChatStatus) + + async def cancel_chat(self, *, request_id: str, credentials: HordeCredentials) -> HordeChatStatus: + """Cancel a chat request through the cancel pool and return its last status. + + Args: + request_id: The Horde request id. + credentials: Who the call is made for. + + Raises: + HordeAPIError: The Horde answered with an error. + HordeUnavailableError: No Horde root accepted the connection. + """ + status_path = CHAT_STATUS_PATH_TEMPLATE.format(request_id=request_id) + response = await self._send( + "DELETE", + status_path, + credentials=credentials, + http_client=self._cancel_http_client, + call_slots=self._cancel_slots, + ) + return _parse_response(response, HordeChatStatus) + + async def list_text_models(self, *, credentials: HordeCredentials) -> list[HordeActiveModel]: + """Return the Horde's active text models. + + Args: + credentials: Who the call is made for. + + Raises: + HordeAPIError: The Horde answered with an error. + HordeUnavailableError: No Horde root accepted the connection. + """ + response = await self._send("GET", MODELS_PATH, credentials=credentials, query={"type": "text"}) + try: + return [HordeActiveModel.model_validate(model_entry) for model_entry in response.json()] + except (ValueError, TypeError, ValidationError) as error: + raise _unexpected_body_error(response) from error + + def _build_headers(self, credentials: HordeCredentials) -> dict[str, str]: + """Return the headers a Horde call carries for ``credentials``.""" + headers = { + APIKEY_HEADER: credentials.apikey, + CLIENT_AGENT_HEADER: credentials.client_agent, + } + if self._proxy_passkey and credentials.client_address: + headers[PROXY_AUTHORIZATION_HEADER] = self._proxy_passkey + headers[PROXIED_FOR_HEADER] = credentials.client_address + return headers + + async def _send( + self, + method: str, + path: str, + *, + credentials: HordeCredentials, + json_body: Mapping[str, object] | None = None, + query: Mapping[str, str] | None = None, + http_client: httpx.AsyncClient | None = None, + call_slots: asyncio.Semaphore | None = None, + ) -> httpx.Response: + """Send one call, trying each healthy root in rotation until one accepts the connection. + + Args: + method: The HTTP method. + path: The route below the API root. + credentials: Who the call is made for. + json_body: The JSON body, if any. + query: The query parameters, if any. + http_client: The client to send through; None selects the main client. + call_slots: The semaphore bounding calls through that client; None selects the main one. + + Returns: + The Horde's successful response. + + Raises: + HordeAPIError: The Horde answered with a status of 400 or above. + HordeUnavailableError: No healthy root accepted the connection, or the connection failed after the call + was sent. + """ + headers = self._build_headers(credentials) + sending_client = http_client if http_client is not None else self._http_client + async with call_slots if call_slots is not None else self._call_slots: + for horde_url in self._healthy_urls_in_rotation(): + try: + response = await sending_client.request( + method, + f"{horde_url}{path}", + headers=headers, + json=json_body, + params=query, + ) + except httpx.ConnectError as error: + # Only a refused connection is retried elsewhere: the call never reached that Horde, so a submit + # cannot be queued twice. + self._mark_unhealthy(horde_url, error) + continue + except httpx.TransportError as error: + # The call may have reached the Horde, so it is not retried elsewhere. A timeout says the Horde + # is slow rather than gone, so it does not take the root out of the rotation. + if not isinstance(error, httpx.TimeoutException): + self._mark_unhealthy(horde_url, error) + raise HordeUnavailableError( + f"The Horde at {horde_url} failed {method} {path}: {error!r}", + request_sent=not isinstance(error, httpx.ConnectTimeout | httpx.PoolTimeout), + ) from error + if response.status_code >= HTTPStatus.BAD_REQUEST: + raise _api_error_from_response(response) + return response + raise HordeUnavailableError(f"No healthy Horde accepted the connection for {method} {path}.", request_sent=False) + + def _healthy_urls_in_rotation(self) -> list[str]: + """Return the roots not marked unhealthy, starting from the next one in the rotation.""" + start_index = next(self._rotation) % len(self._horde_urls) + now = self._clock() + rotated_urls = self._horde_urls[start_index:] + self._horde_urls[:start_index] + return [horde_url for horde_url in rotated_urls if self._unhealthy_until.get(horde_url, 0.0) <= now] + + def _mark_unhealthy(self, horde_url: str, error: httpx.TransportError) -> None: + """Leave ``horde_url`` out of the rotation for the backoff, logging only when it was healthy until now.""" + now = self._clock() + if self._unhealthy_until.get(horde_url, 0.0) <= now: + logger.warning( + f"Horde at {horde_url} failed a connection ({type(error).__name__}: {error}); " + f"leaving it out of the rotation for {UNHEALTHY_URL_BACKOFF_SECONDS:g}s", + ) + self._unhealthy_until[horde_url] = now + UNHEALTHY_URL_BACKOFF_SECONDS + + +def _parse_response[ModelType: BaseModel](response: httpx.Response, model_class: type[ModelType]) -> ModelType: + """Return ``response``'s JSON body validated as ``model_class``. + + Raises: + HordeAPIError: The body is not the expected JSON object. + """ + try: + return model_class.model_validate(response.json()) + except (ValueError, ValidationError) as error: + raise _unexpected_body_error(response) from error + + +def _unexpected_body_error(response: httpx.Response) -> HordeAPIError: + """Return the error for a successful Horde response whose body could not be parsed.""" + return HordeAPIError( + status_code=HTTPStatus.BAD_GATEWAY, + return_code=UNEXPECTED_RESPONSE_CODE, + message=f"The Horde returned an unexpected response body: {response.text[:MAX_ERROR_BODY_CHARACTERS]}", + forwarded_headers={}, + ) + + +def _api_error_from_response(response: httpx.Response) -> HordeAPIError: + """Return the error for a Horde response with a status of 400 or above.""" + forwarded_headers = { + header_name: header_value + for header_name, header_value in response.headers.items() + if header_name.lower() in FORWARDED_RESPONSE_HEADERS or header_name.lower().startswith(RATE_LIMIT_HEADER_PREFIX) + } + return_code: str | None = None + message = response.text[:MAX_ERROR_BODY_CHARACTERS] + try: + error_body = response.json() + except ValueError: + error_body = None + if isinstance(error_body, dict): + body_return_code = error_body.get("rc") + body_message = error_body.get("message") + return_code = body_return_code if isinstance(body_return_code, str) else None + if isinstance(body_message, str): + message = body_message + return HordeAPIError( + status_code=response.status_code, + return_code=return_code, + message=message, + forwarded_headers=forwarded_headers, + ) diff --git a/horde/oai/mapping.py b/horde/oai/mapping.py new file mode 100644 index 00000000..1a597ffc --- /dev/null +++ b/horde/oai/mapping.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Map a chat request to the native body, a finished Horde chat status to the OpenAI completion, and an error to the +OpenAI error envelope. + +Usage counts come from the worker's report when there is one. A worker that does not report counts gets an estimate +of one token per [`CHARACTERS_PER_ESTIMATED_TOKEN`][horde.oai.mapping.CHARACTERS_PER_ESTIMATED_TOKEN] characters, and +``x_horde.estimated_usage`` says so. A generation whose reply holds no visible text gets +[`EMPTY_RETURN_NOTICE`][horde.oai.mapping.EMPTY_RETURN_NOTICE] in ``x_horde.notice``. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from http import HTTPStatus + +from horde.oai.horde_client import HordeChatGeneration, HordeChatStatus +from horde.oai.schema import ( + ChatCompletionChoice, + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseMessage, + ChatCompletionUsage, + ChatMessage, + ErrorMetadata, + FinishReason, + HordeResponseDetails, + OpenAIError, + OpenAIErrorEnvelope, + OpenAIErrorType, +) + +__all__ = [ + "CHARACTERS_PER_ESTIMATED_TOKEN", + "COMPLETION_ID_PREFIX", + "EMPTY_RETURN_NOTICE", + "build_completion", + "build_completion_id", + "build_error", + "build_error_envelope", + "build_native_chat_body", + "estimate_tokens", + "flatten_messages", + "generation_content", +] + +CHARACTERS_PER_ESTIMATED_TOKEN: int = 3 +"""Characters counted as one token when a worker does not report counts. + +A rough ratio for English text; the estimate only fills the usage object, which ``x_horde.estimated_usage`` marks. +""" + +COMPLETION_ID_PREFIX: str = "chatcmpl-" +"""The prefix OpenAI clients expect on a completion id; the Horde request id follows it.""" + +FILTERED_GENERATION_STATES: frozenset[str] = frozenset( + { + "censored", + "csam", + }, +) +"""Generation states whose output a filter withheld; their choice reports ``content_filter``.""" + +EMPTY_RETURN_NOTICE: str = ( + "The worker ran this request and returned no visible text. A sampling setting or the model choice is the likely " + "cause. Quote request id {request_id} when reporting it." +) +"""The ``x_horde.notice`` of a completion whose reply holds no visible text; the request id is filled in.""" + +NATIVE_BODY_EXCLUDED_KEYS: frozenset[str] = frozenset( + { + "stream", + "stream_options", + "user", + "metadata", + }, +) +"""Request keys the front applies or drops itself and never forwards: streaming is the front's, annotations are unused.""" + +ERROR_TYPE_BY_STATUS: dict[int, OpenAIErrorType] = { + HTTPStatus.BAD_REQUEST: OpenAIErrorType.INVALID_REQUEST_ERROR, + HTTPStatus.UNAUTHORIZED: OpenAIErrorType.AUTHENTICATION_ERROR, + HTTPStatus.FORBIDDEN: OpenAIErrorType.PERMISSION_ERROR, + HTTPStatus.NOT_FOUND: OpenAIErrorType.NOT_FOUND_ERROR, + HTTPStatus.REQUEST_TIMEOUT: OpenAIErrorType.TIMEOUT_ERROR, + HTTPStatus.TOO_MANY_REQUESTS: OpenAIErrorType.RATE_LIMIT_ERROR, +} +"""The OpenAI error type for each HTTP status; any status not listed is an ``api_error``.""" + + +def build_completion_id(request_id: str) -> str: + """Return the OpenAI completion id for a Horde request id.""" + return f"{COMPLETION_ID_PREFIX}{request_id}" + + +def build_native_chat_body(chat_request: ChatCompletionRequest) -> dict[str, object]: + """Create the native ``POST /v2/generate/chat/async`` body for a validated request. + + Omitted and null keys stay omitted, so the native route fills its defaults; ``developer`` messages and message + names are forwarded unchanged. + + Args: + chat_request: The validated request, after the default no-op keys were dropped. + """ + return chat_request.model_dump(mode="json", exclude_unset=True, exclude_none=True, exclude=set(NATIVE_BODY_EXCLUDED_KEYS)) + + +def flatten_messages(messages: Sequence[ChatMessage]) -> str: + """Return the messages' roles and texts joined by newlines, the text the prompt token estimate counts.""" + flattened_lines: list[str] = [] + for message in messages: + flattened_lines.append(message.role.value) + if isinstance(message.content, str): + flattened_lines.append(message.content) + else: + flattened_lines.extend(content_part.text for content_part in message.content) + return "\n".join(flattened_lines) + + +def estimate_tokens(text: str) -> int: + """Return the estimated token count of ``text``.""" + return math.ceil(len(text) / CHARACTERS_PER_ESTIMATED_TOKEN) + + +def generation_content(generation: HordeChatGeneration) -> str: + """Return a generation's reply text: the message content when present, else the plain ``text`` field.""" + if generation.message is not None and generation.message.content is not None: + return generation.message.content + return generation.text or "" + + +def build_completion( + *, + request_id: str, + created: int, + chat_request: ChatCompletionRequest, + status: HordeChatStatus, +) -> ChatCompletionResponse: + """Create the OpenAI completion for a finished Horde chat request. + + Args: + request_id: The Horde request id. + created: When the request was submitted, in Unix seconds. + chat_request: The client's request, used for the model fallback and the prompt estimate. + status: The Horde's terminal status of the request. + + Returns: + One choice per generation, in the Horde's order; ``model`` and the ``x_horde`` worker details are the first + generation's. + """ + choices: list[ChatCompletionChoice] = [] + completion_tokens = 0 + estimated_usage = False + prompt_tokens: int | None = None + has_empty_return = False + for generation_index, generation in enumerate(status.generations): + generated_text = generation_content(generation) + is_filtered = generation.state in FILTERED_GENERATION_STATES + if not is_filtered and not generated_text.strip(): + has_empty_return = True + choices.append( + ChatCompletionChoice( + index=generation_index, + message=ChatCompletionResponseMessage(content=generated_text), + finish_reason=FinishReason.CONTENT_FILTER if is_filtered else _finish_reason(generation.finish_reason), + native_finish_reason=generation.native_finish_reason, + ), + ) + if generation.completion_tokens is None: + completion_tokens += estimate_tokens(generated_text) + estimated_usage = True + else: + completion_tokens += generation.completion_tokens + if prompt_tokens is None and generation.prompt_tokens is not None: + prompt_tokens = generation.prompt_tokens + if prompt_tokens is None: + prompt_tokens = estimate_tokens(flatten_messages(chat_request.messages)) + estimated_usage = True + first_generation = status.generations[0] if status.generations else None + return ChatCompletionResponse( + completion_id=build_completion_id(request_id), + created=created, + model=first_generation.model if first_generation is not None and first_generation.model else chat_request.model, + choices=choices, + usage=ChatCompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + x_horde=HordeResponseDetails( + request_id=request_id, + worker_id=first_generation.worker_id if first_generation is not None else None, + worker_name=first_generation.worker_name if first_generation is not None else None, + kudos=status.kudos, + charge_basis=first_generation.charge_basis if first_generation is not None else None, + estimated_usage=estimated_usage, + queue_wait_seconds=first_generation.queue_wait_seconds if first_generation is not None else None, + context_adjustments=status.context_adjustments, + notice=EMPTY_RETURN_NOTICE.format(request_id=request_id) if has_empty_return else None, + ), + ) + + +def _finish_reason(reported_finish_reason: str | None) -> FinishReason: + """Return the finish reason a worker reported, or ``stop`` when it reported none or an unknown one.""" + if reported_finish_reason is None: + return FinishReason.STOP + try: + return FinishReason(reported_finish_reason) + except ValueError: + return FinishReason.STOP + + +def build_error( + *, + status_code: int, + message: str, + code: str | None, + param: str | None = None, + metadata: ErrorMetadata | None = None, +) -> OpenAIError: + """Create the OpenAI error body for an HTTP status. + + Args: + status_code: The HTTP status the error is returned with, or would have been before stream headers. + message: The explanation for the client. + code: The Horde return code, or the front's own code. + param: The request path at fault, when one is. + metadata: The request id, return code and retry details that are known. + """ + return OpenAIError( + message=message, + type=ERROR_TYPE_BY_STATUS.get(status_code, OpenAIErrorType.API_ERROR), + param=param, + code=code, + metadata=metadata if metadata is not None else ErrorMetadata(), + ) + + +def build_error_envelope( + *, + status_code: int, + message: str, + code: str | None, + param: str | None = None, + metadata: ErrorMetadata | None = None, +) -> OpenAIErrorEnvelope: + """Create the OpenAI error envelope for an HTTP status. + + Args: + status_code: The HTTP status the error is returned with, or would have been before stream headers. + message: The explanation for the client. + code: The Horde return code, or the front's own code. + param: The request path at fault, when one is. + metadata: The request id, return code and retry details that are known. + """ + return OpenAIErrorEnvelope( + error=build_error(status_code=status_code, message=message, code=code, param=param, metadata=metadata), + ) diff --git a/horde/oai/schema.py b/horde/oai/schema.py new file mode 100644 index 00000000..f761f8c4 --- /dev/null +++ b/horde/oai/schema.py @@ -0,0 +1,830 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""The OpenAI chat completions request, response and error models the front accepts and returns. + +The request side is screened in two steps before validation. +[`find_refused_parameter`][horde.oai.schema.find_refused_parameter] returns the first key, role, content part or +``horde`` option the service does not serve, so the client gets a 400 naming it rather than a silently degraded +answer. [`drop_default_no_ops`][horde.oai.schema.drop_default_no_ops] then removes the keys listed in +[`DEFAULT_NO_OP_PARAMETERS`][horde.oai.schema.DEFAULT_NO_OP_PARAMETERS] when they hold their documented default, +because stock clients send them unprompted. The table is a closed list; any other key outside +[`ChatCompletionRequest`][horde.oai.schema.ChatCompletionRequest] is refused. + +[`ChatCompletionRequest`][horde.oai.schema.ChatCompletionRequest] then validates what remains with ``extra="forbid"``. +Values outside the service contract (``n`` other than 1, a token limit outside 16 to 4096, more than four stop +sequences) raise the error type [`UNSUPPORTED_PARAMETER_ERROR_TYPE`][horde.oai.schema.UNSUPPORTED_PARAMETER_ERROR_TYPE] +so the front reports them as ``UnsupportedChatParameter``; contradictory values raise +[`CHAT_REQUEST_CONFLICT_ERROR_TYPE`][horde.chat_contract.CHAT_REQUEST_CONFLICT_ERROR_TYPE]. A request that sets no +token limit is forwarded without one and the native route applies its default of 512. + +The response side follows the OpenAI ``chat.completion``, ``chat.completion.chunk``, error and ``model`` shapes; +``x_horde`` carries the Horde-specific details. Error envelopes always carry ``message``, ``type``, ``param`` and +``code``, with null written explicitly, plus a ``metadata`` object holding only the details that are known. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from enum import StrEnum +from types import MappingProxyType +from typing import Annotated, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + StringConstraints, + ValidationInfo, + field_validator, + model_serializer, +) +from pydantic_core import PydanticCustomError + +from horde.chat_contract import CHAT_REQUEST_CONFLICT_ERROR_TYPE, INTERNAL_HORDE_OPTION_KEYS, ChatHordeOptions + +__all__ = [ + "BAD_REQUEST_CODE", + "CHAT_REQUEST_CONFLICT_CODE", + "DEFAULT_NO_OP_PARAMETERS", + "UNSUPPORTED_PARAMETER_CODE", + "UNSUPPORTED_PARAMETER_ERROR_TYPE", + "ChatCompletionChoice", + "ChatCompletionChunk", + "ChatCompletionChunkChoice", + "ChatCompletionChunkDelta", + "ChatCompletionRequest", + "ChatCompletionResponse", + "ChatCompletionResponseMessage", + "ChatCompletionUsage", + "ChatMessage", + "ChatRole", + "ErrorMetadata", + "FinishReason", + "HordeModelDetails", + "HordeResponseDetails", + "ModelArchitecture", + "ModelCard", + "ModelList", + "OpenAIError", + "OpenAIErrorEnvelope", + "OpenAIErrorType", + "RefusedParameter", + "StreamOptions", + "SubmitOutcome", + "TextContentPart", + "drop_default_no_ops", + "find_refused_parameter", +] + +UNSUPPORTED_PARAMETER_CODE: str = "UnsupportedChatParameter" +"""The error code for a key, role, part or value the service does not serve; the native route returns the same rc.""" + +BAD_REQUEST_CODE: str = "BadRequest" +"""The error code for a body that is not valid JSON or fails validation; the Horde's generic rc for a 400.""" + +CHAT_REQUEST_CONFLICT_CODE: str = "ChatRequestConflict" +"""The error code for two keys whose values contradict each other; the native route returns the same rc.""" + +UNSUPPORTED_PARAMETER_ERROR_TYPE: str = "unsupported_chat_parameter" +"""The pydantic error type for a value outside the service contract, reported as ``UnsupportedChatParameter``.""" + +SUPPORTED_CHOICE_COUNT: int = 1 +"""The only ``n`` served: each further choice would need its own pinned model and stored reply.""" + +MIN_MAX_TOKENS: int = 16 +"""The smallest token limit served; the native minimum for ``max_length``.""" + +MAX_MAX_TOKENS: int = 4096 +"""The largest token limit served; the native cap for ``max_length``.""" + +MAX_STOP_SEQUENCES: int = 4 +"""The most stop sequences a request may carry, as OpenAI documents.""" + +MAX_USER_CHARACTERS: int = 256 +"""The longest ``user`` annotation accepted; it is unused and only bounded so it cannot carry bulk data.""" + +MAX_METADATA_PAIRS: int = 16 +"""The most ``metadata`` pairs accepted, as OpenAI documents; unused and not persisted.""" + +MAX_METADATA_KEY_CHARACTERS: int = 64 +"""The longest ``metadata`` key accepted, as OpenAI documents.""" + +MAX_METADATA_VALUE_CHARACTERS: int = 512 +"""The longest ``metadata`` value accepted, as OpenAI documents.""" + +TEXT_CONTENT_PART_TYPE: str = "text" +"""The only content part type served.""" + + +class ChatRole(StrEnum): + """The message roles a chat accepts.""" + + SYSTEM = "system" + """Instructions for the model.""" + + DEVELOPER = "developer" + """Instructions for the model from the application; the native side stores it as ``system``.""" + + USER = "user" + """The end user's turn.""" + + ASSISTANT = "assistant" + """A previous reply of the model.""" + + +class FinishReason(StrEnum): + """The OpenAI reasons a generation stopped.""" + + STOP = "stop" + """The model ended the reply or hit a stop sequence.""" + + LENGTH = "length" + """The generation reached its token limit.""" + + CONTENT_FILTER = "content_filter" + """The output was withheld by a filter.""" + + +class OpenAIErrorType(StrEnum): + """The OpenAI error ``type`` values the front returns, chosen by HTTP status.""" + + INVALID_REQUEST_ERROR = "invalid_request_error" + AUTHENTICATION_ERROR = "authentication_error" + PERMISSION_ERROR = "permission_error" + NOT_FOUND_ERROR = "not_found_error" + TIMEOUT_ERROR = "timeout_error" + RATE_LIMIT_ERROR = "rate_limit_error" + API_ERROR = "api_error" + + +class SubmitOutcome(StrEnum): + """What the front knows about whether a submit created a Horde request.""" + + UNKNOWN = "unknown" + """The submit was sent and no answer arrived, so a request may exist.""" + + +class RefusedParameter(BaseModel): + """Represents the first part of a raw chat request the service does not serve.""" + + model_config = ConfigDict(frozen=True) + + parameter: str + """The request path of the part, such as ``tools`` or ``messages[1].content[0].type=image_url``; a refused content + part's path names its type the way the native route does.""" + + message: str + """The explanation returned to the client.""" + + +NoOpPredicate = Callable[[object, Mapping[str, object]], bool] +"""Return whether a key's value, given the whole request body, is the default that may be dropped.""" + + +def _is_empty_or_null(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is null or an empty list.""" + return value is None or (isinstance(value, list) and not value) + + +def _is_null_or_none_choice(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is null or the string ``none``.""" + return value is None or value == "none" + + +def _is_text_response_format(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is exactly ``{"type": "text"}``.""" + return value == {"type": "text"} + + +def _is_false_or_null(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is false or null.""" + return value is None or value is False + + +def _is_zero_or_null(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is the integer 0 or null; ``false`` is not 0 here.""" + return value is None or (type(value) is int and value == 0) + + +def _is_parallel_tool_calls_without_tools(value: object, request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is a boolean or null and the request carries no tools.""" + is_boolean_or_null = value is None or isinstance(value, bool) + return is_boolean_or_null and _is_empty_or_null(request_body.get("tools"), request_body) + + +def _is_false(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is false.""" + return value is False + + +def _is_default_service_tier(value: object, _request_body: Mapping[str, object]) -> bool: + """Return whether ``value`` is ``auto`` or ``default``.""" + return value == "auto" or value == "default" + + +DEFAULT_NO_OP_PARAMETERS: Mapping[str, NoOpPredicate] = MappingProxyType( + { + "tools": _is_empty_or_null, + "tool_choice": _is_null_or_none_choice, + "response_format": _is_text_response_format, + "logprobs": _is_false_or_null, + "top_logprobs": _is_zero_or_null, + "parallel_tool_calls": _is_parallel_tool_calls_without_tools, + "store": _is_false, + "service_tier": _is_default_service_tier, + }, +) +"""Keys accepted and dropped when they hold their documented default, each with the test for that default. + +Stock clients send these unprompted. Any other value of one of these keys is refused by name, as is any key that is +neither here nor a field of [`ChatCompletionRequest`][horde.oai.schema.ChatCompletionRequest]. +""" + +ACCEPTED_MESSAGE_KEYS: frozenset[str] = frozenset( + { + "role", + "content", + "name", + }, +) +"""The message keys a chat message carries; others (``tool_calls``, ``audio``, ...) belong to features not served.""" + +ACCEPTED_ROLES: frozenset[str] = frozenset(chat_role.value for chat_role in ChatRole) +"""The role strings a message may carry; ``tool`` and ``function`` belong to features not served.""" + + +def find_refused_parameter(request_body: Mapping[str, object]) -> RefusedParameter | None: + """Return the first key, role, content part or ``horde`` option of a raw chat request that is not served, or None. + + Malformed values of served keys are left to [`ChatCompletionRequest`][horde.oai.schema.ChatCompletionRequest] + validation; this only finds what the service does not serve at all. + + Args: + request_body: The decoded JSON request body. + + Returns: + The refused part and an explanation, or None when every part is served. + """ + for request_key, value in request_body.items(): + if request_key in ChatCompletionRequest.model_fields: + continue + no_op_predicate = DEFAULT_NO_OP_PARAMETERS.get(request_key) + if no_op_predicate is None: + return RefusedParameter( + parameter=request_key, + message=f"The parameter '{request_key}' is not supported by the AI Horde chat completions endpoint.", + ) + if not no_op_predicate(value, request_body): + return RefusedParameter( + parameter=request_key, + message=f"The parameter '{request_key}' is accepted only at its default value; other values are not served.", + ) + messages = request_body.get("messages") + if isinstance(messages, list): + for message_index, message in enumerate(messages): + if not isinstance(message, dict): + continue + refused_in_message = _find_refused_in_message(message, message_index=message_index) + if refused_in_message is not None: + return refused_in_message + horde_options = request_body.get("horde") + if isinstance(horde_options, dict): + return _find_refused_horde_option(horde_options) + return None + + +def _find_refused_in_message(message: Mapping[str, object], *, message_index: int) -> RefusedParameter | None: + """Return the first refused key, role or content part of one raw message, or None.""" + for message_key in message: + if message_key not in ACCEPTED_MESSAGE_KEYS: + return RefusedParameter( + parameter=f"messages[{message_index}].{message_key}", + message=f"The message key '{message_key}' is not supported; only text messages are served.", + ) + role = message.get("role") + if isinstance(role, str) and role not in ACCEPTED_ROLES: + return RefusedParameter( + parameter=f"messages[{message_index}].role", + message=f"The role '{role}' is not supported; use one of {', '.join(chat_role.value for chat_role in ChatRole)}.", + ) + content = message.get("content") + if not isinstance(content, list): + return None + for part_index, content_part in enumerate(content): + if not isinstance(content_part, dict): + continue + part_type = content_part.get("type") + if part_type == TEXT_CONTENT_PART_TYPE: + continue + return RefusedParameter( + parameter=f"messages[{message_index}].content[{part_index}].type={part_type}", + message=f"The content part type '{part_type}' is not supported; only 'text' parts are served.", + ) + return None + + +def _find_refused_horde_option(horde_options: Mapping[str, object]) -> RefusedParameter | None: + """Return the first ``horde`` key a public request may not send, or None.""" + for option_key in horde_options: + if option_key in INTERNAL_HORDE_OPTION_KEYS: + return RefusedParameter( + parameter=f"horde.{option_key}", + message=f"The horde option '{option_key}' is set by the front and cannot be sent.", + ) + if option_key not in ChatHordeOptions.model_fields: + return RefusedParameter( + parameter=f"horde.{option_key}", + message=f"The horde option '{option_key}' is not supported.", + ) + return None + + +def drop_default_no_ops(request_body: Mapping[str, object]) -> dict[str, object]: + """Return a copy of ``request_body`` without the keys of the default no-op table. + + Call it only after [`find_refused_parameter`][horde.oai.schema.find_refused_parameter] found nothing, which + guarantees every such key holds its default. + + Args: + request_body: The decoded JSON request body. + """ + return {request_key: value for request_key, value in request_body.items() if request_key not in DEFAULT_NO_OP_PARAMETERS} + + +class TextContentPart(BaseModel): + """Represents one text part of a message's content array.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + type: Literal["text"] + """Always ``text``.""" + + text: StrictStr + """The text of this part.""" + + +class ChatMessage(BaseModel): + """Represents one message of a chat request.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + role: ChatRole + """Who wrote the message.""" + + content: StrictStr | Annotated[list[TextContentPart], Field(min_length=1)] + """The message text, either a string or an array of text parts.""" + + name: StrictStr | None = None + """An optional participant name, passed through to the worker.""" + + +class StreamOptions(BaseModel): + """Represents the ``stream_options`` object of a chat request.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + include_usage: StrictBool | None = None + """Send a usage chunk with ``choices: []`` before ``[DONE]``.""" + + +MetadataKey = Annotated[str, StringConstraints(strict=True, max_length=MAX_METADATA_KEY_CHARACTERS)] +"""A ``metadata`` key within OpenAI's documented bound.""" + +MetadataValue = Annotated[str, StringConstraints(strict=True, max_length=MAX_METADATA_VALUE_CHARACTERS)] +"""A ``metadata`` value within OpenAI's documented bound.""" + + +def _unsupported_value_error(message: str) -> PydanticCustomError: + """Create the validation error for a value outside the service contract.""" + return PydanticCustomError(UNSUPPORTED_PARAMETER_ERROR_TYPE, message) + + +class ChatCompletionRequest(BaseModel): + """Represents an OpenAI chat completions request in the subset the service serves. + + A null value means the key was omitted. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + model: StrictStr = Field(min_length=1) + """The Horde text model name.""" + + messages: list[ChatMessage] = Field(min_length=1) + """The conversation so far, in order.""" + + stream: StrictBool | None = None + """Answer as server-sent events; the whole text arrives in one content chunk.""" + + stream_options: StreamOptions | None = None + """Options for a streamed answer; only allowed with ``stream: true``.""" + + max_tokens: StrictInt | None = None + """The most tokens to generate, 16 to 4096.""" + + max_completion_tokens: StrictInt | None = None + """The newer name for ``max_tokens``; both must agree when both are set.""" + + temperature: StrictFloat | StrictInt | None = None + """The sampling temperature.""" + + top_p: StrictFloat | StrictInt | None = None + """The nucleus sampling probability.""" + + frequency_penalty: StrictFloat | StrictInt | None = None + """The frequency penalty; a non-default value needs a worker that declares it.""" + + presence_penalty: StrictFloat | StrictInt | None = None + """The presence penalty; a non-default value needs a worker that declares it.""" + + stop: StrictStr | list[StrictStr] | None = None + """One stop sequence or up to four.""" + + seed: StrictInt | None = None + """The sampling seed.""" + + n: StrictInt | None = None + """How many choices to generate; only 1 is served.""" + + user: Annotated[StrictStr, Field(max_length=MAX_USER_CHARACTERS)] | None = None + """An end-user identifier; accepted, unused and not persisted.""" + + metadata: Annotated[dict[MetadataKey, MetadataValue], Field(max_length=MAX_METADATA_PAIRS)] | None = None + """Free-form annotations; accepted, unused and not persisted.""" + + horde: ChatHordeOptions | None = None + """The Horde extension object.""" + + @field_validator("stream_options") + @classmethod + def validate_stream_options_need_stream(cls, stream_options: StreamOptions | None, info: ValidationInfo) -> StreamOptions | None: + """Validate that ``stream_options`` is only set on a streamed request. + + Raises: + ValueError: ``stream_options`` is set and ``stream`` is not true. + """ + if stream_options is not None and info.data.get("stream") is not True: + raise ValueError("'stream_options' is only allowed when 'stream' is true.") + return stream_options + + @field_validator("max_tokens", "max_completion_tokens") + @classmethod + def validate_token_limit_range(cls, token_limit: int | None) -> int | None: + """Validate that a token limit is within the served range; it is never clamped. + + Raises: + PydanticCustomError: The limit is outside 16 to 4096. + """ + if token_limit is not None and not MIN_MAX_TOKENS <= token_limit <= MAX_MAX_TOKENS: + raise _unsupported_value_error(f"The token limit must be between {MIN_MAX_TOKENS} and {MAX_MAX_TOKENS}.") + return token_limit + + @field_validator("max_completion_tokens") + @classmethod + def validate_token_limits_agree(cls, max_completion_tokens: int | None, info: ValidationInfo) -> int | None: + """Validate that ``max_completion_tokens`` equals ``max_tokens`` when both are set. + + Raises: + PydanticCustomError: The two limits differ. + """ + max_tokens = info.data.get("max_tokens") + if max_completion_tokens is not None and max_tokens is not None and max_completion_tokens != max_tokens: + raise PydanticCustomError( + CHAT_REQUEST_CONFLICT_ERROR_TYPE, + "'max_tokens' and 'max_completion_tokens' must be equal when both are set.", + ) + return max_completion_tokens + + @field_validator("stop") + @classmethod + def validate_stop_count(cls, stop: str | list[str] | None) -> str | list[str] | None: + """Validate that at most four stop sequences are sent. + + Raises: + PydanticCustomError: More than four sequences are sent. + """ + if isinstance(stop, list) and len(stop) > MAX_STOP_SEQUENCES: + raise _unsupported_value_error(f"At most {MAX_STOP_SEQUENCES} stop sequences are served.") + return stop + + @field_validator("n") + @classmethod + def validate_single_choice(cls, choice_count: int | None) -> int | None: + """Validate that one choice is asked for. + + Raises: + PydanticCustomError: ``n`` is set to anything but 1. + """ + if choice_count is not None and choice_count != SUPPORTED_CHOICE_COUNT: + raise _unsupported_value_error(f"Only {SUPPORTED_CHOICE_COUNT} choice is served; 'n' must be 1 or omitted.") + return choice_count + + +def _drop_absent_fields(model: BaseModel, handler: SerializerFunctionWrapHandler) -> dict[str, object]: + """Return the serialised ``model`` without its null fields.""" + serialised: dict[str, object] = handler(model) + return {field_name: value for field_name, value in serialised.items() if value is not None} + + +class ChatCompletionResponseMessage(BaseModel): + """Represents the assistant message of one completion choice.""" + + model_config = ConfigDict(frozen=True) + + role: Literal["assistant"] = "assistant" + """Always ``assistant``.""" + + content: str + """The generated text.""" + + +class ChatCompletionChoice(BaseModel): + """Represents one completion choice, built from one Horde generation.""" + + model_config = ConfigDict(frozen=True) + + index: int + """The position of the choice, in the order the Horde lists its generations.""" + + message: ChatCompletionResponseMessage + """The generated message.""" + + finish_reason: FinishReason + """Why the generation stopped; ``stop`` when the worker did not say.""" + + native_finish_reason: str | None = None + """The backend's own reason, as the worker reported it; null when it did not.""" + + logprobs: None = None + """Always null; log probabilities are not served.""" + + +class ChatCompletionUsage(BaseModel): + """Represents the token counts of a completion.""" + + model_config = ConfigDict(frozen=True) + + prompt_tokens: int + """Tokens in the prompt, as the worker reported or as estimated.""" + + completion_tokens: int + """Tokens generated, as the worker reported or as estimated.""" + + total_tokens: int + """The sum of the prompt and completion tokens.""" + + +class HordeResponseDetails(BaseModel): + """Represents the ``x_horde`` details of a completion; fields the Horde did not supply are left out.""" + + model_config = ConfigDict(frozen=True) + + request_id: str + """The Horde request id; also the completion id without its prefix.""" + + worker_id: str | None = None + """The worker that generated the choice.""" + + worker_name: str | None = None + """The name of that worker.""" + + kudos: float + """The kudos the request consumed.""" + + charge_basis: str | None = None + """Which count the kudos were computed on, as the Horde reports it.""" + + estimated_usage: bool + """True when any usage count is an estimate from text length because the worker did not report it.""" + + queue_wait_seconds: float | None = None + """Seconds from submit to the first assignment to a worker, when the Horde supplies an assignment time.""" + + context_adjustments: list[object] | None = None + """The context changes the Horde applied to fit the messages, when it reports any.""" + + notice: str | None = None + """Present on an empty return: the worker ran the job and returned no visible text.""" + + @model_serializer(mode="wrap") + def serialize_without_absent_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]: + """Convert the details to a dict without the fields the Horde did not supply.""" + return _drop_absent_fields(self, handler) + + +class ChatCompletionResponse(BaseModel): + """Represents an OpenAI ``chat.completion`` response.""" + + model_config = ConfigDict(frozen=True, serialize_by_alias=True) + + completion_id: str = Field(serialization_alias="id") + """``chatcmpl-`` followed by the Horde request id.""" + + object_type: Literal["chat.completion"] = Field(default="chat.completion", serialization_alias="object") + """Always ``chat.completion``.""" + + created: int + """When the request was submitted, in Unix seconds.""" + + model: str + """The model that generated the first choice.""" + + choices: list[ChatCompletionChoice] + """One choice per Horde generation.""" + + usage: ChatCompletionUsage + """The token counts.""" + + x_horde: HordeResponseDetails + """The Horde-specific details.""" + + +class ErrorMetadata(BaseModel): + """Represents the ``metadata`` object of an error; only the details that are known are written.""" + + model_config = ConfigDict(frozen=True) + + request_id: str | None = None + """The Horde request id, once one exists.""" + + rc: str | None = None + """The Horde return code, when the Horde produced the error.""" + + retry_after: int | None = None + """Seconds to wait before retrying, when waiting helps.""" + + submit_outcome: SubmitOutcome | None = None + """Set when the front cannot tell whether the submit created a request.""" + + @model_serializer(mode="wrap") + def serialize_without_absent_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]: + """Convert the metadata to a dict without the details that are not known.""" + return _drop_absent_fields(self, handler) + + +class OpenAIError(BaseModel): + """Represents the body of an OpenAI error envelope; the four OpenAI fields are always written, null included.""" + + model_config = ConfigDict(frozen=True) + + message: str + """The explanation for the client.""" + + type: OpenAIErrorType + """The error class, chosen by HTTP status.""" + + param: str | None = None + """The request path at fault, when one is.""" + + code: str | None + """The Horde return code when the Horde produced the error, or the front's own code.""" + + metadata: ErrorMetadata = Field(default_factory=ErrorMetadata) + """The request id, Horde return code and retry details that are known.""" + + +class OpenAIErrorEnvelope(BaseModel): + """Represents an OpenAI error response, ``{"error": {...}}``, before or after stream headers.""" + + model_config = ConfigDict(frozen=True) + + error: OpenAIError + """The error details.""" + + +class ModelArchitecture(BaseModel): + """Represents the modalities a model list entry serves.""" + + model_config = ConfigDict(frozen=True) + + modality: str = "text->text" + """The input and output modalities in OpenRouter's notation.""" + + input_modalities: list[str] = Field(default_factory=lambda: ["text"]) + """The content the model accepts.""" + + output_modalities: list[str] = Field(default_factory=lambda: ["text"]) + """The content the model produces.""" + + +class HordeModelDetails(BaseModel): + """Represents the ``x_horde`` details of a model list entry.""" + + model_config = ConfigDict(frozen=True) + + workers: int + """How many worker threads serve the model, as the Horde's model list counts them.""" + + queued: float | None = None + """The queued work for the model, as the Horde's model list reports it.""" + + @model_serializer(mode="wrap") + def serialize_without_absent_fields(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]: + """Convert the details to a dict without the values the Horde did not report.""" + return _drop_absent_fields(self, handler) + + +class ModelCard(BaseModel): + """Represents one entry of the OpenAI model list.""" + + model_config = ConfigDict(frozen=True, serialize_by_alias=True) + + model_id: str = Field(serialization_alias="id") + """The Horde text model name.""" + + object_type: Literal["model"] = Field(default="model", serialization_alias="object") + """Always ``model``.""" + + created: int = 0 + """Always 0 until the Horde supplies a stable creation time for a model.""" + + owned_by: str = "aihorde" + """Always ``aihorde``.""" + + architecture: ModelArchitecture = Field(default_factory=ModelArchitecture) + """The modalities served; text only.""" + + x_horde: HordeModelDetails + """The Horde-specific details.""" + + +class ModelList(BaseModel): + """Represents the OpenAI model list response.""" + + model_config = ConfigDict(frozen=True, serialize_by_alias=True) + + object_type: Literal["list"] = Field(default="list", serialization_alias="object") + """Always ``list``.""" + + data: list[ModelCard] + """The text models with at least one worker serving them.""" + + +class ChatCompletionChunkDelta(BaseModel): + """Represents the ``delta`` of one streamed choice; unset fields are left out of the chunk.""" + + model_config = ConfigDict(frozen=True) + + role: Literal["assistant"] | None = None + """``assistant`` in the first chunk of a choice, otherwise absent.""" + + content: str | None = None + """The generated text in the content chunk, otherwise absent.""" + + +class ChatCompletionChunkChoice(BaseModel): + """Represents one choice of a streamed chunk.""" + + model_config = ConfigDict(frozen=True) + + index: int + """The position of the choice, in the order the Horde lists its generations.""" + + delta: ChatCompletionChunkDelta + """What this chunk adds to the choice.""" + + finish_reason: FinishReason | None = None + """Why the generation stopped, in the finish chunk; null before it.""" + + native_finish_reason: str | None = None + """The backend's own reason, in the finish chunk when the worker reported it; null otherwise.""" + + logprobs: None = None + """Always null; log probabilities are not served.""" + + +class ChatCompletionChunk(BaseModel): + """Represents an OpenAI ``chat.completion.chunk`` server-sent event payload.""" + + model_config = ConfigDict(frozen=True, serialize_by_alias=True) + + completion_id: str = Field(serialization_alias="id") + """``chatcmpl-`` followed by the Horde request id; the same in every chunk of a stream.""" + + object_type: Literal["chat.completion.chunk"] = Field( + default="chat.completion.chunk", + serialization_alias="object", + ) + """Always ``chat.completion.chunk``.""" + + created: int + """When the request was submitted, in Unix seconds.""" + + model: str + """The model that generated the first choice.""" + + choices: list[ChatCompletionChunkChoice] + """The choices this chunk adds to; empty in the usage chunk.""" + + usage: ChatCompletionUsage | None = None + """The token counts, in the usage chunk only.""" + + x_horde: HordeResponseDetails | None = None + """The Horde-specific details, in the finish chunk only.""" diff --git a/horde/oai/settings.py b/horde/oai/settings.py new file mode 100644 index 00000000..b3242d12 --- /dev/null +++ b/horde/oai/settings.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Read the chat front's settings from the environment. + +[`FrontSettings`][horde.oai.settings.FrontSettings] is a ``pydantic-settings`` model reading every ``HORDE_OAI_*`` +variable, and a ``.env`` file in the working directory when one exists. +[`load_settings`][horde.oai.settings.load_settings] is the entry point ``python -m horde.oai`` uses. A variable that +is set but invalid fails the start with a validation error naming it. +""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + +from horde.chat_contract import MAX_CHAT_BODY_BYTES, MAX_CHAT_MESSAGE_COUNT, MAX_CHAT_TEXT_BYTES + +__all__ = [ + "DEFAULT_HORDE_URL", + "SETTINGS_ENV_PREFIX", + "FrontSettings", + "load_settings", +] + +SETTINGS_ENV_PREFIX: str = "HORDE_OAI_" +"""The prefix of every environment variable the front reads.""" + +DEFAULT_HORDE_URL: str = "http://127.0.0.1:7001/api" +"""The Horde API root the front calls when ``HORDE_OAI_HORDE_URLS`` is unset: a Horde on the same host, default port.""" + +HORDE_URL_SEPARATOR: str = "," +"""The separator between roots in ``HORDE_OAI_HORDE_URLS``.""" + +HIGHEST_PORT: int = 65535 +"""The highest TCP port number.""" + + +class FrontSettings(BaseSettings): + """Represents the chat front's configuration, read from the ``HORDE_OAI_*`` environment variables.""" + + model_config = SettingsConfigDict( + env_prefix=SETTINGS_ENV_PREFIX, + env_file=".env", + extra="ignore", + frozen=True, + ) + + horde_urls: Annotated[tuple[str, ...], NoDecode] = Field(default=(DEFAULT_HORDE_URL,), min_length=1) + """The private Horde API roots (each ending in ``/api``), comma-separated, used round-robin. + + More than one root lets a refused connection move to another instance instead of failing the request. + """ + + listen: str = "0.0.0.0" + """The address the front binds; the same default as the Horde's ``--listen``, since the front runs in a container + reached only through the listener.""" + + port: int = Field(default=7002, ge=1, le=HIGHEST_PORT) + """The port the front binds; the port after the Horde's default 7001.""" + + redis_url: str | None = Field(default=None, repr=False) + """The primary Redis the Horde publishes chat signals on. + + Unset selects poll-only mode, where every request wakes on the fallback poll alone. The URL can carry a password, + so it is left out of the settings' representation. + """ + + redis_liveness_seconds: float = Field(default=30.0, gt=0) + """How long the signal subscription may receive nothing before it sends a PING, and how long the PONG may take. + + An idle subscription otherwise sends nothing, so a connection a network fault black-holes looks like a quiet one + and ``/healthz`` keeps reporting it subscribed. A PONG that does not arrive in time drops the subscription, which + is then made again; tens of seconds keeps the extra traffic negligible and a dead connection short-lived. + """ + + proxy_passkey: str | None = Field(default=None, repr=False) + """The front operator's service-account proxy passkey. + + When set, each Horde call carries it with the end-user address as ``Proxied-For``, so the Horde's limiter and + suspicion apply to the user rather than to the front. It is a credential, so it is left out of the representation. + """ + + max_inflight: int = Field(default=100, ge=1) + """Open requests per process before 503 ``FrontAtCapacity``. + + Every open request costs native status reads; the cap stays low until the native call budget is measured. + """ + + max_inflight_per_source: int = Field(default=5, ge=1) + """Open requests per end-user address per process before 503 ``FrontAtCapacity``. + + Stops one client from filling the process. Users behind one egress address share it, so the value needs measuring + before it is raised or lowered. + """ + + max_body_bytes: int = Field(default=MAX_CHAT_BODY_BYTES, ge=1) + """The largest request body accepted, counted while it is received and before JSON is parsed. + + Defaults to the native route's limit, so a body the front accepts is not refused for size after the submit; the + bound keeps one body from holding memory the process needs for thousands of waits. + """ + + max_message_count: int = Field(default=MAX_CHAT_MESSAGE_COUNT, ge=1) + """The most messages one request may carry; defaults to the native route's limit.""" + + max_text_bytes: int = Field(default=MAX_CHAT_TEXT_BYTES, ge=1) + """The most UTF-8 bytes of message text one request may carry; defaults to the native route's limit.""" + + body_read_timeout_seconds: float = Field(default=30.0, gt=0) + """How long receiving the request body may take, so a slow sender cannot hold a connection indefinitely.""" + + nonstream_timeout_seconds: float = Field(default=300.0, gt=0) + """The total budget of a non-streamed request, starting before the submit; a request still waiting after it is 408.""" + + stream_timeout_seconds: float = Field(default=600.0, gt=0) + """The total budget of a streamed request, starting before the submit. + + Longer than the non-streamed budget because keepalive comments hold the connection open, but finite so a request + the Horde keeps alive cannot hold a delivery slot forever. + """ + + final_status_timeout_seconds: float = Field(default=2.0, gt=0) + """The allowance for the one status read made at the deadline, so an outcome reached just in time is returned.""" + + cancel_timeout_seconds: float = Field(default=10.0, gt=0) + """The budget of one cancel, which runs shielded from the client's disconnect.""" + + shutdown_cancel_seconds: float = Field(default=15.0, gt=0) + """The process-wide bound for cancelling every open request on a clean stop, not one cancel budget per request.""" + + shutdown_drain_seconds: float = Field(default=30.0, gt=0) + """How long uvicorn waits for open requests to finish on a clean stop before cancelling them.""" + + heartbeat_seconds: float = Field(default=5.0, gt=0) + """The keepalive comment interval of a streamed response; well under the listener's inactivity timeouts.""" + + fallback_poll_seconds: float = Field(default=15.0, gt=0) + """The base status poll interval when no signal arrives; it stretches to half the Horde's ``wait_time``, at most 60.""" + + signal_recheck_seconds: float = Field(default=0.5, gt=0) + """How soon a request reads its status once more after a terminal signal its first read did not yet show. + + The signal is published after the commit, but the status read can reach an instance or replica that lags; one short + recheck avoids waiting a whole poll interval for that. + """ + + fail_fast_grace_seconds: float = Field(default=20.0, gt=0) + """How long the Horde may report no eligible worker, with nothing processing, before the request is cancelled and + answered 503 ``no_eligible_worker``; long enough for a worker restart or a model load.""" + + native_max_connections: int = Field(default=200, ge=1) + """The most simultaneous calls to the Horde and the size of the main connection pool; cancels use their own pool.""" + + forwarded_allow_ips: str = "127.0.0.1" + """The listener addresses uvicorn trusts for ``X-Forwarded-For``; the ASGI client address is then the end user.""" + + @field_validator("horde_urls", mode="before") + @classmethod + def parse_horde_urls(cls, horde_urls: object) -> object: + """Parse a comma-separated string of roots into a tuple, without trailing slashes or empty entries.""" + if isinstance(horde_urls, str): + horde_urls = horde_urls.split(HORDE_URL_SEPARATOR) + if isinstance(horde_urls, list | tuple): + return tuple(str(horde_url).strip().rstrip("/") for horde_url in horde_urls if str(horde_url).strip()) + return horde_urls + + +def load_settings() -> FrontSettings: + """Return the front's settings read from the ``HORDE_OAI_*`` environment variables and ``.env``. + + Raises: + pydantic.ValidationError: A variable is set but invalid. + """ + return FrontSettings() diff --git a/horde/oai/sse.py b/horde/oai/sse.py new file mode 100644 index 00000000..de446c3b --- /dev/null +++ b/horde/oai/sse.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Render a streamed chat answer as server-sent events. + +The Horde produces the whole text before it reports a generation, so a stream is buffered: keepalive comments while +the request waits (``: horde queue_position=7 wait_time=42``), then the finished completion as a role chunk, one +content chunk, a finish chunk carrying ``x_horde``, the usage chunk when the client asked for it, and ``[DONE]``. +Nullable choice fields remain explicit; absent delta members are omitted. + +An error after the headers is one ``data:`` event carrying the same ``{"error": {...}}`` envelope as a response +before headers, with null fields written explicitly and no choice, followed by ``[DONE]``. +""" + +from __future__ import annotations + +import json + +from sse_starlette import ServerSentEvent + +from horde.oai.horde_client import HordeChatStatus +from horde.oai.schema import ( + ChatCompletionChunk, + ChatCompletionChunkChoice, + ChatCompletionChunkDelta, + ChatCompletionResponse, + OpenAIErrorEnvelope, +) + +__all__ = [ + "DONE_MARKER", + "KEEPALIVE_COMMENT_PREFIX", + "build_completion_events", + "build_error_events", + "build_keepalive_event", +] + +DONE_MARKER: str = "[DONE]" +"""The data of the last event of every stream, which OpenAI clients read as the end.""" + +KEEPALIVE_COMMENT_PREFIX: str = "horde" +"""The first word of a keepalive comment; the queue position and wait estimate follow it as ``key=value`` pairs.""" + + +def build_keepalive_event(status: HordeChatStatus | None) -> ServerSentEvent: + """Create the keepalive comment for the request's latest status. + + Args: + status: The latest status read, or None before the first read succeeded. + + Returns: + A comment event, ``: horde queue_position=7 wait_time=42``; a value the Horde did not report is left out. + """ + comment_parts = [KEEPALIVE_COMMENT_PREFIX] + if status is not None and status.queue_position is not None: + comment_parts.append(f"queue_position={status.queue_position}") + if status is not None and status.wait_time is not None: + comment_parts.append(f"wait_time={status.wait_time}") + return ServerSentEvent(comment=" ".join(comment_parts)) + + +def build_completion_events(completion: ChatCompletionResponse, *, include_usage: bool) -> list[ServerSentEvent]: + """Create the events that deliver a finished completion, ending with ``[DONE]``. + + Args: + completion: The finished completion. + include_usage: Whether to send the usage chunk the client asked for with ``stream_options.include_usage``. + """ + role_chunk = _build_chunk( + completion, + choices=[ + ChatCompletionChunkChoice(index=choice.index, delta=ChatCompletionChunkDelta(role="assistant")) for choice in completion.choices + ], + ) + content_chunk = _build_chunk( + completion, + choices=[ + ChatCompletionChunkChoice(index=choice.index, delta=ChatCompletionChunkDelta(content=choice.message.content)) + for choice in completion.choices + ], + ) + finish_chunk = _build_chunk( + completion, + choices=[ + ChatCompletionChunkChoice( + index=choice.index, + delta=ChatCompletionChunkDelta(), + finish_reason=choice.finish_reason, + native_finish_reason=choice.native_finish_reason, + ) + for choice in completion.choices + ], + x_horde=True, + ) + events = [ + _chunk_event(role_chunk, include_usage=include_usage), + _chunk_event(content_chunk, include_usage=include_usage), + _chunk_event(finish_chunk, include_usage=include_usage), + ] + if include_usage: + usage_chunk = ChatCompletionChunk( + completion_id=completion.completion_id, + created=completion.created, + model=completion.model, + choices=[], + usage=completion.usage, + ) + events.append(_chunk_event(usage_chunk, include_usage=True)) + events.append(ServerSentEvent(data=DONE_MARKER)) + return events + + +def build_error_events(envelope: OpenAIErrorEnvelope) -> list[ServerSentEvent]: + """Create the events that report an error after the stream has started, ending with ``[DONE]``. + + The envelope is serialised with its null fields, so ``param`` and ``code`` are present even when null. + + Args: + envelope: The error to report. + """ + return [ + ServerSentEvent(data=envelope.model_dump_json()), + ServerSentEvent(data=DONE_MARKER), + ] + + +def _build_chunk( + completion: ChatCompletionResponse, + *, + choices: list[ChatCompletionChunkChoice], + x_horde: bool = False, +) -> ChatCompletionChunk: + """Create a chunk of ``completion`` carrying ``choices``, and the Horde details when ``x_horde`` is set.""" + return ChatCompletionChunk( + completion_id=completion.completion_id, + created=completion.created, + model=completion.model, + choices=choices, + x_horde=completion.x_horde if x_horde else None, + ) + + +def _chunk_event(chunk: ChatCompletionChunk, *, include_usage: bool) -> ServerSentEvent: + """Create the data event for one chunk.""" + payload = chunk.model_dump(mode="json") + if chunk.x_horde is None: + payload.pop("x_horde") + if not include_usage: + payload.pop("usage") + for choice in payload["choices"]: + choice["delta"] = {key: value for key, value in choice["delta"].items() if value is not None} + return ServerSentEvent(data=json.dumps(payload)) diff --git a/horde/oai/waiter.py b/horde/oai/waiter.py new file mode 100644 index 00000000..6243c3e2 --- /dev/null +++ b/horde/oai/waiter.py @@ -0,0 +1,632 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Wait for Horde chat requests to finish, woken by the Horde's Redis completion signal or a fallback poll. + +The Horde publishes ``{"id": , "event": ...}`` on channel ``horde:chat:`` when a chat +generation lands and when the request finishes, faults or is cancelled. +[`CompletionSignalListener`][horde.oai.waiter.CompletionSignalListener] holds one pattern subscription to every such +channel for the whole process and hands each message to the request registered for it, so thousands of waiting +requests share one Redis connection; messages for requests this process is not waiting on are dropped. A +subscription that receives nothing is pinged, and one whose PONG does not arrive is dropped and made again. + +[`ChatRequestWatcher`][horde.oai.waiter.ChatRequestWatcher] is the one wait implementation both response modes use. +After the submit it reads the status once (the request may have finished before the registration), then sleeps until +a signal, the jittered fallback poll, the fail-fast check or the deadline, and reads the status once on each wake. +A terminal signal whose outcome the next read does not yet show earns one short recheck. The poll covers a lost +message and a Redis outage; its interval stretches with the request's reported wait (see +[`compute_fallback_poll_seconds`][horde.oai.waiter.compute_fallback_poll_seconds]). + +The deadline bounds the wait, not the outcome: a status read that returns a terminal outcome is returned even when it +completes after the deadline. At the deadline one final read runs under its own allowance; only a request still +waiting after it raises [`ChatWaitTimeoutError`][horde.oai.waiter.ChatWaitTimeoutError]. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import math +import random +import time +import uuid +from collections.abc import Callable, Iterator, Sequence +from enum import StrEnum +from http import HTTPStatus + +import httpx +import redis.asyncio +import redis.exceptions +from loguru import logger +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from horde.oai.horde_client import HordeAPIError, HordeChatStatus, HordeClient, HordeCredentials, HordeUnavailableError + +__all__ = [ + "CHAT_SIGNAL_CHANNEL_PREFIX", + "MAX_STRETCHED_POLL_SECONDS", + "ChatRequestWatcher", + "ChatWaitTimeoutError", + "CompletionEvent", + "CompletionEventKind", + "CompletionSignalListener", + "CompletionSubscription", + "NoEligibleWorkerError", + "WaitBudget", + "WakeSource", + "compute_fallback_poll_seconds", + "jitter_poll_seconds", +] + +CHAT_SIGNAL_CHANNEL_PREFIX: str = "horde:chat:" +"""The prefix of the channel the Horde publishes a chat request's events on; the request id follows it.""" + +CHAT_SIGNAL_CHANNEL_PATTERN: str = f"{CHAT_SIGNAL_CHANNEL_PREFIX}*" +"""The pattern the listener subscribes to, covering every chat request's channel.""" + +LISTENER_READ_TIMEOUT_SECONDS: float = 1.0 +"""How long one read of the subscription waits for a message, so a stop request is noticed within this time.""" + +PONG_MESSAGE_TYPE: str = "pong" +"""The pub/sub message type redis-py gives the answer to a PING sent on a subscribed connection.""" + +LISTENER_RECONNECT_DELAY_SECONDS: float = 2.0 +"""How long the listener waits before subscribing again after a Redis error, so an outage does not spin the loop.""" + +MAX_STRETCHED_POLL_SECONDS: float = 60.0 +"""The longest a stretched fallback poll interval grows, so a lost signal costs at most about a minute of extra wait.""" + +WAIT_TIME_POLL_DIVISOR: float = 2.0 +"""The fraction of the Horde's ``wait_time`` estimate a poll interval stretches to: half, so a request finishing on +schedule is still polled about twice before its estimate runs out.""" + +POLL_JITTER_FRACTION: float = 0.2 +"""The most a poll interval is shortened at random, so requests submitted together do not poll the Horde in step. + +Jitter only shortens the interval, so a lost signal still costs at most the stretched maximum. +""" + +TRANSIENT_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504}) +"""Horde statuses that a status read survives: the request keeps waiting and tries again at the next poll.""" + + +class CompletionEventKind(StrEnum): + """The events the Horde publishes for a chat request.""" + + GENERATION = "generation" + DONE = "done" + FAULTED = "faulted" + CANCELLED = "cancelled" + + +FINISHING_EVENT_KINDS: frozenset[CompletionEventKind] = frozenset( + { + CompletionEventKind.DONE, + CompletionEventKind.FAULTED, + CompletionEventKind.CANCELLED, + }, +) +"""The events after which the request's status should show it finished.""" + + +class WakeSource(StrEnum): + """What woke a waiting request when it found the request finished.""" + + IMMEDIATE = "immediate" + SIGNAL = "signal" + RECHECK = "recheck" + POLL = "poll" + DEADLINE = "deadline" + + +class CompletionEvent(BaseModel): + """Represents one completion signal message.""" + + model_config = ConfigDict(frozen=True) + + request_id: str = Field(validation_alias="id") + """The Horde request id.""" + + event: CompletionEventKind + """What happened to the request.""" + + +class ChatWaitTimeoutError(Exception): + """The request was still waiting after its deadline and the final status read.""" + + +class NoEligibleWorkerError(Exception): + """The Horde reported no eligible worker, with nothing processing, for longer than the fail-fast grace.""" + + def __init__(self, message: str, *, status: HordeChatStatus | None) -> None: + """Create the error. + + Args: + message: What happened. + status: The last status read, whose ``wait_time`` hints when to retry. + """ + super().__init__(message) + self.status: HordeChatStatus | None = status + """The last status read.""" + + +class WaitBudget(BaseModel): + """Represents the time limits of one request's wait.""" + + model_config = ConfigDict(frozen=True) + + deadline: float + """The ``clock`` time the wait ends at; it bounds the wait, not the outcome.""" + + final_status_timeout_seconds: float + """The allowance for the one status read made at the deadline.""" + + fallback_poll_seconds: float + """The base poll interval when no signal arrives.""" + + signal_recheck_seconds: float + """How soon to read once more after a terminal signal the first read did not yet show.""" + + fail_fast_grace_seconds: float + """How long no eligible worker, with nothing processing, is tolerated.""" + + +class CompletionSubscription: + """Receive the completion events of one request from the listener.""" + + def __init__(self, request_id: str) -> None: + """Create the subscription. + + Args: + request_id: The Horde request id. + """ + self.request_id: str = request_id + """The Horde request id.""" + self._events: asyncio.Queue[CompletionEvent] = asyncio.Queue() + + def deliver(self, event: CompletionEvent) -> None: + """Mutate the subscription by queueing ``event`` for the waiting request.""" + self._events.put_nowait(event) + + def requeue(self, events: Sequence[CompletionEvent]) -> None: + """Mutate the subscription by putting ``events``, taken from it in this order, back ahead of any queued since. + + Args: + events: Events the waiting request took from the subscription without acting on them, oldest first. + """ + if not events: + return + queued_since: list[CompletionEvent] = [] + with contextlib.suppress(asyncio.QueueEmpty): + while True: + queued_since.append(self._events.get_nowait()) + for event in (*events, *queued_since): + self._events.put_nowait(event) + + async def wait(self, timeout: float) -> CompletionEvent | None: + """Return the next event, waiting up to ``timeout`` seconds, or None when none arrives in time. + + Args: + timeout: The most seconds to wait; 0 or less only takes an event already queued. + """ + if timeout <= 0: + with contextlib.suppress(asyncio.QueueEmpty): + return self._events.get_nowait() + return None + try: + return await asyncio.wait_for(self._events.get(), timeout) + except TimeoutError: + return None + + +class CompletionSignalListener: + """Hold the process's Redis subscription to every chat channel and wake the registered requests. + + With no Redis client the listener never delivers anything and requests wait on their fallback poll alone. + + Concurrency: + Registration and delivery run on the event loop, so the registry needs no lock. + """ + + def __init__(self, redis_client: redis.asyncio.Redis | None, *, liveness_seconds: float) -> None: + """Create the listener. + + Args: + redis_client: The client for the Redis database the Horde publishes chat signals to, or None. + liveness_seconds: How long the subscription may receive nothing before a PING is sent, and how long its + PONG may take before the subscription is dropped and made again. + """ + self._redis_client = redis_client + self._liveness_seconds = liveness_seconds + self._subscriptions: dict[str, set[CompletionSubscription]] = {} + self._reader_task: asyncio.Task[None] | None = None + self._subscribed = asyncio.Event() + + @property + def subscribed(self) -> bool: + """Return whether the Redis subscription is currently established.""" + return self._subscribed.is_set() + + async def start(self) -> None: + """Start reading the subscription in a background task; does nothing without a Redis client.""" + if self._redis_client is None or self._reader_task is not None: + return + self._reader_task = asyncio.create_task(self._read_forever(self._redis_client), name="horde-oai-signal-listener") + + async def wait_until_subscribed(self, timeout: float) -> bool: + """Return whether the subscription is established, waiting up to ``timeout`` seconds for it.""" + try: + await asyncio.wait_for(self._subscribed.wait(), timeout) + except TimeoutError: + return False + return True + + async def stop(self) -> None: + """Stop the background reader and wait for it to close the subscription.""" + if self._reader_task is None: + return + self._reader_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._reader_task + self._reader_task = None + + @contextlib.contextmanager + def register(self, request_id: str) -> Iterator[CompletionSubscription]: + """Context manager that routes ``request_id``'s events to a new subscription until it exits. + + Args: + request_id: The Horde request id. + """ + subscription = CompletionSubscription(request_id) + self._subscriptions.setdefault(request_id, set()).add(subscription) + try: + yield subscription + finally: + request_subscriptions = self._subscriptions.get(request_id) + if request_subscriptions is not None: + request_subscriptions.discard(subscription) + if not request_subscriptions: + del self._subscriptions[request_id] + + def dispatch(self, channel: str, payload: bytes | str) -> None: + """Hand one published message to the subscriptions registered for its request. + + Args: + channel: The channel the message arrived on. + payload: The message body. + """ + try: + event = CompletionEvent.model_validate(json.loads(payload)) + except (ValueError, ValidationError) as error: + logger.warning(f"Ignoring a malformed chat signal on {channel}: {error}") + return + for subscription in tuple(self._subscriptions.get(event.request_id, ())): + subscription.deliver(event) + + async def _read_forever(self, redis_client: redis.asyncio.Redis) -> None: + """Read the pattern subscription until cancelled, subscribing again after any Redis error.""" + while True: + pubsub = redis_client.pubsub() + try: + await pubsub.psubscribe(CHAT_SIGNAL_CHANNEL_PATTERN) + self._subscribed.set() + logger.info(f"Subscribed to chat signals on {CHAT_SIGNAL_CHANNEL_PATTERN}") + await self._read_messages(pubsub) + except (redis.exceptions.RedisError, OSError) as error: + logger.warning(f"Chat signal subscription failed; requests fall back to polling until it returns: {error}") + finally: + self._subscribed.clear() + with contextlib.suppress(redis.exceptions.RedisError, OSError): + await pubsub.close() + await asyncio.sleep(LISTENER_RECONNECT_DELAY_SECONDS) + + async def _read_messages(self, pubsub: redis.asyncio.client.PubSub) -> None: + """Dispatch messages from ``pubsub`` until it errors, stops answering or is no longer subscribed. + + A connection a network fault black-holes delivers nothing and raises nothing, the same as a quiet one. After + ``liveness_seconds`` with no message a PING is sent; a PONG carrying its token that does not arrive within + ``liveness_seconds`` more marks the listener unsubscribed and raises, so the caller subscribes again. + + Raises: + redis.exceptions.ConnectionError: The PING went unanswered, or the subscription ended. + """ + read_timeout_seconds = min(LISTENER_READ_TIMEOUT_SECONDS, self._liveness_seconds) + last_message_at = time.monotonic() + ping_token: str | None = None + ping_sent_at = -math.inf + while pubsub.subscribed: + message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=read_timeout_seconds) + now = time.monotonic() + if message is not None: + last_message_at = now + if message is not None and message.get("type") == PONG_MESSAGE_TYPE: + if _decode_text(message.get("data")) == ping_token: + ping_token = None + continue + if ping_token is not None and now - ping_sent_at >= self._liveness_seconds: + self._subscribed.clear() + raise redis.exceptions.ConnectionError( + f"The chat signal subscription did not answer a PING within {self._liveness_seconds:g}s.", + ) + if ping_token is None and now - last_message_at >= self._liveness_seconds: + ping_token = uuid.uuid4().hex + ping_sent_at = now + await pubsub.ping(ping_token) + if message is None: + continue + channel = message.get("channel") + channel_name = channel.decode("utf-8", "replace") if isinstance(channel, bytes) else str(channel) + self.dispatch(channel_name, message.get("data", b"")) + raise redis.exceptions.ConnectionError("The chat signal subscription ended.") + + +class ChatRequestWatcher: + """Follow one queued Horde chat request until it finishes, fails fast or outlives its budget. + + Both response modes wait through this class; the streamed mode's heartbeat reads + [`latest_status`][horde.oai.waiter.ChatRequestWatcher.latest_status]. + """ + + def __init__( + self, + *, + horde_client: HordeClient, + credentials: HordeCredentials, + subscription: CompletionSubscription, + budget: WaitBudget, + clock: Callable[[], float] = time.monotonic, + random_fraction: Callable[[], float] = random.random, + ) -> None: + """Create the watcher. + + Args: + horde_client: The Horde client. + credentials: Who the status calls are made for. + subscription: The request's registration with the completion listener. + budget: The deadline and the intervals of the wait. + clock: The monotonic clock the deadline is measured on. + random_fraction: Return a number in ``[0, 1)`` for the poll jitter. + """ + self.request_id: str = subscription.request_id + """The Horde request id.""" + self.latest_status: HordeChatStatus | None = None + """The last status read, or None before the first successful read.""" + self.woke_by: WakeSource | None = None + """What woke the request when it was found finished.""" + self.woke_at: float | None = None + """The wall-clock time the request was found finished, in Unix seconds.""" + self._horde_client = horde_client + self._credentials = credentials + self._subscription = subscription + self._budget = budget + self._impossible_since: float | None = None + self._faulted_signal_seen = False + self._clock = clock + self._random_fraction = random_fraction + self._next_poll_at: float = clock() + self._recheck_at: float | None = None + self._recheck_earned = False + self._last_read_at: float | None = None + self._fail_fast_retry_at: float = -math.inf + + async def wait_until_finished(self) -> HordeChatStatus: + """Return the request's terminal status once it is done or faulted. + + A status observed terminal at or after the deadline is returned, never raised. A request whose ``faulted`` signal + is followed by a 404 status read expired on the Horde and is returned as faulted. + + Raises: + ChatWaitTimeoutError: The request was still waiting after the deadline and the final status read. + NoEligibleWorkerError: No eligible worker, with nothing processing, for longer than the grace. + HordeAPIError: A status read before the deadline failed in a way waiting cannot fix, such as 404. + """ + status = await self._read_before_deadline(WakeSource.IMMEDIATE) + while True: + if status is not None and _is_finished(status): + return status + if status is not None: + self._check_fail_fast(status) + now = self._clock() + if now >= self._budget.deadline: + break + event = await self._subscription.wait(self._next_wake_at() - now) + if event is not None: + if event.event in FINISHING_EVENT_KINDS: + self._recheck_earned = True + if event.event == CompletionEventKind.FAULTED: + self._faulted_signal_seen = True + status = await self._read_before_deadline(WakeSource.SIGNAL) + continue + now = self._clock() + if now >= self._budget.deadline: + break + if self._recheck_at is not None and now >= self._recheck_at: + self._recheck_at = None + status = await self._read_before_deadline(WakeSource.RECHECK) + elif now >= self._next_poll_at or now >= self._fail_fast_read_due_at(): + status = await self._read_before_deadline(WakeSource.POLL) + return await self._read_final_status() + + def _next_wake_at(self) -> float: + """Return the ``clock`` time of the earliest scheduled wake.""" + return min( + self._next_poll_at, + self._budget.deadline, + self._recheck_at if self._recheck_at is not None else math.inf, + self._fail_fast_read_due_at(), + ) + + def _fail_fast_read_due_at(self) -> float: + """Return the ``clock`` time of the read that decides fail-fast, or infinity once a read after the grace ran. + + One read after the grace is enough: later reads are ordinary polls, so failing reads cannot spin the loop. + """ + fail_fast_due_at = self._fail_fast_due_at() + if self._last_read_at is not None and self._last_read_at >= fail_fast_due_at: + return math.inf + return max(fail_fast_due_at, self._fail_fast_retry_at) + + def _fail_fast_due_at(self) -> float: + """Return the ``clock`` time the fail-fast grace ends, or infinity while a worker is eligible.""" + if self._impossible_since is None: + return math.inf + return self._impossible_since + self._budget.fail_fast_grace_seconds + + def _check_fail_fast(self, status: HordeChatStatus) -> None: + """Mutate the fail-fast clock from ``status`` and raise once the grace has passed. + + Raises: + NoEligibleWorkerError: No eligible worker and nothing processing for at least the grace. + """ + if status.is_possible or status.processing > 0: + self._impossible_since = None + return + now = self._clock() + if self._impossible_since is None: + self._impossible_since = now + if now >= self._fail_fast_due_at(): + raise NoEligibleWorkerError( + f"No worker is eligible for chat request {self.request_id} and none is processing it.", + status=status, + ) + + async def _read_before_deadline(self, wake_source: WakeSource) -> HordeChatStatus | None: + """Return a status read bounded by the time left before the deadline, or None when the read did not finish. + + A read that finishes is returned whatever the clock shows by then. + + Raises: + HordeAPIError: The Horde answered with a status waiting cannot fix. + """ + remaining = self._budget.deadline - self._clock() + if remaining <= 0: + return None + try: + async with asyncio.timeout(remaining): + status = await self._refresh(wake_source) + except TimeoutError: + status = None + if status is None: + # Retry the deciding read after a bounded pause; stale observations cannot cancel live work. + self._fail_fast_retry_at = self._clock() + self._budget.signal_recheck_seconds + return status + + async def _read_final_status(self) -> HordeChatStatus: + """Return the terminal status from one read under the final allowance. + + Raises: + ChatWaitTimeoutError: The read failed, did not finish in its allowance, or showed the request waiting. + """ + status: HordeChatStatus | None = None + try: + async with asyncio.timeout(self._budget.final_status_timeout_seconds): + status = await self._refresh(WakeSource.DEADLINE) + except TimeoutError: + logger.warning(f"The final status read of chat request {self.request_id} did not finish in its allowance") + except HordeAPIError as error: + logger.warning(f"The final status read of chat request {self.request_id} failed: {error}") + if status is not None and _is_finished(status): + return status + raise ChatWaitTimeoutError(f"Chat request {self.request_id} was still waiting at its deadline.") + + async def _refresh(self, wake_source: WakeSource) -> HordeChatStatus | None: + """Return a freshly read status, or None when the read failed in a way the next poll may fix. + + Raises: + HordeAPIError: The Horde answered with a status waiting cannot fix. + """ + read_started_at = self._clock() + self._next_poll_at = read_started_at + jitter_poll_seconds(self._budget.fallback_poll_seconds, self._random_fraction) + try: + status = await self._horde_client.get_chat_status(request_id=self.request_id, credentials=self._credentials) + except HordeAPIError as error: + if error.status_code == HTTPStatus.NOT_FOUND and self.latest_status is not None and not self._faulted_signal_seen: + await self._reconcile_missing_status() + if error.status_code == HTTPStatus.NOT_FOUND and self._faulted_signal_seen: + # The cleanup thread deletes an expired request and then publishes faulted, so the read finds no row. + logger.info(f"Chat request {self.request_id} expired: faulted signal and no status row") + status = HordeChatStatus(faulted=True) + self.latest_status = status + self.woke_by = wake_source + self.woke_at = time.time() + return status + if error.status_code not in TRANSIENT_STATUS_CODES: + raise + logger.warning(f"Status of chat request {self.request_id} unavailable, retrying at the next poll: {error}") + return None + except (HordeUnavailableError, httpx.TransportError) as error: + logger.warning(f"Status of chat request {self.request_id} unreachable, retrying at the next poll: {error}") + return None + self._last_read_at = read_started_at + self.latest_status = status + stretched_poll_seconds = compute_fallback_poll_seconds(base_seconds=self._budget.fallback_poll_seconds, wait_time=status.wait_time) + self._next_poll_at = read_started_at + jitter_poll_seconds(stretched_poll_seconds, self._random_fraction) + if _is_finished(status): + self.woke_by = wake_source + self.woke_at = time.time() + elif self._recheck_earned: + self._recheck_earned = False + self._recheck_at = self._clock() + self._budget.signal_recheck_seconds + return status + + async def _reconcile_missing_status(self) -> None: + """Wait once for expiry's post-commit signal after a previously visible row disappears. + + Storage cleanup happens between deletion and publication. An initial unknown id stays a 404, and a missing + row without a fault signal stays a 404 after this bounded allowance. The enclosing read budget still applies. + Other events taken from the subscription while waiting are queued again in their order, so the main loop + still sees them. + """ + reconcile_until = self._clock() + self._budget.signal_recheck_seconds + other_events: list[CompletionEvent] = [] + try: + while not self._faulted_signal_seen: + remaining = reconcile_until - self._clock() + if remaining <= 0: + return + event = await self._subscription.wait(remaining) + if event is None: + return + if event.event == CompletionEventKind.FAULTED: + self._faulted_signal_seen = True + else: + other_events.append(event) + finally: + self._subscription.requeue(other_events) + + +def compute_fallback_poll_seconds(*, base_seconds: float, wait_time: int | None) -> float: + """Return the fallback poll interval for a request whose latest status estimates ``wait_time`` seconds. + + The interval is ``max(base_seconds, min(MAX_STRETCHED_POLL_SECONDS, wait_time / WAIT_TIME_POLL_DIVISOR))``: never + shorter than the configured poll, and stretched for a request the Horde expects to wait long. + + Args: + base_seconds: The configured fallback poll interval. + wait_time: The Horde's wait estimate in seconds, or None when it reported none. + """ + if wait_time is None or wait_time <= 0: + return base_seconds + return max(base_seconds, min(MAX_STRETCHED_POLL_SECONDS, wait_time / WAIT_TIME_POLL_DIVISOR)) + + +def jitter_poll_seconds(poll_seconds: float, random_fraction: Callable[[], float] = random.random) -> float: + """Return ``poll_seconds`` shortened by up to [`POLL_JITTER_FRACTION`][horde.oai.waiter.POLL_JITTER_FRACTION]. + + Args: + poll_seconds: The interval before jitter. + random_fraction: Return a number in ``[0, 1)``. + """ + return poll_seconds * (1 - POLL_JITTER_FRACTION * random_fraction()) + + +def _decode_text(text: object) -> str: + """Return a pub/sub field as text, whether the client decodes responses or not.""" + return text.decode("utf-8", "replace") if isinstance(text, bytes) else str(text) + + +def _is_finished(status: HordeChatStatus) -> bool: + """Return whether ``status`` is terminal.""" + return status.done or status.faulted diff --git a/pyproject.toml b/pyproject.toml index 4f9c1890..ed0969ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,20 @@ dependencies = [ "logfire[flask,sqlalchemy,redis,requests]==4.32.1", "chardet<6", "pycurl>=7.45.7,<8", + # The OpenAI-compatible chat front (horde/oai) is an ASGI app, so one process holds thousands of waiting requests and streams. + "fastapi", + # Serves the horde/oai ASGI app. + "uvicorn", + # The horde/oai front calls the Horde API asynchronously through it. + "httpx", + # Server-sent events for the horde/oai streaming responses. + "sse-starlette", + # Reads the horde/oai front's HORDE_OAI_* settings into a validated model. + "pydantic-settings", + # httpcore tries to import sniffio on every request to detect the async library. Current anyio no longer + # depends on sniffio, and a failed import is never cached, so without it every Horde call from the front + # walked the whole import path and cost more CPU than the request itself. + "sniffio", ] license = {file = "LICENSE"} classifiers = [ @@ -93,6 +107,10 @@ dev = [ "pre-commit", "locust>=2.43.4", "types-python-dateutil>=2.9.0.20260716", + # Mocks the Horde API under the horde/oai front tests. + "respx", + # Official OpenAI client; the chat front's conformance tests drive it against the front. + "openai>=1.109,<2", # Issues a throwaway CA and server certificate for the loopback HTTPS outbound tests. "trustme", ] diff --git a/tests/unit/oai/__init__.py b/tests/unit/oai/__init__.py new file mode 100644 index 00000000..ba0b6ebe --- /dev/null +++ b/tests/unit/oai/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the OpenAI-compatible chat front, with the Horde mocked over HTTP and Redis faked in process.""" diff --git a/tests/unit/oai/conftest.py b/tests/unit/oai/conftest.py new file mode 100644 index 00000000..1d4ef987 --- /dev/null +++ b/tests/unit/oai/conftest.py @@ -0,0 +1,349 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Fixtures running the chat front in process against a mocked Horde and an in-process fake Redis. + +[`FakeHorde`][tests.unit.oai.conftest.FakeHorde] answers the Horde routes the front calls through ``respx`` and lets a +test change the status a request reports and publish its completion signal. The front is driven through +``httpx.ASGITransport``, which runs the app to completion before returning, so a streamed response arrives whole. +[`call_raw`][tests.unit.oai.conftest.call_raw] drives the app over raw ASGI where a test controls the body's arrival or +the client's disconnect. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import AsyncIterator, Callable, MutableMapping, Sequence +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import Any + +import fakeredis +import httpx +import pytest_asyncio +import respx +from fastapi import FastAPI + +from horde.oai.app import create_app +from horde.oai.settings import FrontSettings +from horde.oai.waiter import CHAT_SIGNAL_CHANNEL_PREFIX + +HORDE_URL: str = "http://horde.test/api" +"""The Horde API root the mocked routes live under.""" + +FRONT_URL: str = "http://front.test" +"""The base URL the test client addresses the front with.""" + +REQUEST_ID: str = "00000000-0000-4000-8000-000000000001" +"""The Horde request id every mocked submit returns.""" + +MODEL_NAME: str = "koboldcpp/LLaMA2-13B-Tiefighter" +"""The model every test request asks for.""" + +GENERATED_TEXT: str = "Hello from the horde." +"""The text the mocked worker generated.""" + +CHARGE_BASIS: str = "output_estimate" +"""The charge basis the mocked Horde reports; the front passes it through unchanged.""" + +SUBSCRIBE_TIMEOUT_SECONDS: float = 2.0 +"""How long a test waits for the front's signal listener to subscribe before failing.""" + +CALL_WAIT_SECONDS: float = 2.0 +"""How long a test waits for an expected Horde call or application exit.""" + +RAW_CLIENT_ADDRESS: tuple[str, int] = ("203.0.113.9", 50000) +"""The client address raw ASGI calls come from.""" + +FAST_TEST_SETTINGS: FrontSettings = FrontSettings( + horde_urls=(HORDE_URL,), + max_inflight=50, + max_inflight_per_source=50, + nonstream_timeout_seconds=5.0, + stream_timeout_seconds=5.0, + final_status_timeout_seconds=0.5, + cancel_timeout_seconds=1.0, + shutdown_cancel_seconds=1.0, + heartbeat_seconds=0.05, + fallback_poll_seconds=10.0, + signal_recheck_seconds=0.1, + fail_fast_grace_seconds=5.0, +) +"""Settings with a fallback poll long enough that a test finishing quickly proves another wake-up path.""" + + +def build_chat_body(**overrides: Any) -> dict[str, Any]: + """Return a minimal valid chat request body with ``overrides`` applied.""" + chat_body: dict[str, Any] = { + "model": MODEL_NAME, + "messages": [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Say hello."}, + ], + "max_tokens": 64, + } + chat_body.update(overrides) + return chat_body + + +def waiting_status(*, queue_position: int = 3, wait_time: int = 20, is_possible: bool = True, processing: int = 0) -> dict[str, Any]: + """Return the native status of a request still waiting for a worker.""" + return { + "done": False, + "faulted": False, + "finished": 0, + "processing": processing, + "waiting": 1 - processing, + "restarted": 0, + "queue_position": queue_position, + "wait_time": wait_time, + "kudos": 0, + "is_possible": is_possible, + "generations": [], + } + + +def done_generation( + *, + content: str = GENERATED_TEXT, + finish_reason: str | None = "length", + native_finish_reason: str | None = "length", + prompt_tokens: int | None = 21, + completion_tokens: int | None = 6, +) -> dict[str, Any]: + """Return one finished generation in the native chat status shape.""" + return { + "text": content, + "message": {"role": "assistant", "content": content}, + "model": MODEL_NAME, + "worker_id": "worker-1", + "worker_name": "Test Worker", + "state": "ok", + "seed": 7, + "finish_reason": finish_reason, + "native_finish_reason": native_finish_reason, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "charge_basis": CHARGE_BASIS, + "queue_wait_seconds": 1.5, + } + + +def done_status(**generation_overrides: Any) -> dict[str, Any]: + """Return the native status of a request whose one generation is finished.""" + return { + "done": True, + "faulted": False, + "finished": 1, + "processing": 0, + "waiting": 0, + "restarted": 0, + "queue_position": 0, + "wait_time": 0, + "kudos": 12.5, + "is_possible": True, + "generations": [done_generation(**generation_overrides)], + } + + +class FakeHorde: + """A mocked Horde whose status for the one request a test changes as it goes.""" + + def __init__(self, router: respx.MockRouter, redis_client: fakeredis.FakeAsyncRedis) -> None: + """Mock the Horde routes on ``router``; signals are published on ``redis_client``.""" + self.status_payload: dict[str, Any] = waiting_status() + """The body the status route returns.""" + self.status_code: int = 200 + """The status code the status route returns.""" + self._redis_client = redis_client + status_path = f"{HORDE_URL}/v2/generate/chat/status/{REQUEST_ID}" + self.submit_route: respx.Route = router.post(f"{HORDE_URL}/v2/generate/chat/async").mock( + return_value=httpx.Response(202, json={"id": REQUEST_ID, "kudos": 12.5}), + ) + """The submit route.""" + self.status_route: respx.Route = router.get(status_path).mock(side_effect=self._answer_status) + """The status route.""" + self.cancel_gate: asyncio.Event | None = None + """When set, the cancel route answers only once this event is set.""" + self.cancels_received: int = 0 + """How many cancels reached the cancel route, answered or not.""" + self.on_cancel: Callable[[], None] | None = None + """Called when a cancel arrives, before it is answered.""" + self.cancel_route: respx.Route = router.delete(status_path).mock(side_effect=self._answer_cancel) + """The cancel route.""" + self.models_route: respx.Route = router.get(f"{HORDE_URL}/v2/status/models").mock( + return_value=httpx.Response(200, json=[]), + ) + """The model list route.""" + + def _answer_status(self, _request: httpx.Request) -> httpx.Response: + """Return the current status response.""" + return httpx.Response(self.status_code, json=self.status_payload) + + async def _answer_cancel(self, request: httpx.Request) -> httpx.Response: + """Return the current status response, after ``cancel_gate`` opens when one is set.""" + self.cancels_received += 1 + if self.on_cancel is not None: + self.on_cancel() + if self.cancel_gate is not None: + await self.cancel_gate.wait() + return self._answer_status(request) + + def submitted_body(self) -> dict[str, Any]: + """Return the JSON body of the last submit.""" + submitted: dict[str, Any] = json.loads(self.submit_route.calls.last.request.content) + return submitted + + async def publish(self, event: str) -> None: + """Publish a completion signal for the request, in the native body shape.""" + await self._redis_client.publish( + f"{CHAT_SIGNAL_CHANNEL_PREFIX}{REQUEST_ID}", + json.dumps({"id": REQUEST_ID, "event": event, "generations": 1}), + ) + + async def finish_later(self, delay_seconds: float, *, signal: bool = True, **generation_overrides: Any) -> None: + """Mark the request done after ``delay_seconds`` and publish ``done`` when ``signal`` is set.""" + await asyncio.sleep(delay_seconds) + self.status_payload = done_status(**generation_overrides) + if signal: + await self.publish("done") + + +class RunningFront: + """A started front with its test client and mocked Horde.""" + + def __init__(self, *, app: FastAPI, client: httpx.AsyncClient, horde: FakeHorde) -> None: + """Bundle the running pieces.""" + self.app: FastAPI = app + """The front application.""" + self.client: httpx.AsyncClient = client + """A client addressing the front through ASGI.""" + self.horde: FakeHorde = horde + """The mocked Horde.""" + + @property + def inflight(self) -> int: + """Return the front's open request count.""" + inflight: int = self.app.state.front.inflight_limiter.inflight + return inflight + + +StartFront = Callable[..., AbstractAsyncContextManager[RunningFront]] +"""Start a front with optional settings and Redis overrides; see the ``start_front`` fixture.""" + + +@pytest_asyncio.fixture +async def start_front() -> AsyncIterator[StartFront]: + """Return a context manager factory that runs a front against a mocked Horde for one test.""" + + @asynccontextmanager + async def run_front(settings: FrontSettings = FAST_TEST_SETTINGS, *, with_redis: bool = True) -> AsyncIterator[RunningFront]: + redis_client = fakeredis.FakeAsyncRedis() + with respx.mock(assert_all_called=False) as router: + horde = FakeHorde(router, redis_client) + async with httpx.AsyncClient() as horde_http_client: + app = create_app( + settings, + http_client=horde_http_client, + cancel_http_client=horde_http_client, + redis_client=redis_client if with_redis else None, + ) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=FRONT_URL) as front_client: + async with app.router.lifespan_context(app): + if with_redis: + assert await app.state.front.listener.wait_until_subscribed(SUBSCRIBE_TIMEOUT_SECONDS) + yield RunningFront(app=app, client=front_client, horde=horde) + await redis_client.close() + + yield run_front + + +async def wait_for_call(call_count: Callable[[], int], *, minimum: int = 1) -> None: + """Return once ``call_count()`` reaches ``minimum``, failing after ``CALL_WAIT_SECONDS``.""" + deadline = time.monotonic() + CALL_WAIT_SECONDS + while call_count() < minimum: + assert time.monotonic() < deadline, "The expected Horde call did not happen." + await asyncio.sleep(0.01) + + +async def call_raw( + app: FastAPI, + *, + body_chunks: Sequence[bytes], + chunk_delay_seconds: float = 0, + disconnect_after_seconds: float | None = None, + response_start_delay_seconds: float = 0, + headers: Sequence[tuple[bytes, bytes]] = (), + wait_for_app: bool = True, +) -> tuple[asyncio.Task[None], list[dict[str, Any]]]: + """Send ``POST /v1/chat/completions`` over raw ASGI. + + Args: + app: The front application. + body_chunks: The body, sent as these chunks. + chunk_delay_seconds: The pause before each chunk. + disconnect_after_seconds: When set, the client disconnects this long after the call starts. + response_start_delay_seconds: How long sending the response start takes, which holds a streamed response + before its event generator first runs. + headers: Extra request headers. + wait_for_app: Whether to wait for the application to finish handling the request before returning. + + Returns: + The task running the application and the ASGI messages it sent. + """ + pending_chunks = list(body_chunks) + disconnected = asyncio.Event() + sent_messages: list[dict[str, Any]] = [] + + async def receive() -> dict[str, Any]: + if pending_chunks: + await asyncio.sleep(chunk_delay_seconds) + body_chunk = pending_chunks.pop(0) + return {"type": "http.request", "body": body_chunk, "more_body": bool(pending_chunks)} + await disconnected.wait() + return {"type": "http.disconnect"} + + async def send(message: MutableMapping[str, Any]) -> None: + sent_messages.append(dict(message)) + if message["type"] == "http.response.start" and response_start_delay_seconds > 0: + await asyncio.sleep(response_start_delay_seconds) + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"host", b"front.test"), *headers], + "client": RAW_CLIENT_ADDRESS, + "server": ("front.test", 80), + "state": {}, + } + app_task = asyncio.create_task(app(scope, receive, send)) + if disconnect_after_seconds is not None: + await asyncio.sleep(disconnect_after_seconds) + disconnected.set() + if wait_for_app: + await asyncio.wait_for(app_task, timeout=CALL_WAIT_SECONDS) + return app_task, sent_messages + + +def response_status(sent_messages: Sequence[dict[str, Any]]) -> int | None: + """Return the status of a raw ASGI response, or None when no response started.""" + for message in sent_messages: + if message["type"] == "http.response.start": + status: int = message["status"] + return status + return None + + +def response_body(sent_messages: Sequence[dict[str, Any]]) -> bytes: + """Return the body of a raw ASGI response.""" + return b"".join(message.get("body", b"") for message in sent_messages if message["type"] == "http.response.body") diff --git a/tests/unit/oai/test_admission_and_table.py b/tests/unit/oai/test_admission_and_table.py new file mode 100644 index 00000000..9b55a2f5 --- /dev/null +++ b/tests/unit/oai/test_admission_and_table.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's synchronous building blocks: the no-op table's coverage and the inflight limiter.""" + +from __future__ import annotations + +from horde.oai.app import InflightLimiter +from horde.oai.schema import DEFAULT_NO_OP_PARAMETERS +from tests.unit.oai.test_request_screening import NO_OP_ROWS + + +def test_no_op_rows_cover_every_key_of_the_table() -> None: + assert {parameter for parameter, _value in NO_OP_ROWS} == set(DEFAULT_NO_OP_PARAMETERS) + + +def test_per_source_cap_refuses_before_the_global_cap() -> None: + limiter = InflightLimiter(max_inflight=2, max_inflight_per_source=1) + first_slot = limiter.try_acquire("198.51.100.1") + + assert first_slot is not None + assert limiter.has_capacity + assert limiter.try_acquire("198.51.100.1") is None + second_slot = limiter.try_acquire("198.51.100.2") + assert second_slot is not None + assert limiter.try_acquire("198.51.100.3") is None + + first_slot.release() + first_slot.release() + assert limiter.inflight == 1 + assert limiter.inflight_for("198.51.100.1") == 0 + assert limiter.try_acquire("198.51.100.1") is not None diff --git a/tests/unit/oai/test_chat_front.py b/tests/unit/oai/test_chat_front.py new file mode 100644 index 00000000..8a9d6f42 --- /dev/null +++ b/tests/unit/oai/test_chat_front.py @@ -0,0 +1,371 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's successful paths: completions, streams, models, health and the Horde call headers.""" + +from __future__ import annotations + +import asyncio +import json +import time + +import pytest + +from horde.consts import ANONYMOUS_API_KEY +from tests.unit.oai.conftest import ( + CHARGE_BASIS, + FAST_TEST_SETTINGS, + GENERATED_TEXT, + HORDE_URL, + MODEL_NAME, + REQUEST_ID, + StartFront, + build_chat_body, + done_status, + waiting_status, +) + +pytestmark = pytest.mark.asyncio + +COMPLETION_ID: str = f"chatcmpl-{REQUEST_ID}" +"""The completion id the front derives from the mocked request id.""" + +QUICK_FINISH_SECONDS: float = 2.0 +"""An upper bound well below ``FAST_TEST_SETTINGS.fallback_poll_seconds``: finishing within it proves the request did +not wait for the fallback poll.""" + + +def parse_sse_events(body: str) -> list[str]: + """Return the raw events of a server-sent event body, comments included, whichever line separator it uses.""" + normalized_body = body.replace("\r\n", "\n") + return [raw_event for raw_event in normalized_body.split("\n\n") if raw_event.strip()] + + +def data_payloads(events: list[str]) -> list[str]: + """Return the ``data:`` payloads of ``events``, in order.""" + return [raw_event.removeprefix("data: ") for raw_event in events if raw_event.startswith("data: ")] + + +async def test_non_streamed_request_returns_the_completion_when_the_signal_arrives(start_front: StartFront) -> None: + async with start_front() as front: + finisher = asyncio.create_task(front.horde.finish_later(0.1)) + started_at = time.monotonic() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + elapsed = time.monotonic() - started_at + await finisher + + assert response.status_code == 200 + assert elapsed < QUICK_FINISH_SECONDS + assert "x-should-retry" not in response.headers + assert response.json() == { + "id": COMPLETION_ID, + "object": "chat.completion", + "created": response.json()["created"], + "model": MODEL_NAME, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": GENERATED_TEXT}, + "finish_reason": "length", + "native_finish_reason": "length", + "logprobs": None, + }, + ], + "usage": {"prompt_tokens": 21, "completion_tokens": 6, "total_tokens": 27}, + "x_horde": { + "request_id": REQUEST_ID, + "worker_id": "worker-1", + "worker_name": "Test Worker", + "kudos": 12.5, + "charge_basis": CHARGE_BASIS, + "estimated_usage": False, + "queue_wait_seconds": 1.5, + }, + } + + +async def test_native_body_carries_the_request_without_front_only_keys(start_front: StartFront) -> None: + chat_body = build_chat_body( + messages=[ + {"role": "developer", "content": "Answer in French."}, + {"role": "user", "content": [{"type": "text", "text": "Say hello."}], "name": "alice"}, + ], + temperature=None, + user="end-user-7", + metadata={"trace": "abc"}, + horde={"only": ["worker-1"], "trusted_workers": True}, + ) + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=chat_body) + + assert response.status_code == 200 + assert front.horde.submitted_body() == { + "model": MODEL_NAME, + "messages": [ + {"role": "developer", "content": "Answer in French."}, + {"role": "user", "content": [{"type": "text", "text": "Say hello."}], "name": "alice"}, + ], + "max_tokens": 64, + "horde": {"only": ["worker-1"], "trusted_workers": True}, + } + + +async def test_non_streamed_request_returns_at_once_when_the_status_is_already_done(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + started_at = time.monotonic() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + elapsed = time.monotonic() - started_at + + assert response.status_code == 200 + assert elapsed < QUICK_FINISH_SECONDS + assert front.horde.status_route.call_count == 1 + + +async def test_signal_before_the_outcome_is_visible_earns_one_short_recheck(start_front: StartFront) -> None: + async def signal_then_settle() -> None: + await asyncio.sleep(0.1) + await front.horde.publish("done") + await asyncio.sleep(0.05) + front.horde.status_payload = done_status() + + async with start_front() as front: + settler = asyncio.create_task(signal_then_settle()) + started_at = time.monotonic() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + elapsed = time.monotonic() - started_at + await settler + + assert response.status_code == 200 + assert elapsed < QUICK_FINISH_SECONDS + assert front.horde.status_route.call_count == 3 + + +async def test_polling_without_reported_counts_estimates_usage(start_front: StartFront) -> None: + polling_settings = FAST_TEST_SETTINGS.model_copy(update={"fallback_poll_seconds": 0.05}) + async with start_front(polling_settings, with_redis=False) as front: + front.horde.status_payload = waiting_status(wait_time=0) + finisher = asyncio.create_task( + front.horde.finish_later( + 0.2, + signal=False, + finish_reason=None, + native_finish_reason=None, + prompt_tokens=None, + completion_tokens=None, + ), + ) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + await finisher + + assert response.status_code == 200 + completion = response.json() + assert completion["choices"][0]["finish_reason"] == "stop" + assert completion["choices"][0]["native_finish_reason"] is None + assert completion["x_horde"]["estimated_usage"] is True + assert completion["usage"]["completion_tokens"] == 7 + assert completion["usage"]["total_tokens"] == completion["usage"]["prompt_tokens"] + 7 + assert front.horde.status_route.call_count >= 2 + + +async def test_message_content_is_preferred_over_the_plain_text_field(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + front.horde.status_payload["generations"][0]["text"] = "flattened" + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.json()["choices"][0]["message"]["content"] == GENERATED_TEXT + + +async def test_plain_text_field_is_used_when_the_generation_has_no_message(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + del front.horde.status_payload["generations"][0]["message"] + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.json()["choices"][0]["message"]["content"] == GENERATED_TEXT + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_empty_return_carries_the_notice_with_the_request_id(start_front: StartFront, stream: bool) -> None: + async with start_front() as front: + front.horde.status_payload = done_status(content="") + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=stream)) + + assert response.status_code == 200 + if stream: + chunks = [json.loads(payload) for payload in data_payloads(parse_sse_events(response.text))[:-1]] + x_horde = chunks[-1]["x_horde"] + else: + x_horde = response.json()["x_horde"] + assert REQUEST_ID in x_horde["notice"] + assert "no visible text" in x_horde["notice"] + + +async def test_non_empty_return_carries_no_notice(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert "notice" not in response.json()["x_horde"] + + +@pytest.mark.parametrize("include_usage", [True, False]) +async def test_streamed_request_sends_keepalives_then_the_completion_chunks(start_front: StartFront, include_usage: bool) -> None: + chat_body = build_chat_body(stream=True, stream_options={"include_usage": include_usage}) + async with start_front() as front: + finisher = asyncio.create_task(front.horde.finish_later(0.3)) + response = await front.client.post("/v1/chat/completions", json=chat_body) + await finisher + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.headers["cache-control"] == "no-cache" + assert response.headers["x-accel-buffering"] == "no" + assert response.headers["x-should-retry"] == "false" + events = parse_sse_events(response.text) + assert ": horde queue_position=3 wait_time=20" in events + payloads = data_payloads(events) + assert payloads[-1] == "[DONE]" + chunks = [json.loads(payload) for payload in payloads[:-1]] + assert all(chunk["id"] == COMPLETION_ID and chunk["object"] == "chat.completion.chunk" for chunk in chunks) + assert chunks[0]["choices"] == [ + {"index": 0, "delta": {"role": "assistant"}, "finish_reason": None, "native_finish_reason": None, "logprobs": None}, + ] + assert chunks[1]["choices"] == [ + {"index": 0, "delta": {"content": GENERATED_TEXT}, "finish_reason": None, "native_finish_reason": None, "logprobs": None}, + ] + assert chunks[2]["choices"] == [ + {"index": 0, "delta": {}, "finish_reason": "length", "native_finish_reason": "length", "logprobs": None}, + ] + assert chunks[2]["x_horde"]["request_id"] == REQUEST_ID + assert chunks[2]["x_horde"]["charge_basis"] == CHARGE_BASIS + if include_usage: + assert all(chunk["usage"] is None for chunk in chunks[:3]) + assert len(chunks) == 4 + assert chunks[3]["choices"] == [] + assert chunks[3]["usage"] == {"prompt_tokens": 21, "completion_tokens": 6, "total_tokens": 27} + else: + assert all("usage" not in chunk for chunk in chunks) + assert len(chunks) == 3 + keepalive_positions = [index for index, raw_event in enumerate(events) if raw_event.startswith(":")] + first_data_position = next(index for index, raw_event in enumerate(events) if raw_event.startswith("data: ")) + assert min(keepalive_positions) < first_data_position + + +@pytest.mark.parametrize("state", ["censored", "csam"]) +@pytest.mark.parametrize("stream", [False, True]) +async def test_filtered_generations_report_content_filter(start_front: StartFront, state: str, stream: bool) -> None: + async with start_front() as front: + front.horde.status_payload = done_status(finish_reason="length") + front.horde.status_payload["generations"][0]["state"] = state + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=stream)) + if stream: + chunks = [json.loads(payload) for payload in data_payloads(parse_sse_events(response.text))[:-1]] + assert chunks[-1]["choices"][0]["finish_reason"] == "content_filter" + else: + assert response.json()["choices"][0]["finish_reason"] == "content_filter" + + +@pytest.mark.parametrize( + ("authorization", "expected_apikey"), + [ + ("Bearer my-horde-key", "my-horde-key"), + ("bearer my-horde-key", "my-horde-key"), + (None, ANONYMOUS_API_KEY), + ], +) +async def test_bearer_token_becomes_the_horde_apikey_and_a_missing_one_is_anonymous( + start_front: StartFront, + authorization: str | None, + expected_apikey: str, +) -> None: + headers = {"User-Agent": "test-client/1.0"} + if authorization is not None: + headers["Authorization"] = authorization + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=build_chat_body(), headers=headers) + + assert response.status_code == 200 + horde_headers = front.horde.submit_route.calls.last.request.headers + assert horde_headers["apikey"] == expected_apikey + assert horde_headers["client-agent"].startswith("horde-oai-front:") + assert horde_headers["client-agent"].endswith(":test-client/1.0") + assert "proxy-authorization" not in horde_headers + assert "proxied-for" not in horde_headers + + +async def test_proxy_passkey_forwards_the_client_address(start_front: StartFront) -> None: + passkey_settings = FAST_TEST_SETTINGS.model_copy(update={"proxy_passkey": "operator-passkey"}) + async with start_front(passkey_settings) as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 200 + for horde_call in (front.horde.submit_route.calls.last, front.horde.status_route.calls.last): + assert horde_call.request.headers["proxy-authorization"] == "operator-passkey" + assert horde_call.request.headers["proxied-for"] == "127.0.0.1" + + +MODEL_LIST: list[dict[str, object]] = [ + {"name": MODEL_NAME, "count": 3, "performance": 10.0, "queued": 7, "jobs": 0, "eta": 0, "type": "text"}, + {"name": "aphrodite/other-model", "count": 1, "type": "text"}, +] +"""The Horde's active text model list the model route tests serve.""" + + +async def test_models_lists_the_horde_text_models_in_the_openai_shape(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.models_route.respond(200, json=MODEL_LIST) + response = await front.client.get("/v1/models") + + assert response.status_code == 200 + text_architecture = {"modality": "text->text", "input_modalities": ["text"], "output_modalities": ["text"]} + assert response.json() == { + "object": "list", + "data": [ + { + "id": MODEL_NAME, + "object": "model", + "created": 0, + "owned_by": "aihorde", + "architecture": text_architecture, + "x_horde": {"workers": 3, "queued": 7}, + }, + { + "id": "aphrodite/other-model", + "object": "model", + "created": 0, + "owned_by": "aihorde", + "architecture": text_architecture, + "x_horde": {"workers": 1}, + }, + ], + } + assert front.horde.models_route.calls.last.request.url == f"{HORDE_URL}/v2/status/models?type=text" + + +async def test_model_route_resolves_an_id_containing_slashes(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.models_route.respond(200, json=MODEL_LIST) + found = await front.client.get(f"/v1/models/{MODEL_NAME}") + missing = await front.client.get("/v1/models/koboldcpp/Absent") + + assert found.status_code == 200 + assert found.json()["id"] == MODEL_NAME + assert missing.status_code == 404 + assert missing.json()["error"]["code"] == "model_not_found" + assert front.horde.models_route.call_count == 1 + + +async def test_health_reports_liveness_readiness_and_subscription_separately(start_front: StartFront) -> None: + async with start_front() as front: + subscribed = await front.client.get("/healthz") + async with start_front(with_redis=False) as front: + poll_only = await front.client.get("/healthz") + + assert subscribed.json() == {"live": True, "ready": True, "signal_subscribed": True, "inflight": 0} + assert poll_only.json() == {"live": True, "ready": True, "signal_subscribed": False, "inflight": 0} diff --git a/tests/unit/oai/test_errors.py b/tests/unit/oai/test_errors.py new file mode 100644 index 00000000..0e7d7928 --- /dev/null +++ b/tests/unit/oai/test_errors.py @@ -0,0 +1,426 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's error scenarios: status, headers, ``code``, ``param`` and ``metadata`` per scenario.""" + +from __future__ import annotations + +import asyncio +import json + +import httpx +import pytest +import respx + +from horde.oai.app import create_app +from horde.oai.settings import FrontSettings +from tests.unit.oai.conftest import ( + FAST_TEST_SETTINGS, + FRONT_URL, + HORDE_URL, + REQUEST_ID, + StartFront, + build_chat_body, + call_raw, + done_status, + response_body, + response_status, + wait_for_call, + waiting_status, +) +from tests.unit.oai.test_chat_front import data_payloads, parse_sse_events + +pytestmark = pytest.mark.asyncio + +SHOULD_RETRY: str = "x-should-retry" +"""The retry-suppression header.""" + + +def chunked_body(chat_body: dict[str, object], *, chunk_bytes: int = 64) -> list[bytes]: + """Return ``chat_body`` encoded and split into chunks with no declared length.""" + encoded_body = json.dumps(chat_body).encode() + return [encoded_body[offset : offset + chunk_bytes] for offset in range(0, len(encoded_body), chunk_bytes)] + + +async def test_body_over_the_declared_limit_is_413_before_parsing(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"max_body_bytes": 100})) as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 413 + assert response.json()["error"]["code"] == "RequestTooLarge" + assert SHOULD_RETRY not in response.headers + assert front.horde.submit_route.call_count == 0 + + +async def test_chunked_body_over_the_limit_is_413_while_received(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"max_body_bytes": 100})) as front: + _app_task, sent_messages = await call_raw(front.app, body_chunks=chunked_body(build_chat_body())) + + assert response_status(sent_messages) == 413 + assert json.loads(response_body(sent_messages))["error"]["code"] == "RequestTooLarge" + assert front.horde.submit_route.call_count == 0 + + +async def test_slow_body_is_408_after_the_read_timeout(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"body_read_timeout_seconds": 0.1})) as front: + _app_task, sent_messages = await call_raw( + front.app, + body_chunks=chunked_body(build_chat_body()), + chunk_delay_seconds=0.05, + ) + + assert response_status(sent_messages) == 408 + error = json.loads(response_body(sent_messages))["error"] + assert error["code"] == "RequestBodyTimeout" + assert error["type"] == "timeout_error" + assert all(header_name != SHOULD_RETRY.encode() for header_name, _ in sent_messages[0]["headers"]) + assert front.horde.submit_route.call_count == 0 + assert front.inflight == 0 + + +@pytest.mark.parametrize("raw_body", [b"{not json", b"[1, 2]"]) +async def test_invalid_json_or_a_non_object_is_a_bad_request(start_front: StartFront, raw_body: bytes) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", content=raw_body, headers={"Content-Type": "application/json"}) + + assert response.status_code == 400 + assert response.json() == { + "error": { + "message": response.json()["error"]["message"], + "type": "invalid_request_error", + "param": None, + "code": "BadRequest", + "metadata": {}, + }, + } + + +@pytest.mark.parametrize("authorization", ["Basic dXNlcjpwYXNz", "Bearer", "Bearer ", "Bearer two words"]) +async def test_malformed_authorization_is_401_not_anonymous(start_front: StartFront, authorization: str) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body(), headers={"Authorization": authorization}) + + assert response.status_code == 401 + assert response.json()["error"]["type"] == "authentication_error" + assert response.json()["error"]["code"] == "InvalidAuthorizationHeader" + assert front.horde.submit_route.call_count == 0 + + +@pytest.mark.parametrize( + ("horde_status", "horde_rc", "expected_type", "expected_code"), + [ + (401, "InvalidAPIKey", "authentication_error", "InvalidAPIKey"), + (402, "KudosUpfront", "api_error", "KudosUpfront"), + (403, "CorruptPrompt", "permission_error", "CorruptPrompt"), + (404, "RequestNotFound", "not_found_error", "RequestNotFound"), + ], +) +async def test_horde_refusal_at_submit_keeps_its_status_and_rc( + start_front: StartFront, + horde_status: int, + horde_rc: str, + expected_type: str, + expected_code: str, +) -> None: + async with start_front() as front: + front.horde.submit_route.respond(horde_status, json={"message": "Refused.", "rc": horde_rc}) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == horde_status + error = response.json()["error"] + assert error["type"] == expected_type + assert error["code"] == expected_code + assert error["metadata"] == {"rc": horde_rc} + assert SHOULD_RETRY not in response.headers + assert front.inflight == 0 + + +async def test_horde_rate_limit_passes_through_with_its_headers(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.submit_route.respond( + 429, + json={"message": "Too many prompts.", "rc": "TooManyPrompts"}, + headers={"Retry-After": "7", "X-RateLimit-Limit": "2", "X-RateLimit-Remaining": "0"}, + ) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 429 + assert response.headers["retry-after"] == "7" + assert response.headers["x-ratelimit-limit"] == "2" + assert response.headers["x-ratelimit-remaining"] == "0" + assert SHOULD_RETRY not in response.headers + assert response.json() == { + "error": { + "message": "Too many prompts.", + "type": "rate_limit_error", + "param": None, + "code": "TooManyPrompts", + "metadata": {"rc": "TooManyPrompts", "retry_after": 7}, + }, + } + + +async def test_chat_admission_disabled_is_503_with_retry_after(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.submit_route.respond(503, json={"message": "Chat is switched off.", "rc": "ChatAdmissionDisabled"}) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 503 + assert response.headers["retry-after"] == "30" + assert response.json()["error"]["code"] == "ChatAdmissionDisabled" + assert SHOULD_RETRY not in response.headers + + +async def test_horde_unreachable_before_send_is_a_retriable_502() -> None: + unreachable_settings = FrontSettings(horde_urls=(HORDE_URL, "http://horde-second.test/api")) + with respx.mock(assert_all_called=False) as router: + first_route = router.post(f"{HORDE_URL}/v2/generate/chat/async").mock(side_effect=httpx.ConnectError("refused")) + second_route = router.post("http://horde-second.test/api/v2/generate/chat/async").mock(side_effect=httpx.ConnectError("refused")) + async with httpx.AsyncClient() as horde_http_client: + app = create_app(unreachable_settings, http_client=horde_http_client, cancel_http_client=horde_http_client) + async with app.router.lifespan_context(app): + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=FRONT_URL) as front_client: + response = await front_client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "HordeUnavailable" + assert response.json()["error"]["metadata"] == {} + assert SHOULD_RETRY not in response.headers + assert first_route.call_count == 1 + assert second_route.call_count == 1 + + +@pytest.mark.parametrize("transport_error", [httpx.ReadTimeout("slow"), httpx.ReadError("reset")]) +async def test_submit_lost_after_send_is_502_without_retry(start_front: StartFront, transport_error: httpx.TransportError) -> None: + async with start_front() as front: + front.horde.submit_route.mock(side_effect=transport_error) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 502 + assert response.headers[SHOULD_RETRY] == "false" + error = response.json()["error"] + assert error["code"] == "HordeUnavailable" + assert error["metadata"] == {"submit_outcome": "unknown"} + assert front.horde.submit_route.call_count == 1 + assert front.inflight == 0 + + +async def test_front_at_capacity_is_503_with_retry_after_and_no_submit(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"max_inflight": 1})) as front: + first_request = asyncio.create_task(front.client.post("/v1/chat/completions", json=build_chat_body())) + await wait_for_call(lambda: front.horde.status_route.call_count) + refused = await front.client.post("/v1/chat/completions", json=build_chat_body()) + await front.horde.finish_later(0) + first_response = await first_request + + assert refused.status_code == 503 + assert refused.headers["retry-after"] == "5" + assert SHOULD_RETRY not in refused.headers + assert refused.json()["error"]["code"] == "FrontAtCapacity" + assert refused.json()["error"]["metadata"] == {"retry_after": 5} + assert first_response.status_code == 200 + assert front.horde.submit_route.call_count == 1 + + +async def test_deadline_with_the_request_still_waiting_is_408_without_retry_and_cancels(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"nonstream_timeout_seconds": 0.3})) as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 408 + assert response.headers[SHOULD_RETRY] == "false" + error = response.json()["error"] + assert error["code"] == "ChatRequestTimeout" + assert error["type"] == "timeout_error" + assert error["metadata"] == {"request_id": REQUEST_ID} + assert front.horde.cancel_route.call_count == 1 + + +async def test_faulted_request_is_502_without_retry(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = {**waiting_status(), "faulted": True} + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 502 + assert response.headers[SHOULD_RETRY] == "false" + assert response.json()["error"]["code"] == "ChatRequestFaulted" + assert response.json()["error"]["metadata"] == {"request_id": REQUEST_ID} + assert front.horde.cancel_route.call_count == 0 + + +async def test_faulted_signal_then_missing_status_is_an_expired_faulted_request(start_front: StartFront) -> None: + async def expire() -> None: + await asyncio.sleep(0.1) + front.horde.status_code = 404 + front.horde.status_payload = {"message": "Request not found.", "rc": "RequestNotFound"} + await front.horde.publish("faulted") + + async with start_front() as front: + expirer = asyncio.create_task(expire()) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + await expirer + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "ChatRequestFaulted" + + +async def test_status_404_without_a_faulted_signal_passes_through_without_retry(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_code = 404 + front.horde.status_payload = {"message": "Request not found.", "rc": "RequestNotFound"} + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 404 + assert response.headers[SHOULD_RETRY] == "false" + assert response.json()["error"]["metadata"] == {"request_id": REQUEST_ID, "rc": "RequestNotFound"} + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_no_eligible_worker_past_the_grace_is_503_and_cancels(start_front: StartFront, stream: bool) -> None: + fail_fast_settings = FAST_TEST_SETTINGS.model_copy(update={"fail_fast_grace_seconds": 0.2}) + async with start_front(fail_fast_settings) as front: + front.horde.status_payload = waiting_status(is_possible=False) + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=stream)) + + assert front.horde.cancel_route.call_count == 1 + assert response.headers[SHOULD_RETRY] == "false" + if stream: + assert response.status_code == 200 + payloads = data_payloads(parse_sse_events(response.text)) + assert json.loads(payloads[0])["error"]["code"] == "no_eligible_worker" + assert payloads[-1] == "[DONE]" + else: + assert response.status_code == 503 + assert response.headers["retry-after"] == "30" + assert response.json()["error"]["code"] == "no_eligible_worker" + assert response.json()["error"]["metadata"] == {"request_id": REQUEST_ID, "retry_after": 30} + + +async def test_no_eligible_worker_does_not_fail_while_a_worker_is_processing(start_front: StartFront) -> None: + fail_fast_settings = FAST_TEST_SETTINGS.model_copy(update={"fail_fast_grace_seconds": 0.1, "fallback_poll_seconds": 0.05}) + async with start_front(fail_fast_settings, with_redis=False) as front: + front.horde.status_payload = waiting_status(is_possible=False, processing=1) + finisher = asyncio.create_task(front.horde.finish_later(0.4, signal=False)) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + await finisher + + assert response.status_code == 200 + assert front.horde.cancel_route.call_count == 0 + + +async def test_stream_error_event_writes_explicit_nulls_and_no_choice(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_code = 404 + front.horde.status_payload = {"detail": "gone"} + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=True)) + + assert response.status_code == 200 + payloads = data_payloads(parse_sse_events(response.text)) + assert payloads[-1] == "[DONE]" + assert len(payloads) == 2 + assert '"param":null' in payloads[0] + assert '"code":null' in payloads[0] + error_event = json.loads(payloads[0]) + assert error_event == { + "error": { + "message": error_event["error"]["message"], + "type": "not_found_error", + "param": None, + "code": None, + "metadata": {"request_id": REQUEST_ID}, + }, + } + assert "choices" not in error_event + + +@pytest.mark.parametrize("failure", ["faulted", "timeout"]) +async def test_stream_failure_after_headers_is_an_error_event_then_done(start_front: StartFront, failure: str) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"stream_timeout_seconds": 0.3})) as front: + if failure == "faulted": + front.horde.status_payload = {**waiting_status(), "faulted": True} + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=True)) + + assert response.status_code == 200 + assert response.headers[SHOULD_RETRY] == "false" + payloads = data_payloads(parse_sse_events(response.text)) + expected_code = "ChatRequestFaulted" if failure == "faulted" else "ChatRequestTimeout" + assert json.loads(payloads[0])["error"]["code"] == expected_code + assert payloads[-1] == "[DONE]" + assert front.horde.cancel_route.call_count == (1 if failure == "timeout" else 0) + + +async def test_completed_non_streamed_response_carries_no_retry_header(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 200 + assert SHOULD_RETRY not in response.headers + + +@pytest.mark.parametrize("failure", [httpx.Response(503, json={"message": "Unavailable"}), httpx.ReadTimeout("slow")]) +async def test_failed_fail_fast_recheck_does_not_cancel_from_stale_status(start_front: StartFront, failure: object) -> None: + settings = FAST_TEST_SETTINGS.model_copy(update={"fail_fast_grace_seconds": 0.02, "signal_recheck_seconds": 0.02}) + async with start_front(settings, with_redis=False) as front: + front.horde.status_route.mock( + side_effect=[ + httpx.Response(200, json=waiting_status(is_possible=False)), + failure, + httpx.Response(200, json=done_status()), + ] + ) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + assert response.status_code == 200 + assert front.horde.status_route.call_count == 3 + assert front.horde.cancel_route.call_count == 0 + + +@pytest.mark.parametrize("token", [b"caf\xe9", b"key\x00", b"key\x7f", b"key\tpart"]) +async def test_non_ascii_or_control_bearer_token_is_401(start_front: StartFront, token: bytes) -> None: + async with start_front() as front: + _, messages = await call_raw( + front.app, body_chunks=[json.dumps(build_chat_body()).encode()], headers=[(b"authorization", b"Bearer " + token)] + ) + assert response_status(messages) == 401 + assert json.loads(response_body(messages))["error"]["code"] == "InvalidAuthorizationHeader" + assert front.horde.submit_route.call_count == 0 + assert front.inflight == 0 + + +@pytest.mark.parametrize("field", ["temperature", "top_p", "frequency_penalty", "presence_penalty"]) +@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")]) +async def test_nonfinite_sampling_value_is_refused_before_forwarding(start_front: StartFront, field: str, value: float) -> None: + async with start_front() as front: + body = {**build_chat_body(), field: value} + response = await front.client.post("/v1/chat/completions", content=json.dumps(body), headers={"Content-Type": "application/json"}) + assert response.status_code == 400 + assert front.horde.submit_route.call_count == 0 + assert front.inflight == 0 + + +@pytest.mark.parametrize("publish_fault", [False, True]) +async def test_missing_previously_visible_status_reconciles_expiry_within_a_bound(start_front: StartFront, publish_fault: bool) -> None: + settings = FAST_TEST_SETTINGS.model_copy(update={"fallback_poll_seconds": 0.02, "signal_recheck_seconds": 0.1}) + missing_read = asyncio.Event() + reads = 0 + + async def answer(_request: httpx.Request) -> httpx.Response: + nonlocal reads + reads += 1 + if reads == 1: + return httpx.Response(200, json=waiting_status(wait_time=0)) + missing_read.set() + return httpx.Response(404, json={"message": "Gone", "rc": "RequestNotFound"}) + + async with start_front(settings) as front: + front.horde.status_route.mock(side_effect=answer) + request = asyncio.create_task(front.client.post("/v1/chat/completions", json=build_chat_body())) + await asyncio.wait_for(missing_read.wait(), timeout=1) + if publish_fault: + await asyncio.sleep(0.02) + await front.horde.publish("faulted") + response = await asyncio.wait_for(request, timeout=0.5) + assert response.status_code == (502 if publish_fault else 404) + assert response.json()["error"]["code"] == ("ChatRequestFaulted" if publish_fault else "RequestNotFound") + assert response.headers[SHOULD_RETRY] == "false" diff --git a/tests/unit/oai/test_horde_client.py b/tests/unit/oai/test_horde_client.py new file mode 100644 index 00000000..9c105027 --- /dev/null +++ b/tests/unit/oai/test_horde_client.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's Horde client: bounded concurrent calls and the backoff of a failing Horde root.""" + +from __future__ import annotations + +import asyncio +import time + +import httpx +import pytest +import respx + +from horde.oai.horde_client import ( + HTTP_KEEPALIVE_EXPIRY_SECONDS, + HTTP_MAX_CONNECTIONS, + UNHEALTHY_URL_BACKOFF_SECONDS, + HordeClient, + HordeCredentials, + HordeUnavailableError, + create_http_client, +) +from horde.oai.waiter import ChatRequestWatcher, ChatWaitTimeoutError, CompletionSubscription, WaitBudget +from tests.unit.oai.conftest import REQUEST_ID, done_status + +pytestmark = pytest.mark.asyncio + +FIRST_URL: str = "http://horde-first.test/api" +"""The first Horde root in the rotation.""" + +SECOND_URL: str = "http://horde-second.test/api" +"""The second Horde root in the rotation.""" + +CREDENTIALS: HordeCredentials = HordeCredentials(apikey="test-key", client_agent="test:1:test") +"""The identity every call in these tests is made with.""" + +STATUS_PATH: str = f"/v2/generate/chat/status/{REQUEST_ID}" +"""The status route of the one request these tests ask about.""" + + +async def test_watcher_deadline_includes_wait_for_client_capacity() -> None: + entered = asyncio.Event() + release = asyncio.Event() + + async def hold_slot(_request: httpx.Request) -> httpx.Response: + entered.set() + await release.wait() + return httpx.Response(200, json=done_status()) + + with respx.mock() as router: + route = router.get(f"{FIRST_URL}{STATUS_PATH}").mock(side_effect=hold_slot) + async with httpx.AsyncClient() as http_client: + client = HordeClient(horde_urls=(FIRST_URL,), http_client=http_client, proxy_passkey=None, max_concurrent_calls=1) + occupied = asyncio.create_task(client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS)) + await asyncio.wait_for(entered.wait(), timeout=2) + watcher = ChatRequestWatcher( + horde_client=client, + credentials=CREDENTIALS, + subscription=CompletionSubscription(REQUEST_ID), + budget=WaitBudget( + deadline=time.monotonic() + 0.05, + final_status_timeout_seconds=0.05, + fallback_poll_seconds=1, + signal_recheck_seconds=0.1, + fail_fast_grace_seconds=10, + ), + ) + try: + with pytest.raises(ChatWaitTimeoutError): + await asyncio.wait_for(watcher.wait_until_finished(), timeout=2) + assert route.call_count == 0 # The occupied mock response has not returned yet. + finally: + release.set() + await occupied + # The timed-out waiter must not leak a slot or send a delayed status call. + assert (await client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS)).done + assert route.call_count == 2 + + +class ManualClock: + """A monotonic clock a test advances by hand.""" + + def __init__(self) -> None: + """Start the clock at zero.""" + self.now: float = 0.0 + """The current reading.""" + + def __call__(self) -> float: + """Return the current reading.""" + return self.now + + +async def test_concurrent_calls_beyond_the_limit_wait_outside_the_pool() -> None: + in_flight = 0 + peak_in_flight = 0 + release_calls = asyncio.Event() + + async def answer_slowly(_request: httpx.Request) -> httpx.Response: + nonlocal in_flight, peak_in_flight + in_flight += 1 + peak_in_flight = max(peak_in_flight, in_flight) + await release_calls.wait() + in_flight -= 1 + return httpx.Response(200, json=done_status()) + + with respx.mock() as router: + router.get(f"{FIRST_URL}{STATUS_PATH}").mock(side_effect=answer_slowly) + async with httpx.AsyncClient() as http_client: + horde_client = HordeClient( + horde_urls=(FIRST_URL,), + http_client=http_client, + proxy_passkey=None, + max_concurrent_calls=2, + ) + calls = [asyncio.create_task(horde_client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS)) for _ in range(6)] + await asyncio.sleep(0.1) + peak_before_release = peak_in_flight + release_calls.set() + statuses = await asyncio.gather(*calls) + + assert peak_before_release == 2 + assert peak_in_flight == 2 + assert all(status.done for status in statuses) + + +async def test_refused_root_is_left_out_of_the_rotation_until_its_backoff_ends() -> None: + clock = ManualClock() + with respx.mock() as router: + first_route = router.get(f"{FIRST_URL}{STATUS_PATH}").mock(side_effect=httpx.ConnectError("refused")) + second_route = router.get(f"{SECOND_URL}{STATUS_PATH}").mock( + return_value=httpx.Response(200, json=done_status()), + ) + async with httpx.AsyncClient() as http_client: + horde_client = HordeClient( + horde_urls=(FIRST_URL, SECOND_URL), + http_client=http_client, + proxy_passkey=None, + clock=clock, + ) + for _ in range(4): + await horde_client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS) + calls_to_first_during_backoff = first_route.call_count + + clock.now += UNHEALTHY_URL_BACKOFF_SECONDS + 0.1 + for _ in range(2): + await horde_client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS) + + assert calls_to_first_during_backoff == 1 + assert first_route.call_count == 2 + assert second_route.call_count == 6 + + +async def test_every_root_in_backoff_fails_without_connecting() -> None: + clock = ManualClock() + with respx.mock() as router: + only_route = router.get(f"{FIRST_URL}{STATUS_PATH}").mock(side_effect=httpx.ConnectError("refused")) + async with httpx.AsyncClient() as http_client: + horde_client = HordeClient( + horde_urls=(FIRST_URL,), + http_client=http_client, + proxy_passkey=None, + clock=clock, + ) + for _ in range(3): + with pytest.raises(HordeUnavailableError): + await horde_client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS) + + assert only_route.call_count == 1 + + +async def test_connection_dropped_after_sending_is_not_retried_elsewhere() -> None: + with respx.mock(assert_all_called=False) as router: + first_route = router.post(f"{FIRST_URL}/v2/generate/chat/async").mock( + side_effect=httpx.ReadError("connection reset"), + ) + second_route = router.post(f"{SECOND_URL}/v2/generate/chat/async").mock( + return_value=httpx.Response(202, json={"id": REQUEST_ID}), + ) + async with httpx.AsyncClient() as http_client: + horde_client = HordeClient(horde_urls=(FIRST_URL, SECOND_URL), http_client=http_client, proxy_passkey=None) + with pytest.raises(HordeUnavailableError): + await horde_client.submit_chat(request_body={"model": "m"}, credentials=CREDENTIALS) + + assert first_route.call_count == 1 + assert second_route.call_count == 0 + + +async def test_http_client_pool_matches_the_call_limit_and_expires_idle_connections() -> None: + async with create_http_client() as http_client: + pool = http_client._transport._pool # type: ignore[attr-defined] + + assert pool._max_connections == HTTP_MAX_CONNECTIONS + assert pool._max_keepalive_connections == HTTP_MAX_CONNECTIONS + assert pool._keepalive_expiry == HTTP_KEEPALIVE_EXPIRY_SECONDS + + +async def test_cancels_use_their_own_pool_while_status_reads_fill_the_main_one() -> None: + release_status = asyncio.Event() + + async def hold_status(_request: httpx.Request) -> httpx.Response: + await release_status.wait() + return httpx.Response(200, json=done_status()) + + with respx.mock() as router: + router.get(f"{FIRST_URL}{STATUS_PATH}").mock(side_effect=hold_status) + cancel_route = router.delete(f"{FIRST_URL}{STATUS_PATH}").mock(return_value=httpx.Response(200, json=done_status())) + async with httpx.AsyncClient() as http_client, httpx.AsyncClient() as cancel_http_client: + horde_client = HordeClient( + horde_urls=(FIRST_URL,), + http_client=http_client, + cancel_http_client=cancel_http_client, + proxy_passkey=None, + max_concurrent_calls=1, + ) + held_read = asyncio.create_task(horde_client.get_chat_status(request_id=REQUEST_ID, credentials=CREDENTIALS)) + await asyncio.sleep(0.05) + cancelled = await asyncio.wait_for(horde_client.cancel_chat(request_id=REQUEST_ID, credentials=CREDENTIALS), timeout=1) + release_status.set() + await held_read + + assert cancelled.done + assert cancel_route.call_count == 1 + + +async def test_credentials_representation_omits_the_api_key() -> None: + assert "test-key" not in repr(CREDENTIALS) + assert "test-key" not in str(CREDENTIALS) diff --git a/tests/unit/oai/test_lifecycle.py b/tests/unit/oai/test_lifecycle.py new file mode 100644 index 00000000..4757182b --- /dev/null +++ b/tests/unit/oai/test_lifecycle.py @@ -0,0 +1,450 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's request lifecycle: the deadline rule, admission slots, cancels and shutdown.""" + +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any + +import anyio +import httpx +import pytest +import redis.asyncio +from loguru import logger + +from horde.oai import waiter +from horde.oai.app import create_app +from horde.oai.waiter import CompletionSignalListener +from tests.unit.oai.conftest import ( + CALL_WAIT_SECONDS, + FAST_TEST_SETTINGS, + REQUEST_ID, + RunningFront, + StartFront, + build_chat_body, + call_raw, + done_status, + response_status, + wait_for_call, + waiting_status, +) + +pytestmark = pytest.mark.asyncio + +DEADLINE_SETTINGS = FAST_TEST_SETTINGS.model_copy( + update={"nonstream_timeout_seconds": 0.3, "fallback_poll_seconds": 0.05, "final_status_timeout_seconds": 0.2}, +) +"""A short budget with frequent polls, so the deadline arrives within a test.""" + + +def encoded_chat_body(**overrides: Any) -> list[bytes]: + """Return a chat body as one raw ASGI chunk.""" + return [json.dumps(build_chat_body(**overrides)).encode()] + + +def assert_slot_released(front: RunningFront) -> None: + """Assert the front holds no slot and no open request.""" + assert front.app.state.front.inflight_limiter.inflight == 0 + assert front.app.state.front.inflight_limiter.inflight_for("127.0.0.1") == 0 + assert front.app.state.front.inflight_limiter.inflight_for("203.0.113.9") == 0 + assert not front.app.state.front.open_requests + + +async def test_status_terminal_at_the_deadline_is_returned_not_408(start_front: StartFront) -> None: + async with start_front(DEADLINE_SETTINGS, with_redis=False) as front: + budget_ends_at = time.monotonic() + DEADLINE_SETTINGS.nonstream_timeout_seconds + + def answer_status(_request: httpx.Request) -> httpx.Response: + if time.monotonic() < budget_ends_at: + return httpx.Response(200, json=waiting_status(wait_time=0)) + return httpx.Response(200, json=done_status()) + + front.horde.status_route.mock(side_effect=answer_status) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 200 + assert front.horde.cancel_route.call_count == 0 + + +async def test_read_finishing_after_the_deadline_with_a_terminal_status_is_returned(start_front: StartFront) -> None: + async def answer_status(_request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.1) + return httpx.Response(200, json=done_status()) + + settings = FAST_TEST_SETTINGS.model_copy(update={"nonstream_timeout_seconds": 0.05, "final_status_timeout_seconds": 0.5}) + async with start_front(settings, with_redis=False) as front: + front.horde.status_route.mock(side_effect=answer_status) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 200 + assert front.horde.cancel_route.call_count == 0 + + +@pytest.mark.parametrize("final_read", ["error", "slow", "unanswered"]) +async def test_failed_final_read_still_cancels_and_answers_408(start_front: StartFront, final_read: str) -> None: + async with start_front(DEADLINE_SETTINGS, with_redis=False) as front: + budget_ends_at = time.monotonic() + DEADLINE_SETTINGS.nonstream_timeout_seconds + + async def answer_status(_request: httpx.Request) -> httpx.Response: + if time.monotonic() < budget_ends_at: + return httpx.Response(200, json=waiting_status(wait_time=0)) + if final_read == "slow": + await asyncio.sleep(1) + return httpx.Response(200, json=done_status()) + if final_read == "unanswered": + raise httpx.ReadTimeout("no answer") + return httpx.Response(404, json={"message": "Request not found.", "rc": "RequestNotFound"}) + + front.horde.status_route.mock(side_effect=answer_status) + started_at = time.monotonic() + response = await asyncio.wait_for(front.client.post("/v1/chat/completions", json=build_chat_body()), timeout=CALL_WAIT_SECONDS) + elapsed = time.monotonic() - started_at + + assert response.status_code == 408 + assert response.headers["x-should-retry"] == "false" + assert front.horde.cancel_route.call_count == 1 + assert elapsed < DEADLINE_SETTINGS.nonstream_timeout_seconds + DEADLINE_SETTINGS.final_status_timeout_seconds + 0.5 + assert_slot_released(front) + + +@pytest.mark.parametrize("slow_read", [1, 2]) +async def test_slow_status_read_is_bounded_by_the_deadline(start_front: StartFront, slow_read: int) -> None: + settings = FAST_TEST_SETTINGS.model_copy( + update={"nonstream_timeout_seconds": 0.1, "fallback_poll_seconds": 0.01, "final_status_timeout_seconds": 0.1}, + ) + read_cancelled = asyncio.Event() + calls = 0 + + async def answer_status(_request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + if calls == slow_read: + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + read_cancelled.set() + raise + return httpx.Response(200, json=done_status()) + return httpx.Response(200, json=waiting_status(wait_time=0)) + + async with start_front(settings, with_redis=False) as front: + front.horde.status_route.mock(side_effect=answer_status) + response = await asyncio.wait_for(front.client.post("/v1/chat/completions", json=build_chat_body()), timeout=2) + + assert response.status_code == 408 + assert read_cancelled.is_set() + assert front.horde.cancel_route.call_count == 1 + assert_slot_released(front) + + +async def test_streamed_request_has_its_own_finite_budget(start_front: StartFront) -> None: + settings = FAST_TEST_SETTINGS.model_copy(update={"nonstream_timeout_seconds": 5.0, "stream_timeout_seconds": 0.2}) + async with start_front(settings) as front: + started_at = time.monotonic() + response = await front.client.post("/v1/chat/completions", json=build_chat_body(stream=True)) + elapsed = time.monotonic() - started_at + + assert '"ChatRequestTimeout"' in response.text + assert elapsed < 2 + assert front.horde.cancel_route.call_count == 1 + + +async def test_per_source_cap_refuses_a_second_request_from_one_address(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"max_inflight_per_source": 1})) as front: + first_request = asyncio.create_task(front.client.post("/v1/chat/completions", json=build_chat_body())) + await wait_for_call(lambda: front.horde.status_route.call_count) + refused = await front.client.post("/v1/chat/completions", json=build_chat_body()) + _app_task, other_source_messages = await call_raw( + front.app, + body_chunks=encoded_chat_body(), + disconnect_after_seconds=None, + wait_for_app=False, + ) + await wait_for_call(lambda: front.horde.submit_route.call_count, minimum=2) + await front.horde.finish_later(0) + first_response = await first_request + await asyncio.wait_for(_app_task, timeout=CALL_WAIT_SECONDS) + + assert refused.status_code == 503 + assert refused.json()["error"]["code"] == "FrontAtCapacity" + assert first_response.status_code == 200 + assert response_status(other_source_messages) == 200 + assert_slot_released(front) + + +@pytest.mark.parametrize( + "exit_path", + ["completed", "refused", "unreachable", "lost_submit", "timeout", "faulted", "fail_fast", "status_error"], +) +@pytest.mark.parametrize("stream", [False, True]) +async def test_slot_is_released_once_on_every_exit_path(start_front: StartFront, exit_path: str, stream: bool) -> None: + settings = FAST_TEST_SETTINGS.model_copy( + update={"nonstream_timeout_seconds": 0.2, "stream_timeout_seconds": 0.2, "fail_fast_grace_seconds": 0.05}, + ) + async with start_front(settings) as front: + if exit_path == "completed": + front.horde.status_payload = done_status() + elif exit_path == "refused": + front.horde.submit_route.respond(403, json={"message": "No.", "rc": "Forbidden"}) + elif exit_path == "unreachable": + front.horde.submit_route.mock(side_effect=httpx.ConnectError("refused")) + elif exit_path == "lost_submit": + front.horde.submit_route.mock(side_effect=httpx.ReadError("reset")) + elif exit_path == "faulted": + front.horde.status_payload = {**waiting_status(), "faulted": True} + elif exit_path == "fail_fast": + front.horde.status_payload = waiting_status(is_possible=False) + elif exit_path == "status_error": + front.horde.status_code = 410 + front.horde.status_payload = {"message": "Gone.", "rc": "Gone"} + await front.client.post("/v1/chat/completions", json=build_chat_body(stream=stream)) + await asyncio.sleep(0.05) + assert_slot_released(front) + + +async def test_disconnect_before_the_id_is_known_releases_the_slot_and_cancels_once(start_front: StartFront) -> None: + async def answer_submit_slowly(_request: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.2) + return httpx.Response(202, json={"id": REQUEST_ID}) + + async with start_front() as front: + front.horde.submit_route.mock(side_effect=answer_submit_slowly) + await call_raw(front.app, body_chunks=encoded_chat_body(), disconnect_after_seconds=0.05) + await wait_for_call(lambda: front.horde.cancel_route.call_count) + await asyncio.sleep(0.05) + + assert front.horde.cancel_route.call_count == 1 + assert_slot_released(front) + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_disconnect_during_the_wait_releases_the_slot_before_the_cancel_answers(start_front: StartFront, stream: bool) -> None: + async with start_front() as front: + front.horde.cancel_gate = asyncio.Event() + app_task, _sent_messages = await call_raw( + front.app, + body_chunks=encoded_chat_body(stream=stream), + disconnect_after_seconds=0.2, + wait_for_app=False, + ) + await wait_for_call(lambda: front.horde.cancels_received) + inflight_while_cancel_pending = front.inflight + front.horde.cancel_gate.set() + await asyncio.wait_for(app_task, timeout=CALL_WAIT_SECONDS) + + assert inflight_while_cancel_pending == 0 + assert front.horde.cancel_route.call_count == 1 + assert_slot_released(front) + + +async def test_streamed_client_gone_before_the_stream_starts_is_cancelled_once_and_freed(start_front: StartFront) -> None: + async with start_front() as front: + await call_raw( + front.app, + body_chunks=encoded_chat_body(stream=True), + disconnect_after_seconds=0, + response_start_delay_seconds=0.3, + ) + await wait_for_call(lambda: front.horde.cancel_route.call_count) + await asyncio.sleep(0.05) + + assert front.horde.status_route.call_count == 0 + assert front.horde.cancel_route.call_count == 1 + assert_slot_released(front) + + +async def test_cancel_that_does_not_complete_in_time_is_logged(start_front: StartFront) -> None: + warnings: list[str] = [] + sink_id = logger.add(lambda message: warnings.append(message.record["message"]), level="WARNING") + try: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"cancel_timeout_seconds": 0.1})) as front: + front.horde.cancel_gate = asyncio.Event() + await call_raw(front.app, body_chunks=encoded_chat_body(), disconnect_after_seconds=0.2) + front.horde.cancel_gate.set() + finally: + logger.remove(sink_id) + + assert f"Cancelling chat request {REQUEST_ID} did not complete within 0.1s" in warnings + + +@pytest.mark.parametrize("cancel_mode", ["asyncio", "repeated", "anyio"]) +async def test_cancelled_nonstream_handler_cleans_up_the_job(start_front: StartFront, cancel_mode: str) -> None: + scope = anyio.CancelScope() + + async with start_front() as front: + front.horde.cancel_gate = asyncio.Event() + + async def request_completion() -> None: + with scope: + await front.client.post("/v1/chat/completions", json=build_chat_body()) + + request_task = asyncio.create_task(request_completion()) + await wait_for_call(lambda: front.horde.status_route.call_count) + if cancel_mode == "anyio": + scope.cancel() + else: + request_task.cancel() + await wait_for_call(lambda: front.horde.cancels_received) + assert front.inflight == 0 + if cancel_mode == "repeated": + request_task.cancel() + await asyncio.sleep(0) + assert not request_task.done() + front.horde.cancel_gate.set() + if cancel_mode == "anyio": + await asyncio.wait_for(request_task, timeout=2) + else: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(request_task, timeout=2) + assert front.horde.cancel_route.call_count == 1 + assert not front.app.state.front.listener._subscriptions + + +async def test_task_cancellation_cleanup_is_bounded(start_front: StartFront) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update={"cancel_timeout_seconds": 0.05})) as front: + front.horde.cancel_gate = asyncio.Event() + request_task = asyncio.create_task(front.client.post("/v1/chat/completions", json=build_chat_body())) + await wait_for_call(lambda: front.horde.status_route.call_count) + request_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(request_task, timeout=2) + assert front.horde.cancels_received == 1 + assert front.inflight == 0 + + +async def test_shutdown_cancels_open_requests_before_the_subscription_closes_within_the_bound(start_front: StartFront) -> None: + subscribed_at_cancel: list[bool] = [] + shutdown_settings = FAST_TEST_SETTINGS.model_copy(update={"shutdown_cancel_seconds": 0.2, "cancel_timeout_seconds": 5.0}) + async with start_front(shutdown_settings) as front: + front.horde.cancel_gate = asyncio.Event() + front.horde.on_cancel = lambda: subscribed_at_cancel.append(front.app.state.front.listener.subscribed) + request_task = asyncio.create_task(front.client.post("/v1/chat/completions", json=build_chat_body())) + await wait_for_call(lambda: front.horde.status_route.call_count) + shutdown_started_at = time.monotonic() + shutdown_seconds = time.monotonic() - shutdown_started_at + request_task.cancel() + with pytest.raises(asyncio.CancelledError): + await request_task + + assert subscribed_at_cancel == [True] + assert front.horde.cancels_received == 1 + assert shutdown_seconds < shutdown_settings.shutdown_cancel_seconds + 1.0 + assert not front.app.state.front.listener.subscribed + + +async def test_owned_redis_client_enables_keepalive_without_the_client_health_check(monkeypatch: pytest.MonkeyPatch) -> None: + created_clients: list[redis.asyncio.Redis] = [] + create_from_url = redis.asyncio.Redis.from_url + + def record_from_url(url: str, **kwargs: Any) -> redis.asyncio.Redis: + redis_client = create_from_url(url, **kwargs) + created_clients.append(redis_client) + return redis_client + + monkeypatch.setattr(redis.asyncio.Redis, "from_url", record_from_url) + settings = FAST_TEST_SETTINGS.model_copy(update={"redis_url": "redis://redis.test:6379/7", "redis_liveness_seconds": 12.5}) + async with httpx.AsyncClient() as horde_http_client: + create_app(settings, http_client=horde_http_client, cancel_http_client=horde_http_client) + + assert len(created_clients) == 1 + connection_kwargs = created_clients[0].connection_pool.connection_kwargs + await created_clients[0].close() + assert connection_kwargs["socket_keepalive"] is True + assert connection_kwargs.get("health_check_interval", 0) == 0 + assert created_clients[0].connection_pool.get_encoder().decode_responses is False + + +class FakePubSub: + """A pattern subscription that delivers no signal and answers each PING only when ``answers_pings`` is set.""" + + def __init__(self, *, answers_pings: bool) -> None: + """Create the subscription.""" + self.answers_pings: bool = answers_pings + """Whether a PING is answered with its PONG.""" + self.pings_sent: int = 0 + """How many PINGs the listener sent.""" + self.closed: bool = False + """Whether the listener closed the subscription.""" + self._replies: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + + @property + def subscribed(self) -> bool: + """Return whether the subscription is open.""" + return not self.closed + + async def psubscribe(self, _pattern: str) -> None: + """Accept the pattern subscription.""" + + async def ping(self, token: str) -> None: + """Record a PING and queue its PONG when the subscription answers.""" + self.pings_sent += 1 + if self.answers_pings: + self._replies.put_nowait({"type": "pong", "pattern": None, "channel": None, "data": token.encode()}) + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, Any] | None: + """Return the next queued reply, or None after ``timeout`` seconds.""" + try: + return await asyncio.wait_for(self._replies.get(), timeout) + except TimeoutError: + return None + + async def close(self) -> None: + """Close the subscription.""" + self.closed = True + + +class FakeSignalRedis: + """A Redis client whose every subscription is a new [`FakePubSub`][tests.unit.oai.test_lifecycle.FakePubSub].""" + + def __init__(self, *, answers_pings: bool) -> None: + """Create the client.""" + self._answers_pings = answers_pings + self.pubsubs: list[FakePubSub] = [] + """Every subscription the listener made, oldest first.""" + + def pubsub(self) -> FakePubSub: + """Create and record a subscription.""" + fake_pubsub = FakePubSub(answers_pings=self._answers_pings) + self.pubsubs.append(fake_pubsub) + return fake_pubsub + + +LIVENESS_SECONDS: float = 0.1 +"""A liveness interval short enough that several pass within a test.""" + + +async def test_unanswered_liveness_ping_drops_the_subscription_and_subscribes_again(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(waiter, "LISTENER_RECONNECT_DELAY_SECONDS", 0.3) + fake_redis = FakeSignalRedis(answers_pings=False) + listener = CompletionSignalListener(fake_redis, liveness_seconds=LIVENESS_SECONDS) + await listener.start() + try: + assert await listener.wait_until_subscribed(CALL_WAIT_SECONDS) + await wait_for_call(lambda: fake_redis.pubsubs[0].pings_sent) + await wait_for_call(lambda: int(not listener.subscribed)) + assert fake_redis.pubsubs[0].pings_sent == 1 + assert len(fake_redis.pubsubs) == 1 + await wait_for_call(lambda: len(fake_redis.pubsubs), minimum=2) + assert await listener.wait_until_subscribed(CALL_WAIT_SECONDS) + assert fake_redis.pubsubs[0].closed + finally: + await listener.stop() + + +async def test_answered_liveness_pings_keep_the_subscription(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(waiter, "LISTENER_RECONNECT_DELAY_SECONDS", 0.05) + fake_redis = FakeSignalRedis(answers_pings=True) + listener = CompletionSignalListener(fake_redis, liveness_seconds=LIVENESS_SECONDS) + await listener.start() + try: + assert await listener.wait_until_subscribed(CALL_WAIT_SECONDS) + await wait_for_call(lambda: fake_redis.pubsubs[0].pings_sent, minimum=3) + assert listener.subscribed + assert len(fake_redis.pubsubs) == 1 + finally: + await listener.stop() diff --git a/tests/unit/oai/test_poll_cadence.py b/tests/unit/oai/test_poll_cadence.py new file mode 100644 index 00000000..d08cfe86 --- /dev/null +++ b/tests/unit/oai/test_poll_cadence.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the fallback poll interval, which stretches with the Horde's wait estimate.""" + +from __future__ import annotations + +import pytest + +from horde.oai.waiter import MAX_STRETCHED_POLL_SECONDS, compute_fallback_poll_seconds, jitter_poll_seconds + +BASE_POLL_SECONDS: float = 15.0 +"""The default configured fallback poll interval.""" + + +@pytest.mark.parametrize( + ("wait_time", "expected_seconds"), + [ + (None, BASE_POLL_SECONDS), + (0, BASE_POLL_SECONDS), + (10, BASE_POLL_SECONDS), + (30, BASE_POLL_SECONDS), + (50, 25.0), + (120, MAX_STRETCHED_POLL_SECONDS), + (3600, MAX_STRETCHED_POLL_SECONDS), + ], +) +def test_poll_interval_stretches_with_the_wait_estimate_between_the_base_and_the_cap( + wait_time: int | None, + expected_seconds: float, +) -> None: + assert compute_fallback_poll_seconds(base_seconds=BASE_POLL_SECONDS, wait_time=wait_time) == expected_seconds + + +def test_poll_interval_never_drops_below_a_base_above_the_cap() -> None: + assert compute_fallback_poll_seconds(base_seconds=90.0, wait_time=3600) == 90.0 + + +@pytest.mark.parametrize(("random_fraction", "expected_seconds"), [(0.0, 60.0), (0.5, 54.0), (0.999, 48.012)]) +def test_jitter_only_shortens_the_interval_so_the_cap_still_holds(random_fraction: float, expected_seconds: float) -> None: + jittered = jitter_poll_seconds(MAX_STRETCHED_POLL_SECONDS, lambda: random_fraction) + + assert jittered == pytest.approx(expected_seconds) + assert jittered <= MAX_STRETCHED_POLL_SECONDS diff --git a/tests/unit/oai/test_request_screening.py b/tests/unit/oai/test_request_screening.py new file mode 100644 index 00000000..5cef2067 --- /dev/null +++ b/tests/unit/oai/test_request_screening.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's request screening: the default no-op table, refusals by name, and contract limits.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests.unit.oai.conftest import FAST_TEST_SETTINGS, StartFront, build_chat_body, done_status + +pytestmark = pytest.mark.asyncio + +NO_OP_ROWS: list[tuple[str, object]] = [ + ("tools", []), + ("tools", None), + ("tool_choice", None), + ("tool_choice", "none"), + ("response_format", {"type": "text"}), + ("logprobs", False), + ("logprobs", None), + ("top_logprobs", 0), + ("top_logprobs", None), + ("parallel_tool_calls", True), + ("parallel_tool_calls", False), + ("store", False), + ("service_tier", "auto"), + ("service_tier", "default"), +] +"""Every accepted value of the default no-op table.""" + + +@pytest.mark.parametrize(("parameter", "value"), NO_OP_ROWS) +async def test_default_no_op_is_accepted_and_not_forwarded(start_front: StartFront, parameter: str, value: object) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post("/v1/chat/completions", json=build_chat_body(**{parameter: value})) + + assert response.status_code == 200 + assert parameter not in front.horde.submitted_body() + + +@pytest.mark.parametrize( + ("body_overrides", "expected_param"), + [ + ({"tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}]}, "tools"), + ({"tool_choice": "auto"}, "tool_choice"), + ({"response_format": {"type": "json_object"}}, "response_format"), + ({"logprobs": True}, "logprobs"), + ({"top_logprobs": 2}, "top_logprobs"), + ({"top_logprobs": False}, "top_logprobs"), + ({"parallel_tool_calls": True, "tools": [{"type": "function"}]}, "parallel_tool_calls"), + ({"store": True}, "store"), + ({"service_tier": "flex"}, "service_tier"), + ({"logit_bias": {"50256": -100}}, "logit_bias"), + ({"functions": []}, "functions"), + ({"function_call": "none"}, "function_call"), + ({"audio": {"voice": "alloy"}}, "audio"), + ({"modalities": ["text"]}, "modalities"), + ({"prediction": {"type": "content", "content": "x"}}, "prediction"), + ({"web_search_options": {}}, "web_search_options"), + ({"models": ["a", "b"]}, "models"), + ({"prompt_cache_key": "k"}, "prompt_cache_key"), + ({"session_id": "s"}, "session_id"), + ({"some_future_key": 1}, "some_future_key"), + ({"messages": [{"role": "tool", "content": "x", "tool_call_id": "1"}]}, "messages[0].tool_call_id"), + ({"messages": [{"role": "tool", "content": "x"}]}, "messages[0].role"), + ({"messages": [{"role": "assistant", "content": "", "tool_calls": []}]}, "messages[0].tool_calls"), + ( + { + "messages": [ + {"role": "system", "content": "Describe."}, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.test/a.png"}}]}, + ], + }, + "messages[1].content[0].type=image_url", + ), + ( + {"messages": [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]}]}, + "messages[0].content[0].type=input_audio", + ), + ( + {"messages": [{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "file", "file": {"file_id": "f"}}]}]}, + "messages[0].content[1].type=file", + ), + ({"horde": {"dry_run": True}}, "horde.dry_run"), + ({"horde": {"operation_key": "00000000-0000-4000-8000-000000000002"}}, "horde.operation_key"), + ({"horde": {"workers": ["w"]}}, "horde.workers"), + ({"n": 2}, "n"), + ({"max_tokens": 8}, "max_tokens"), + ({"max_tokens": 5000}, "max_tokens"), + ({"max_completion_tokens": 5000, "max_tokens": None}, "max_completion_tokens"), + ({"stop": ["a", "b", "c", "d", "e"]}, "stop"), + ], +) +async def test_unsupported_parameter_is_refused_with_param_before_submitting( + start_front: StartFront, + body_overrides: dict[str, Any], + expected_param: str, +) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body(**body_overrides)) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["code"] == "UnsupportedChatParameter" + assert error["type"] == "invalid_request_error" + assert error["param"] == expected_param + assert front.horde.submit_route.call_count == 0 + + +@pytest.mark.parametrize( + ("body_overrides", "expected_param"), + [ + ({"max_tokens": 64, "max_completion_tokens": 128}, "max_completion_tokens"), + ({"horde": {"only": ["a"], "ignore": ["b"]}}, "horde.ignore"), + ], +) +async def test_contradictory_values_are_refused_as_a_conflict( + start_front: StartFront, + body_overrides: dict[str, Any], + expected_param: str, +) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body(**body_overrides)) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "ChatRequestConflict" + assert response.json()["error"]["param"] == expected_param + assert front.horde.submit_route.call_count == 0 + + +async def test_agreeing_token_limits_and_n_of_one_are_accepted(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() + response = await front.client.post( + "/v1/chat/completions", + json=build_chat_body(max_tokens=128, max_completion_tokens=128, n=1, stop=["a", "b", "c", "d"]), + ) + + assert response.status_code == 200 + + +@pytest.mark.parametrize( + ("body_overrides", "expected_param"), + [ + ({"messages": []}, "messages"), + ({"stream_options": {"include_usage": True}}, "stream_options"), + ({"stream": True, "stream_options": {"include_obfuscation": True}}, "stream_options.include_obfuscation"), + ({"temperature": "hot"}, None), + ({"user": "u" * 300}, "user"), + ({"horde": {"context_fit": "truncate_oldest"}}, "horde.context_fit"), + ], +) +async def test_invalid_values_are_bad_requests( + start_front: StartFront, + body_overrides: dict[str, Any], + expected_param: str | None, +) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body(**body_overrides)) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["code"] == "BadRequest" + if expected_param is not None: + assert error["param"] == expected_param + else: + assert error["param"].startswith("temperature") + assert front.horde.submit_route.call_count == 0 + + +async def test_model_id_with_a_variant_suffix_is_not_found(start_front: StartFront) -> None: + async with start_front() as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body(model="koboldcpp/Tiefighter:free")) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "model_not_found" + assert front.horde.submit_route.call_count == 0 + + +@pytest.mark.parametrize( + "settings_update", + [ + {"max_message_count": 1}, + {"max_text_bytes": 10}, + ], +) +async def test_message_count_and_text_size_limits_refuse_before_submitting( + start_front: StartFront, settings_update: dict[str, int] +) -> None: + async with start_front(FAST_TEST_SETTINGS.model_copy(update=settings_update)) as front: + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "UnsupportedChatParameter" + assert response.json()["error"]["param"] == "messages" + assert front.horde.submit_route.call_count == 0 diff --git a/tests/unit/oai/test_settings.py b/tests/unit/oai/test_settings.py new file mode 100644 index 00000000..30a84e57 --- /dev/null +++ b/tests/unit/oai/test_settings.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Behavior of the chat front's settings: environment loading, validation and credential-free diagnostics.""" + +from __future__ import annotations + +import os + +import pytest +from pydantic import ValidationError + +from horde.chat_contract import MAX_CHAT_BODY_BYTES, MAX_CHAT_MESSAGE_COUNT, MAX_CHAT_TEXT_BYTES +from horde.oai.settings import DEFAULT_HORDE_URL, FrontSettings, load_settings + + +@pytest.fixture(autouse=True) +def clear_front_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """Remove every ``HORDE_OAI_*`` variable and leave the working directory's ``.env`` unread.""" + for variable_name in list(os.environ): + if variable_name.startswith("HORDE_OAI_"): + monkeypatch.delenv(variable_name) + monkeypatch.setitem(FrontSettings.model_config, "env_file", None) + + +def test_settings_repr_omits_secrets_without_changing_runtime_values() -> None: + settings = FrontSettings(proxy_passkey="secret-passkey", redis_url="redis://user:secret-password@localhost:6379/0") + for display in (str(settings), repr(settings)): + assert "secret-passkey" not in display + assert "secret-password" not in display + assert settings.proxy_passkey == "secret-passkey" + assert settings.redis_url == "redis://user:secret-password@localhost:6379/0" + + +def test_defaults_match_the_documented_values() -> None: + settings = load_settings() + + assert settings.horde_urls == (DEFAULT_HORDE_URL,) + assert settings.max_inflight == 100 + assert settings.max_inflight_per_source == 5 + assert settings.nonstream_timeout_seconds == 300 + assert settings.stream_timeout_seconds == 600 + assert settings.final_status_timeout_seconds == 2 + assert settings.cancel_timeout_seconds == 10 + assert settings.heartbeat_seconds == 5 + assert settings.fallback_poll_seconds == 15 + assert settings.fail_fast_grace_seconds == 20 + assert settings.max_body_bytes == MAX_CHAT_BODY_BYTES + assert settings.max_message_count == MAX_CHAT_MESSAGE_COUNT + assert settings.max_text_bytes == MAX_CHAT_TEXT_BYTES + assert settings.redis_url is None + assert settings.redis_liveness_seconds == 30 + assert settings.proxy_passkey is None + + +def test_settings_load_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HORDE_OAI_HORDE_URLS", " http://a.test/api/ , http://b.test/api,") + monkeypatch.setenv("HORDE_OAI_MAX_INFLIGHT", "7") + monkeypatch.setenv("HORDE_OAI_STREAM_TIMEOUT_SECONDS", "42.5") + monkeypatch.setenv("HORDE_OAI_FORWARDED_ALLOW_IPS", "10.0.0.1,10.0.0.2") + monkeypatch.setenv("HORDE_OAI_PROXY_PASSKEY", "operator-passkey") + monkeypatch.setenv("HORDE_OAI_REDIS_LIVENESS_SECONDS", "12.5") + + settings = load_settings() + + assert settings.horde_urls == ("http://a.test/api", "http://b.test/api") + assert settings.max_inflight == 7 + assert settings.stream_timeout_seconds == 42.5 + assert settings.forwarded_allow_ips == "10.0.0.1,10.0.0.2" + assert settings.proxy_passkey == "operator-passkey" + assert settings.redis_liveness_seconds == 12.5 + + +@pytest.mark.parametrize( + ("variable_name", "invalid_value"), + [ + ("HORDE_OAI_MAX_INFLIGHT", "many"), + ("HORDE_OAI_MAX_INFLIGHT", "0"), + ("HORDE_OAI_PORT", "70000"), + ("HORDE_OAI_NONSTREAM_TIMEOUT_SECONDS", "-1"), + ("HORDE_OAI_REDIS_LIVENESS_SECONDS", "0"), + ("HORDE_OAI_HORDE_URLS", " , "), + ], +) +def test_invalid_environment_value_fails_naming_the_setting( + monkeypatch: pytest.MonkeyPatch, variable_name: str, invalid_value: str +) -> None: + monkeypatch.setenv(variable_name, invalid_value) + + with pytest.raises(ValidationError) as raised: + load_settings() + + assert variable_name.removeprefix("HORDE_OAI_").lower() in str(raised.value) diff --git a/uv.lock b/uv.lock index 8ce25434..e07d97b5 100644 --- a/uv.lock +++ b/uv.lock @@ -14,6 +14,7 @@ dependencies = [ { name = "chardet" }, { name = "emoji" }, { name = "environs" }, + { name = "fastapi" }, { name = "flask" }, { name = "flask-caching" }, { name = "flask-dance", extra = ["sqla"] }, @@ -22,6 +23,7 @@ dependencies = [ { name = "flask-sqlalchemy" }, { name = "horde-model-reference" }, { name = "horde-sdk" }, + { name = "httpx" }, { name = "logfire", extra = ["flask", "redis", "requests", "sqlalchemy"] }, { name = "loguru" }, { name = "markdown" }, @@ -32,6 +34,7 @@ dependencies = [ { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pycurl" }, + { name = "pydantic-settings" }, { name = "python-dateutil" }, { name = "python-dotenv" }, { name = "redis" }, @@ -40,9 +43,12 @@ dependencies = [ { name = "safetext" }, { name = "semver" }, { name = "setuptools" }, + { name = "sniffio" }, { name = "sqlalchemy" }, + { name = "sse-starlette" }, { name = "stripe" }, { name = "unidecode" }, + { name = "uvicorn" }, { name = "waitress" }, { name = "werkzeug" }, ] @@ -53,11 +59,13 @@ dev = [ { name = "freezegun" }, { name = "locust" }, { name = "mypy" }, + { name = "openai" }, { name = "pip-audit" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "respx" }, { name = "reuse" }, { name = "ruff" }, { name = "sqlparse" }, @@ -83,6 +91,7 @@ requires-dist = [ { name = "chardet", specifier = "<6" }, { name = "emoji" }, { name = "environs" }, + { name = "fastapi" }, { name = "flask", specifier = "~=2.2.2" }, { name = "flask-caching" }, { name = "flask-dance", extras = ["sqla"] }, @@ -91,6 +100,7 @@ requires-dist = [ { name = "flask-sqlalchemy", specifier = "~=3.1.1" }, { name = "horde-model-reference", specifier = ">=8.7.0,<9.0.0" }, { name = "horde-sdk", specifier = "~=0.33.0" }, + { name = "httpx" }, { name = "logfire", extras = ["flask", "sqlalchemy", "redis", "requests"], specifier = "==4.32.1" }, { name = "loguru" }, { name = "markdown", specifier = ">=3.7.0,<4.0.0" }, @@ -101,6 +111,7 @@ requires-dist = [ { name = "pillow", specifier = ">=10.0.1" }, { name = "psycopg2-binary" }, { name = "pycurl", specifier = ">=7.45.7,<8" }, + { name = "pydantic-settings" }, { name = "python-dateutil", specifier = "~=2.8.2" }, { name = "python-dotenv" }, { name = "redis", specifier = "~=4.3.5" }, @@ -109,9 +120,12 @@ requires-dist = [ { name = "safetext" }, { name = "semver", specifier = ">=3.0.2" }, { name = "setuptools", specifier = "<82" }, + { name = "sniffio" }, { name = "sqlalchemy", specifier = "~=2.0.35" }, + { name = "sse-starlette" }, { name = "stripe", specifier = ">=15,<16" }, { name = "unidecode" }, + { name = "uvicorn" }, { name = "waitress", specifier = "~=2.1.2" }, { name = "werkzeug", specifier = "~=2.2.2" }, ] @@ -122,11 +136,13 @@ dev = [ { name = "freezegun" }, { name = "locust", specifier = ">=2.43.4" }, { name = "mypy" }, + { name = "openai", specifier = ">=1.109,<2" }, { name = "pip-audit" }, { name = "pre-commit" }, { name = "pytest", specifier = "==9.0.2" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "respx" }, { name = "reuse" }, { name = "ruff", specifier = "==0.15.12" }, { name = "sqlparse" }, @@ -265,6 +281,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/75/e0e10dc7ed1408c28e03a6cb2d7a407f99320eb953f229d008a7a6d05546/aniso8601-10.0.1-py2.py3-none-any.whl", hash = "sha256:eb19717fd4e0db6de1aab06f12450ab92144246b257423fe020af5748c0cb89e", size = 52848, upload-time = "2025-04-18T17:29:41.492Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -794,6 +819,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "docker" version = "7.2.0" @@ -852,6 +886,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/e0/fe1c760abd9e9cb5dfbce057b6582cb1ac36dfd8b5ce202e1445605c922d/fakeredis-2.37.1-py3-none-any.whl", hash = "sha256:f15c41be151c1e9206416dece764369a4dedb6b1341df3734c3c2c000e405508", size = 152738, upload-time = "2026-08-18T15:22:00.1Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "filelock" version = "3.32.3" @@ -1325,6 +1375,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/1f/8176d92e001f86505424b41664032ae26a882bc9ca41a32c803f373f9195/jiter-0.17.0.tar.gz", hash = "sha256:03e432f226a453851079fb84cd17c6da9991eab723e28d716f14ae3d906e0c12", size = 229037, upload-time = "2026-09-12T15:14:14.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/f8/07bd8c3a23f7a8a6875e6a820bbffe1483a18f18f9398a91b5495123176e/jiter-0.17.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ebf918dfd6a74adc1b9ad71f63c4ab00902fcd3b7fd39f2e24d871db8d713b91", size = 291633, upload-time = "2026-09-12T15:11:49.431Z" }, + { url = "https://files.pythonhosted.org/packages/0e/5e/0de4c6f84ffefa6809ffc2d550b9a314365acf7e7ec9b6c7375d49047900/jiter-0.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61aed66ee042b3b49ef85fdf75714234d055d89d8496ac1c6e47f89e7a30d5e4", size = 321695, upload-time = "2026-09-12T15:11:52.727Z" }, + { url = "https://files.pythonhosted.org/packages/20/ac/befe2e82065bee37a0252081666ed2f48c1ac5f5c6c318c2de8168ba393d/jiter-0.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76eb4a5c20e86f9f848286f167024890f2862258a965d254774deb7fc1545ca1", size = 341967, upload-time = "2026-09-12T15:11:54.231Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cd/9797c1e529746750ae589da7c1a8c24373f00d88e11a989f9e5eb1959079/jiter-0.17.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bcc064f99183a9cbe7f26ed648c352031a74145cd61ed75d34632c73eb46a5a8", size = 326546, upload-time = "2026-09-12T15:11:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fd/e6914c38d6347bab4ebff2b1f0c0f191db276e7a1d5c376176757da42fe3/jiter-0.17.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73b64e69c4150748e020356d958af94bec33c70a0a93d665cfa8f6d580fe1a63", size = 340995, upload-time = "2026-09-12T15:11:58.211Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7d/611b3abf6f88945b5474da5cdc6d1a185e805ac9bf446bb7766dcda6ea87/jiter-0.17.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0bc7f684b65bcda9c20434267577db71bf9905ceddd32b60d1d93278d8c8d3a", size = 352188, upload-time = "2026-09-12T15:11:59.414Z" }, + { url = "https://files.pythonhosted.org/packages/52/f8/b6e513ecbdf3b3cebe587c2279281ecf775b729a58cf4cc7bdf898ded029/jiter-0.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c21265b251d99bbb40080d178a8953e35601d3a1564e05c4de4c0d2ca616797", size = 345025, upload-time = "2026-09-12T15:12:00.697Z" }, + { url = "https://files.pythonhosted.org/packages/28/a8/fe26d06c5a6c5a4cfe703c5154c8a140da1305671eb3681aba9422d4f393/jiter-0.17.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:f3d7f7b34114f7ddc6d72a8e882d49de636b35d9fd12b4d420d3c5729f6c9812", size = 329180, upload-time = "2026-09-12T15:12:01.831Z" }, + { url = "https://files.pythonhosted.org/packages/e1/58/e6d66a26af40a20e62486feb7e222fd50f6e7aaa4f107abd89675dcc835b/jiter-0.17.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5078ab00664307fab2019b522a93aeb191122789f085daf5fd9e362154021d4a", size = 335805, upload-time = "2026-09-12T15:12:03.056Z" }, + { url = "https://files.pythonhosted.org/packages/ef/3e/96520aa2fef5ef831d95483a902140bfab83dcac9eaa74f7df61b5e50a1b/jiter-0.17.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:470e1b1e4c42f1ead2189166a299691871a2df5056c976e7fb96feafaf5f9d44", size = 484121, upload-time = "2026-09-12T15:12:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8f/5d9d92fe538bf36ff481a2278c48147e59c1cf8eb2f7be665260665febe5/jiter-0.17.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6eb6aedeb7352b8f3b6af9cbd67983840165c00428e63f1b420a85885128ea31", size = 521310, upload-time = "2026-09-12T15:12:05.612Z" }, + { url = "https://files.pythonhosted.org/packages/50/06/a09f979b22e652afbc3de66c709b2ba92edcef555f7535ab937c86b4f21a/jiter-0.17.0-cp312-cp312-win32.whl", hash = "sha256:362bb47423886d45a9f705d2d9d4008c6eedd4e41eb1bab4e96fb6daa06b33fd", size = 185029, upload-time = "2026-09-12T15:12:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d9/98265a005b2473ec2be5a84e2b64c2f65382c673879f1574845cd4bcd77c/jiter-0.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:9bd3caac219df476dd0cc3fe01d2f1581ed588906feac767abd9614c1c12f8b3", size = 227381, upload-time = "2026-09-12T15:12:08.823Z" }, + { url = "https://files.pythonhosted.org/packages/a8/11/2e05bf5a56e57a543ebb8f585074adf09383e99d7b062dac92eab1f4d57f/jiter-0.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:36ee6e69027396664e59995b9a635a947a5304ee9837279584a0bb8145c8f6b8", size = 183610, upload-time = "2026-09-12T15:12:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/40/eb/2c4a8075ed5ea02b56911e9375d4c8d7784572ff4af32e5a99ae0d071044/jiter-0.17.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b18434638228c0c184281609bf3d9459026a0f1ea48fb76c205e3ef72069caa", size = 290991, upload-time = "2026-09-12T15:12:11.641Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b1/34bfa29599d420423baac6ff7cada6674fe63d5a7a2ccb3900b904678783/jiter-0.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ec89771f4272b989487a6364e519db6bbaba323e8bbf949ac89a45ea9c18b7a3", size = 321425, upload-time = "2026-09-12T15:12:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/11/71/a5ac64a62a04aebd556afadab14a6b730001e16df87266ded943a100a1d9/jiter-0.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3f052c671d5f425cca5ea5901cf11a831369fba4a55a3862cab93c323b4c3b", size = 343138, upload-time = "2026-09-12T15:12:15.046Z" }, + { url = "https://files.pythonhosted.org/packages/01/dd/f761e320ea473314cb68612bc6a435393464dbd198051399b36848b4ebf3/jiter-0.17.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:785a216bbaf8f15fc974e964ced7322cd3d774bb0e86949edd78c6bffd6ba35b", size = 325805, upload-time = "2026-09-12T15:12:16.506Z" }, + { url = "https://files.pythonhosted.org/packages/19/1a/27d8e40f0fb29bbc7a5adf30907144396a115dbe93d5d8976c054a6dfe96/jiter-0.17.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d85c558c9f8532bba287a990ac63767c7daf756f0d8c030219f62499b1fa228a", size = 340230, upload-time = "2026-09-12T15:12:17.682Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c0/30bcde78a28155461f965d16b7aca4ffca6d17494d905f7a0bb072e6c64e/jiter-0.17.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c23849235d2142ce444b2b8c6eceee9f82f4cc0bd5c9081602e4155c6197807", size = 351343, upload-time = "2026-09-12T15:12:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/91420b156315ae22732f5ee1a7b5725a030aab9dc8fd7dcdacfb4aa588d3/jiter-0.17.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58df29268a95e910f17db7ec9178eb7f15aa8619aaca3575275c4e6b3f4fe4c5", size = 344990, upload-time = "2026-09-12T15:12:20.705Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a2/ae6d5672644cc11127970277c9aeb0fa6fae376845587f5b0a8e8828167c/jiter-0.17.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:a277f97eba7d66b1ee27eb5dab5b774ff46a10c78d89a1d3dcce04ce1357c8ca", size = 328624, upload-time = "2026-09-12T15:12:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/04/62/45cb1162f6aa586536e4a973fc339d72dc6b08cca030d70a838a307aa778/jiter-0.17.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe15ddf316f1f1f643347d3a474e74ce61880c79a11ec5dca53df20c071bd3e8", size = 334731, upload-time = "2026-09-12T15:12:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5f/45c1574b644da7deda0b7591c349520dcf83ce45b24d7ca19922dab1fc27/jiter-0.17.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:02adebb7ce6413c44d40af9ad59d1c1cd79630ccdcb6f7bdd2d461e48c03d8f9", size = 483649, upload-time = "2026-09-12T15:12:27.557Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d3/ebea1ecb5b241c519f192b30215c79a8e47f42f1621acbcd6f8830728416/jiter-0.17.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:55d0e0e613a3f9ad600cf436e0e2b8057d1b52bcf1d91b2d36ac53451231e6a8", size = 520758, upload-time = "2026-09-12T15:12:28.99Z" }, + { url = "https://files.pythonhosted.org/packages/64/e6/682b641ff0765ea9bdc349dbc7d223de5c8af8ec1abda0db3406992f92fe/jiter-0.17.0-cp313-cp313-win32.whl", hash = "sha256:2c45ad7c973ef33fe5114a953377b35a95240f4542c0724d9f781e47dc24bac7", size = 184334, upload-time = "2026-09-12T15:12:30.813Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d2/9a49aac2b27af4cc5015e368c0cc3588491a532f717a668ffce1f1ac57da/jiter-0.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:a3cebb1fe4a1abb00465f3f8a17e09112603e8b7c59e5c3adbcd9f7815a64acd", size = 226601, upload-time = "2026-09-12T15:12:32.096Z" }, + { url = "https://files.pythonhosted.org/packages/b4/ce/9a43e9f614608eafa78de22aedcff54cd21324467b5d442d5c9b00244145/jiter-0.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:96b8b0c6dc5d78682f54a450785e075aa929cde768304cad363cd4efba5a82ac", size = 183103, upload-time = "2026-09-12T15:12:34.396Z" }, + { url = "https://files.pythonhosted.org/packages/17/31/4bb27f54333d3b9ef1e5bd3312dc0b4bbe59c68bb0885fdb40583a6b1567/jiter-0.17.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:454c4997d73cc466c71fd565d91e603b0274e48ea0c6b0b7a7aee6967e4ceb7c", size = 288415, upload-time = "2026-09-12T15:14:08.455Z" }, + { url = "https://files.pythonhosted.org/packages/28/30/879570ecf82574eaea77c5eb10309f4b630dece5f2a556e9814a90ba3f2d/jiter-0.17.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:40d2c240f8f80b5b0f201b29f0ae129c81448c60c772227a41747b5e0026f6a2", size = 279113, upload-time = "2026-09-12T15:14:10.117Z" }, + { url = "https://files.pythonhosted.org/packages/77/7a/1f0b8a35fbd079a4f1752c31a15dc99cf277f863747c459be0af39e900e5/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e05f5adbf68c4bd11e1610f394034d984152988e84be6f8314235ce6f2139e5", size = 303708, upload-time = "2026-09-12T15:14:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/d76219ebdbcf3d4209d9d21a0810db4c8d0a6f88e3ee87d30bdea4e90d30/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c0bf24c72fd0491405dce5d40194f2070e9021ce648c1a1d46234b93d848ff", size = 307147, upload-time = "2026-09-12T15:14:12.897Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -1816,6 +1906,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/80/cab10959dc1faead58dc8384a781dfbf93cb4d33d50988f7a69f1b7c9bbe/oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", size = 151688, upload-time = "2022-10-17T20:04:24.037Z" }, ] +[[package]] +name = "openai" +version = "1.109.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.40.0" @@ -2836,6 +2945,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "respx" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/98/4e55c9c486404ec12373708d015ebce157966965a5ebe7f28ff2c784d41b/respx-0.23.1.tar.gz", hash = "sha256:242dcc6ce6b5b9bf621f5870c82a63997e8e82bc7c947f9ffe272b8f3dd5a780", size = 29243, upload-time = "2026-04-08T14:37:16.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/4a/221da6ca167db45693d8d26c7dc79ccfc978a440251bf6721c9aaf251ac0/respx-0.23.1-py2.py3-none-any.whl", hash = "sha256:b18004b029935384bccfa6d7d9d74b4ec9af73a081cc28600fffc0447f4b8c1a", size = 25557, upload-time = "2026-04-08T14:37:14.613Z" }, +] + [[package]] name = "reuse" version = "6.2.0" @@ -3051,6 +3172,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -3105,6 +3235,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/50/f00935da0ec7cbf325f8dc4f772ae46fbc7b672dd62876e73f0a94adda57/sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f", size = 50070, upload-time = "2026-08-13T19:16:04.062Z" }, ] +[[package]] +name = "sse-starlette" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/be/0123026f719d1a7936f214a88b553bb5701e04ff2511147c1dab0c5035eb/sse_starlette-3.5.0.tar.gz", hash = "sha256:75de713aa8a9441513cc283220826da079d982770965b951e9437720e8bafdb2", size = 36794, upload-time = "2026-09-28T17:48:14.7Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/e4/cdda14023c316d71493bc54fdffc3dd006631b88866145c9d3cc33e0f1df/sse_starlette-3.5.0-py3-none-any.whl", hash = "sha256:3e6e1070df3f0f5d9cea81496de92dbb72f6721871d99748ece67441dd8b7997", size = 17407, upload-time = "2026-09-28T17:48:13.228Z" }, +] + +[[package]] +name = "starlette" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/2b/3850dc6bf7ef71b088962eba31dafc6cffd2f96e577ebb0bb316df96da3e/starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d", size = 2736246, upload-time = "2026-09-23T07:30:26.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, +] + [[package]] name = "strenum" version = "0.4.15" @@ -3231,6 +3387,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/7e/496c1b338a9ef244a03a654fcb9e922c37a9c547d00e73f99f3a94ac838e/tox-4.14.2-py3-none-any.whl", hash = "sha256:2900c4eb7b716af4a928a7fdc2ed248ad6575294ed7cfae2ea41203937422847", size = 155382, upload-time = "2024-03-22T15:44:43.079Z" }, ] +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + [[package]] name = "trustme" version = "1.2.1" @@ -3341,6 +3509,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/38/18c4bbe751a7357b3f6a33352e3af3305ad78f3e72ab7e3d667de4663ed9/urlobject-3.0.0-py3-none-any.whl", hash = "sha256:fd2465520d0a8c5ed983aa47518a2c5bcde0c276a4fd0eb28b0de5dcefd93b1e", size = 16261, upload-time = "2025-07-11T17:53:21.989Z" }, ] +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + [[package]] name = "virtualenv" version = "21.7.4"