diff --git a/docs/reference/chat_front.md b/docs/reference/chat_front.md index d362a9cd..69a2dab9 100644 --- a/docs/reference/chat_front.md +++ b/docs/reference/chat_front.md @@ -383,15 +383,15 @@ anything. ## Metrics Attribute keys carry the `horde.` prefix. Every attribute is bounded: call types, outcomes, wake sources, HTTP -statuses, error codes, refusal reasons, booleans and the model id. The model id comes from a delivered completion, so it is a model a +statuses, error codes, outcome classes, refusal reasons, booleans and the model id. The model id comes from a delivered completion, so it is a model a worker served. Client addresses, API keys, prompts, client titles and request ids are never attributes. | Name | Type | Attributes | Meaning | | --- | --- | --- | --- | | `horde.oai.requests.admitted` | counter | `streamed` | Chat requests that took an in-flight slot. | -| `horde.oai.requests.refused` | counter | `status_code`, `code`, `parameter` | Chat requests refused before an in-flight slot: body size (413), body read timeout (408 `RequestBodyTimeout`), validation (400, 404), authorization (401), `FrontAtCapacity` (503). For `UnsupportedChatParameter`, `parameter` is the top-level request key of the refused path (`store`, `messages` for a refused role or content part, `horde` for a refused horde option) when the request schema, the default no-op table or the OpenAI chat completions request keys the front does not define (`logit_bias`, `functions` and the others in `UNDEFINED_OPENAI_PARAMETERS`) or the common non-OpenAI sampler keys in `COMMON_NON_OPENAI_SAMPLER_PARAMETERS` (`top_k`, `min_p`, `rep_pen` and the others) list it. It is `other` for any other key, so a client-chosen key does not become a label value. It is empty for every other code. Each such refusal also logs its path (see [Refused parameter log line](#refused-parameter-log-line)), and the log shows the keys counted as `other`. | -| `horde.oai.requests.errors` | counter | `status_code`, `code`, `streamed`, `reason`, `stage`, `max_tokens_bucket` | Admitted requests that got an error status before any header: horde refusals at submit (429 `rate_limit_exceeded` when the horde's 429 does not include an `rc`), 408 `ChatRequestTimeout`, 404 `no_worker_supports_features`, 400 `no_worker_supports_limit`, 503 `no_eligible_worker`, 503 `queue_unavailable`, 502 `ChatRequestFaulted`, `ChatRequestNoGeneration` and `HordeUnavailable`. `reason` is `no_eligible_threads` or `wait_exceeds_budget` for `queue_unavailable` and empty for every other code. `stage` is `submit` for a `no_eligible_worker` refused by the Horde at creation, such as for a model without an online chat worker. It is `grace` for a `no_eligible_worker` returned when the fail-fast grace ended, and empty for every other code. `max_tokens_bucket` is the request's completion limit, as in the finished line's `max_tokens`, reported as the upper bound of its `BUCKETS_MAX_TOKENS` bucket (`16`, `64`, `128`, `256`, `512`, `768`, `1024`, `2048`, `4096`, `8192`, `16384`), or `over` above 16384. A limit of 600 is `768`. It shows refusals by the limit clients asked for. | -| `horde.oai.stream.failures` | counter | `code`, `stage` | Streamed requests whose 200 headers were sent and which ended in an SSE error event. `stage` is as for `horde.oai.requests.errors`. | +| `horde.oai.requests.refused` | counter | `status_code`, `code`, `parameter`, `outcome_class` | Chat requests refused before an in-flight slot: body size (413), body read timeout (408 `RequestBodyTimeout`), validation (400, 404), authorization (401), `FrontAtCapacity` (503). For `UnsupportedChatParameter`, `parameter` is the top-level request key of the refused path (`store`, `messages` for a refused role or content part, `horde` for a refused horde option) when the request schema, the default no-op table or the OpenAI chat completions request keys the front does not define (`logit_bias`, `functions` and the others in `UNDEFINED_OPENAI_PARAMETERS`) or the common non-OpenAI sampler keys in `COMMON_NON_OPENAI_SAMPLER_PARAMETERS` (`top_k`, `min_p`, `rep_pen` and the others) list it. It is `other` for any other key, so a client-chosen key does not become a label value. It is empty for every other code. Each such refusal also logs its path (see [Refused parameter log line](#refused-parameter-log-line)), and the log shows the keys counted as `other`. | +| `horde.oai.requests.errors` | counter | `status_code`, `code`, `streamed`, `reason`, `stage`, `max_tokens_bucket`, `outcome_class` | Admitted requests that got an error status before any header: horde refusals at submit (429 `rate_limit_exceeded` when the horde's 429 does not include an `rc`), 408 `ChatRequestTimeout`, 404 `no_worker_supports_features`, 400 `no_worker_supports_limit`, 503 `no_eligible_worker`, 503 `queue_unavailable`, 502 `ChatRequestFaulted`, `ChatRequestNoGeneration` and `HordeUnavailable`. `reason` is `no_eligible_threads` or `wait_exceeds_budget` for `queue_unavailable` and empty for every other code. `stage` is `submit` for a `no_eligible_worker` refused by the Horde at creation, such as for a model without an online chat worker. It is `grace` for a `no_eligible_worker` returned when the fail-fast grace ended, and empty for every other code. `max_tokens_bucket` is the request's completion limit, as in the finished line's `max_tokens`, reported as the upper bound of its `BUCKETS_MAX_TOKENS` bucket (`16`, `64`, `128`, `256`, `512`, `768`, `1024`, `2048`, `4096`, `8192`, `16384`), or `over` above 16384. A limit of 600 is `768`. It shows refusals by the limit clients asked for. | +| `horde.oai.stream.failures` | counter | `code`, `stage`, `outcome_class` | Streamed requests whose 200 headers were sent and which ended in an SSE error event. `stage` is as for `horde.oai.requests.errors`. | | `horde.oai.fail_fast.rescues` | counter | none | Waits in which one status read showed no eligible worker and nothing processing, and a later read within `fail_fast_grace_seconds` showed an eligible worker or the request processing. Compared with the `no_eligible_worker` count at the `grace` stage, it shows how many requests the grace kept from a refusal. | | `horde.oai.inflight` | observable gauge | none | Requests holding an in-flight slot. | | `horde.oai.inflight.sources` | observable gauge | none | Distinct client addresses holding at least one slot. | @@ -400,7 +400,7 @@ worker served. Client addresses, API keys, prompts, client titles and request id | `horde.oai.native.duration` | histogram, seconds | `call_type`, `outcome` | Latency of each native call from taking a call slot to the answer. The histogram count is the call count. `call_type` is `submit`, `recovery`, `status`, `cancel` or `models`; `outcome` is `ok`, `refused` (horde status 400 or above), `unreachable` or `abandoned` (the request's budget ended first). | | `horde.oai.queue_wait` | histogram, seconds | `model` | The horde's `queue_wait_seconds` of each delivered generation: request creation to assignment. | | `horde.oai.first_content.duration` | histogram, seconds | `streamed`, `relayed` | From the submit acknowledgement, when a stream's headers go out, to the first content sent: the first relayed delta (`relayed` true), the first completion chunk, or the non-streamed body. | -| `horde.oai.request.duration` | histogram, seconds | `streamed`, `status_code`, `code` | From an admitted request's arrival to its end. A stream records 200 whatever its outcome; `code` is the error code, `ok` for a delivered completion or `client_closed` for a client that disconnected before delivery. | +| `horde.oai.request.duration` | histogram, seconds | `streamed`, `status_code`, `code`, `outcome_class` | From an admitted request's arrival to its end. A stream records 200 whatever its outcome; `code` is the error code, `ok` for a delivered completion or `client_closed` for a client that disconnected before delivery. | | `horde.oai.wakes` | counter | `wake_source`, `found_finished`, `found_done_without_generation` | Each status read of a waiting request, by what caused it: `immediate` (the read after the submit), `signal`, `recheck`, `poll`, `deadline`; and `disconnect`, recorded without a read when the client disconnects. `found_done_without_generation` marks a read showing the request done, not faulted and without a generation. | | `horde.oai.submit.outcome_unknown` | counter | none | Requests answered with `submit_outcome: "unknown"`. | | `horde.oai.submit.recoveries` | counter | `existing_claim` | Lost submits recovered by re-sending; `existing_claim` true is a claim match (horde answered 200). | @@ -427,6 +427,36 @@ to 10 seconds), and pieces per post uses `horde.oai.metrics.BUCKETS_PIECES_PER_P uses `horde.oai.metrics.BUCKETS_MAX_TOKENS` (16 to 16384 tokens), and the reasoning share uses `horde.oai.metrics.BUCKETS_REASONING_SHARE` (0 to 1 in steps of 0.1). +### Outcome class + +Every finished front request has one `outcome_class` on `horde.oai.requests.refused`, `horde.oai.requests.errors`, +`horde.oai.stream.failures` and `horde.oai.request.duration`. `outcome_class_for` in `horde/oai/app.py` derives it +from the request's `code` and, for `no_eligible_worker`, its `stage`, through the table `OUTCOME_CLASS_BY_CODE`. + +| Class | Codes | +| --- | --- | +| `served` | `ok` | +| `client_gave_up` | `client_closed` | +| `capacity` | `no_eligible_worker` at stage `grace`, `ChatRequestTimeout`, `queue_unavailable` | +| `service_fault` | `HordeUnavailable`, `HordeUnexpectedResponse`, `ChatRequestFaulted`, `ChatRequestNoGeneration`, `ChatStreamDiverged`, `ChatWorkerFault` | +| `unservable` | `no_eligible_worker` at stage `submit`, `UnknownChatModel`, `model_not_found`, `no_worker_supports_features`, `no_worker_supports_limit`, `PromptExceedsContext` | +| `client_error` | `UnsupportedChatParameter`, `InvalidAPIKey`, `InvalidAuthorizationHeader`, `RequestTooLarge`, `RequestBodyTimeout`, `rate_limit_exceeded`, `max_kudos_exceeded`, `KudosUpfront`, `OperationKeyConflict`, `SourceImageSizeExceeded`, `TooManyExtraSourceImages`, `ChatRequestConflict` | +| `front_capacity` | `FrontAtCapacity` | +| `unclassified` | any other code | + +The class of `no_eligible_worker` depends on its `stage`. At `grace` the Horde accepted the request and no eligible +worker took it within the fail-fast grace, so it is `capacity`. At `submit` the Horde refused it because no online +worker could serve it, so it is `unservable`. That describes the moment, since a volunteer worker coming online can +make the same request servable. + +The front passes the Horde's `rc` through as the code, so the code set is open. A code not in the table is +`unclassified` and is not given a default class, so a new return code stays visible. + +`FrontAtCapacity` is the in-flight cap per client address or in total. It rises with client concurrency, and also when +served requests wait longer, so it has its own class. + +The served share of servable requests is `served` divided by the sum of `served`, `capacity` and `service_fault`. + ## Refused parameter log line The front logs one Loguru `INFO` record for each `UnsupportedChatParameter` refusal: diff --git a/horde/oai/app.py b/horde/oai/app.py index 87750dd8..79c982b2 100644 --- a/horde/oai/app.py +++ b/horde/oai/app.py @@ -81,6 +81,7 @@ from horde.consts import ANONYMOUS_API_KEY, HORDE_VERSION from horde.oai.attribution import build_client_agent from horde.oai.horde_client import ( SUBMIT_MAX_CONCURRENT_CALLS, + UNEXPECTED_RESPONSE_CODE, CancelSlotUnavailableError, HordeActiveModel, HordeAPIError, @@ -116,6 +117,7 @@ from horde.oai.metrics import ( FrontMetrics, MaxTokensSource, NoEligibleWorkerStage, + OutcomeClass, PacingFlushReason, RelayOutcome, get_front_meter, @@ -428,6 +430,95 @@ more than the native total cap; the native rc for the same.""" TOO_MANY_IMAGES_CODE: str = "TooManyExtraSourceImages" """The error code of a request with more image parts than the native cap; the native rc for the same.""" +UNKNOWN_CHAT_MODEL_RC: str = "UnknownChatModel" +"""The Horde rc for a model that is not listed in the text model reference and has no online chat worker. The front +passes it through as the code. The Horde raises it from ``horde.exceptions.UnknownChatModel``, which the front does not +import.""" + +PROMPT_EXCEEDS_CONTEXT_RC: str = "PromptExceedsContext" +"""The Horde rc for a prompt that, with the requested completion, does not fit the request's usable context length. +The front passes it through as the code. The Horde builds it in ``horde.classes.kobold.request_fit``.""" + +INVALID_API_KEY_RC: str = "InvalidAPIKey" +"""The Horde rc for an API key that does not identify a user. The front passes it through as the code. The Horde +raises it from ``horde.exceptions.InvalidAPIKey``, which the front does not import.""" + +KUDOS_UPFRONT_RC: str = "KudosUpfront" +"""The Horde rc for a requester who cannot cover the request's kudos hold. The front passes it through as the code. +The Horde raises it from ``horde.exceptions.KudosUpfront``, which the front does not import.""" + +OUTCOME_CLASS_BY_CODE: Mapping[str, OutcomeClass] = MappingProxyType( + { + COMPLETED_CODE: OutcomeClass.SERVED, + CLIENT_CLOSED_CODE: OutcomeClass.CLIENT_GAVE_UP, + TIMEOUT_CODE: OutcomeClass.CAPACITY, + QUEUE_UNAVAILABLE_CODE: OutcomeClass.CAPACITY, + HORDE_UNAVAILABLE_CODE: OutcomeClass.SERVICE_FAULT, + UNEXPECTED_RESPONSE_CODE: OutcomeClass.SERVICE_FAULT, + FAULTED_CODE: OutcomeClass.SERVICE_FAULT, + NO_GENERATION_CODE: OutcomeClass.SERVICE_FAULT, + STREAM_DIVERGED_CODE: OutcomeClass.SERVICE_FAULT, + WORKER_FAULT_RC: OutcomeClass.SERVICE_FAULT, + UNKNOWN_CHAT_MODEL_RC: OutcomeClass.UNSERVABLE, + MODEL_NOT_FOUND_CODE: OutcomeClass.UNSERVABLE, + NO_WORKER_SUPPORTS_FEATURES_CODE: OutcomeClass.UNSERVABLE, + NO_WORKER_SUPPORTS_LIMIT_CODE: OutcomeClass.UNSERVABLE, + PROMPT_EXCEEDS_CONTEXT_RC: OutcomeClass.UNSERVABLE, + UNSUPPORTED_PARAMETER_CODE: OutcomeClass.CLIENT_ERROR, + INVALID_API_KEY_RC: OutcomeClass.CLIENT_ERROR, + INVALID_AUTHORIZATION_CODE: OutcomeClass.CLIENT_ERROR, + REQUEST_TOO_LARGE_CODE: OutcomeClass.CLIENT_ERROR, + REQUEST_BODY_TIMEOUT_CODE: OutcomeClass.CLIENT_ERROR, + RATE_LIMIT_EXCEEDED_CODE: OutcomeClass.CLIENT_ERROR, + MAX_KUDOS_EXCEEDED_CODE: OutcomeClass.CLIENT_ERROR, + KUDOS_UPFRONT_RC: OutcomeClass.CLIENT_ERROR, + OPERATION_KEY_CONFLICT_CODE: OutcomeClass.CLIENT_ERROR, + IMAGE_SIZE_EXCEEDED_CODE: OutcomeClass.CLIENT_ERROR, + TOO_MANY_IMAGES_CODE: OutcomeClass.CLIENT_ERROR, + CHAT_REQUEST_CONFLICT_CODE: OutcomeClass.CLIENT_ERROR, + FRONT_FULL_CODE: OutcomeClass.FRONT_CAPACITY, + }, +) +"""The outcome class of each code whose class does not depend on a stage. Classes are decided in this table and +nowhere else. ``no_eligible_worker`` is not listed, because its class depends on the stage, which +[`outcome_class_for`][horde.oai.app.outcome_class_for] reads. The front passes the Horde's ``rc`` through as the +code, so the code set is open. A code that is not listed here is ``unclassified`` rather than given a default class, +so a new return code stays visible instead of being counted in a class that may be wrong for it.""" + +OUTCOME_CLASS_BY_NO_ELIGIBLE_WORKER_STAGE: Mapping[NoEligibleWorkerStage, OutcomeClass] = MappingProxyType( + { + NoEligibleWorkerStage.GRACE: OutcomeClass.CAPACITY, + NoEligibleWorkerStage.SUBMIT: OutcomeClass.UNSERVABLE, + }, +) +"""The outcome class of a ``no_eligible_worker`` code at each stage. At ``grace`` the Horde accepted the request and no +eligible worker took it in time, so the request was servable. At ``submit`` the Horde refused it because no online +worker could serve it. The request is unservable only at that time, since a volunteer coming online can make the same +request servable.""" + + +def outcome_class_for(code: str | None, stage: NoEligibleWorkerStage | None) -> OutcomeClass: + """Return the outcome class of a finished request, the ``horde.outcome_class`` label of its outcome metrics. + + Args: + code: The request's final code: an error code, [`COMPLETED_CODE`][horde.oai.app.COMPLETED_CODE] or + [`CLIENT_CLOSED_CODE`][horde.oai.app.CLIENT_CLOSED_CODE]. + stage: The stage of a ``no_eligible_worker`` error, and None for every other code. + + Returns: + The class from [`OUTCOME_CLASS_BY_CODE`][horde.oai.app.OUTCOME_CLASS_BY_CODE], or for ``no_eligible_worker`` + from [`OUTCOME_CLASS_BY_NO_ELIGIBLE_WORKER_STAGE`][horde.oai.app.OUTCOME_CLASS_BY_NO_ELIGIBLE_WORKER_STAGE]. + ``unclassified`` for any other code, for None, and for ``no_eligible_worker`` without a stage. + """ + if code is None: + return OutcomeClass.UNCLASSIFIED + if code == NO_ELIGIBLE_WORKER_CODE: + if stage is None: + return OutcomeClass.UNCLASSIFIED + return OUTCOME_CLASS_BY_NO_ELIGIBLE_WORKER_STAGE[stage] + return OUTCOME_CLASS_BY_CODE.get(code, OutcomeClass.UNCLASSIFIED) + + DATA_URL_PREFIX: str = "data:" """The scheme prefix of an inline image URL.""" @@ -1835,6 +1926,7 @@ class RequestTrace: streamed=self.streamed, reason=error.queue_refusal.reason if error.queue_refusal is not None else None, max_tokens=effective_completion_limit(self.chat_request, self._final_context_adjustments(watcher)), + outcome_class=outcome_class_for(error.code, error.no_eligible_worker_stage), stage=error.no_eligible_worker_stage, ) if error.metadata.submit_outcome == SubmitOutcome.UNKNOWN: @@ -1844,7 +1936,11 @@ class RequestTrace: def error_events(self, error: ChatFrontError, *, watcher: ChatRequestWatcher | None) -> list[ServerSentEvent]: """Return the error events sent after a stream's 200 headers, recording the failure and the request's end.""" - self._front_metrics.record_stream_failure(code=error.code, stage=error.no_eligible_worker_stage) + self._front_metrics.record_stream_failure( + code=error.code, + outcome_class=outcome_class_for(error.code, error.no_eligible_worker_stage), + stage=error.no_eligible_worker_stage, + ) self.finish(status_code=HTTPStatus.OK, code=error.code or "", watcher=watcher, front_error=error) return error.to_events() @@ -1887,6 +1983,10 @@ class RequestTrace: streamed=self.streamed, status_code=int(status_code), code=code, + outcome_class=outcome_class_for( + code, + front_error.no_eligible_worker_stage if front_error is not None else None, + ), ) self._record_max_tokens(watcher) context_adjustments = self._final_context_adjustments(watcher) @@ -2426,7 +2526,11 @@ async def _chat_completions(request: Request) -> Response: # the front buffers any of it. inflight_slot = front_state.inflight_limiter.try_acquire(_request_source(request)) if inflight_slot is None: - front_metrics.record_refusal(status_code=HTTPStatus.SERVICE_UNAVAILABLE, code=FRONT_FULL_CODE) + front_metrics.record_refusal( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + code=FRONT_FULL_CODE, + outcome_class=outcome_class_for(FRONT_FULL_CODE, None), + ) return ChatFrontError( status_code=HTTPStatus.SERVICE_UNAVAILABLE, message="This front holds its maximum of open requests for this client or in total; retry shortly.", @@ -2439,7 +2543,11 @@ async def _chat_completions(request: Request) -> Response: chat_request = await _read_chat_request(request, front_state) except ClientDisconnect: inflight_slot.release() - front_metrics.record_refusal(status_code=CLIENT_CLOSED_REQUEST_STATUS, code=CLIENT_CLOSED_CODE) + front_metrics.record_refusal( + status_code=CLIENT_CLOSED_REQUEST_STATUS, + code=CLIENT_CLOSED_CODE, + outcome_class=outcome_class_for(CLIENT_CLOSED_CODE, None), + ) return Response(status_code=CLIENT_CLOSED_REQUEST_STATUS) except ChatFrontError as error: inflight_slot.release() @@ -2450,6 +2558,7 @@ async def _chat_completions(request: Request) -> Response: front_metrics.record_refusal( status_code=error.status_code, code=error.code, + outcome_class=outcome_class_for(error.code, error.no_eligible_worker_stage), parameter=refused_parameter_label, ) return error.to_response() diff --git a/horde/oai/metrics.py b/horde/oai/metrics.py index 69b1c7b8..2dc7781e 100644 --- a/horde/oai/metrics.py +++ b/horde/oai/metrics.py @@ -47,6 +47,7 @@ __all__ = [ "NativeCallOutcome", "NativeCallType", "NoEligibleWorkerStage", + "OutcomeClass", "PacingFlushReason", "RelayOutcome", "get_front_meter", @@ -263,6 +264,7 @@ class MetricAttribute(StrEnum): REASONING_EFFORT = "horde.reasoning_effort" SOURCE = "horde.source" MAX_TOKENS_BUCKET = "horde.max_tokens_bucket" + OUTCOME_CLASS = "horde.outcome_class" class CompletionContent(StrEnum): @@ -325,6 +327,40 @@ class NoEligibleWorkerStage(StrEnum): """For the whole fail-fast grace, the status reads showed no eligible worker and nothing processing.""" +class OutcomeClass(StrEnum): + """How a finished front request ended, recorded as the ``horde.outcome_class`` label on the instruments that count + outcomes. [`outcome_class_for`][horde.oai.app.outcome_class_for] derives it from the request's code. + + The served share of servable requests is ``served`` divided by the sum of ``served``, ``capacity`` and + ``service_fault``. The other classes are left out of it, because the front could not have served those requests or + the client ended them. + """ + + SERVED = "served" + """The completion was delivered. Counts as served in the served share of servable requests.""" + CLIENT_GAVE_UP = "client_gave_up" + """The client disconnected before delivery. Left out of the served share of servable requests.""" + CAPACITY = "capacity" + """The request was servable, but no worker served it in time, or the queue could not take it. Counts against the + served share of servable requests.""" + SERVICE_FAULT = "service_fault" + """The Horde, a worker or the front failed the request: an unreachable or unparsable Horde, a faulted or empty + generation, or a relayed stream that diverged. Counts against the served share of servable requests.""" + UNSERVABLE = "unservable" + """No online worker could serve the request at the time, such as for an unknown model, a model without an online + chat worker, or a limit or feature that is not declared by any online worker. A volunteer coming online can change + this, so the class describes the moment and not the request. Left out of the served share of servable requests.""" + CLIENT_ERROR = "client_error" + """The request itself was refused: an invalid key or header, an unsupported parameter, a body too large or too + slow, a rate limit, or a kudos limit. Left out of the served share of servable requests.""" + FRONT_CAPACITY = "front_capacity" + """The front held its maximum of open requests for the client's address or in total. It rises with client + concurrency, and also when served requests wait longer. Left out of the served share of servable requests.""" + UNCLASSIFIED = "unclassified" + """A code that is not in the class table, such as a new Horde return code passed through by the front. Left out of + the served share of servable requests.""" + + class CancelOutcome(StrEnum): """How a cancel sent to the Horde ended.""" @@ -584,12 +620,20 @@ class FrontMetrics: """Record a chat request that took an in-flight slot.""" self._admitted.add(1, {MetricAttribute.STREAMED: streamed}) - def record_refusal(self, *, status_code: int, code: str | None, parameter: str | None = None) -> None: + def record_refusal( + self, + *, + status_code: int, + code: str | None, + outcome_class: OutcomeClass, + parameter: str | None = None, + ) -> None: """Record a chat request refused before it took an in-flight slot. Args: status_code: The HTTP status sent. code: The error code. + outcome_class: The class of the refusal's code. parameter: The bounded label of the refused parameter of an ``UnsupportedChatParameter`` refusal; None for every other refusal. """ @@ -599,6 +643,7 @@ class FrontMetrics: MetricAttribute.STATUS_CODE: int(status_code), MetricAttribute.CODE: code or "", MetricAttribute.PARAMETER: parameter or "", + MetricAttribute.OUTCOME_CLASS: str(outcome_class), }, ) @@ -610,6 +655,7 @@ class FrontMetrics: streamed: bool, reason: str | None, max_tokens: int, + outcome_class: OutcomeClass, stage: NoEligibleWorkerStage | None = None, ) -> None: """Record an admitted chat request that got an error status before any header was sent. @@ -617,6 +663,7 @@ class FrontMetrics: Args: status_code: The HTTP status sent. code: The error code. + outcome_class: The class of the error's code and stage. streamed: Whether the request is streamed. reason: Which rule refused a ``queue_unavailable`` request; None for every other error. max_tokens: The request's completion limit, recorded as its @@ -633,14 +680,22 @@ class FrontMetrics: MetricAttribute.REASON: reason or "", MetricAttribute.STAGE: str(stage) if stage is not None else "", MetricAttribute.MAX_TOKENS_BUCKET: max_tokens_bucket(max_tokens), + MetricAttribute.OUTCOME_CLASS: str(outcome_class), }, ) - def record_stream_failure(self, *, code: str | None, stage: NoEligibleWorkerStage | None = None) -> None: + def record_stream_failure( + self, + *, + code: str | None, + outcome_class: OutcomeClass, + stage: NoEligibleWorkerStage | None = None, + ) -> None: """Record a streamed request that ended in an error event after its 200 headers. Args: code: The error code. + outcome_class: The class of the error's code and stage. stage: The stage at which a ``no_eligible_worker`` request was found unservable. None for every other error. """ @@ -649,6 +704,7 @@ class FrontMetrics: { MetricAttribute.CODE: code or "", MetricAttribute.STAGE: str(stage) if stage is not None else "", + MetricAttribute.OUTCOME_CLASS: str(outcome_class), }, ) @@ -684,7 +740,15 @@ class FrontMetrics: }, ) - def record_request_duration(self, *, duration_seconds: float, streamed: bool, status_code: int, code: str) -> None: + def record_request_duration( + self, + *, + duration_seconds: float, + streamed: bool, + status_code: int, + code: str, + outcome_class: OutcomeClass, + ) -> None: """Record an admitted request's time from arrival to its end. Args: @@ -692,6 +756,7 @@ class FrontMetrics: streamed: Whether the request was streamed. status_code: The HTTP status sent, 200 for every stream whose headers went out. code: The error code the request ended with, or the success or disconnect code. + outcome_class: The class of the request's code and, for an error, its stage. """ self._request_duration.record( duration_seconds, @@ -699,6 +764,7 @@ class FrontMetrics: MetricAttribute.STREAMED: streamed, MetricAttribute.STATUS_CODE: int(status_code), MetricAttribute.CODE: code, + MetricAttribute.OUTCOME_CLASS: str(outcome_class), }, ) diff --git a/tests/unit/oai/test_observability.py b/tests/unit/oai/test_observability.py index 3579464e..1f66f5e0 100644 --- a/tests/unit/oai/test_observability.py +++ b/tests/unit/oai/test_observability.py @@ -15,7 +15,8 @@ import httpx import pytest from loguru import logger -from horde.oai.app import REFUSED_PARAMETER_LOG_LENGTH +from horde.oai import app as front_app +from horde.oai.app import NO_ELIGIBLE_WORKER_CODE, REFUSED_PARAMETER_LOG_LENGTH, outcome_class_for from horde.oai.metrics import ( ADMITTED_NAME, CANCELS_NAME, @@ -37,6 +38,8 @@ from horde.oai.metrics import ( STREAM_FAILURES_NAME, SUBMIT_OUTCOME_UNKNOWN_NAME, WAKES_NAME, + NoEligibleWorkerStage, + OutcomeClass, max_tokens_bucket, ) from tests.unit.oai.conftest import ( @@ -163,6 +166,7 @@ async def test_completed_request_records_inflight_native_latency_wake_and_comple assert front.metric_total(WAKES_NAME, horde_wake_source="immediate", horde_found_finished=False) == 1 assert front.metric_total(WAKES_NAME, horde_wake_source="signal", horde_found_finished=True) == 1 assert front.metric_total(REQUEST_DURATION_NAME, horde_streamed=False, horde_status_code=200, horde_code="ok") == 1 + assert front.metric_total(REQUEST_DURATION_NAME, horde_code="ok", horde_outcome_class="served") == 1 assert front.metric_total(FIRST_CONTENT_NAME, horde_streamed=False) == 1 assert front.metric_total(COMPLETIONS_NAME, horde_model=MODEL_NAME, horde_estimated_usage=False) == 1 assert front.metric_total(EMPTY_RETURNS_NAME) == 0 @@ -191,6 +195,7 @@ async def test_front_at_capacity_records_the_refusal(start_front: StartFront) -> assert refused.status_code == 503 assert front.metric_total(REFUSED_NAME, horde_status_code=503, horde_code="FrontAtCapacity") == 1 + assert front.metric_total(REFUSED_NAME, horde_code="FrontAtCapacity", horde_outcome_class="front_capacity") == 1 assert front.metric_total(ADMITTED_NAME) == 1 assert front.metric_total(ERRORS_NAME) == 0 @@ -266,6 +271,8 @@ async def test_a_no_worker_refusal_at_submit_records_the_submit_stage(start_fron assert response.status_code == 503 assert front.metric_total(ERRORS_NAME, horde_code="no_eligible_worker", horde_stage="submit") == 1 assert front.metric_total(ERRORS_NAME, horde_code="no_eligible_worker", horde_stage="grace") == 0 + assert front.metric_total(ERRORS_NAME, horde_code="no_eligible_worker", horde_outcome_class="unservable") == 1 + assert front.metric_total(REQUEST_DURATION_NAME, horde_code="no_eligible_worker", horde_outcome_class="unservable") == 1 @pytest.mark.parametrize( @@ -324,6 +331,62 @@ async def test_the_completion_limit_bucket_is_the_smallest_bound_at_or_above_the assert max_tokens_bucket(max_tokens) == expected_bucket +async def test_every_front_code_has_an_outcome_class() -> None: + """Each ``*_CODE`` value in the front's module, including the imported conflict and unsupported-parameter codes, + has a class other than ``unclassified``. ``no_eligible_worker`` has one at both stages.""" + front_codes = {value for name, value in vars(front_app).items() if name.endswith("_CODE") and isinstance(value, str)} + assert front_app.CHAT_REQUEST_CONFLICT_CODE in front_codes + assert front_app.UNSUPPORTED_PARAMETER_CODE in front_codes + assert NO_ELIGIBLE_WORKER_CODE in front_codes + unclassified_codes = { + (code, stage) + for code in front_codes + for stage in ([NoEligibleWorkerStage.SUBMIT, NoEligibleWorkerStage.GRACE] if code == NO_ELIGIBLE_WORKER_CODE else [None]) + if outcome_class_for(code, stage) == OutcomeClass.UNCLASSIFIED + } + assert unclassified_codes == set() + + +@pytest.mark.parametrize( + ("code", "stage", "expected_class"), + [ + ("ok", None, OutcomeClass.SERVED), + ("client_closed", None, OutcomeClass.CLIENT_GAVE_UP), + ("no_eligible_worker", NoEligibleWorkerStage.GRACE, OutcomeClass.CAPACITY), + ("no_eligible_worker", NoEligibleWorkerStage.SUBMIT, OutcomeClass.UNSERVABLE), + ("ChatWorkerFault", None, OutcomeClass.SERVICE_FAULT), + ("HordeUnexpectedResponse", None, OutcomeClass.SERVICE_FAULT), + ("UnknownChatModel", None, OutcomeClass.UNSERVABLE), + ("PromptExceedsContext", None, OutcomeClass.UNSERVABLE), + ("InvalidAPIKey", None, OutcomeClass.CLIENT_ERROR), + ("KudosUpfront", None, OutcomeClass.CLIENT_ERROR), + ("FrontAtCapacity", None, OutcomeClass.FRONT_CAPACITY), + ], +) +async def test_a_code_maps_to_its_outcome_class( + code: str, + stage: NoEligibleWorkerStage | None, + expected_class: OutcomeClass, +) -> None: + """The class of each listed code, including the Horde return codes passed through by the front.""" + assert outcome_class_for(code, stage) == expected_class + + +@pytest.mark.parametrize( + ("code", "stage"), + [ + ("SomeNewReturnCode", None), + (None, None), + ("no_eligible_worker", None), + ], + ids=["unknown_code", "no_code", "no_eligible_worker_without_a_stage"], +) +async def test_a_code_without_a_class_maps_to_unclassified(code: str | None, stage: NoEligibleWorkerStage | None) -> None: + """A code not in the class table is ``unclassified`` rather than a default class, so a new Horde return code + passed through by the front stays visible.""" + assert outcome_class_for(code, stage) == OutcomeClass.UNCLASSIFIED + + async def test_the_finished_line_records_a_lowered_completion_limit_and_its_source( start_front: StartFront, captured_logs: CapturedLogs, @@ -352,6 +415,8 @@ async def test_a_no_worker_refusal_after_the_grace_records_the_grace_stage(start failure_name = STREAM_FAILURES_NAME if stream else ERRORS_NAME assert front.metric_total(failure_name, horde_code="no_eligible_worker", horde_stage="grace") == 1 + assert front.metric_total(failure_name, horde_code="no_eligible_worker", horde_outcome_class="capacity") == 1 + assert front.metric_total(REQUEST_DURATION_NAME, horde_code="no_eligible_worker", horde_outcome_class="capacity") == 1 assert front.metric_total(GRACE_RESCUES_NAME) == 0 @@ -639,6 +704,7 @@ async def test_streamed_fault_after_headers_is_a_stream_failure_not_an_error_sta assert response.status_code == 200 assert front.metric_total(STREAM_FAILURES_NAME, horde_code="ChatRequestFaulted") == 1 + assert front.metric_total(STREAM_FAILURES_NAME, horde_code="ChatRequestFaulted", horde_outcome_class="service_fault") == 1 assert front.metric_total(ERRORS_NAME) == 0 assert front.metric_total(REQUEST_DURATION_NAME, horde_streamed=True, horde_status_code=200, horde_code="ChatRequestFaulted") == 1 @@ -665,6 +731,7 @@ async def test_disconnect_during_the_wait_records_a_disconnect_wake(start_front: assert front.metric_total(WAKES_NAME, horde_wake_source="disconnect") == 1 assert front.metric_total(REQUEST_DURATION_NAME, horde_status_code=499, horde_code="client_closed") == 1 + assert front.metric_total(REQUEST_DURATION_NAME, horde_code="client_closed", horde_outcome_class="client_gave_up") == 1 assert front.metric_total(CANCELS_NAME, horde_outcome="acknowledged") == 1