From 993e45c6cf0e71eca3097c1d659aba575ea16b5c Mon Sep 17 00:00:00 2001 From: tazlin Date: Sun, 1 Nov 2026 15:17:57 -0500 Subject: [PATCH] fix(chat): count reasoning in the charge, the reward and the speed sample Reasoning is output the worker generated, and OpenAI bills it as completion tokens. A reply whose whole budget went to reasoning was charged and rewarded at the 20-token floor, and its speed sample was the floor over the full elapsed time, so a reasoning model pushed its own worker under the chat speed gate. - `chat_reply_characters` counts `reasoning_content` with the content and the tool calls. `get_things_count` keeps the floor of 20 and the cap at the requested `max_length`, so a reasoning-only reply is charged and rewarded for the budget it used - `TextJobSubmit.get_submitted_things_count` is the speed sample's count. The chat submit override passes the verified reply as `message` to `get_things_count`, so the sample and the charge are one count. Text jobs and faulted chat submits keep the generation string path - `build_completion` estimates `usage.completion_tokens_details.reasoning_tokens` from the reasoning characters when the worker did not report a count, with the formula of the other estimated counts. An estimated `completion_tokens` includes the reasoning, as OpenAI's does. Both estimates are capped at the effective completion limit from `effective_completion_limit`: the lowered `worker_max_length`, else `max_completion_tokens`, else `max_tokens`, else the native default. Reported counts are used as reported, and the stored `reasoning_tokens` column keeps its reported-only meaning - `DEFAULT_CHAT_MAX_TOKENS` moves to `horde/chat_contract.py` so the front can read the native default. `horde/classes/kobold/chat_request.py` re-exports it - The integration test that asserted reasoning is not charged now asserts that a reasoning reply earns and samples what a same-length plain reply does - ADR 0027 and the chat pricing, reasoning and usage docs state the charge basis, the speed sample path and the estimate --- ...generations-by-the-text-output-estimate.md | 4 +- docs/explanation/chat_requests.md | 12 +- docs/how-to/serve_chat_jobs.md | 3 +- docs/reference/chat_front.md | 15 ++- docs/reference/chat_requests.md | 12 +- horde/apis/v2/base.py | 11 +- horde/apis/v2/kobold.py | 18 ++- horde/chat_contract.py | 5 + horde/classes/kobold/chat_request.py | 17 +-- horde/classes/kobold/processing_generation.py | 18 ++- horde/oai/app.py | 7 +- horde/oai/mapping.py | 101 ++++++++++++++-- horde/oai/schema.py | 10 +- tests/integration/test_chat.py | 55 +++++---- tests/unit/oai/test_chat_front.py | 110 +++++++++++++++++- tests/unit/test_chat_request.py | 66 ++++++++++- 16 files changed, 386 insertions(+), 78 deletions(-) diff --git a/docs/decisions/0027-price-chat-generations-by-the-text-output-estimate.md b/docs/decisions/0027-price-chat-generations-by-the-text-output-estimate.md index a388f88f..f31efff6 100644 --- a/docs/decisions/0027-price-chat-generations-by-the-text-output-estimate.md +++ b/docs/decisions/0027-price-chat-generations-by-the-text-output-estimate.md @@ -40,8 +40,8 @@ Chosen option: "The text output estimate, with the charge basis recorded per gen A chat generation is charged and rewarded as a text generation with the same reply: the text kudos formula over ceil(output characters / 4), floored at 20 and capped at the requested `max_length`. The reply's characters are its -content plus each tool call's function name and arguments (`chat_reply_characters`), so tool calls are charged as -output. `TextProcessingGeneration.get_charge_basis` returns the basis, `output_estimate`, for every text generation, +content, its reasoning, and each tool call's function name and arguments (`chat_reply_characters`), so tool calls and +reasoning are charged as output. `TextProcessingGeneration.get_charge_basis` returns the basis, `output_estimate`, for every text generation, and a submitted chat generation stores it in the `charge_basis` column. The reported prompt, completion and reasoning token counts are optional, stored, and shown on the status route and in the OpenAI `usage` object. They do not change the charge or the reward. diff --git a/docs/explanation/chat_requests.md b/docs/explanation/chat_requests.md index c85799f7..3fe1a6cf 100644 --- a/docs/explanation/chat_requests.md +++ b/docs/explanation/chat_requests.md @@ -298,8 +298,8 @@ as a deliberate change. ### Pricing by the text output estimate -A chat generation is charged and rewarded through the text formula, on the characters of the reply (content plus -each tool call's name and arguments) divided by four, floored at 20 tokens and capped at the requested `max_length` +A chat generation is charged and rewarded through the text formula, on the characters of the reply (content, reasoning, +and each tool call's name and arguments) divided by four, floored at 20 tokens and capped at the requested `max_length` ([ADR 0027](../decisions/0027-price-chat-generations-by-the-text-output-estimate.md), [Pricing](../reference/chat_requests.md#pricing)). A short reply to a large `max_tokens` is charged for the short reply. The basis is computed in one function and stored on every generation as `charge_basis: output_estimate`. @@ -325,9 +325,11 @@ A reasoning model writes its thinking before the answer. The worker submits it i from `content`, so a client that does not expect reasoning receives a clean answer, and a block cut off by `max_tokens` cannot become the reply. The front returns it as `reasoning_content`, so a client that reads that field shows progress during a long reasoning phase. A request that sets `horde.include_reasoning` to false does not -receive it. The native status shows it either way. It is not priced: the charge and reward are computed on the content -and tool calls, and the worker's `reasoning_tokens` count is shown and never charged, as with every other reported -count ([Reasoning](../reference/chat_requests.md#reasoning)). +receive it. The native status shows it either way. Reasoning is output the worker generated, and OpenAI bills it as +completion tokens, so its characters are counted with the content and tool calls, capped at the requested +`max_length`. A reply whose whole budget went to reasoning is charged and rewarded for that budget, and the same count +is the worker's speed sample. The worker's `reasoning_tokens` count is shown and never charged, as with every other +reported count ([Reasoning](../reference/chat_requests.md#reasoning)). ### Admission against the online pool diff --git a/docs/how-to/serve_chat_jobs.md b/docs/how-to/serve_chat_jobs.md index dbd3e556..e3bc2996 100644 --- a/docs/how-to/serve_chat_jobs.md +++ b/docs/how-to/serve_chat_jobs.md @@ -307,7 +307,8 @@ Send `message`, `finish_reason` and `model_id` in place of `generation` the answer. Send it in this field. Do not put it inside `content`: split a closed block off the reply, and when generation stops at `max_length` inside a block that never closed, submit that text as `reasoning_content` with an empty `content` and `finish_reason` `length`. Leave the field out when the model produced no reasoning. It is - stored and shown beside the reply, is not charged, and may take up to 1 MiB of UTF-8. + stored and shown beside the reply, counted with the content in the charge and the reward, and may take up to + 1 MiB of UTF-8. - `finish_reason` is one of `stop`, `length`, `content_filter` and `tool_calls`. Map the backend's reason onto them: `length` when generation reached `max_length`, `tool_calls` when the reply carries calls, `content_filter` when the backend withheld or cut the reply on a content rule, and `stop` otherwise. `tool_calls` needs a non-empty diff --git a/docs/reference/chat_front.md b/docs/reference/chat_front.md index 88173428..0d8861d2 100644 --- a/docs/reference/chat_front.md +++ b/docs/reference/chat_front.md @@ -132,9 +132,16 @@ calls. `x_horde.context_adjustments` is the native status's list, passed through dropped (see [Context adjustments](chat_requests.md#context-adjustments)). It is left out when the Horde did not make any of these changes. `reasoning_content`, the model's reasoning as DeepSeek and OpenRouter return it, is written when the worker submitted reasoning, unless the request sets `horde.include_reasoning` to false. It is never part of `content`. -`usage.completion_tokens_details.reasoning_tokens`, OpenAI's shape, is the reasoning count the worker reported, -written whether or not the reasoning is returned and left out when no worker reported one. Like the other reported -counts it is informational: the Horde charges and rewards on the reply it measures. The response `model` is the first +`usage.completion_tokens_details.reasoning_tokens`, OpenAI's shape, is the reasoning count the worker reported or, +when the worker did not report one, an estimate from the reasoning characters made like the other estimated counts, +which sets `x_horde.estimated_usage`. It is written whether or not the reasoning is returned, and left out when the +reply has no reasoning and no worker reported a count. An estimated `usage.completion_tokens` counts the reasoning +with the reply text and tool calls, so it includes the reasoning tokens as OpenAI's does. Each generation's estimated +completion and reasoning counts are capped at the request's effective completion limit: the lowered value of a +`worker_max_length` entry, else `max_completion_tokens`, else `max_tokens`, else the native default of 512 +(`DEFAULT_CHAT_MAX_TOKENS`, `effective_completion_limit` in `horde/oai/mapping.py`). A reported count is used as +reported. The counts are informational: the Horde charges and rewards on the reply it measures, and the reasoning +characters are part of that reply. The response `model` is the first generation's model, the one that served the request, which under `models` is whichever listed model a worker ran; the request's `model`, or its first listed model, stands in only when the generation does not carry a model. `system_fingerprint` is `fp_` and the first 10 hex characters of the SHA-256 of the first generation's worker @@ -242,7 +249,7 @@ returns 502 `ChatRequestNoGeneration`. The front does not return a completion wi | First streamed chunk | `delta` holds only `role` | `delta` also holds `content` | | Delta granularity | For plain text requests, each worker progress post (about one a second) is paced over the interval until the next post, in up to 20 chunks a second. Other requests get the whole text at once. | One chunk per token or few tokens | | `json_object` without "json" in the messages | Served, unless `horde.openai_guards` is set | Refused | -| Reasoning text | Returned as `message.reasoning_content` and `delta.reasoning_content`, which the OpenAI SDK keeps in `model_extra`; `horde.include_reasoning: false` hides it | Not returned; `usage.completion_tokens_details.reasoning_tokens` only | +| Reasoning text | Returned as `message.reasoning_content` and `delta.reasoning_content`, which the OpenAI SDK keeps in `model_extra`; `horde.include_reasoning: false` hides it. `usage.completion_tokens_details.reasoning_tokens` is the worker's count or an estimate from the reasoning characters | Not returned; `usage.completion_tokens_details.reasoning_tokens` only | ## No eligible worker diff --git a/docs/reference/chat_requests.md b/docs/reference/chat_requests.md index 27572b96..a509dcf2 100644 --- a/docs/reference/chat_requests.md +++ b/docs/reference/chat_requests.md @@ -787,8 +787,9 @@ the reply carries tool calls, which are a complete reply. A reasoning model, such as Qwen3, emits a `` block before the answer. The worker submits it as `message.reasoning_content` and never inside `content`, an unclosed block cut off at `max_length` included. The Horde stores it in `message` beside `content`, with NUL replaced, only when it was submitted, and the status shows it there. -It is not the reply: `generation` holds the content alone, the charge and reward are computed on the content and tool -calls (`chat_reply_characters`), and the conformance rules do not read it. Its UTF-8 length is capped at the SDK's +It is not the reply: `generation` holds the content alone, and the conformance rules do not read it. It is model +output, so its characters are counted with the content and tool calls in the charge, the reward and the worker's speed +sample (`chat_reply_characters`), capped at the requested `max_length`. Its UTF-8 length is capped at the SDK's `MAX_CHAT_TEXT_BYTES` (1 MiB), the most message text a request may carry, since the kudos cost does not bound what is stored as reasoning; a longer one returns `400 ChatSubmitInvalid`. `reasoning_tokens` is stored in a separate column and shown on the status. Like the other reported counts it never changes the charge or reward, since the Horde pays only for the @@ -908,8 +909,11 @@ horde stored is not counted, and the worker's fault after it is counted once, wi ### Pricing A chat generation is charged and rewarded exactly as a text generation with the same reply, through the text kudos -formula, and stores `charge_basis` `output_estimate`. The reply's characters are the content plus each tool call's -function name and arguments (`chat_reply_characters`, read by `TextProcessingGeneration.get_things_count`). The +formula, and stores `charge_basis` `output_estimate`. The reply's characters are the content, the reasoning, and each tool +call's function name and arguments (`chat_reply_characters`, read by `TextProcessingGeneration.get_things_count`). The +same count is the worker's speed sample: the submit passes the verified message to `get_things_count` before it is +stored (`TextJobSubmit.get_submitted_things_count`), since `generation` holds the content alone. A reply whose budget +went to reasoning is not recorded as a slow one. The formula's token count is those characters divided by 4, rounded up, at least 20 and at most the requested `max_length`, so a short reply to a large `max_tokens` is not charged the full request. Reported token counts are stored and shown on the status route and never change the charge. The basis is computed in one place, `TextProcessingGeneration.get_charge_basis`. The status route returns, per diff --git a/horde/apis/v2/base.py b/horde/apis/v2/base.py index a40717b3..f3833469 100644 --- a/horde/apis/v2/base.py +++ b/horde/apis/v2/base.py @@ -1417,7 +1417,7 @@ class JobSubmitTemplate(Resource): def set_generation(self): """Set to its own function to it can be overwritten depending on the class""" record_fulfilment_t0 = time.monotonic() - things_per_sec = stats.record_fulfilment(self.procgen, self.procgen.get_things_count(self.args["generation"])) + things_per_sec = stats.record_fulfilment(self.procgen, self.get_submitted_things_count()) submit_record_fulfilment_stat_duration.record(time.monotonic() - record_fulfilment_t0, {"horde.gentype": self.gentype}) gen_metadata = getattr(self.args, "gen_metadata", None) kwargs: dict[str, object] = { @@ -1439,6 +1439,15 @@ class JobSubmitTemplate(Resource): """ return {} + def get_submitted_things_count(self) -> int: + """Return the things count of the submitted generation, which is the worker's speed sample. + + Returns: + int: The generation's count of the submitted ``generation``. + """ + things_count: int = self.procgen.get_things_count(self.args["generation"]) + return things_count + def get_submission_details(self): """Return what a gentype's submission carries beyond the shared fields, passed on to ``set_generation``. diff --git a/horde/apis/v2/kobold.py b/horde/apis/v2/kobold.py index c02de259..46ca4aa5 100644 --- a/horde/apis/v2/kobold.py +++ b/horde/apis/v2/kobold.py @@ -7,7 +7,7 @@ import json import math import os from datetime import datetime -from typing import NoReturn +from typing import Any, NoReturn import logfire import redis @@ -1839,6 +1839,8 @@ class TextJobSubmit(JobSubmitTemplate): dropped_tool_call_count = 0 """The tool calls verification dropped from a chat reply under ``parallel_tool_calls: false``, for the submit log; zero for every other submit.""" + chat_message: dict[str, Any] | None = None + """A chat job's verified reply as it is stored; None for a text job and a faulted chat submit.""" @api.expect(parsers.job_submit_parser, models.input_model_job_submit, validate=True) @api.marshal_with(models.response_model_job_submit, code=200, description="Generation Submitted") @@ -2021,6 +2023,20 @@ class TextJobSubmit(JobSubmitTemplate): self.chat_message = build_stored_reply(chat_message) self.args["generation"] = self.chat_message["content"] or "" + def get_submitted_things_count(self) -> int: + """Return a chat reply's things count from its content, tool calls and reasoning, the same count as the charge. + + The ``generation`` of a chat job holds the content alone, so a reply's reasoning would otherwise be missing + from the worker's speed sample. + + Returns: + int: The generation's count of the verified reply for a chat job, or of the ``generation`` otherwise. + """ + if self.chat_message is None: + return super().get_submitted_things_count() + things_count: int = self.procgen.get_things_count(message=self.chat_message) + return things_count + def get_submission_log_details(self) -> dict[str, object]: """Return how many tool calls verification dropped from a chat reply, when it dropped any. diff --git a/horde/chat_contract.py b/horde/chat_contract.py index b24275ef..48ee8609 100644 --- a/horde/chat_contract.py +++ b/horde/chat_contract.py @@ -51,6 +51,7 @@ __all__ = [ "CHAT_KUDOS_CAP_EXCEEDED_RC", "CHAT_REQUEST_CONFLICT_ERROR_TYPE", "CHAT_REQUEST_CONFLICT_RC", + "DEFAULT_CHAT_MAX_TOKENS", "INTERNAL_HORDE_OPTION_KEYS", "MAX_CHAT_BODY_BYTES", "MAX_CHAT_IMAGE_BYTES", @@ -150,6 +151,10 @@ of images as data URLs, which base64 grows by 4/3 to about 33.3 MiB, plus the SD of text: about 34.3 MiB. 36 MiB leaves about 1.7 MiB for JSON structure, escaping and the data URL prefixes on top of that.""" +DEFAULT_CHAT_MAX_TOKENS: int = 512 +"""``max_length`` for a chat request that sets neither token limit. OpenAI clients usually omit it, and the text +default of 80 tokens is too short for a chat reply.""" + CHAT_REQUEST_CONFLICT_ERROR_TYPE: str = "chat_request_conflict" """The pydantic error type raised for two keys whose values contradict each other. diff --git a/horde/classes/kobold/chat_request.py b/horde/classes/kobold/chat_request.py index 146ed2cb..0e7c7328 100644 --- a/horde/classes/kobold/chat_request.py +++ b/horde/classes/kobold/chat_request.py @@ -95,6 +95,7 @@ from horde.chat_contract import ( CHAT_IMAGE_PROMPT_TOKENS, CHAT_REQUEST_CONFLICT_ERROR_TYPE, CHAT_REQUEST_CONFLICT_RC, + DEFAULT_CHAT_MAX_TOKENS, MAX_CHAT_IMAGE_BYTES, MAX_CHAT_IMAGE_COUNT, MAX_CHAT_IMAGE_TOTAL_BYTES, @@ -218,10 +219,6 @@ MAX_CHAT_FAULT_REASON_LENGTH = 1000 sentence, and a schema mismatch adds only the schema name, a keyword and a JSON path, so only a deeply nested path reaches this width; a longer message is stored cut to it.""" -DEFAULT_CHAT_MAX_TOKENS = 512 -"""``max_length`` for a chat request that sets neither token limit. OpenAI clients usually omit it, and the text -default of 80 tokens is too short for a chat reply.""" - CHAT_STREAM_PROGRESS_KEY = "stream_progress" """The stored extras key, and the job payload key, of the stream hint: ``true`` when the client streams the reply, so the worker posts deltas to the progress route while it generates. Absent otherwise.""" @@ -1680,18 +1677,22 @@ def build_stored_reply(message: ChatSubmittedMessage) -> dict[str, Any]: def chat_reply_characters(message: Mapping[str, Any]) -> int: """Return the characters a submitted chat reply is charged on. - The text formula prices a reply by its characters. A tool call is output the model generated as well, so its - function name and arguments count with the content. + The text formula prices a reply by its characters. Tool calls and reasoning are output the model generated as + well, so a tool call's function name and arguments and the reasoning count with the content. OpenAI bills + reasoning as completion tokens in the same way. Args: message: The stored assistant message a worker submitted. Returns: - The length of the content, plus the length of each tool call's function name and arguments. + The length of the content, plus the length of each tool call's function name and arguments, plus the length + of the reasoning. """ content = message.get("content") or "" + reasoning = message.get("reasoning_content") or "" tool_calls = message.get("tool_calls") or () - return len(content) + sum(len(tool_call["function"]["name"]) + len(tool_call["function"]["arguments"]) for tool_call in tool_calls) + tool_call_characters = sum(len(tool_call["function"]["name"]) + len(tool_call["function"]["arguments"]) for tool_call in tool_calls) + return len(content) + tool_call_characters + len(reasoning) def rewrite_message_texts( diff --git a/horde/classes/kobold/processing_generation.py b/horde/classes/kobold/processing_generation.py index 70fb617e..826356b7 100644 --- a/horde/classes/kobold/processing_generation.py +++ b/horde/classes/kobold/processing_generation.py @@ -6,7 +6,9 @@ import enum import math import os import time +from collections.abc import Mapping from datetime import UTC, datetime +from typing import Any from sqlalchemy.sql import expression @@ -675,19 +677,25 @@ class TextProcessingGeneration(ProcessingGeneration): db.session.commit() return kudos - def get_things_count(self, generation=None): + def get_things_count(self, generation: str | None = None, message: Mapping[str, Any] | None = None) -> int: """Return the output estimate the generation is charged on: its characters divided by 4, rounded up, at least 20 and at most the requested ``max_length``. Args: generation: Text to estimate in place of the stored generation. + message: A chat reply to estimate in place of the stored generation, before the reply is stored. Returns: - The estimate. A stored chat reply is counted by - [`chat_reply_characters`][horde.classes.kobold.chat_request.chat_reply_characters], so tool calls are - charged as output. A cancelled streamed generation is counted by the posted characters it was settled on. + The estimate. A chat reply is counted by + [`chat_reply_characters`][horde.classes.kobold.chat_request.chat_reply_characters], so tool calls and + reasoning are charged as output, and a reply whose budget went to reasoning is charged up to the requested + ``max_length``. A chat submit passes its reply as ``message`` for the worker's speed sample, so the speed + sample and the charge are the same count. A cancelled streamed generation is counted by the posted + characters it was settled on. """ - if generation is not None: + if message is not None: + character_count = chat_reply_characters(message) + elif generation is not None: character_count = len(generation) elif self.message is not None: character_count = chat_reply_characters(self.message) diff --git a/horde/oai/app.py b/horde/oai/app.py index 362a8ca7..f3fca5dc 100644 --- a/horde/oai/app.py +++ b/horde/oai/app.py @@ -64,7 +64,6 @@ from horde.chat_contract import ( MAX_CHAT_IMAGE_TOTAL_BYTES, NO_ELIGIBLE_WORKER_RC, ChatAvailabilityMode, - ChatContextAdjustmentReason, ChatImpossibleReason, ChatRequestRefusal, KudosCapExcess, @@ -102,6 +101,7 @@ from horde.oai.mapping import ( build_model_card, build_native_chat_body, build_system_fingerprint, + lowered_max_length, relays_deltas, ) from horde.oai.metrics import ( @@ -1137,10 +1137,7 @@ def _log_refused_parameter(request_path: str | None, *, label: str) -> None: def _max_length_was_lowered(context_adjustments: Sequence[object] | None) -> bool: """Return whether the Horde's ``context_adjustments`` hold a ``worker_max_length`` entry.""" - return any( - isinstance(adjustment, Mapping) and adjustment.get("reason") == ChatContextAdjustmentReason.WORKER_MAX_LENGTH.value - for adjustment in context_adjustments or () - ) + return lowered_max_length(context_adjustments) is not None def _refused_parameter_label(request_path: str | None) -> str: diff --git a/horde/oai/mapping.py b/horde/oai/mapping.py index b307c235..fbc9a5c1 100644 --- a/horde/oai/mapping.py +++ b/horde/oai/mapping.py @@ -17,13 +17,19 @@ import hashlib import json import math import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from http import HTTPStatus from typing import cast from horde_sdk.generation_parameters.text.chat_policy import ChatFeature, ChatGuarantee -from horde.chat_contract import ChatTextResponseFormat, ChatTool, ChatToolCall +from horde.chat_contract import ( + DEFAULT_CHAT_MAX_TOKENS, + ChatContextAdjustmentReason, + ChatTextResponseFormat, + ChatTool, + ChatToolCall, +) from horde.oai.horde_client import HordeActiveModel, HordeChatGeneration, HordeChatStatus from horde.oai.schema import ( ChatCompletionChoice, @@ -58,11 +64,13 @@ __all__ = [ "build_error_envelope", "build_model_card", "build_native_chat_body", + "effective_completion_limit", "estimate_tokens", "flatten_messages", "generation_content", "generation_reasoning", "generation_tool_calls", + "lowered_max_length", "relays_deltas", ] @@ -355,6 +363,46 @@ def estimate_tokens(text: str) -> int: return math.ceil(len(text) / CHARACTERS_PER_ESTIMATED_TOKEN) +def lowered_max_length(context_adjustments: Sequence[object] | None) -> int | None: + """Return the lowered ``max_tokens`` of the Horde's ``worker_max_length`` adjustment. + + Args: + context_adjustments: The ``context_adjustments`` of the Horde's chat status. + + Returns: + The ``to`` value of the ``worker_max_length`` entry, or None when the Horde did not lower the limit. + """ + for adjustment in context_adjustments or (): + if not isinstance(adjustment, Mapping) or adjustment.get("reason") != ChatContextAdjustmentReason.WORKER_MAX_LENGTH.value: + continue + lowered_limit = adjustment.get("to") + if isinstance(lowered_limit, int): + return lowered_limit + return None + + +def effective_completion_limit(chat_request: ChatCompletionRequest, context_adjustments: Sequence[object] | None) -> int: + """Return the completion limit the request ran under. + + Args: + chat_request: The client's request. + context_adjustments: The ``context_adjustments`` of the Horde's chat status. + + Returns: + The lowered limit from [`lowered_max_length`][horde.oai.mapping.lowered_max_length] when the Horde lowered + it, else ``max_completion_tokens``, else ``max_tokens``, else + [`DEFAULT_CHAT_MAX_TOKENS`][horde.chat_contract.DEFAULT_CHAT_MAX_TOKENS], the default of the native route. + """ + lowered_limit = lowered_max_length(context_adjustments) + if lowered_limit is not None: + return lowered_limit + if chat_request.max_completion_tokens is not None: + return chat_request.max_completion_tokens + if chat_request.max_tokens is not None: + return chat_request.max_tokens + return DEFAULT_CHAT_MAX_TOKENS + + def generation_content(generation: HordeChatGeneration) -> str | None: """Return a generation's reply text: the message content when present, else the plain ``text`` field. @@ -405,8 +453,14 @@ def build_completion( Returns: One choice per generation, in the Horde's order; ``model``, ``system_fingerprint`` and the ``x_horde`` worker details are the first generation's. A message has ``reasoning_content`` unless - ``horde.include_reasoning`` is false. ``usage.completion_tokens_details.reasoning_tokens`` is the sum of the - reasoning counts the workers reported, present when any reported one, whether or not the reasoning is returned. + ``horde.include_reasoning`` is false. ``usage.completion_tokens_details.reasoning_tokens`` is the sum of each + generation's reasoning count, as its worker reported it or, when the worker did not report it, as estimated + from the reasoning characters. It is present when any generation has a count, whether or not the reasoning is + returned. An estimated ``usage.completion_tokens`` counts the reasoning with the reply text and tool calls, so + it includes the reasoning tokens as OpenAI's does. Each generation's estimated completion and reasoning + counts are capped at the [`effective_completion_limit`][horde.oai.mapping.effective_completion_limit], + since the worker could not generate more than that, and an estimated reasoning count is at most the + generation's completion count. A reported count is used as reported. Raises: CompletionWithoutGenerationError: ``status`` has no finished generation. A completion built from it would be a @@ -421,6 +475,9 @@ def build_completion( reasoning_tokens: int | None = None has_empty_return = False include_reasoning = chat_request.horde is None or chat_request.horde.include_reasoning + # The characters-per-token estimate overcounts dense text, so a reply that used its whole budget would report + # more tokens than the limit allowed. + completion_limit = effective_completion_limit(chat_request, status.context_adjustments) for generation_index, generation in enumerate(status.generations): generated_text = generation_content(generation) tool_calls = generation_tool_calls(generation) @@ -439,15 +496,27 @@ def build_completion( native_finish_reason=generation.native_finish_reason, ), ) + generated_reasoning = generation_reasoning(generation) if generation.completion_tokens is None: - completion_tokens += estimate_tokens(_generated_output_text(generated_text, tool_calls)) + generation_completion_tokens = min( + estimate_tokens(_generated_output_text(generated_text, tool_calls, generated_reasoning)), + completion_limit, + ) estimated_usage = True else: - completion_tokens += generation.completion_tokens + generation_completion_tokens = generation.completion_tokens + completion_tokens += generation_completion_tokens if prompt_tokens is None and generation.prompt_tokens is not None: prompt_tokens = generation.prompt_tokens if generation.reasoning_tokens is not None: reasoning_tokens = (reasoning_tokens or 0) + generation.reasoning_tokens + elif generated_reasoning: + reasoning_tokens = (reasoning_tokens or 0) + min( + estimate_tokens(generated_reasoning), + completion_limit, + generation_completion_tokens, + ) + estimated_usage = True if prompt_tokens is None: prompt_tokens = estimate_tokens( flatten_messages(chat_request.messages, tools=chat_request.tools), @@ -481,12 +550,26 @@ def build_completion( ) -def _generated_output_text(generated_text: str | None, tool_calls: Sequence[ChatToolCall] | None) -> str: - """Return the text the completion token estimate counts: the reply text and the compact JSON of its tool calls.""" +def _generated_output_text( + generated_text: str | None, + tool_calls: Sequence[ChatToolCall] | None, + generated_reasoning: str | None, +) -> str: + """Return the text the completion token estimate counts: the reasoning, the reply text and the compact JSON of its + tool calls. + + OpenAI counts reasoning tokens among the completion tokens, so the estimate includes the reasoning. A reply with + reasoning and no other output is the reasoning alone, so its completion estimate equals its reasoning estimate. + """ output_parts = [generated_text or ""] if tool_calls: output_parts.append(_compact_json([tool_call.model_dump(mode="json") for tool_call in tool_calls])) - return "\n".join(output_parts) + output_text = "\n".join(output_parts) + if not generated_reasoning: + return output_text + if not output_text: + return generated_reasoning + return f"{generated_reasoning}\n{output_text}" def _finish_reason(reported_finish_reason: str | None) -> FinishReason: diff --git a/horde/oai/schema.py b/horde/oai/schema.py index 81058dcd..128a5697 100644 --- a/horde/oai/schema.py +++ b/horde/oai/schema.py @@ -1015,11 +1015,13 @@ class CompletionTokensDetails(BaseModel): model_config = ConfigDict(frozen=True) reasoning_tokens: int - """How many of the generated tokens were reasoning, as the worker reported them.""" + """How many of the generated tokens were reasoning, as the worker reported them or as estimated from the reasoning + characters.""" class ChatCompletionUsage(BaseModel): - """Represents the token counts of a completion; ``completion_tokens_details`` only when a worker reported them.""" + """Represents the token counts of a completion; ``completion_tokens_details`` only when a reply has reasoning or a reported + reasoning count.""" model_config = ConfigDict(frozen=True) @@ -1033,8 +1035,8 @@ class ChatCompletionUsage(BaseModel): """The sum of the prompt and completion tokens.""" completion_tokens_details: CompletionTokensDetails | None = None - """The reasoning tokens among the completion tokens, when the worker reported them. Informational: the Horde - charges and rewards on the reply it measures.""" + """The reasoning tokens among the completion tokens, when a worker reported them or submitted reasoning. + Informational: the Horde charges and rewards on the reply it measures.""" @model_serializer(mode="wrap") def serialize_without_absent_details(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]: diff --git a/tests/integration/test_chat.py b/tests/integration/test_chat.py index 3d245e2b..2ec78998 100644 --- a/tests/integration/test_chat.py +++ b/tests/integration/test_chat.py @@ -3033,50 +3033,59 @@ def test_a_request_on_its_third_attempt_is_not_done_while_the_job_is_processing( _cancel(client, request_headers, "chat", request_id) -def test_reasoning_is_stored_and_shown_beside_the_reply_and_neither_it_nor_its_count_is_charged( +def test_reasoning_is_stored_beside_the_reply_and_its_characters_are_charged_rewarded_and_sampled_as_content( app, client, request_headers: dict[str, str], chat_worker: FakeChatWorker, ) -> None: - """The Horde charges and rewards on the reply it measures, so a reply with reasoning earns what the same reply - without it earns.""" + """The Horde charges, rewards and samples the worker's speed on the characters it measures, reasoning included, + so a reply with reasoning earns and samples what a plain reply of the same total length does. The reported + ``reasoning_tokens`` count is stored and shown without being charged.""" + from horde.classes.base.stats import FulfillmentPerformance from horde.classes.kobold.processing_generation import TextProcessingGeneration from horde.flask import db - reasoning_request_id = _submit_chat_request(client, request_headers, max_tokens=400) - reasoning_pop = chat_worker.pop() - reasoning_submit = chat_worker.submit( - build_chat_submit_body( - generation_id=reasoning_pop["id"], - content="Blue.", - model_id=CHAT_MODEL, - completion_tokens=300, - reasoning_content="The user asked for a colour. " * 40, - reasoning_tokens=297, - ), - ) - assert reasoning_submit.status_code == 200, reasoning_submit.get_data(as_text=True) + reasoning_content = "The user asked for a colour. " * 40 - _submit_chat_request(client, request_headers, max_tokens=400) - plain_pop = chat_worker.pop() - plain_submit = chat_worker.submit(build_chat_submit_body(generation_id=plain_pop["id"], content="Blue.", model_id=CHAT_MODEL)) - assert plain_submit.status_code == 200, plain_submit.get_data(as_text=True) + def submit_reply(**submit_fields: Any) -> tuple[str, str, float, float]: + """Submit a reply to a new request of 400 tokens, so the cap does not apply, and return the request id, the + generation id, the reward and the things count of the speed sample recorded by the submit.""" + request_id = _submit_chat_request(client, request_headers, max_tokens=400) + pop_result = chat_worker.pop() + response = chat_worker.submit(build_chat_submit_body(generation_id=pop_result["id"], model_id=CHAT_MODEL, **submit_fields)) + assert response.status_code == 200, response.get_data(as_text=True) + with app.app_context(): + sampled_things = db.session.query(FulfillmentPerformance.things).order_by(FulfillmentPerformance.id.desc()).limit(1).scalar() + return request_id, pop_result["id"], response.get_json()["reward"], sampled_things + + reasoning_request_id, reasoning_generation_id, reasoning_reward, reasoning_sample = submit_reply( + content="Blue.", + completion_tokens=300, + reasoning_content=reasoning_content, + reasoning_tokens=297, + ) + _, _, same_length_reward, same_length_sample = submit_reply(content="Blue." + "x" * len(reasoning_content)) + _, _, plain_reward, plain_sample = submit_reply(content="Blue.") [generation] = _chat_status(client, request_headers, reasoning_request_id)["generations"] assert generation["text"] == "Blue." assert generation["message"] == { "role": "assistant", "content": "Blue.", - "reasoning_content": "The user asked for a colour. " * 40, + "reasoning_content": reasoning_content, } assert generation["completion_tokens"] == 300 assert generation["reasoning_tokens"] == 297 with app.app_context(): - stored = db.session.get(TextProcessingGeneration, uuid.UUID(reasoning_pop["id"])) + stored = db.session.get(TextProcessingGeneration, uuid.UUID(reasoning_generation_id)) assert stored.generation == "Blue." assert stored.reasoning_tokens == 297 - assert reasoning_submit.get_json()["reward"] == plain_submit.get_json()["reward"] + assert reasoning_reward == same_length_reward + assert reasoning_reward > plain_reward + # 1165 characters over 4, rounded up; the plain reply's 5 characters get the 20 token floor. + assert reasoning_sample == same_length_sample == 292 + assert plain_sample == 20 def test_a_submit_with_reasoning_over_the_text_cap_is_refused_as_invalid( diff --git a/tests/unit/oai/test_chat_front.py b/tests/unit/oai/test_chat_front.py index 3d1bec5c..1a72d341 100644 --- a/tests/unit/oai/test_chat_front.py +++ b/tests/unit/oai/test_chat_front.py @@ -8,13 +8,16 @@ from __future__ import annotations import asyncio import json +import math import time import uuid +from typing import Any import pytest +from horde.chat_contract import DEFAULT_CHAT_MAX_TOKENS from horde.consts import ANONYMOUS_API_KEY -from horde.oai.mapping import build_system_fingerprint +from horde.oai.mapping import CHARACTERS_PER_ESTIMATED_TOKEN, build_system_fingerprint from tests.unit.oai.conftest import ( CHARGE_BASIS, FAST_TEST_SETTINGS, @@ -755,9 +758,110 @@ async def test_reported_reasoning_tokens_appear_in_the_usage_details_whether_or_ } -async def test_usage_omits_completion_details_when_the_worker_did_not_report_reasoning_tokens(start_front: StartFront) -> None: +@pytest.mark.parametrize("include_reasoning", [True, False]) +async def test_reasoning_tokens_are_estimated_from_the_reasoning_when_the_worker_did_not_report_them( + start_front: StartFront, + include_reasoning: bool, +) -> None: + """Without a reported count, the reasoning tokens are estimated from the reasoning characters as the other + unreported counts are, and the usage is marked as estimated.""" async with start_front() as front: - front.horde.status_payload = done_status(reasoning_content=REASONING_TEXT) + front.horde.status_payload = done_status(reasoning_content=REASONING_TEXT, completion_tokens=40) + response = await front.client.post( + "/v1/chat/completions", + json=build_chat_body(horde={"include_reasoning": include_reasoning}), + ) + + expected_reasoning_tokens = math.ceil(len(REASONING_TEXT) / CHARACTERS_PER_ESTIMATED_TOKEN) + assert response.json()["usage"]["completion_tokens_details"] == {"reasoning_tokens": expected_reasoning_tokens} + assert response.json()["x_horde"]["estimated_usage"] is True + + +@pytest.mark.parametrize("content", ["", GENERATED_TEXT]) +async def test_an_estimated_completion_count_includes_the_estimated_reasoning_count( + start_front: StartFront, + content: str, +) -> None: + """Without reported counts, the estimated completion tokens include the reasoning, so they are at least the + estimated reasoning tokens, and equal to them when the reply is the reasoning alone.""" + async with start_front() as front: + front.horde.status_payload = done_status(content=content, reasoning_content=REASONING_TEXT, completion_tokens=None) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + usage = response.json()["usage"] + reasoning_tokens = usage["completion_tokens_details"]["reasoning_tokens"] + if content: + assert usage["completion_tokens"] > reasoning_tokens + else: + assert usage["completion_tokens"] == reasoning_tokens + + +def _chat_body_with_token_limits(**token_limits: int) -> dict[str, Any]: + """Return a chat request body that sets only the given token limits.""" + chat_body = build_chat_body() + del chat_body["max_tokens"] + chat_body.update(token_limits) + return chat_body + + +@pytest.mark.parametrize( + ("chat_body", "context_adjustments", "expected_limit"), + [ + (_chat_body_with_token_limits(max_tokens=40), None, 40), + (_chat_body_with_token_limits(max_completion_tokens=40), None, 40), + (_chat_body_with_token_limits(), None, DEFAULT_CHAT_MAX_TOKENS), + ( + _chat_body_with_token_limits(max_tokens=64), + [{"from": 64, "to": 32, "reason": "worker_max_length"}], + 32, + ), + ], + ids=["max-tokens", "max-completion-tokens", "native-default", "lowered-worker-max-length"], +) +async def test_estimated_completion_and_reasoning_counts_are_capped_at_the_effective_completion_limit( + start_front: StartFront, + chat_body: dict[str, Any], + context_adjustments: list[dict[str, Any]] | None, + expected_limit: int, +) -> None: + """A reasoning-only reply whose estimate exceeds the limit the request ran under reports the limit as both its + completion and reasoning count.""" + long_reasoning = "x" * (expected_limit * CHARACTERS_PER_ESTIMATED_TOKEN * 2) + async with start_front() as front: + front.horde.status_payload = done_status(content="", reasoning_content=long_reasoning, completion_tokens=None) + front.horde.status_payload["context_adjustments"] = context_adjustments + response = await front.client.post("/v1/chat/completions", json=chat_body) + + usage = response.json()["usage"] + assert usage["completion_tokens"] == expected_limit + assert usage["completion_tokens_details"] == {"reasoning_tokens": expected_limit} + assert response.json()["x_horde"]["estimated_usage"] is True + + +async def test_estimated_counts_below_the_effective_completion_limit_are_not_capped(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status(content="", reasoning_content=REASONING_TEXT, completion_tokens=None) + response = await front.client.post("/v1/chat/completions", json=build_chat_body()) + + expected_tokens = math.ceil(len(REASONING_TEXT) / CHARACTERS_PER_ESTIMATED_TOKEN) + usage = response.json()["usage"] + assert usage["completion_tokens"] == expected_tokens + assert usage["completion_tokens_details"] == {"reasoning_tokens": expected_tokens} + + +async def test_reported_counts_over_the_effective_completion_limit_are_used_as_reported(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status(reasoning_content=REASONING_TEXT, completion_tokens=100, reasoning_tokens=90) + response = await front.client.post("/v1/chat/completions", json=_chat_body_with_token_limits(max_tokens=40)) + + usage = response.json()["usage"] + assert usage["completion_tokens"] == 100 + assert usage["completion_tokens_details"] == {"reasoning_tokens": 90} + + +async def test_usage_omits_completion_details_when_the_reply_has_no_reasoning(start_front: StartFront) -> None: + async with start_front() as front: + front.horde.status_payload = done_status() response = await front.client.post("/v1/chat/completions", json=build_chat_body(horde={"include_reasoning": True})) assert "completion_tokens_details" not in response.json()["usage"] diff --git a/tests/unit/test_chat_request.py b/tests/unit/test_chat_request.py index 9783a379..de3ad22d 100644 --- a/tests/unit/test_chat_request.py +++ b/tests/unit/test_chat_request.py @@ -7,8 +7,10 @@ from __future__ import annotations import base64 +import functools import gzip import json +import math import uuid from io import BytesIO from types import SimpleNamespace @@ -92,7 +94,7 @@ from horde.classes.kobold.chat_request import ( truncate_oldest_messages, verify_chat_reply, ) -from horde.classes.kobold.processing_generation import is_empty_submission +from horde.classes.kobold.processing_generation import TextProcessingGeneration, is_empty_submission from horde.classes.kobold.request_fit import estimate_prompt_tokens from horde.exceptions import KNOWN_RC from horde.image import MAX_CHAT_IMAGE_SIDE, load_chat_image @@ -1516,12 +1518,39 @@ class TestSubmittedReply: with pytest.raises(ValidationError): ChatSubmittedMessage.model_validate({"role": "assistant", "content": None, "reasoning_content": "Thinking."}) - def test_reasoning_is_not_charged(self) -> None: + def test_reasoning_is_charged_with_the_content(self) -> None: reply = build_stored_reply( ChatSubmittedMessage.model_validate({"role": "assistant", "content": "four", "reasoning_content": "x" * 400}) ) - assert chat_reply_characters(reply) == 4 + assert chat_reply_characters(reply) == 404 + + def test_a_reply_whose_budget_went_to_reasoning_is_charged_on_the_reasoning_up_to_the_requested_length(self) -> None: + """A reply with empty content and long reasoning is charged on the reasoning characters over 4, capped at the + requested ``max_length``, so it is not charged the 20 token floor.""" + reply = build_stored_reply( + ChatSubmittedMessage.model_validate({"role": "assistant", "content": "", "reasoning_content": "x" * 1200}) + ) + + assert _things_count(reply, max_length=512) == 300 + assert _things_count(reply, max_length=256) == 256 + + @pytest.mark.parametrize("max_length", [400, 256]) + def test_a_chat_submit_speed_sample_is_the_charged_count_with_the_reasoning(self, max_length: int) -> None: + """The submitted ``generation`` of a chat job holds the content alone, and the speed sample still counts the + reasoning, capped at the requested ``max_length`` as the charge is.""" + reply = build_stored_reply( + ChatSubmittedMessage.model_validate({"role": "assistant", "content": "Blue.", "reasoning_content": "x" * 1160}) + ) + + speed_sample = _submitted_things_count(chat_message=reply, generation="Blue.", max_length=max_length) + + assert speed_sample == _things_count(reply, max_length=max_length) + assert speed_sample == min(math.ceil(1165 / 4), max_length) + + def test_a_text_submit_speed_sample_is_the_count_of_the_submitted_generation(self) -> None: + assert _submitted_things_count(chat_message=None, generation="x" * 400, max_length=512) == 100 + assert _submitted_things_count(chat_message=None, generation="short", max_length=512) == 20 def test_the_charged_characters_include_each_tool_call_name_and_arguments(self) -> None: reply = {"role": "assistant", "content": "Checking.", "tool_calls": [LOOKUP_CALL, LOOKUP_CALL]} @@ -1556,6 +1585,37 @@ def _schema_format(schema: dict[str, Any], *, strict: bool | None = True) -> dic return {"type": "json_schema", "json_schema": json_schema} +def _things_count(stored_reply: dict[str, Any], *, max_length: int) -> int: + """Return the output estimate a generation with ``stored_reply`` is charged on, for a request of ``max_length``.""" + generation = SimpleNamespace( + message=stored_reply, + generation=None, + settled_character_count=None, + wp=SimpleNamespace(things=max_length), + ) + things_count: int = TextProcessingGeneration.get_things_count(generation) # type: ignore[arg-type,no-untyped-call] + return things_count + + +def _submitted_things_count(*, chat_message: dict[str, Any] | None, generation: str, max_length: int) -> int: + """Return the speed sample count of a text submit resource holding ``chat_message`` and ``generation``, for a + request of ``max_length`` whose reply is not stored yet.""" + from horde.apis.v2.kobold import TextJobSubmit + + procgen = SimpleNamespace( + message=None, + generation=None, + settled_character_count=None, + wp=SimpleNamespace(things=max_length), + ) + procgen.get_things_count = functools.partial(TextProcessingGeneration.get_things_count, procgen) # type: ignore[arg-type] + submit = TextJobSubmit.__new__(TextJobSubmit) + submit.procgen = procgen + submit.args = {"generation": generation} + submit.chat_message = chat_message + return submit.get_submitted_things_count() + + def _call(name: str = "lookup_weather", call_id: str = "call_abc") -> dict[str, Any]: return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}