Files
AI-Horde/horde/exceptions.py
T
tazlin 2a7361dabb fix(text): refuse NoEligibleWorker in place of a kudos refusal when no worker can serve the request
With no eligible worker the free allowance takes its floor, and a request over it was refused for kudos that do not help until a worker that can serve it comes online.
- A text or chat request the kudos rules would refuse while no online worker passes the dispatch gates is refused 403 NoEligibleWorker (TextAsyncGenerate.refuse_when_no_worker_is_eligible). The waiting prompt is deleted and the decision counter records refused.
- The NoEligibleWorker message states the missing declarations or exceeded limits when they are the cause. The body carries impossible_reason in the status route's shape.
- handle_bad_requests merges an exception's body_fields into the error body. Exceptions without body_fields keep the message and rc body.
- Funded requests and requests within the allowance are created and queued whether or not a worker can serve them.
- The front reads impossible_reason from a native error body onto HordeAPIError. _horde_error maps NoEligibleWorker to no_worker_supports_features, no_worker_supports_limit or no_eligible_worker.
- The front's gap and no-eligible-worker errors take request_id and job_may_exist. A refusal at submit carries no request id and does not state that the request was cancelled.
- _size_limit_gap_chat_offers takes the gap directly, so a submit refused on a size limit gap states the largest declared limit.
- NoEligibleWorker is in KNOWN_RC, README_return_codes.md, ADR 0037, chat_requests.md and chat_front.md.
2026-10-31 01:14:53 -04:00

872 lines
31 KiB
Python

# SPDX-FileCopyrightText: 2022 Konstantinos Thoukydidis <mail@dbzer0.com>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
from flask import has_request_context, request
from werkzeug import exceptions as wze
from horde import metrics
from horde.chat_contract import CHAT_KUDOS_CAP_EXCEEDED_RC, NO_ELIGIBLE_WORKER_RC, ChatImpossibleReason, KudosCapExcess
from horde.logger import logger
from horde.vars import horde_title
UNLISTED_RETURN_CODE = "Unlisted"
"""The return code label a refusal is counted under when its code is not in ``KNOWN_RC``, which keeps the label bounded."""
UNMATCHED_ROUTE = "unmatched"
"""The route label a refusal is counted under when it was raised outside a matched route."""
KNOWN_RC = [
"MissingPrompt",
"CorruptPrompt",
"KudosValidationError",
"NoValidActions",
"InvalidSize",
"InvalidPromptSize",
"TooManySteps",
"Profanity",
"ProfaneWorkerName",
"ProfaneBridgeAgent",
"ProfaneWorkerInfo",
"ProfaneUserName",
"ProfaneUserContact",
"ProfaneAdminComment",
"ProfaneTeamName",
"ProfaneTeamInfo",
"TooLong",
"TooLongWorkerName",
"TooLongUserName",
"NameAlreadyExists",
"WorkerNameAlreadyExists",
"TeamNameAlreadyExists",
"PolymorphicNameConflict",
"ImageValidationFailed",
"SourceImageResolutionExceeded",
"SourceImageSizeExceeded",
"SourceImageUrlInvalid",
"SourceImageUrlRefused",
"SourceImageUnreadable",
"InpaintingMissingMask",
"SourceMaskUnnecessary",
"UnsupportedSampler",
"UnsupportedModel",
"ControlNetUnsupported",
"ControlNetSourceMissing",
"ControlNetInvalidPayload",
"SourceImageRequiredForModel",
"UnexpectedModelName",
"TooManyUpscalers",
"ProcGenNotFound",
"InvalidAestheticAttempt",
"AestheticsNotCompleted",
"AestheticsNotPublic",
"AestheticsDuplicate",
"AestheticsMissing",
"AestheticsSolo",
"AestheticsConfused",
"AestheticsAlreadyExist",
"AestheticsServerRejected",
"AestheticsServerError",
"AestheticsServerDown",
"AestheticsServerTimeout",
"InvalidAPIKey",
"WrongCredentials",
"NotAdmin",
"NotModerator",
"NotOwner",
"NotRequestOwner",
"NotPrivileged",
"AnonForbidden",
"AnonForbiddenWorker",
"AnonForbiddenUserMod",
"NotTrusted",
"UntrustedTeamCreation",
"UntrustedUnsafeIP",
"WorkerMaintenance",
"WorkerFlaggedMaintenance",
"TooManySameIPs",
"WorkerInviteOnly",
"UnsafeIP",
"TimeoutIP",
"TooManyNewIPs",
"KudosUpfront",
"SharedKeyInvalid",
"SharedKeyEmpty",
"SharedKeyExpired",
"SharedKeyInsufficientKudos",
"SharedKeyAssignedStyles",
"InvalidJobID",
"RequestNotFound",
"WorkerNotFound",
"TeamNotFound",
"FilterNotFound",
"UserNotFound",
"DuplicateGen",
"AbortedGen",
"RequestExpired",
"TooManyPrompts",
"NoValidWorkers",
"MaintenanceMode",
"TargetAccountFlagged",
"SourceAccountFlagged",
"FaultWhenKudosReceiving",
"FaultWhenKudosSending",
"TooFastKudosTransfers",
"KudosTransferToAnon",
"KudosTransferToSelf",
"KudosTransferNotEnough",
"NegativeKudosTransfer",
"KudosTransferFromAnon",
"InvalidAwardUsername",
"KudosAwardToAnon",
"NotAllowedAwards",
"NoWorkerModSelected",
"NoUserModSelected",
"NoHordeModSelected",
"NoTeamModSelected",
"NoFilterModSelected",
"NoSharedKeyModSelected",
"BadRequest",
"Forbidden",
"Locked",
"ControlNetMismatch",
"HiResFixMismatch",
"TooManyLoras",
"BadLoraVersion",
"TooManyTIs",
"BetaAnonForbidden",
"BetaComparisonFault",
"BadCFGDecimals",
"BadCFGNumber",
"BadClientAgent",
"SpecialMissingPayload",
"SpecialForbidden",
"SpecialMissingUsername",
"SpecialModelNeedsSpecialUser",
"SpecialFieldNeedsSpecialUser",
"Img2ImgMismatch",
"TilingMismatch",
"EducationCannotSendKudos",
"InvalidPriorityUsername",
"OnlyServiceAccountProxy",
"RequiresTrust",
"InvalidRemixModel",
"InvalidExtraSourceImages",
"TooManyExtraSourceImages",
"MissingFullSamplerOrder",
"TooManyStopSequences",
"ExcessiveStopSequence",
"TokenOverflow",
"MoreThanMinExtraSourceImage",
"InvalidExtraTexts",
"MissingExtraTexts",
"InvalidTransparencyModel",
"InvalidTransparencyImg2Img",
"InvalidTransparencyCN",
"HiResMismatch",
"StylesAnonForbidden",
"StylePromptMissingVars",
"StylePromptFieldInvalid",
"TemplateFieldsMatchNoCollectionStyle",
"StylesRequiresCustomizer",
"StyleMismatch",
"StyleGetMistmatch",
"TooManyStyleExamples",
"ExampleURLAlreadyInUse",
"MessagesOnlyOwnWorkers",
"NotUserOrMod",
"CannotDeleteMod",
"UserNotDeleted",
"DeletedUser",
"CannotWipeActiveUser",
"NonServiceForbidden",
"NulByteInPayload",
"SamplerKnobInapplicable",
"SamplerKnobOutOfRange",
"SamplerSolverTypeUnsupported",
"SamplerSchedulerMismatch",
"InvalidOperationsLimit",
"InvalidOperationsCursor",
"InvalidModerationTimeRange",
"InvalidModerationAddressFilter",
"InvalidModerationEventID",
"InvalidModerationReason",
"InvalidModerationNote",
"ModerationEventNotFound",
"InvalidWorkerID",
"InvalidSuspicionID",
"SchedulerBaselineMismatch",
"FlowShiftInapplicable",
"FlowShiftOutOfRange",
"ControlStrengthWithoutControlType",
"ControlStrengthOutOfRange",
"InvalidPromptEncoding",
"ControlNetInpaintingMismatch",
"InvalidControlType",
"ThingNotFound",
"TooManyWorkers",
"TooManyWorkersTrusted",
"StylePolicyOverridableMisplaced",
"StyleParameterAboveCeiling",
"StyleDeclarationInvalid",
"TemplateFieldsRequireStyle",
"TemplateFieldUnknown",
"TemplateFieldMissing",
"PromptExceedsContext",
"UnsupportedChatParameter",
"ChatRequestConflict",
"ChatAdmissionDisabled",
"ChatDeclarationInvalid",
"ChatSubmitInvalid",
"ChatSubmitNonconformant",
"ChatWorkerFault",
"ChatModelMismatch",
"UnknownChatModel",
"ChatKudosCapExceeded",
"ChatImageStorageUnavailable",
"OperationKeyConflict",
"RequestTooLarge",
"ChatProgressNotStreaming",
"ChatProgressOutOfSequence",
"ChatProgressCapReached",
"ChatProgressUnavailable",
"ChatJobAborted",
"NoEligibleWorker",
]
class BadRequest(wze.BadRequest):
def __init__(self, message: str, log: str | None = None, rc: str = "BadRequest") -> None:
self.specific = message
self.log = log
self.rc = rc
class Forbidden(wze.Forbidden):
def __init__(self, message, log=None, rc="Forbidden"):
self.specific = message
self.log = log
self.rc = rc
class ServiceUnavailable(wze.ServiceUnavailable):
"""A route that is switched off on this horde, refused with 503 so a client can retry elsewhere or later."""
def __init__(self, message: str, log: str | None = None, rc: str = "ServiceUnavailable") -> None:
self.specific = message
self.log = log
self.rc = rc
class NotFound(wze.NotFound):
"""Something a route acts on that does not exist or no longer accepts the call, refused with 404."""
def __init__(self, message: str, log: str | None = None, rc: str = "NotFound") -> None:
self.specific = message
self.log = log
self.rc = rc
class RequestTooLarge(wze.RequestEntityTooLarge):
"""A request body above the size its route accepts, refused with 413 before the body is parsed."""
def __init__(self, message: str, rc: str = "RequestTooLarge") -> None:
self.specific = message
self.log = None
self.rc = rc
class Locked(wze.Locked):
def __init__(self, message, rc="Locked"):
self.specific = message
self.log = None
self.rc = rc
class MissingPrompt(wze.BadRequest):
def __init__(self, username, rc="MissingPrompt"):
self.specific = "You cannot specify an empty prompt."
self.log = f"User '{username}' sent an empty prompt. Aborting!"
self.rc = rc
class CorruptPrompt(wze.BadRequest):
def __init__(self, username, ip, prompt, message=None, rc="CorruptPrompt", proxy_service_ip=None):
if message:
self.specific = message
else:
self.specific = (
"This prompt appears to violate our terms of service and will be reported. Please contact us if you think this is an error."
)
self.log = f"User '{username}' with IP '{ip}' sent an a corrupt prompt: '{prompt}'. Aborting!"
if proxy_service_ip:
self.log += f" Service Proxy IP: {proxy_service_ip}"
self.rc = rc
class KudosValidationError(wze.BadRequest):
def __init__(self, username, error_message, action="transfer", rc="KudosValidationError"):
self.specific = error_message
self.log = f"User '{username}' Failed to {action} Kudos."
self.rc = rc
class NoValidActions(wze.BadRequest):
def __init__(self, error_message, rc="NoValidActions"):
self.specific = error_message
self.log = None
self.rc = rc
class InvalidSize(wze.BadRequest):
def __init__(self, username, rc="InvalidSize"):
self.specific = "Invalid size. The image dimensions have to be multiples of 64."
self.log = f"User '{username}' sent an invalid size. Aborting!"
self.rc = rc
class InvalidPromptSize(wze.BadRequest):
def __init__(self, rc="InvalidPromptSize"):
self.specific = "Too large prompt. Please reduce the amount of tokens contained."
self.log = None
self.rc = rc
class TooManySteps(wze.BadRequest):
def __init__(self, username, steps, rc="TooManySteps"):
self.specific = "Too many sampling steps. To allow resources for everyone, we allow only up to 500 steps."
self.log = f"User '{username}' sent too many steps ({steps}). Aborting!"
self.rc = rc
class Profanity(wze.BadRequest):
def __init__(self, username, text, text_type, rc="Profanity"):
self.specific = f"As our API is public, we do not allow profanity in the {text_type}, please try again."
self.log = f"User '{username}' tried to submit profanity for {text_type} ({text}). Aborting!"
self.rc = rc
class TooLong(wze.BadRequest):
def __init__(self, username, chars, limit, text_type, rc="TooLong"):
self.specific = f"The specified {text_type} is too long. Please stay below {limit}"
self.log = f"User '{username}' tried to submit {chars} chars for {text_type}. Aborting!"
self.rc = rc
class NameAlreadyExists(wze.BadRequest):
def __init__(self, username, old_name, new_name, object_type="worker", rc="NameAlreadyExists"):
self.specific = f"The specified {object_type} name '{new_name}' is already taken!"
self.log = f"User '{username}' tried to change {object_type} name from {old_name} to {new_name}. Aborting!"
self.rc = rc
class PolymorphicNameConflict(wze.BadRequest):
def __init__(self, name, object_type="worker", rc="PolymorphicNameConflict"):
self.specific = (
f"The specified name '{name}' is already taken by a different type of {object_type}. Please choose a different name!"
)
self.log = None
self.rc = rc
class ImageValidationFailed(wze.BadRequest):
def __init__(
self,
message: str = ("Please ensure the source image payload is either a URL containing an image or a valid base64 encoded image."),
rc: str = "ImageValidationFailed",
) -> None:
self.specific = f"Image validation failed. {message}"
self.log = "Source image validation failed"
self.rc = rc
class SourceMaskUnnecessary(wze.BadRequest):
def __init__(self, rc="SourceMaskUnnecessary"):
self.specific = "Please do not pass a source_mask unless you are sending a source_image as well"
self.log = "Tried to pass source_mask with txt2img"
self.rc = rc
class UnsupportedSampler(wze.BadRequest):
def __init__(self, rc="UnsupportedSampler"):
self.specific = "This sampler is not supported in this mode the moment"
self.log = None
self.rc = rc
class UnsupportedModel(wze.BadRequest):
def __init__(self, message=None, rc="UnsupportedModel"):
if message:
self.specific = message
else:
self.specific = "This model is not supported in this mode the moment"
self.log = None
self.rc = rc
class ProcGenNotFound(wze.BadRequest):
def __init__(self, procgen_id, rc="ProcGenNotFound"):
self.specific = f"Image with ID '{procgen_id}' not found in this request."
self.log = f"Attempted to log aesthetic rating with non-existent image ID '{procgen_id}'"
self.rc = rc
class InvalidAestheticAttempt(wze.BadRequest):
def __init__(self, message, rc="InvalidAestheticAttempt"):
self.specific = message
self.log = None
self.rc = rc
class InvalidAPIKey(wze.Unauthorized):
def __init__(self, subject, keytype="API", rc="InvalidAPIKey"):
if keytype == "Shared":
self.specific = "No user matching sent Shared Key."
else:
self.specific = "No user matching sent API Key. Have you remembered to register at https://stablehorde.net/register ?"
self.log = f"Invalid {keytype} Key sent for {subject}"
self.rc = rc
class WrongCredentials(wze.Forbidden):
def __init__(self, username, worker, rc="WrongCredentials"):
self.specific = "Wrong credentials to submit as this worker."
self.log = f"User '{username}' sent wrong credentials for utilizing worker {worker}"
self.rc = rc
class NotAdmin(wze.Forbidden):
def __init__(self, username, endpoint, rc="NotAdmin"):
self.specific = "You're not an admin. Sod off!"
self.log = f"Non-admin user '{username}' tried to use admin endpoint: '{endpoint}. Aborting!"
self.rc = rc
class NotModerator(wze.Forbidden):
def __init__(self, username, endpoint, rc="NotModerator"):
self.specific = "You're not a mod. BTFO!"
self.log = f"Non-mod user '{username}' tried to use mod endpoint: '{endpoint}. Aborting!"
self.rc = rc
class NotOwner(wze.Forbidden):
def __init__(self, username, worker_name, rc="NotOwner"):
self.specific = "You're not an admin. Sod off!"
self.log = f"User '{username}'' tried to modify worker they do not own '{worker_name}'. Aborting!"
self.rc = rc
class NotRequestOwner(wze.Forbidden):
def __init__(self, req_id, rc="NotRequestOwner"):
self.specific = "Only the API key that submitted this request can retrieve its parameters."
self.log = f"Key mismatch when retrieving the parameters of request {req_id}."
self.rc = rc
class NotPrivileged(wze.Forbidden):
def __init__(self, username, message, action, rc="NotPrivileged"):
self.specific = message
self.log = f"Non-Privileged user '{username}' tried to take privileged action '{action}'. Aborting!"
self.rc = rc
class AnonForbidden(wze.Forbidden):
def __init__(self, rc="AnonForbidden"):
self.specific = "Anonymous user is forbidden from performing this operation"
self.log = None
self.rc = rc
class NotTrusted(wze.Forbidden):
def __init__(self, rc="NotTrusted"):
self.specific = "Only Trusted users are allowed to perform this operation"
self.log = None
self.rc = rc
class WorkerMaintenance(wze.Forbidden):
def __init__(self, maintenance_msg, rc="WorkerMaintenance"):
self.specific = maintenance_msg
self.log = None
self.rc = rc
class TooManySameIPs(wze.Forbidden):
def __init__(self, username, rc="TooManySameIPs"):
self.specific = (
"You are running too many workers from the same location. To prevent abuse, "
"please contact us on Discord to allow you to join more workers from the same IP: https://discord.gg/aG68kk3Qpz"
)
self.log = f"User '{username} is trying to onboard too many workers from the same IP Address. Aborting!"
self.rc = rc
class WorkerInviteOnly(wze.Forbidden):
def __init__(self, current_workers, rc="WorkerInviteOnly"):
if current_workers == 0:
self.specific = (
"This horde has been switched to worker invite-only mode. "
"Please contact us on Discord to allow you to join your worker: https://discord.gg/aG68kk3Qpz "
)
else:
self.specific = (
"This horde has been switched to worker invite-only mode and "
f"you already have {current_workers} workers. "
"Please contact us on Discord to allow you to join more workers: https://discord.gg/aG68kk3Qpz "
)
self.log = None
self.rc = rc
class UnsafeIP(wze.Forbidden):
def __init__(self, ipaddr, rc="UnsafeIP"):
self.specific = (
"Due to abuse prevention, we cannot accept more workers from VPNs. Please contact us on Discord if you feel this is a mistake."
)
self.log = f"Worker attempted to pop from unsafe IP: {ipaddr}"
self.rc = rc
class TimeoutIP(wze.Forbidden):
def __init__(self, ipaddr, ttl, connect_type="Client", rc="TimeoutIP"):
base_message = (
"Due to abuse prevention, your IP address has been put into timeout for {ttl} more seconds. "
"Please try again later, or contact us on discord if you think this was an error."
)
non_atomic_message = (
"Due to abuse prevention, your IP address has been put into timeout. "
"Please try again later, or contact us on discord if you think this was an error."
)
try:
ttl = int(ttl)
except ValueError:
logger.warning(f"Invalid TTL value: {ttl} during timeout IP for {ipaddr}")
ttl = None
if ttl is None or ttl > (60 * 60 * 24 * 4):
self.specific = non_atomic_message
else:
self.specific = base_message.format(ttl=ttl)
self.log = f"{connect_type} attempted to connect from {ipaddr} while in {ttl} seconds timeout"
self.rc = rc
class TooManyNewIPs(wze.Forbidden):
def __init__(self, ipaddr, rc="TooManyNewIPs"):
self.specific = (
"We are getting too many new workers from unknown IPs. To prevent abuse, please try again later. "
"If this persists, please contact us on discord https://discord.gg/3DxrhksKzn"
)
self.log = f"Too many new IPs to check: {ipaddr}. Asked to retry"
self.rc = rc
class KudosUpfront(wze.Forbidden):
def __init__(self, kudos_required, username, message, rc="KudosUpfront"):
self.specific = message
self.log = f"{username} attempted request for {kudos_required} kudos without having enough."
self.rc = rc
class NoEligibleWorker(wze.Forbidden):
"""A request the kudos rules would refuse while no online worker can serve it.
Kudos do not help such a request until a worker that can serve it comes online, so it is refused with the
declaration gap, when there is one, in place of ``KudosUpfront``. The status stays 403, as for ``KudosUpfront``.
"""
body_fields: dict[str, object]
"""Fields added to the error body beside ``message`` and ``rc``: ``impossible_reason`` in the shape the status
route reports it, when there is a declaration gap."""
def __init__(self, impossible_reason: ChatImpossibleReason | None, rc: str = NO_ELIGIBLE_WORKER_RC) -> None:
"""Create the refusal.
Args:
impossible_reason: The missing declarations and exceeded limits that leave no worker eligible; None when
the cause is not a declaration gap, such as no online worker serving the model.
rc: The return code.
"""
causes: list[str] = []
if impossible_reason is not None and impossible_reason.missing_declarations:
causes.append(
"no online worker serving the model declares " + ", ".join(impossible_reason.missing_declarations),
)
if impossible_reason is not None and impossible_reason.exceeded_limits:
causes.append(
f"the request exceeds the {', '.join(impossible_reason.exceeded_limits)} limit of every online worker serving the model",
)
self.specific = "No online worker can serve this request"
self.specific += f": {'; '.join(causes)}." if causes else "."
self.log = None
self.rc = rc
self.body_fields = {}
if impossible_reason is not None:
self.body_fields["impossible_reason"] = impossible_reason.model_dump(mode="json")
class InvalidJobID(wze.NotFound):
def __init__(self, job_id, rc="InvalidJobID"):
self.specific = f"Processing Job with ID {job_id} does not exist."
self.log = f"Worker attempted to provide job for {job_id} but it did not exist"
self.rc = rc
class RequestNotFound(wze.NotFound):
def __init__(
self,
req_id,
request_type="Waiting Prompt",
client_agent="unknown",
ipaddr="unknown",
rc="RequestNotFound",
):
self.specific = f"{request_type} with ID '{req_id}' not found."
if request_type != "Interrogation": # FIXME: Figure out why there's so many
self.log = f"{request_type} with ID '{req_id}' does not exist. Client agent: {client_agent}@{ipaddr}"
else:
self.log = None
self.rc = rc
class WorkerNotFound(wze.NotFound):
def __init__(self, worker_id, rc="WorkerNotFound"):
self.specific = f"Worker with ID '{worker_id}' not found."
self.log = f"Attempted to retrieve worker with non-existent ID '{worker_id}'"
self.rc = rc
class TeamNotFound(wze.NotFound):
def __init__(self, team_id, rc="TeamNotFound"):
self.specific = f"Team with ID '{team_id}' not found."
self.log = f"Attempted to retrieve team with non-existent ID '{team_id}'"
self.rc = rc
class UnknownChatModel(wze.NotFound):
"""A chat request for a model that neither the text model reference lists nor an online chat worker serves."""
def __init__(self, model_name: str, rc: str = "UnknownChatModel", *, request_path: str | None = None) -> None:
"""Create the refusal.
Args:
model_name: The unknown model.
rc: The return code.
request_path: The entry of ``models`` holding the model, such as ``models[1]``, written quoted at the start
of the message so a client can tell which entry to change; None for ``model``.
"""
self.specific = (
f"The model '{model_name}' is not known to this horde: the text model reference does not list it and "
"no online worker serves chat jobs with it."
)
if request_path is not None:
self.specific = f"'{request_path}': {self.specific}"
self.log = None
self.rc = rc
class ChatKudosCapExceeded(wze.BadRequest):
"""A chat request whose kudos estimate exceeds the ``horde.max_kudos`` its client set."""
def __init__(self, kudos_cap_excess: KudosCapExcess, rc: str = CHAT_KUDOS_CAP_EXCEEDED_RC) -> None:
self.specific = kudos_cap_excess.to_message()
self.log = None
self.rc = rc
class ModerationEventNotFound(wze.NotFound):
def __init__(self, event_id, rc="ModerationEventNotFound"):
self.specific = f"Moderation event with ID '{event_id}' not found."
self.log = None
self.rc = rc
class ThingNotFound(wze.NotFound):
def __init__(self, thing_type, thing_id, message=None, rc="ThingNotFound"):
if message is not None:
self.specific = message
else:
self.specific = f"{thing_type.capitalize()} with ID '{thing_id}' not found."
self.log = f"Attempted to retrieve {thing_type} with non-existent ID '{thing_id}'"
self.rc = f"{thing_type.capitalize()}NotFound"
class UserNotFound(wze.NotFound):
def __init__(self, user_id, lookup_type="ID", message=None, rc="UserNotFound"):
if message:
self.specific = message
else:
self.specific = f"User with {lookup_type} '{user_id}' not found."
self.log = f"Attempted to retrieve user with non-existent {lookup_type} '{user_id}'"
self.rc = rc
class DuplicateGen(wze.BadRequest):
def __init__(self, worker, gen_id, rc="DuplicateGen"):
self.specific = f"Processing Generation with ID {gen_id} already submitted."
self.log = f"Worker '{worker}' attempted to provide duplicate generation for {gen_id}"
self.rc = rc
class AbortedGen(wze.BadRequest):
def __init__(self, worker, gen_id, ttl, rc="AbortedGen"):
self.specific = (
f"Processing Generation with ID {gen_id} took too long to process and has been aborted! "
f"Please check your worker speed - this generation had to complete within {ttl} seconds!"
)
self.log = f"Worker '{worker}' attempted to provide aborted generation for {gen_id}."
self.rc = rc
class RequestExpired(wze.Gone):
def __init__(self, username, rc="RequestExpired"):
self.specific = "Prompt Request Expired"
self.log = f"Request from '{username}' took too long to complete and has been cancelled."
self.rc = rc
class TooManyPrompts(wze.TooManyRequests):
def __init__(self, username, count, concurrency, msg=None, rc="TooManyPrompts"):
if msg is None:
self.specific = (
f"Parallel requests ({count}) exceeded user limit ({concurrency}). "
"Please try again later or request to increase your concurrency."
)
else:
self.specific = msg
self.log = f"User '{username}' has already requested too many parallel requests ({count}/{concurrency}). Aborting!"
self.rc = rc
class NoValidWorkers(wze.BadRequest):
retry_after = 600
def __init__(self, username, rc="NoValidWorkers"):
self.specific = "No active worker found to fulfill this request. Please Try again later..."
self.log = f"No active worker found to match the request from '{username}'. Aborting!"
self.rc = rc
class MaintenanceMode(wze.BadRequest):
retry_after = 60
def __init__(self, endpoint, rc="MaintenanceMode"):
self.specific = f"{horde_title} has entered maintenance mode. Please try again later."
self.log = f"Rejecting endpoint '{endpoint}' because horde in maintenance mode."
self.rc = rc
def rejection_attributes(error: wze.HTTPException) -> dict[str, str | int]:
"""Return the metric attributes a refusal is counted under.
The route is the matched URL rule, not the request path, so the label stays bounded by the number of
routes.
Args:
error: The refusal being returned.
Returns:
The return code, HTTP status, route and method.
"""
return_code = getattr(error, "rc", None)
route = UNMATCHED_ROUTE
method = UNMATCHED_ROUTE
if has_request_context():
method = request.method
if request.url_rule is not None:
route = request.url_rule.rule
return {
"horde.rc": return_code if return_code in KNOWN_RC else UNLISTED_RETURN_CODE,
"http.response.status_code": error.code,
"http.route": route,
"http.request.method": method,
}
def handle_bad_requests(error):
"""Namespace error handler"""
metrics.api_rejections.add(1, rejection_attributes(error))
if error.log:
logger.warning(f"{error.rc}: {error.log}")
else:
# The counter carries the volume; the line is for following one refusal while debugging.
logger.debug(f"{error.rc}: {error.specific}")
return (
{
"message": error.specific,
"rc": error.rc,
**getattr(error, "body_fields", {}),
},
error.code,
)
# PostgreSQL text/varchar columns cannot store NUL (0x00) bytes. When a
# user-supplied string carrying a NUL reaches a flush, psycopg2 rejects it
# *client-side* with a bare ``ValueError`` (NOT a DBAPIError) before any SQL is
# sent, so SQLAlchemy neither wraps it nor fires its ``handle_error`` event,
# and the session is left in a ``PendingRollbackError`` state. NUL has no
# legitimate place in our JSON payloads, so rather than proactively scanning
# every request body, we detect this specific failure reactively and translate
# it into a clean client error. Matching on the message keeps the detection
# robust across psycopg2 / SQLAlchemy versions and wrap/no-wrap behaviour.
_NUL_BYTE_VALUE_ERROR_SIGNATURE = "NUL (0x00)"
# PostgreSQL SQLSTATE for a detected deadlock (``psycopg2.errors.DeadlockDetected``).
_DEADLOCK_PGCODE = "40P01"
def is_nul_byte_value_error(error: BaseException) -> bool:
"""Report whether ``error`` is the psycopg2 NUL-byte flush rejection."""
return isinstance(error, ValueError) and _NUL_BYTE_VALUE_ERROR_SIGNATURE in str(error)
def find_nul_byte_location(obj, _path: str = "") -> str | None:
"""Return a dotted path to the first NUL-containing string in a parsed JSON
value, or ``None``. Used only on the error path to enrich logging, never on
the request hot path."""
if isinstance(obj, str):
return (_path or "<root>") if "\x00" in obj else None
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(k, str) and "\x00" in k:
return f"{_path}.<key {k!r}>" if _path else f"<key {k!r}>"
found = find_nul_byte_location(v, f"{_path}.{k}" if _path else str(k))
if found:
return found
elif isinstance(obj, (list, tuple)):
for i, item in enumerate(obj):
found = find_nul_byte_location(item, f"{_path}[{i}]")
if found:
return found
return None
def is_deadlock_error(error: BaseException) -> bool:
"""Report whether ``error`` is a PostgreSQL deadlock (SQLSTATE 40P01).
PostgreSQL aborts exactly one transaction in a lock cycle with this error.
Deadlocks are transient by construction: the surviving transactions proceed,
and the aborted victim can retry and succeed. We detect it structurally via
the DBAPI ``pgcode`` so the check survives message-wording changes across
psycopg2 / SQLAlchemy versions, and fall back to a substring match for the
wrapped/no-wrap edge cases.
"""
orig = getattr(error, "orig", None)
if getattr(orig, "pgcode", None) == _DEADLOCK_PGCODE:
return True
return "deadlock detected" in str(error).lower()
def nul_byte_error_response():
"""The 400 returned when a payload's NUL byte is rejected."""
return (
{
"message": "Request payload may not contain NUL (0x00) bytes.",
"rc": "NulByteInPayload",
},
400,
)