mirror of
https://github.com/Haidra-Org/AI-Horde.git
synced 2026-11-06 05:17:31 +00:00
With no eligible worker the free allowance takes its floor, and a request over it was refused for kudos that do not help until a worker that can serve it comes online. - A text or chat request the kudos rules would refuse while no online worker passes the dispatch gates is refused 403 NoEligibleWorker (TextAsyncGenerate.refuse_when_no_worker_is_eligible). The waiting prompt is deleted and the decision counter records refused. - The NoEligibleWorker message states the missing declarations or exceeded limits when they are the cause. The body carries impossible_reason in the status route's shape. - handle_bad_requests merges an exception's body_fields into the error body. Exceptions without body_fields keep the message and rc body. - Funded requests and requests within the allowance are created and queued whether or not a worker can serve them. - The front reads impossible_reason from a native error body onto HordeAPIError. _horde_error maps NoEligibleWorker to no_worker_supports_features, no_worker_supports_limit or no_eligible_worker. - The front's gap and no-eligible-worker errors take request_id and job_may_exist. A refusal at submit carries no request id and does not state that the request was cancelled. - _size_limit_gap_chat_offers takes the gap directly, so a submit refused on a size limit gap states the largest declared limit. - NoEligibleWorker is in KNOWN_RC, README_return_codes.md, ADR 0037, chat_requests.md and chat_front.md.
872 lines
31 KiB
Python
872 lines
31 KiB
Python
# SPDX-FileCopyrightText: 2022 Konstantinos Thoukydidis <mail@dbzer0.com>
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
from flask import has_request_context, request
|
|
from werkzeug import exceptions as wze
|
|
|
|
from horde import metrics
|
|
from horde.chat_contract import CHAT_KUDOS_CAP_EXCEEDED_RC, NO_ELIGIBLE_WORKER_RC, ChatImpossibleReason, KudosCapExcess
|
|
from horde.logger import logger
|
|
from horde.vars import horde_title
|
|
|
|
UNLISTED_RETURN_CODE = "Unlisted"
|
|
"""The return code label a refusal is counted under when its code is not in ``KNOWN_RC``, which keeps the label bounded."""
|
|
|
|
UNMATCHED_ROUTE = "unmatched"
|
|
"""The route label a refusal is counted under when it was raised outside a matched route."""
|
|
|
|
KNOWN_RC = [
|
|
"MissingPrompt",
|
|
"CorruptPrompt",
|
|
"KudosValidationError",
|
|
"NoValidActions",
|
|
"InvalidSize",
|
|
"InvalidPromptSize",
|
|
"TooManySteps",
|
|
"Profanity",
|
|
"ProfaneWorkerName",
|
|
"ProfaneBridgeAgent",
|
|
"ProfaneWorkerInfo",
|
|
"ProfaneUserName",
|
|
"ProfaneUserContact",
|
|
"ProfaneAdminComment",
|
|
"ProfaneTeamName",
|
|
"ProfaneTeamInfo",
|
|
"TooLong",
|
|
"TooLongWorkerName",
|
|
"TooLongUserName",
|
|
"NameAlreadyExists",
|
|
"WorkerNameAlreadyExists",
|
|
"TeamNameAlreadyExists",
|
|
"PolymorphicNameConflict",
|
|
"ImageValidationFailed",
|
|
"SourceImageResolutionExceeded",
|
|
"SourceImageSizeExceeded",
|
|
"SourceImageUrlInvalid",
|
|
"SourceImageUrlRefused",
|
|
"SourceImageUnreadable",
|
|
"InpaintingMissingMask",
|
|
"SourceMaskUnnecessary",
|
|
"UnsupportedSampler",
|
|
"UnsupportedModel",
|
|
"ControlNetUnsupported",
|
|
"ControlNetSourceMissing",
|
|
"ControlNetInvalidPayload",
|
|
"SourceImageRequiredForModel",
|
|
"UnexpectedModelName",
|
|
"TooManyUpscalers",
|
|
"ProcGenNotFound",
|
|
"InvalidAestheticAttempt",
|
|
"AestheticsNotCompleted",
|
|
"AestheticsNotPublic",
|
|
"AestheticsDuplicate",
|
|
"AestheticsMissing",
|
|
"AestheticsSolo",
|
|
"AestheticsConfused",
|
|
"AestheticsAlreadyExist",
|
|
"AestheticsServerRejected",
|
|
"AestheticsServerError",
|
|
"AestheticsServerDown",
|
|
"AestheticsServerTimeout",
|
|
"InvalidAPIKey",
|
|
"WrongCredentials",
|
|
"NotAdmin",
|
|
"NotModerator",
|
|
"NotOwner",
|
|
"NotRequestOwner",
|
|
"NotPrivileged",
|
|
"AnonForbidden",
|
|
"AnonForbiddenWorker",
|
|
"AnonForbiddenUserMod",
|
|
"NotTrusted",
|
|
"UntrustedTeamCreation",
|
|
"UntrustedUnsafeIP",
|
|
"WorkerMaintenance",
|
|
"WorkerFlaggedMaintenance",
|
|
"TooManySameIPs",
|
|
"WorkerInviteOnly",
|
|
"UnsafeIP",
|
|
"TimeoutIP",
|
|
"TooManyNewIPs",
|
|
"KudosUpfront",
|
|
"SharedKeyInvalid",
|
|
"SharedKeyEmpty",
|
|
"SharedKeyExpired",
|
|
"SharedKeyInsufficientKudos",
|
|
"SharedKeyAssignedStyles",
|
|
"InvalidJobID",
|
|
"RequestNotFound",
|
|
"WorkerNotFound",
|
|
"TeamNotFound",
|
|
"FilterNotFound",
|
|
"UserNotFound",
|
|
"DuplicateGen",
|
|
"AbortedGen",
|
|
"RequestExpired",
|
|
"TooManyPrompts",
|
|
"NoValidWorkers",
|
|
"MaintenanceMode",
|
|
"TargetAccountFlagged",
|
|
"SourceAccountFlagged",
|
|
"FaultWhenKudosReceiving",
|
|
"FaultWhenKudosSending",
|
|
"TooFastKudosTransfers",
|
|
"KudosTransferToAnon",
|
|
"KudosTransferToSelf",
|
|
"KudosTransferNotEnough",
|
|
"NegativeKudosTransfer",
|
|
"KudosTransferFromAnon",
|
|
"InvalidAwardUsername",
|
|
"KudosAwardToAnon",
|
|
"NotAllowedAwards",
|
|
"NoWorkerModSelected",
|
|
"NoUserModSelected",
|
|
"NoHordeModSelected",
|
|
"NoTeamModSelected",
|
|
"NoFilterModSelected",
|
|
"NoSharedKeyModSelected",
|
|
"BadRequest",
|
|
"Forbidden",
|
|
"Locked",
|
|
"ControlNetMismatch",
|
|
"HiResFixMismatch",
|
|
"TooManyLoras",
|
|
"BadLoraVersion",
|
|
"TooManyTIs",
|
|
"BetaAnonForbidden",
|
|
"BetaComparisonFault",
|
|
"BadCFGDecimals",
|
|
"BadCFGNumber",
|
|
"BadClientAgent",
|
|
"SpecialMissingPayload",
|
|
"SpecialForbidden",
|
|
"SpecialMissingUsername",
|
|
"SpecialModelNeedsSpecialUser",
|
|
"SpecialFieldNeedsSpecialUser",
|
|
"Img2ImgMismatch",
|
|
"TilingMismatch",
|
|
"EducationCannotSendKudos",
|
|
"InvalidPriorityUsername",
|
|
"OnlyServiceAccountProxy",
|
|
"RequiresTrust",
|
|
"InvalidRemixModel",
|
|
"InvalidExtraSourceImages",
|
|
"TooManyExtraSourceImages",
|
|
"MissingFullSamplerOrder",
|
|
"TooManyStopSequences",
|
|
"ExcessiveStopSequence",
|
|
"TokenOverflow",
|
|
"MoreThanMinExtraSourceImage",
|
|
"InvalidExtraTexts",
|
|
"MissingExtraTexts",
|
|
"InvalidTransparencyModel",
|
|
"InvalidTransparencyImg2Img",
|
|
"InvalidTransparencyCN",
|
|
"HiResMismatch",
|
|
"StylesAnonForbidden",
|
|
"StylePromptMissingVars",
|
|
"StylePromptFieldInvalid",
|
|
"TemplateFieldsMatchNoCollectionStyle",
|
|
"StylesRequiresCustomizer",
|
|
"StyleMismatch",
|
|
"StyleGetMistmatch",
|
|
"TooManyStyleExamples",
|
|
"ExampleURLAlreadyInUse",
|
|
"MessagesOnlyOwnWorkers",
|
|
"NotUserOrMod",
|
|
"CannotDeleteMod",
|
|
"UserNotDeleted",
|
|
"DeletedUser",
|
|
"CannotWipeActiveUser",
|
|
"NonServiceForbidden",
|
|
"NulByteInPayload",
|
|
"SamplerKnobInapplicable",
|
|
"SamplerKnobOutOfRange",
|
|
"SamplerSolverTypeUnsupported",
|
|
"SamplerSchedulerMismatch",
|
|
"InvalidOperationsLimit",
|
|
"InvalidOperationsCursor",
|
|
"InvalidModerationTimeRange",
|
|
"InvalidModerationAddressFilter",
|
|
"InvalidModerationEventID",
|
|
"InvalidModerationReason",
|
|
"InvalidModerationNote",
|
|
"ModerationEventNotFound",
|
|
"InvalidWorkerID",
|
|
"InvalidSuspicionID",
|
|
"SchedulerBaselineMismatch",
|
|
"FlowShiftInapplicable",
|
|
"FlowShiftOutOfRange",
|
|
"ControlStrengthWithoutControlType",
|
|
"ControlStrengthOutOfRange",
|
|
"InvalidPromptEncoding",
|
|
"ControlNetInpaintingMismatch",
|
|
"InvalidControlType",
|
|
"ThingNotFound",
|
|
"TooManyWorkers",
|
|
"TooManyWorkersTrusted",
|
|
"StylePolicyOverridableMisplaced",
|
|
"StyleParameterAboveCeiling",
|
|
"StyleDeclarationInvalid",
|
|
"TemplateFieldsRequireStyle",
|
|
"TemplateFieldUnknown",
|
|
"TemplateFieldMissing",
|
|
"PromptExceedsContext",
|
|
"UnsupportedChatParameter",
|
|
"ChatRequestConflict",
|
|
"ChatAdmissionDisabled",
|
|
"ChatDeclarationInvalid",
|
|
"ChatSubmitInvalid",
|
|
"ChatSubmitNonconformant",
|
|
"ChatWorkerFault",
|
|
"ChatModelMismatch",
|
|
"UnknownChatModel",
|
|
"ChatKudosCapExceeded",
|
|
"ChatImageStorageUnavailable",
|
|
"OperationKeyConflict",
|
|
"RequestTooLarge",
|
|
"ChatProgressNotStreaming",
|
|
"ChatProgressOutOfSequence",
|
|
"ChatProgressCapReached",
|
|
"ChatProgressUnavailable",
|
|
"ChatJobAborted",
|
|
"NoEligibleWorker",
|
|
]
|
|
|
|
|
|
class BadRequest(wze.BadRequest):
|
|
def __init__(self, message: str, log: str | None = None, rc: str = "BadRequest") -> None:
|
|
self.specific = message
|
|
self.log = log
|
|
self.rc = rc
|
|
|
|
|
|
class Forbidden(wze.Forbidden):
|
|
def __init__(self, message, log=None, rc="Forbidden"):
|
|
self.specific = message
|
|
self.log = log
|
|
self.rc = rc
|
|
|
|
|
|
class ServiceUnavailable(wze.ServiceUnavailable):
|
|
"""A route that is switched off on this horde, refused with 503 so a client can retry elsewhere or later."""
|
|
|
|
def __init__(self, message: str, log: str | None = None, rc: str = "ServiceUnavailable") -> None:
|
|
self.specific = message
|
|
self.log = log
|
|
self.rc = rc
|
|
|
|
|
|
class NotFound(wze.NotFound):
|
|
"""Something a route acts on that does not exist or no longer accepts the call, refused with 404."""
|
|
|
|
def __init__(self, message: str, log: str | None = None, rc: str = "NotFound") -> None:
|
|
self.specific = message
|
|
self.log = log
|
|
self.rc = rc
|
|
|
|
|
|
class RequestTooLarge(wze.RequestEntityTooLarge):
|
|
"""A request body above the size its route accepts, refused with 413 before the body is parsed."""
|
|
|
|
def __init__(self, message: str, rc: str = "RequestTooLarge") -> None:
|
|
self.specific = message
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class Locked(wze.Locked):
|
|
def __init__(self, message, rc="Locked"):
|
|
self.specific = message
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class MissingPrompt(wze.BadRequest):
|
|
def __init__(self, username, rc="MissingPrompt"):
|
|
self.specific = "You cannot specify an empty prompt."
|
|
self.log = f"User '{username}' sent an empty prompt. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class CorruptPrompt(wze.BadRequest):
|
|
def __init__(self, username, ip, prompt, message=None, rc="CorruptPrompt", proxy_service_ip=None):
|
|
if message:
|
|
self.specific = message
|
|
else:
|
|
self.specific = (
|
|
"This prompt appears to violate our terms of service and will be reported. Please contact us if you think this is an error."
|
|
)
|
|
self.log = f"User '{username}' with IP '{ip}' sent an a corrupt prompt: '{prompt}'. Aborting!"
|
|
if proxy_service_ip:
|
|
self.log += f" Service Proxy IP: {proxy_service_ip}"
|
|
self.rc = rc
|
|
|
|
|
|
class KudosValidationError(wze.BadRequest):
|
|
def __init__(self, username, error_message, action="transfer", rc="KudosValidationError"):
|
|
self.specific = error_message
|
|
self.log = f"User '{username}' Failed to {action} Kudos."
|
|
self.rc = rc
|
|
|
|
|
|
class NoValidActions(wze.BadRequest):
|
|
def __init__(self, error_message, rc="NoValidActions"):
|
|
self.specific = error_message
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class InvalidSize(wze.BadRequest):
|
|
def __init__(self, username, rc="InvalidSize"):
|
|
self.specific = "Invalid size. The image dimensions have to be multiples of 64."
|
|
self.log = f"User '{username}' sent an invalid size. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class InvalidPromptSize(wze.BadRequest):
|
|
def __init__(self, rc="InvalidPromptSize"):
|
|
self.specific = "Too large prompt. Please reduce the amount of tokens contained."
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class TooManySteps(wze.BadRequest):
|
|
def __init__(self, username, steps, rc="TooManySteps"):
|
|
self.specific = "Too many sampling steps. To allow resources for everyone, we allow only up to 500 steps."
|
|
self.log = f"User '{username}' sent too many steps ({steps}). Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class Profanity(wze.BadRequest):
|
|
def __init__(self, username, text, text_type, rc="Profanity"):
|
|
self.specific = f"As our API is public, we do not allow profanity in the {text_type}, please try again."
|
|
self.log = f"User '{username}' tried to submit profanity for {text_type} ({text}). Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class TooLong(wze.BadRequest):
|
|
def __init__(self, username, chars, limit, text_type, rc="TooLong"):
|
|
self.specific = f"The specified {text_type} is too long. Please stay below {limit}"
|
|
self.log = f"User '{username}' tried to submit {chars} chars for {text_type}. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class NameAlreadyExists(wze.BadRequest):
|
|
def __init__(self, username, old_name, new_name, object_type="worker", rc="NameAlreadyExists"):
|
|
self.specific = f"The specified {object_type} name '{new_name}' is already taken!"
|
|
self.log = f"User '{username}' tried to change {object_type} name from {old_name} to {new_name}. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class PolymorphicNameConflict(wze.BadRequest):
|
|
def __init__(self, name, object_type="worker", rc="PolymorphicNameConflict"):
|
|
self.specific = (
|
|
f"The specified name '{name}' is already taken by a different type of {object_type}. Please choose a different name!"
|
|
)
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class ImageValidationFailed(wze.BadRequest):
|
|
def __init__(
|
|
self,
|
|
message: str = ("Please ensure the source image payload is either a URL containing an image or a valid base64 encoded image."),
|
|
rc: str = "ImageValidationFailed",
|
|
) -> None:
|
|
self.specific = f"Image validation failed. {message}"
|
|
self.log = "Source image validation failed"
|
|
self.rc = rc
|
|
|
|
|
|
class SourceMaskUnnecessary(wze.BadRequest):
|
|
def __init__(self, rc="SourceMaskUnnecessary"):
|
|
self.specific = "Please do not pass a source_mask unless you are sending a source_image as well"
|
|
self.log = "Tried to pass source_mask with txt2img"
|
|
self.rc = rc
|
|
|
|
|
|
class UnsupportedSampler(wze.BadRequest):
|
|
def __init__(self, rc="UnsupportedSampler"):
|
|
self.specific = "This sampler is not supported in this mode the moment"
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class UnsupportedModel(wze.BadRequest):
|
|
def __init__(self, message=None, rc="UnsupportedModel"):
|
|
if message:
|
|
self.specific = message
|
|
else:
|
|
self.specific = "This model is not supported in this mode the moment"
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class ProcGenNotFound(wze.BadRequest):
|
|
def __init__(self, procgen_id, rc="ProcGenNotFound"):
|
|
self.specific = f"Image with ID '{procgen_id}' not found in this request."
|
|
self.log = f"Attempted to log aesthetic rating with non-existent image ID '{procgen_id}'"
|
|
self.rc = rc
|
|
|
|
|
|
class InvalidAestheticAttempt(wze.BadRequest):
|
|
def __init__(self, message, rc="InvalidAestheticAttempt"):
|
|
self.specific = message
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class InvalidAPIKey(wze.Unauthorized):
|
|
def __init__(self, subject, keytype="API", rc="InvalidAPIKey"):
|
|
if keytype == "Shared":
|
|
self.specific = "No user matching sent Shared Key."
|
|
else:
|
|
self.specific = "No user matching sent API Key. Have you remembered to register at https://stablehorde.net/register ?"
|
|
self.log = f"Invalid {keytype} Key sent for {subject}"
|
|
self.rc = rc
|
|
|
|
|
|
class WrongCredentials(wze.Forbidden):
|
|
def __init__(self, username, worker, rc="WrongCredentials"):
|
|
self.specific = "Wrong credentials to submit as this worker."
|
|
self.log = f"User '{username}' sent wrong credentials for utilizing worker {worker}"
|
|
self.rc = rc
|
|
|
|
|
|
class NotAdmin(wze.Forbidden):
|
|
def __init__(self, username, endpoint, rc="NotAdmin"):
|
|
self.specific = "You're not an admin. Sod off!"
|
|
self.log = f"Non-admin user '{username}' tried to use admin endpoint: '{endpoint}. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class NotModerator(wze.Forbidden):
|
|
def __init__(self, username, endpoint, rc="NotModerator"):
|
|
self.specific = "You're not a mod. BTFO!"
|
|
self.log = f"Non-mod user '{username}' tried to use mod endpoint: '{endpoint}. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class NotOwner(wze.Forbidden):
|
|
def __init__(self, username, worker_name, rc="NotOwner"):
|
|
self.specific = "You're not an admin. Sod off!"
|
|
self.log = f"User '{username}'' tried to modify worker they do not own '{worker_name}'. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class NotRequestOwner(wze.Forbidden):
|
|
def __init__(self, req_id, rc="NotRequestOwner"):
|
|
self.specific = "Only the API key that submitted this request can retrieve its parameters."
|
|
self.log = f"Key mismatch when retrieving the parameters of request {req_id}."
|
|
self.rc = rc
|
|
|
|
|
|
class NotPrivileged(wze.Forbidden):
|
|
def __init__(self, username, message, action, rc="NotPrivileged"):
|
|
self.specific = message
|
|
self.log = f"Non-Privileged user '{username}' tried to take privileged action '{action}'. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class AnonForbidden(wze.Forbidden):
|
|
def __init__(self, rc="AnonForbidden"):
|
|
self.specific = "Anonymous user is forbidden from performing this operation"
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class NotTrusted(wze.Forbidden):
|
|
def __init__(self, rc="NotTrusted"):
|
|
self.specific = "Only Trusted users are allowed to perform this operation"
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class WorkerMaintenance(wze.Forbidden):
|
|
def __init__(self, maintenance_msg, rc="WorkerMaintenance"):
|
|
self.specific = maintenance_msg
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class TooManySameIPs(wze.Forbidden):
|
|
def __init__(self, username, rc="TooManySameIPs"):
|
|
self.specific = (
|
|
"You are running too many workers from the same location. To prevent abuse, "
|
|
"please contact us on Discord to allow you to join more workers from the same IP: https://discord.gg/aG68kk3Qpz"
|
|
)
|
|
self.log = f"User '{username} is trying to onboard too many workers from the same IP Address. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class WorkerInviteOnly(wze.Forbidden):
|
|
def __init__(self, current_workers, rc="WorkerInviteOnly"):
|
|
if current_workers == 0:
|
|
self.specific = (
|
|
"This horde has been switched to worker invite-only mode. "
|
|
"Please contact us on Discord to allow you to join your worker: https://discord.gg/aG68kk3Qpz "
|
|
)
|
|
else:
|
|
self.specific = (
|
|
"This horde has been switched to worker invite-only mode and "
|
|
f"you already have {current_workers} workers. "
|
|
"Please contact us on Discord to allow you to join more workers: https://discord.gg/aG68kk3Qpz "
|
|
)
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class UnsafeIP(wze.Forbidden):
|
|
def __init__(self, ipaddr, rc="UnsafeIP"):
|
|
self.specific = (
|
|
"Due to abuse prevention, we cannot accept more workers from VPNs. Please contact us on Discord if you feel this is a mistake."
|
|
)
|
|
self.log = f"Worker attempted to pop from unsafe IP: {ipaddr}"
|
|
self.rc = rc
|
|
|
|
|
|
class TimeoutIP(wze.Forbidden):
|
|
def __init__(self, ipaddr, ttl, connect_type="Client", rc="TimeoutIP"):
|
|
base_message = (
|
|
"Due to abuse prevention, your IP address has been put into timeout for {ttl} more seconds. "
|
|
"Please try again later, or contact us on discord if you think this was an error."
|
|
)
|
|
non_atomic_message = (
|
|
"Due to abuse prevention, your IP address has been put into timeout. "
|
|
"Please try again later, or contact us on discord if you think this was an error."
|
|
)
|
|
|
|
try:
|
|
ttl = int(ttl)
|
|
except ValueError:
|
|
logger.warning(f"Invalid TTL value: {ttl} during timeout IP for {ipaddr}")
|
|
ttl = None
|
|
|
|
if ttl is None or ttl > (60 * 60 * 24 * 4):
|
|
self.specific = non_atomic_message
|
|
else:
|
|
self.specific = base_message.format(ttl=ttl)
|
|
|
|
self.log = f"{connect_type} attempted to connect from {ipaddr} while in {ttl} seconds timeout"
|
|
self.rc = rc
|
|
|
|
|
|
class TooManyNewIPs(wze.Forbidden):
|
|
def __init__(self, ipaddr, rc="TooManyNewIPs"):
|
|
self.specific = (
|
|
"We are getting too many new workers from unknown IPs. To prevent abuse, please try again later. "
|
|
"If this persists, please contact us on discord https://discord.gg/3DxrhksKzn"
|
|
)
|
|
self.log = f"Too many new IPs to check: {ipaddr}. Asked to retry"
|
|
self.rc = rc
|
|
|
|
|
|
class KudosUpfront(wze.Forbidden):
|
|
def __init__(self, kudos_required, username, message, rc="KudosUpfront"):
|
|
self.specific = message
|
|
self.log = f"{username} attempted request for {kudos_required} kudos without having enough."
|
|
self.rc = rc
|
|
|
|
|
|
class NoEligibleWorker(wze.Forbidden):
|
|
"""A request the kudos rules would refuse while no online worker can serve it.
|
|
|
|
Kudos do not help such a request until a worker that can serve it comes online, so it is refused with the
|
|
declaration gap, when there is one, in place of ``KudosUpfront``. The status stays 403, as for ``KudosUpfront``.
|
|
"""
|
|
|
|
body_fields: dict[str, object]
|
|
"""Fields added to the error body beside ``message`` and ``rc``: ``impossible_reason`` in the shape the status
|
|
route reports it, when there is a declaration gap."""
|
|
|
|
def __init__(self, impossible_reason: ChatImpossibleReason | None, rc: str = NO_ELIGIBLE_WORKER_RC) -> None:
|
|
"""Create the refusal.
|
|
|
|
Args:
|
|
impossible_reason: The missing declarations and exceeded limits that leave no worker eligible; None when
|
|
the cause is not a declaration gap, such as no online worker serving the model.
|
|
rc: The return code.
|
|
"""
|
|
causes: list[str] = []
|
|
if impossible_reason is not None and impossible_reason.missing_declarations:
|
|
causes.append(
|
|
"no online worker serving the model declares " + ", ".join(impossible_reason.missing_declarations),
|
|
)
|
|
if impossible_reason is not None and impossible_reason.exceeded_limits:
|
|
causes.append(
|
|
f"the request exceeds the {', '.join(impossible_reason.exceeded_limits)} limit of every online worker serving the model",
|
|
)
|
|
self.specific = "No online worker can serve this request"
|
|
self.specific += f": {'; '.join(causes)}." if causes else "."
|
|
self.log = None
|
|
self.rc = rc
|
|
self.body_fields = {}
|
|
if impossible_reason is not None:
|
|
self.body_fields["impossible_reason"] = impossible_reason.model_dump(mode="json")
|
|
|
|
|
|
class InvalidJobID(wze.NotFound):
|
|
def __init__(self, job_id, rc="InvalidJobID"):
|
|
self.specific = f"Processing Job with ID {job_id} does not exist."
|
|
self.log = f"Worker attempted to provide job for {job_id} but it did not exist"
|
|
self.rc = rc
|
|
|
|
|
|
class RequestNotFound(wze.NotFound):
|
|
def __init__(
|
|
self,
|
|
req_id,
|
|
request_type="Waiting Prompt",
|
|
client_agent="unknown",
|
|
ipaddr="unknown",
|
|
rc="RequestNotFound",
|
|
):
|
|
self.specific = f"{request_type} with ID '{req_id}' not found."
|
|
if request_type != "Interrogation": # FIXME: Figure out why there's so many
|
|
self.log = f"{request_type} with ID '{req_id}' does not exist. Client agent: {client_agent}@{ipaddr}"
|
|
else:
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class WorkerNotFound(wze.NotFound):
|
|
def __init__(self, worker_id, rc="WorkerNotFound"):
|
|
self.specific = f"Worker with ID '{worker_id}' not found."
|
|
self.log = f"Attempted to retrieve worker with non-existent ID '{worker_id}'"
|
|
self.rc = rc
|
|
|
|
|
|
class TeamNotFound(wze.NotFound):
|
|
def __init__(self, team_id, rc="TeamNotFound"):
|
|
self.specific = f"Team with ID '{team_id}' not found."
|
|
self.log = f"Attempted to retrieve team with non-existent ID '{team_id}'"
|
|
self.rc = rc
|
|
|
|
|
|
class UnknownChatModel(wze.NotFound):
|
|
"""A chat request for a model that neither the text model reference lists nor an online chat worker serves."""
|
|
|
|
def __init__(self, model_name: str, rc: str = "UnknownChatModel", *, request_path: str | None = None) -> None:
|
|
"""Create the refusal.
|
|
|
|
Args:
|
|
model_name: The unknown model.
|
|
rc: The return code.
|
|
request_path: The entry of ``models`` holding the model, such as ``models[1]``, written quoted at the start
|
|
of the message so a client can tell which entry to change; None for ``model``.
|
|
"""
|
|
self.specific = (
|
|
f"The model '{model_name}' is not known to this horde: the text model reference does not list it and "
|
|
"no online worker serves chat jobs with it."
|
|
)
|
|
if request_path is not None:
|
|
self.specific = f"'{request_path}': {self.specific}"
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class ChatKudosCapExceeded(wze.BadRequest):
|
|
"""A chat request whose kudos estimate exceeds the ``horde.max_kudos`` its client set."""
|
|
|
|
def __init__(self, kudos_cap_excess: KudosCapExcess, rc: str = CHAT_KUDOS_CAP_EXCEEDED_RC) -> None:
|
|
self.specific = kudos_cap_excess.to_message()
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class ModerationEventNotFound(wze.NotFound):
|
|
def __init__(self, event_id, rc="ModerationEventNotFound"):
|
|
self.specific = f"Moderation event with ID '{event_id}' not found."
|
|
self.log = None
|
|
self.rc = rc
|
|
|
|
|
|
class ThingNotFound(wze.NotFound):
|
|
def __init__(self, thing_type, thing_id, message=None, rc="ThingNotFound"):
|
|
if message is not None:
|
|
self.specific = message
|
|
else:
|
|
self.specific = f"{thing_type.capitalize()} with ID '{thing_id}' not found."
|
|
self.log = f"Attempted to retrieve {thing_type} with non-existent ID '{thing_id}'"
|
|
self.rc = f"{thing_type.capitalize()}NotFound"
|
|
|
|
|
|
class UserNotFound(wze.NotFound):
|
|
def __init__(self, user_id, lookup_type="ID", message=None, rc="UserNotFound"):
|
|
if message:
|
|
self.specific = message
|
|
else:
|
|
self.specific = f"User with {lookup_type} '{user_id}' not found."
|
|
self.log = f"Attempted to retrieve user with non-existent {lookup_type} '{user_id}'"
|
|
self.rc = rc
|
|
|
|
|
|
class DuplicateGen(wze.BadRequest):
|
|
def __init__(self, worker, gen_id, rc="DuplicateGen"):
|
|
self.specific = f"Processing Generation with ID {gen_id} already submitted."
|
|
self.log = f"Worker '{worker}' attempted to provide duplicate generation for {gen_id}"
|
|
self.rc = rc
|
|
|
|
|
|
class AbortedGen(wze.BadRequest):
|
|
def __init__(self, worker, gen_id, ttl, rc="AbortedGen"):
|
|
self.specific = (
|
|
f"Processing Generation with ID {gen_id} took too long to process and has been aborted! "
|
|
f"Please check your worker speed - this generation had to complete within {ttl} seconds!"
|
|
)
|
|
self.log = f"Worker '{worker}' attempted to provide aborted generation for {gen_id}."
|
|
self.rc = rc
|
|
|
|
|
|
class RequestExpired(wze.Gone):
|
|
def __init__(self, username, rc="RequestExpired"):
|
|
self.specific = "Prompt Request Expired"
|
|
self.log = f"Request from '{username}' took too long to complete and has been cancelled."
|
|
self.rc = rc
|
|
|
|
|
|
class TooManyPrompts(wze.TooManyRequests):
|
|
def __init__(self, username, count, concurrency, msg=None, rc="TooManyPrompts"):
|
|
if msg is None:
|
|
self.specific = (
|
|
f"Parallel requests ({count}) exceeded user limit ({concurrency}). "
|
|
"Please try again later or request to increase your concurrency."
|
|
)
|
|
else:
|
|
self.specific = msg
|
|
self.log = f"User '{username}' has already requested too many parallel requests ({count}/{concurrency}). Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class NoValidWorkers(wze.BadRequest):
|
|
retry_after = 600
|
|
|
|
def __init__(self, username, rc="NoValidWorkers"):
|
|
self.specific = "No active worker found to fulfill this request. Please Try again later..."
|
|
self.log = f"No active worker found to match the request from '{username}'. Aborting!"
|
|
self.rc = rc
|
|
|
|
|
|
class MaintenanceMode(wze.BadRequest):
|
|
retry_after = 60
|
|
|
|
def __init__(self, endpoint, rc="MaintenanceMode"):
|
|
self.specific = f"{horde_title} has entered maintenance mode. Please try again later."
|
|
self.log = f"Rejecting endpoint '{endpoint}' because horde in maintenance mode."
|
|
self.rc = rc
|
|
|
|
|
|
def rejection_attributes(error: wze.HTTPException) -> dict[str, str | int]:
|
|
"""Return the metric attributes a refusal is counted under.
|
|
|
|
The route is the matched URL rule, not the request path, so the label stays bounded by the number of
|
|
routes.
|
|
|
|
Args:
|
|
error: The refusal being returned.
|
|
|
|
Returns:
|
|
The return code, HTTP status, route and method.
|
|
"""
|
|
return_code = getattr(error, "rc", None)
|
|
route = UNMATCHED_ROUTE
|
|
method = UNMATCHED_ROUTE
|
|
if has_request_context():
|
|
method = request.method
|
|
if request.url_rule is not None:
|
|
route = request.url_rule.rule
|
|
return {
|
|
"horde.rc": return_code if return_code in KNOWN_RC else UNLISTED_RETURN_CODE,
|
|
"http.response.status_code": error.code,
|
|
"http.route": route,
|
|
"http.request.method": method,
|
|
}
|
|
|
|
|
|
def handle_bad_requests(error):
|
|
"""Namespace error handler"""
|
|
metrics.api_rejections.add(1, rejection_attributes(error))
|
|
if error.log:
|
|
logger.warning(f"{error.rc}: {error.log}")
|
|
else:
|
|
# The counter carries the volume; the line is for following one refusal while debugging.
|
|
logger.debug(f"{error.rc}: {error.specific}")
|
|
return (
|
|
{
|
|
"message": error.specific,
|
|
"rc": error.rc,
|
|
**getattr(error, "body_fields", {}),
|
|
},
|
|
error.code,
|
|
)
|
|
|
|
|
|
# PostgreSQL text/varchar columns cannot store NUL (0x00) bytes. When a
|
|
# user-supplied string carrying a NUL reaches a flush, psycopg2 rejects it
|
|
# *client-side* with a bare ``ValueError`` (NOT a DBAPIError) before any SQL is
|
|
# sent, so SQLAlchemy neither wraps it nor fires its ``handle_error`` event,
|
|
# and the session is left in a ``PendingRollbackError`` state. NUL has no
|
|
# legitimate place in our JSON payloads, so rather than proactively scanning
|
|
# every request body, we detect this specific failure reactively and translate
|
|
# it into a clean client error. Matching on the message keeps the detection
|
|
# robust across psycopg2 / SQLAlchemy versions and wrap/no-wrap behaviour.
|
|
_NUL_BYTE_VALUE_ERROR_SIGNATURE = "NUL (0x00)"
|
|
|
|
# PostgreSQL SQLSTATE for a detected deadlock (``psycopg2.errors.DeadlockDetected``).
|
|
_DEADLOCK_PGCODE = "40P01"
|
|
|
|
|
|
def is_nul_byte_value_error(error: BaseException) -> bool:
|
|
"""Report whether ``error`` is the psycopg2 NUL-byte flush rejection."""
|
|
return isinstance(error, ValueError) and _NUL_BYTE_VALUE_ERROR_SIGNATURE in str(error)
|
|
|
|
|
|
def find_nul_byte_location(obj, _path: str = "") -> str | None:
|
|
"""Return a dotted path to the first NUL-containing string in a parsed JSON
|
|
value, or ``None``. Used only on the error path to enrich logging, never on
|
|
the request hot path."""
|
|
if isinstance(obj, str):
|
|
return (_path or "<root>") if "\x00" in obj else None
|
|
if isinstance(obj, dict):
|
|
for k, v in obj.items():
|
|
if isinstance(k, str) and "\x00" in k:
|
|
return f"{_path}.<key {k!r}>" if _path else f"<key {k!r}>"
|
|
found = find_nul_byte_location(v, f"{_path}.{k}" if _path else str(k))
|
|
if found:
|
|
return found
|
|
elif isinstance(obj, (list, tuple)):
|
|
for i, item in enumerate(obj):
|
|
found = find_nul_byte_location(item, f"{_path}[{i}]")
|
|
if found:
|
|
return found
|
|
return None
|
|
|
|
|
|
def is_deadlock_error(error: BaseException) -> bool:
|
|
"""Report whether ``error`` is a PostgreSQL deadlock (SQLSTATE 40P01).
|
|
|
|
PostgreSQL aborts exactly one transaction in a lock cycle with this error.
|
|
Deadlocks are transient by construction: the surviving transactions proceed,
|
|
and the aborted victim can retry and succeed. We detect it structurally via
|
|
the DBAPI ``pgcode`` so the check survives message-wording changes across
|
|
psycopg2 / SQLAlchemy versions, and fall back to a substring match for the
|
|
wrapped/no-wrap edge cases.
|
|
"""
|
|
orig = getattr(error, "orig", None)
|
|
if getattr(orig, "pgcode", None) == _DEADLOCK_PGCODE:
|
|
return True
|
|
return "deadlock detected" in str(error).lower()
|
|
|
|
|
|
def nul_byte_error_response():
|
|
"""The 400 returned when a payload's NUL byte is rejected."""
|
|
return (
|
|
{
|
|
"message": "Request payload may not contain NUL (0x00) bytes.",
|
|
"rc": "NulByteInPayload",
|
|
},
|
|
400,
|
|
)
|