From 502f608fde685ce97057df74e61f9aadbdbf220e Mon Sep 17 00:00:00 2001 From: tazlin Date: Sat, 25 Jul 2026 13:18:38 -0400 Subject: [PATCH] docs: zensical docs via github pages and initial kudos ADR set - Introduce a dedicated docs GitHub Actions workflow to validate ADR index generation, code-path references, strict site builds, SPDX/license compliance, and internal links, plus deploy the docs site to GitHub Pages. - Add markdownlint settings - Zensical site configuration for docs navigation/theme. - Seed a new decisions section with an ADR index and eight kudos-accounting architecture records covering event-ledger design, projection, reservations, cutover, reconciliation/repair, FK policy, and trust-promotion behavior. --- .github/workflows/docs.yml | 68 ++++++++++++ .markdownlint.json | 7 ++ .../0001-event-sourced-kudos-accounting.md | 96 +++++++++++++++++ ...ate-currency-and-statistic-event-tables.md | 79 ++++++++++++++ .../0003-single-serialized-projector.md | 78 ++++++++++++++ ...-payer-reservations-for-spend-admission.md | 82 ++++++++++++++ docs/decisions/0005-shadow-mode-cutover.md | 78 ++++++++++++++ ...manent-archive-compensation-only-repair.md | 82 ++++++++++++++ .../0007-accounting-foreign-key-policy.md | 81 ++++++++++++++ .../0008-trust-promotion-in-the-projector.md | 78 ++++++++++++++ docs/decisions/index.md | 58 ++++++++++ zensical.toml | 101 ++++++++++++++++++ 12 files changed, 888 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 .markdownlint.json create mode 100644 docs/decisions/0001-event-sourced-kudos-accounting.md create mode 100644 docs/decisions/0002-separate-currency-and-statistic-event-tables.md create mode 100644 docs/decisions/0003-single-serialized-projector.md create mode 100644 docs/decisions/0004-payer-reservations-for-spend-admission.md create mode 100644 docs/decisions/0005-shadow-mode-cutover.md create mode 100644 docs/decisions/0006-permanent-archive-compensation-only-repair.md create mode 100644 docs/decisions/0007-accounting-foreign-key-policy.md create mode 100644 docs/decisions/0008-trust-promotion-in-the-projector.md create mode 100644 docs/decisions/index.md create mode 100644 zensical.toml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..c4c620dc --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +name: AI-Horde docs + +on: + pull_request: + branches: + - main + paths: + - 'docs/**' + - 'zensical.toml' + - '.github/workflows/docs.yml' + push: + branches: + - main + paths: + - 'docs/**' + - 'zensical.toml' + - '.github/workflows/docs.yml' + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: astral-sh/setup-uv@v5 + + - name: Generated decision index is current + run: python3 docs/haidra-assets/tools/gen_adr_index.py docs/decisions --check + + - name: Cited code paths exist + run: python3 docs/haidra-assets/tools/check_code_refs.py docs/README.md docs/explanation docs/reference docs/how-to docs/decisions + + - name: Site builds strictly + run: uvx --from "zensical==0.0.51" zensical build -s + + - name: License compliance + run: uvx --from "reuse[charset-normalizer]" reuse lint + + - name: Internal links resolve + uses: lycheeverse/lychee-action@v2 + with: + args: --offline --no-progress docs + fail: true + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: docs + steps: + - uses: actions/configure-pages@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: 3.x + - run: pip install zensical + - run: zensical build --clean + - uses: actions/upload-pages-artifact@v5 + with: + path: site + - uses: actions/deploy-pages@v5 + id: deployment diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 00000000..d60f62a9 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,7 @@ +{ + "MD013": { + "line_length": 119, + "code_block_line_length": 140, + "heading_line_length": 119 + } +} diff --git a/docs/decisions/0001-event-sourced-kudos-accounting.md b/docs/decisions/0001-event-sourced-kudos-accounting.md new file mode 100644 index 00000000..694d696a --- /dev/null +++ b/docs/decisions/0001-event-sourced-kudos-accounting.md @@ -0,0 +1,96 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Record kudos movements as events; keep balances as projections + +## Context and Problem Statement + +Kudos mutations edited the materialized balance and aggregate columns inline, inside whatever request +transaction needed them. That made frequently used user rows part of many otherwise unrelated transaction lock +graphs: concurrent activation, settlement, and transfer paths reached the same rows in different orders, which +produced a recurring deadlock class that could only be papered over with retries. Separately, the current +column value was the only accounting record. There was no audit trail, no replay, and no way to repair a bad +balance other than overwriting the one visible number. + +## Decision Drivers + +- Queueing, API responses, leaderboards, and existing integrations need inexpensive reads. Computing balances + from history on demand was never viable at Horde read volume. +- Kudos are spendable. A design with asynchronous projection must still refuse an overspend at admission time. +- The Horde cannot take a maintenance window, so the migration must be observable and reversible online. + +## Considered Options + +- Record kudos movements as events; keep balances as projections +- Fix lock ordering in place +- Synchronous double-write, permanently +- Full event sourcing + +## Decision Outcome + +Chosen option: "Record kudos movements as events; keep balances as projections". + +Every kudos-moving business event appends signed, typed rows to permanent event tables: `kudos_ledger` for +currency, `kudos_stat_events` for display totals and counters. The existing balance and aggregate columns are +retained as denormalized read models, folded from those events. Producers stop editing the materialized +columns inside request transactions. Spend admission is protected by payer reservations +([ADR 4](0004-payer-reservations-for-spend-admission.md)), projection is owned by a single serialized projector +([ADR 3](0003-single-serialized-projector.md)), and cutover happens through shadow mode +([ADR 5](0005-shadow-mode-cutover.md)). + +### Consequences + +- Good: Ordinary producers append rows instead of locking hot user, worker, team, and stats rows, which + removes the motivating deadlock class (the + [explanation](../explanation/kudos_accounting.md#deadlock-mitigation) states the precise, narrower + claim). +- Good: Movements are durable, typed, and correlated by `event_id`. Drift is diagnosable by reconciliation + against a snapshot, and repair is a deterministic compensating posting rather than a hand edit + ([ADR 6](0006-permanent-archive-compensation-only-repair.md)). + +- Bad: Balance and aggregate reads become eventually consistent in `ledger` mode. Every consumer had to be + inventoried and its tolerated lag made explicit in the + [consumer inventory](../reference/kudos_accounting.md#consumer-and-read-model-inventory). +- Bad: The permanent archives grow without pruning and need capacity planning. +- Bad: Projection throughput is bounded by the single serialized projector + ([ADR 3](0003-single-serialized-projector.md)). + +## Pros and Cons of the Options + +### Fix lock ordering in place + +Impose a global acquisition order on the inline design. + +- Bad: Every future kudos-touching change would need to know and preserve the order, the audit/replay gap + remains, and multi-account transfers still couple two hot rows in one transaction. + +### Synchronous double-write, permanently + +Emit events and keep mutating inline. Retained only as the temporary `shadow` migration mechanism +([ADR 5](0005-shadow-mode-cutover.md)). + +- Good: Reads stay current. +- Bad: Keeps the whole lock graph and adds the event writes; the events become a write-amplified log with none + of the concurrency benefit. + +### Full event sourcing + +Drop the columns and rebuild state from events. + +- Bad: Read cost and blast radius: every consumer in the codebase changes at once, and the opening balances + predate the ledger by design. + +## More Information + +The measurements behind this record come from lock-chain and `pg_stat_statements` captures taken on the +production primary on 2026-07-20. They are summarized in +[the problem is shared mutable rows](../explanation/kudos_accounting.md#the-problem-shared-mutable-rows), +including the queue depths that persisted on the statistics rows once the balance rows stopped serializing. diff --git a/docs/decisions/0002-separate-currency-and-statistic-event-tables.md b/docs/decisions/0002-separate-currency-and-statistic-event-tables.md new file mode 100644 index 00000000..c0d3d068 --- /dev/null +++ b/docs/decisions/0002-separate-currency-and-statistic-event-tables.md @@ -0,0 +1,79 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Separate currency postings from statistic postings + +## Context and Problem Statement + +One settlement moves several kinds of value at once: the requester's spendable balance, the worker owner's +spendable or evaluation balance, the worker's display kudos, the worker's contribution and fulfilment +aggregates, the team totals stamped with the membership at settlement time, and the requester's usage records. +These values are denominated differently. Currency is a two-decimal amount against one user account. A +contribution is measured in things. A fulfilment is a count. Worker and team kudos describe attributed work +that nobody can spend. + +## Decision Drivers + +- Reconciliation has to answer whether user currency is conserved. That question only has an answer if every + row in the table it reads is a signed delta against one user balance in one unit. +- Worker and team rows are hard-deletable, while user rows are soft-deleted and wiped. A single table cannot + carry both the `ON DELETE RESTRICT` currency history requires and the deletion tolerance counter history + requires ([ADR 7](0007-accounting-foreign-key-policy.md)). +- Consumers of the two classes differ. Balance readers, spend admission, and snapshots want currency. + Leaderboards, worker detail, team rankings, and user records want counters. + +## Considered Options + +- Separate currency postings from statistic postings +- One posting table for everything +- Treat worker kudos as currency + +## Decision Outcome + +Chosen option: "Separate currency postings from statistic postings". + +Currency postings go to `kudos_ledger`. Every row is a signed `NUMERIC(20,2)` delta against exactly one +`users.id`, with `escrow` selecting the spendable balance or the evaluation balance. Display and counter +postings go to `kudos_stat_events`. Every row carries a `unit` of `kudos`, `things`, or `count`, targets +exactly one of `user_id` or `worker_id`, and records `team_id` and `worker_user_id` as captured attribution. +Rows produced by one business event share an `event_id` across both tables, so the pairing survives the split. + +Worker and team kudos are display attribution. `WorkerTemplate.modify_kudos` emits stat events; the matching +currency credit reaches the owner through `User.modify_kudos`. + +### Consequences + +- Good: Reconciliation reads one table whose every row is comparable, which is what makes the snapshot baseline in + [ADR 6](0006-permanent-archive-compensation-only-repair.md) a checkable statement about user currency. +- Good: The two tables can hold the foreign-key policies their contents need. +- Good: A reviewer classifies a new value before writing code: currency, display attribution, counter, price, or + quota. The reference records the excluded kudos-like state that is none of these. + +- Bad: The projector claims two batches and folds both, and an event can straddle a batch boundary in either table. + Event-wide side effects such as transfer-hold release must query for remaining unapplied rows. +- Bad: Reconciliation covers user currency and evaluation escrow. Counter drift is replayable from + `kudos_stat_events` but is not compared by the snapshot and repair command. + +## Pros and Cons of the Options + +### One posting table for everything + +A nullable target set (`user_id`, `worker_id`, `team_id`) plus a unit column. + +- Bad: conservation, reconciliation, and reporting all become ambiguous once unlike units and + unlike targets share a table, and the foreign-key policy conflict above has no resolution within one schema. + +### Treat worker kudos as currency + +Credit the worker as an account holder. + +- Bad: a worker is not an account. The spendable credit for a worker's job belongs to its owning user in + `users.kudos`, and `workers.kudos` is the attribution total the worker's own pages display. diff --git a/docs/decisions/0003-single-serialized-projector.md b/docs/decisions/0003-single-serialized-projector.md new file mode 100644 index 00000000..d8a46f0a --- /dev/null +++ b/docs/decisions/0003-single-serialized-projector.md @@ -0,0 +1,78 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Fold events with one database-serialized projector + +## Context and Problem Statement + +Appending postings ([ADR 1](0001-event-sourced-kudos-accounting.md)) leaves the question of who updates +`users.kudos`, `workers.kudos`, the team totals, `user_stats`, `user_records`, and the reservation lifecycle. +The projection targets are exactly the hot rows whose uncoordinated acquisition order produced the motivating +deadlock class, so a concurrent set of appliers would rebuild that lock graph one level down. + +## Decision Drivers + +- A fold must be all-or-nothing with the applied flags it sets. A crash between the two would double-apply or + silently drop a movement. +- The Horde runs several replicas. The mechanism that picks the single writer cannot depend on the Redis + process quorum, which does not participate in the database transaction that commits the fold. +- Some rows commit out of ID order. A transaction that acquired a low `kudos_ledger.id` can commit after a + higher ID is already visible and folded. + +## Considered Options + +- Fold events with one database-serialized projector +- Parallel appliers sharded by target +- A high-water mark over `id` + +## Decision Outcome + +Chosen option: "Fold events with one database-serialized projector". + +One projector owns projection in `ledger` mode. Each cycle acquires the PostgreSQL transaction advisory lock +`KUDOS_APPLIER_LOCK` in `horde/database/kudos_db.py`, which is held for the transaction and is independent of +the process quorum. Under that lock it claims bounded, ID-ordered batches of currency and statistic rows with +`FOR UPDATE SKIP LOCKED`, groups deltas by target, visits materialized targets in stable primary-key order, +consumes or releases the matching reservations, marks the exact claimed row IDs applied, and commits all of it +in one transaction. + +`applied = false` is the work queue. A row is selected by its flag, never by `id > watermark`, so a +late-committing lower ID is claimed by a later cycle instead of being skipped. `kudos_ledger_applier_state` +holds a heartbeat for observability and is not consulted when choosing work. Batches are +`KUDOS_APPLIER_BATCH_SIZE` rows; the background task runs every three seconds and keeps folding within a tick +while batches come back full, up to `KUDOS_APPLIER_MAX_CATCHUP_CYCLES`, so a backlog clears at many batches +per tick while each transaction stays small. + +### Consequences + +- Good: Exactly one committed fold per accepted row. If the process dies before commit, neither the target updates + nor the applied flags commit, and a later cycle sees the same rows. +- Good: The accounting lock graph is one writer visiting targets in a documented order, which is what lets the mode + transition ([ADR 5](0005-shadow-mode-cutover.md)) share that order without an applier/control deadlock. +- Good: A stopped projector is a lag incident: rows stay durable and unapplied until it returns. + +- Bad: Projection throughput is bounded by one writer. Batch size, the three-second interval, and the catch-up + cycle limit are the only scaling controls; going wider means a new ordering and reservation design. +- Bad: Queue age, heartbeat age, and reservation age need alerting; `/api/v2/status/heartbeat` reports `DEGRADED` + once the oldest pending event exceeds 30 seconds. + +## Pros and Cons of the Options + +### Parallel appliers sharded by target + +- Bad: sharding needs a new ordering scheme and a reservation design that survives an event whose postings land + in different shards, which is more machinery than the throughput requires today. + +### A high-water mark over `id` + +Track the highest folded ID and claim above it. + +- Bad: a lower ID that commits later is permanently skipped, and the skip is silent. diff --git a/docs/decisions/0004-payer-reservations-for-spend-admission.md b/docs/decisions/0004-payer-reservations-for-spend-admission.md new file mode 100644 index 00000000..71c46ef6 --- /dev/null +++ b/docs/decisions/0004-payer-reservations-for-spend-admission.md @@ -0,0 +1,82 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Admit spends under a payer lock against reserved availability + +## Context and Problem Statement + +Once projection is asynchronous ([ADR 3](0003-single-serialized-projector.md)), `users.kudos` stops being a +sufficient answer to "may this debit be accepted?". A user with 100 visible kudos can submit two simultaneous +80-kudos transfers; if each debit exists only as an unapplied `kudos_ledger` row, both transactions read 100 +and both accept. Overspend has to be refused at admission, before the projector runs. + +## Decision Drivers + +- A transfer involves two accounts. Locking both rebuilds the two-hot-row coupling the event design removed. +- Upfront admission is retryable. A worker re-claiming an interrogation form, or a client retrying a request, + must not stack a second hold for the same business operation. + +## Considered Options + +- Admit spends under a payer lock against reserved availability +- Read the materialized balance and accept +- Row-lock the payer's `users` row for admission +- Authorize against `effective_kudos` + +## Decision Outcome + +Chosen option: "Admit spends under a payer lock against reserved availability". + +Spend admission serializes on the payer alone, through a transaction-scoped advisory lock derived from the +payer's user ID (`acquire_payer_lock`, namespace `KUDOS_PAYER_LOCK_NAMESPACE`). The recipient of a transfer is +never locked for admission. + +Under that lock, `available_kudos` computes capacity as the materialized balance less the account floor, less +active reservations, less ordinary queued debits, reading all three terms in one statement so they come from a +single database snapshot. Queued credits are deliberately excluded: unprojected income cannot fund a spend. + +An accepted operation calls `reserve_kudos` with a stable `business_id` in the same transaction as its +posting, and attaches the reservation ID to the debit's `detail`. The business IDs are `upfront:`, +`interrogation:`, and `transfer::`. Uniqueness on `business_id` makes a +retry reactivate the existing hold rather than add one, and reactivation may not change the payer. The +projector consumes a request hold as its debit folds, and releases a transfer hold once no unapplied posting +remains for the event. + +`effective_kudos` remains available: materialized balance plus all committed unapplied deltas, clamped to the +floor. It answers "what is my new balance?" for an admin adjustment response or a diagnosis, never a spend. + +### Consequences + +- Good: An overspend cannot be authorized on the strength of unmaterialized income, whatever the projector lag is. +- Good: Transfers serialize on one account, so two users gifting the same recipient do not contend. +- Good: Retries are stable, and reservation release is idempotent, returning zero when no active hold remains. + +- Bad: Admission is conservative. A user whose incoming credit is queued is told they cannot afford a spend they + will shortly be able to afford, and projector lag lengthens that window. Relaxing the queued-credit rule + requires a proof covering credit rollback, event ordering, and projector failure. +- Bad: Some legacy reads still consult the materialized balance, including the stable worker's secondary upfront + eligibility check. Those can transiently alter scheduling, though the initial admission hold still stands. + +## Pros and Cons of the Options + +### Read the materialized balance and accept + +- Bad: The two-transfer case in the problem statement accepts both debits. + +### Row-lock the payer's `users` row for admission + +- Bad: it puts the hot user row back into every admission transaction's lock graph, the coupling + [ADR 1](0001-event-sourced-kudos-accounting.md) removed. + +### Authorize against `effective_kudos` + +- Bad: it includes queued credits and represents no holds, so two concurrent admissions can both be funded by + the same unprojected income. diff --git a/docs/decisions/0005-shadow-mode-cutover.md b/docs/decisions/0005-shadow-mode-cutover.md new file mode 100644 index 00000000..31b8da70 --- /dev/null +++ b/docs/decisions/0005-shadow-mode-cutover.md @@ -0,0 +1,78 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Cut over through a temporary shadow mode + +## Context and Problem Statement + +Moving balance ownership from inline mutation to the serialized projector changes who writes `users.kudos` for +every settlement, transfer, award, uptime reward, and adjustment in the service. The Horde cannot take a +maintenance window, so the change has to land on a running fleet and be reversible while it is running. + +## Decision Drivers + +- The event schema, emission primitives, reservations, and projector need production exposure before any + balance depends on them. +- Rollback must not leave accepted postings unfolded behind a writer that has stopped reading them, and + whatever carries the transition has to be removable afterwards without unpicking the accounting flow. + +## Considered Options + +- Cut over through a temporary shadow mode +- Deploy straight into ledger mode +- Feature-flag each producer + +## Decision Outcome + +Chosen option: "Cut over through a temporary shadow mode". + +New and migrated installations start in `shadow` mode, recorded on the single `kudos_ledger_control` row. +Business methods emit the same typed events in both modes. What differs is ownership of the materialized +change and the applied flag chosen by the two emission primitives: + +- In `shadow`, `horde/database/kudos_legacy_projection.py` performs the historical inline mutation and the + rows are written `applied = true`, giving audit evidence without replaying a movement that already reached + its target. That projector also owns trust promotion and the escrow drain + ([ADR 8](0008-trust-promotion-in-the-projector.md)), so shadow changes no balance the old code would not. +- In `ledger`, rows are written `applied = false` and the serialized projector + ([ADR 3](0003-single-serialized-projector.md)) owns every materialized target. + +The mode branch is confined to `kudos_legacy_projection.py`, the control helpers, and the applied-flag choice +inside the two emission primitives. Endpoints and settlement methods contain no mode check, so removing the +cutover period is a deletion: the compatibility module, its direct calls, and the shadow transition. + +Every mutation transaction pins the mode it observed with a key-share lock on the control row until commit. +`set_kudos_ledger_mode` takes the applier advisory lock, then an exclusive control-row lock that waits for +those writers; returning to `shadow` folds the remaining ledger tail in the same transaction. + +### Consequences + +- Good: Cutover is gated on evidence: a peak-load shadow window, a clean `status`, a snapshot, a clean reconcile. +- Good: Rollback is one command with no service freeze, and active upfront reservations survive it because + shadow-mode debits consume the same holds inline. + +- Bad: The migration mechanism is synchronous double-write, so shadow mode keeps the whole original lock graph and + adds the event writes, buying none of the concurrency benefit. +- Bad: Shadow history is a forward audit beginning at deployment. The materialized balances at that moment are the + opening position, recorded by a snapshot alongside the applied-ledger totals visible at the same time. +- Bad: Rolling back to code that predates reservations and shadow audit rows is unsafe at any point after deploy. + +## Pros and Cons of the Options + +### Deploy straight into ledger mode + +- Bad: the first production evidence about the projector arrives at the moment balances already depend on it, + and there is no rollback that does not strand postings. + +### Feature-flag each producer + +- Bad: mode checks inside settlements, transfers, awards, and endpoints spread across the codebase, and + removing them becomes a diffuse edit rather than a deletion. diff --git a/docs/decisions/0006-permanent-archive-compensation-only-repair.md b/docs/decisions/0006-permanent-archive-compensation-only-repair.md new file mode 100644 index 00000000..4a008232 --- /dev/null +++ b/docs/decisions/0006-permanent-archive-compensation-only-repair.md @@ -0,0 +1,82 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Keep the archive permanent and repair only by compensation + +## Context and Problem Statement + +Under inline mutation the current column was the only accounting record, so correcting a wrong balance meant +overwriting the one visible number and leaving no trace of the error or the correction. Event tables make a +better answer possible, and a worse one tempting: an operator facing drift can reach for +`UPDATE kudos_ledger SET applied = true`, a `DELETE` of a suspect posting, or a direct balance assignment. + +## Decision Drivers + +- Reconciliation computes expected current values from a snapshot baseline plus the events applied since, + which is only sound if applied history never changes after the fact. +- Repair runs during incidents, under time pressure, and may be run twice without doubling its correction. +- Some projector rules are not linear. Minimum-balance forgiveness makes the balance change smaller than the + debit's own amount, so replaying the postings alone would not reproduce the projection. + +## Considered Options + +- Keep the archive permanent and repair only by compensation +- Prune applied events after a retention window +- Repair by assigning the correct balance +- Repair by clearing `applied` so the projector refolds + +## Decision Outcome + +Chosen option: "Keep the archive permanent and repair only by compensation". + +Applied currency and statistic history is permanent. Repair happens only by appending. + +`reconcile ` is read-only: it compares the materialized balances against the snapshot plus every +currency posting applied since, and reports drift. `reconcile --apply` serializes repair runs on +`KUDOS_RECONCILIATION_LOCK` and emits one deterministic `RECONCILIATION` posting per affected user. That +computation excludes prior `RECONCILIATION` entries from its movement baseline, so re-running the same repair +before or after projection cannot duplicate it. + +Non-linear projector behavior is represented as its own posting rather than left implicit. When the account +floor forgives part of a debit, the projector emits a `FLOOR_ADJUSTMENT` credit for the amount created, +written already applied, so replaying postings from a snapshot reproduces the projection exactly across +separate batches. No repair edits an `applied` flag, an amount, a target, or a balance column directly. + +### Consequences + +- Good: Drift is a diagnosable difference against a stated baseline, and its correction is a posting with an entry + type, an amount, and a timestamp that later reconciliations account for. Replay stays linear, so a snapshot + plus subsequent postings is a checkable statement about current balances. +- Good: Disaster recovery keeps its evidence: restore to the selected PITR point, retain the archives and snapshots, + start in `shadow` ([ADR 5](0005-shadow-mode-cutover.md)), reconcile, then repeat the cutover proof. + +- Bad: `kudos_ledger` and `kudos_stat_events` grow without bound and need capacity planning. +- Bad: Repairing an incident is a procedure rather than an edit: preserve evidence, snapshot, reconcile against the + last known-good baseline, review the complete drift list, apply compensation, drain, reconcile again. +- Bad: The reconcile and repair command covers user currency and evaluation escrow. Worker, team, and counter rows + are replayable from `kudos_stat_events` but are not compared by it. + +## Pros and Cons of the Options + +### Prune applied events after a retention window + +- Bad: the baseline for reconciliation and the evidence for a post-incident review both live in applied + history. `prune_applied_kudos_ledger` is a no-op. + +### Repair by assigning the correct balance + +- Bad: the assignment is invisible to reconciliation, which then reports the repair itself as new drift on the + next run. + +### Repair by clearing `applied` so the projector refolds + +- Bad: it silently changes what the snapshot baseline means and can refold a movement that already reached its + target. diff --git a/docs/decisions/0007-accounting-foreign-key-policy.md b/docs/decisions/0007-accounting-foreign-key-policy.md new file mode 100644 index 00000000..1e9c4482 --- /dev/null +++ b/docs/decisions/0007-accounting-foreign-key-policy.md @@ -0,0 +1,81 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Match each accounting table's foreign keys to its lifetime + +## Context and Problem Statement + +The accounting tables reference users, workers, and teams, and those three are deleted in different ways. User +accounts are soft-deleted and wiped. Workers and teams can be hard-deleted by their owners, and that deletion +is a supported operation rather than an exceptional one. + +The four tables also differ in what their rows are for: + +- `kudos_ledger` rows are authoritative currency history and the input to reconciliation. +- `kudos_stat_events` rows are display and counter history behind worker, team, and user aggregates. +- `kudos_reservations` rows are temporary holds against one payer. +- `kudos_balance_snapshots` rows are a per-user reconciliation baseline. + +## Considered Options + +- Match each accounting table's foreign keys to its lifetime +- Ownership foreign keys everywhere, cascading +- Ownership foreign keys everywhere, restricting +- No foreign keys on any accounting table + +## Decision Outcome + +Chosen option: "Match each accounting table's foreign keys to its lifetime". + +Each table gets the policy its contents' lifetime requires. + +- `kudos_ledger.user_id` is a required foreign key with `ON DELETE RESTRICT`. Authoritative currency history + must not become orphaned, and users are removed by soft delete and wipe rather than row deletion, so the + restriction does not block a supported operation. +- `kudos_reservations.user_id` and `kudos_balance_snapshots.user_id` are ownership foreign keys with + `ON DELETE CASCADE`. A hold and a per-user reconciliation baseline only have meaning while the user exists. +- `kudos_stat_events` carries `user_id`, `worker_id`, `worker_user_id`, and `team_id` as immutable audit + references with no foreign key. Workers and teams are hard-deletable while their counter history must + survive them, and neither cascade nor restrict can deliver both. The exception is documented on the mapped + model so it reads as a decision rather than an omission. + +### Consequences + +- Good: A `kudos_ledger` row always names a real user, so reconciliation never has to reason about a posting + whose account is gone. +- Good: Deleting a worker or a team stays a supported, unblocked operation, and the totals its work + contributed to keep their explanation in `kudos_stat_events`. +- Good: Holds and snapshots clean themselves up with the account they belong to. + +- Bad: `kudos_stat_events` rows can reference a worker or team that no longer exists, so a consumer joining + against live rows must tolerate a missing target rather than assume one. +- Bad: Every future accounting table must be assigned a policy deliberately, answering the same question: is + this row authoritative history, an audit reference, or operational state? +- Bad: The split reinforces why currency and statistic postings live in separate tables + ([ADR 2](0002-separate-currency-and-statistic-event-tables.md)). Merging them would reopen a conflict with + no schema-level resolution. + +## Pros and Cons of the Options + +### Ownership foreign keys everywhere, cascading + +- Bad: Deleting a worker would erase the postings that explain its team's totals, and reconciliation would + lose the currency history for a wiped user. + +### Ownership foreign keys everywhere, restricting + +- Bad: Worker and team deletion would fail for any worker that had ever earned anything, which is every worker + that has run. + +### No foreign keys on any accounting table + +- Bad: It gives up the guarantee that a currency posting always names a real user, which is the one + referential guarantee reconciliation depends on. diff --git a/docs/decisions/0008-trust-promotion-in-the-projector.md b/docs/decisions/0008-trust-promotion-in-the-projector.md new file mode 100644 index 00000000..64f3892a --- /dev/null +++ b/docs/decisions/0008-trust-promotion-in-the-projector.md @@ -0,0 +1,78 @@ +--- +status: accepted +date: 2026-07-24 +--- + + + +# Promote users to trusted inside the projector + +## Context and Problem Statement + +An untrusted worker owner's rewards go wholly or partly to `users.evaluating_kudos`. Once that escrow crosses +the trust threshold and the account is old enough, the user becomes trusted and the escrow is released as a +paired `EVALUATION_PROMOTION` escrow debit and balance credit. + +With projection asynchronous ([ADR 3](0003-single-serialized-projector.md)), the transaction that emits the +final qualifying escrow credit cannot see its own effect: the credit is an unapplied `kudos_ledger` row, and +`users.evaluating_kudos` still holds the pre-credit value. The threshold has to be evaluated after the fold. + +## Decision Drivers + +- The final qualifying contribution should promote the user even if they never submit another request. +- Promotion is a threshold crossing, and a crossing evaluated twice can release the escrow twice. +- Shadow mode changes no balance the previous inline code would not ([ADR 5](0005-shadow-mode-cutover.md)). + +## Considered Options + +- Promote users to trusted inside the projector +- Promote on the user's next request +- Promote in the emitting transaction + +## Decision Outcome + +Chosen option: "Promote users to trusted inside the projector". + +In `ledger` mode, the projector evaluates promotion after folding a cycle's escrow deltas, inside the same +transaction. `_promote_eligible_users` selects users who are not already trusted, whose projected +`evaluating_kudos` exceeds the trust threshold, and whose account is at least seven days old, skipping anon +and suspicious accounts. It grants the `TRUSTED` role and unpauses the user's workers. + +`_drain_trusted_escrow` then scans for trusted users still carrying positive escrow and emits an +`EVALUATION_PROMOTION` pair for the full amount under one event ID. A later cycle folds that pair, the escrow +reaches zero, and the scan stops finding the user. It reads trust state and the drain amount from committed +columns rather than ORM instances, because the role write does not refresh a loaded role collection. A user +with an unapplied `EVALUATION_PROMOTION` posting is skipped: without that guard, a cycle that emitted a pair +but could not fold it would emit a second pair, over-crediting the balance once the backlog folds. + +In `shadow` mode the compatibility projector owns promotion and the escrow drain (`project_trust_promotion`), +and the asynchronous projector's promotion duties do not run. + +### Consequences + +- Good: The contribution that crosses the threshold promotes the user, with no dependence on a later request. +- Good: The drain scan is self-healing. A trusted user carrying stranded escrow, including one promoted after their + escrow folded in an earlier cycle, is drained by a later cycle with no operator touching a balance. +- Good: The release is a posting pair like any other, reconciling and replaying through the same path as the rest of + the archive ([ADR 6](0006-permanent-archive-compensation-only-repair.md)). + +- Bad: Promotion is eventually consistent: it happens a cycle after the qualifying contribution, and the spendable + credit lands a cycle after that. +- Bad: The projector holds trust policy: threshold, account age, and the anon and suspicious exclusions. Changing + that policy changes the projector and its shadow-mode counterpart, and a promotion test has to say which of + the two it exercises. + +## Pros and Cons of the Options + +### Promote on the user's next request + +- Bad: promotion then depends on the user coming back, and the request path would have to reason about its own + unprojected escrow to get the threshold right. + +### Promote in the emitting transaction + +- Bad: it cannot observe the escrow its own credit will produce. diff --git a/docs/decisions/index.md b/docs/decisions/index.md new file mode 100644 index 00000000..f9be3a5a --- /dev/null +++ b/docs/decisions/index.md @@ -0,0 +1,58 @@ + + + + +# Decisions + +One decision per record. Status and date come from each record's front matter; +regenerate this page with `gen_adr_index.py` after adding or changing a record. + +| Record | Status | Date | +| --- | --- | --- | +| [Record kudos movements as events; keep balances as projections](0001-event-sourced-kudos-accounting.md) | accepted | 2026-07-24 | +| [Separate currency postings from statistic postings](0002-separate-currency-and-statistic-event-tables.md) | accepted | 2026-07-24 | +| [Fold events with one database-serialized projector](0003-single-serialized-projector.md) | accepted | 2026-07-24 | +| [Admit spends under a payer lock against reserved availability](0004-payer-reservations-for-spend-admission.md) | accepted | 2026-07-24 | +| [Cut over through a temporary shadow mode](0005-shadow-mode-cutover.md) | accepted | 2026-07-24 | +| [Keep the archive permanent and repair only by compensation](0006-permanent-archive-compensation-only-repair.md) | accepted | 2026-07-24 | +| [Match each accounting table's foreign keys to its lifetime](0007-accounting-foreign-key-policy.md) | accepted | 2026-07-24 | +| [Promote users to trusted inside the projector](0008-trust-promotion-in-the-projector.md) | accepted | 2026-07-24 | +| [Balance every currency event against system accounts](0009-balanced-events-via-system-accounts.md) | proposed | 2026-07-25 | + +## Relationships + +An edge points from a record to a record it references. + +```mermaid +flowchart TD + r0001["0001: Record kudos movements as events; kee…"] + r0002["0002: Separate currency postings from stati…"] + r0003["0003: Fold events with one database-seriali…"] + r0004["0004: Admit spends under a payer lock again…"] + r0005["0005: Cut over through a temporary shadow m…"] + r0006["0006: Keep the archive permanent and repair…"] + r0007["0007: Match each accounting table's foreign…"] + r0008["0008: Promote users to trusted inside the p…"] + r0009["0009: Balance every currency event against…"] + r0001 --> r0003 + r0001 --> r0004 + r0001 --> r0005 + r0001 --> r0006 + r0002 --> r0006 + r0002 --> r0007 + r0003 --> r0001 + r0003 --> r0005 + r0004 --> r0001 + r0004 --> r0003 + r0005 --> r0003 + r0005 --> r0008 + r0006 --> r0005 + r0007 --> r0002 + r0008 --> r0003 + r0008 --> r0005 + r0008 --> r0006 +``` diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 00000000..2933cd1d --- /dev/null +++ b/zensical.toml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: 2026 Tazlin +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Static documentation site over docs/, built with Zensical (https://zensical.org). +# Navigation is implicit from the directory structure; docs/README.md is the landing +# page. Build with `uvx zensical build`; preview with `uvx zensical serve`. + +[project] +site_name = "AI Horde" +site_description = "Implementation and operation documentation for the AI Horde service." +copyright = "AGPL-3.0-or-later" + +# Top-level sections mirror the Diataxis classes. Decision records deliberately enter +# navigation only through their generated index; the records are reached from the index, +# cross-links, and search, so navigation does not grow with the record count. +nav = [ + { "Home" = "README.md" }, + { "Explanation" = [ + { "Kudos accounting" = "explanation/kudos_accounting.md" }, + ] }, + { "Reference" = [ + { "Overview" = "reference/README.md" }, + { "Caching and freshness" = "reference/caching_and_freshness.md" }, + { "Worker check-in" = "reference/worker_checkin.md" }, + { "Job pop" = "reference/job_pop.md" }, + { "Generation lifecycle" = "reference/generation_lifecycle.md" }, + { "Status reporting" = "reference/status_reporting.md" }, + { "Kudos" = "reference/kudos.md" }, + { "Kudos accounting" = "reference/kudos_accounting.md" }, + { "Users and trust" = "reference/users_and_trust.md" }, + { "Countermeasures" = "reference/countermeasures.md" }, + { "Interrogation" = "reference/interrogation.md" }, + { "Asset storage" = "reference/asset_storage.md" }, + { "Statistics and metrics" = "reference/stats_metrics.md" }, + { "Styles" = "reference/styles.md" }, + ] }, + { "How-to" = [ + { "Kudos ledger operations" = "how-to/kudos_ledger_operations.md" }, + ] }, + { "Decisions" = "decisions/index.md" }, + { "Concepts" = [ + { "Introduction" = "haidra-assets/docs/intro.md" }, + { "Why the Horde exists" = "haidra-assets/docs/why.md" }, + { "Definitions" = "haidra-assets/docs/definitions.md" }, + { "Workers" = "haidra-assets/docs/workers.md" }, + { "Job lifecycle" = "haidra-assets/docs/job_lifecycle.md" }, + { "Kudos" = "haidra-assets/docs/kudos.md" }, + ] }, + { "Standards" = [ + { "Documentation" = "haidra-assets/docs/meta/documentation.md" }, + { "Pull requests" = "haidra-assets/docs/meta/pull_requests.md" }, + { "Python" = "haidra-assets/docs/meta/python.md" }, + ] }, +] + +[project.theme] +language = "en" +logo = "haidra-assets/haidra-logo/haidra-logo-path.svg" +favicon = "haidra-assets/haidra-logo/haidra-transparent-zoomed-in.png" +features = [ + "content.code.copy", + "content.tooltips", + "navigation.footer", + "navigation.indexes", + "navigation.path", + "navigation.tabs", + "navigation.tabs.sticky", + "navigation.top", + "navigation.tracking", + "search.highlight", + "toc.follow", +] + +[[project.theme.palette]] +scheme = "default" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +scheme = "slate" +toggle.icon = "lucide/moon" +toggle.name = "Switch to light mode" + +[project.markdown_extensions.admonition] +[project.markdown_extensions.attr_list] +[project.markdown_extensions.def_list] +[project.markdown_extensions.footnotes] +[project.markdown_extensions.md_in_html] +[project.markdown_extensions.tables] +[project.markdown_extensions.toc] +permalink = true +[project.markdown_extensions.pymdownx.highlight] +anchor_linenums = true +pygments_lang_class = true +[project.markdown_extensions.pymdownx.inlinehilite] +[project.markdown_extensions.pymdownx.magiclink] +[project.markdown_extensions.pymdownx.superfences] +custom_fences = [ + { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, +]