From 27711b500c39169a367033df2970cd2ded1e5ecc Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 22 Jun 2026 18:55:06 -0700 Subject: [PATCH] ci: add maturity scorecard renderer (#94272) (#95901) * ci: add maturity scorecard renderer * ci: render qa scorecard evidence * ci: type maturity docs renderer * ci: tighten maturity artifact inputs * ci: move maturity renderer under qa scripts * ci: share maturity score schema * ci: centralize maturity taxonomy validation * ci: move maturity scores under qa * ci: remove docs maturity score source * docs: simplify maturity scorecard output * docs: commit generated maturity scorecard * docs: group maturity pages * docs: simplify maturity scorecard dates * docs: promote maturity nav tab * docs: clean up maturity pages * docs: remove maturity outline page * docs: filter maturity taxonomy doc links * docs: simplify maturity taxonomy tables * docs: keep artifact taxonomy links * docs: simplify lts scorecard display * docs: clarify maturity score definitions * docs: derive maturity coverage from evidence * docs: hide maturity scorecard until evidence * docs: remove placeholder maturity pages * docs: keep maturity scores out of pr * ci: open maturity scorecard docs pr --- .agents/skills/claw-score/SKILL.md | 56 +- .github/workflows/maturity-scorecard.yml | 242 +++++ extensions/qa-lab/src/coverage-report.test.ts | 12 +- extensions/qa-lab/src/scorecard-taxonomy.ts | 503 +++++++++- package.json | 2 + scripts/qa/render-maturity-docs.ts | 927 ++++++++++++++++++ taxonomy.yaml | 12 +- 7 files changed, 1716 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/maturity-scorecard.yml create mode 100644 scripts/qa/render-maturity-docs.ts diff --git a/.agents/skills/claw-score/SKILL.md b/.agents/skills/claw-score/SKILL.md index 9fc12f3e2327..e57207d382fb 100644 --- a/.agents/skills/claw-score/SKILL.md +++ b/.agents/skills/claw-score/SKILL.md @@ -15,7 +15,7 @@ committed `inventory/` report tree. This skill owns the operational workflow for: - `taxonomy.yaml` -- `docs/maturity-scores.yaml` +- `qa/maturity-scores.yaml` - `docs/concepts/qa-e2e-automation.md` - `qa/scenarios/index.yaml` @@ -37,28 +37,35 @@ out of this repo. If a score needs private evidence, use the redacted coverage IDs. Do not promote generic IDs into standalone feature names. - Avoid duplicate coverage-ID bundles under different feature names in one category. -- `docs/maturity-scores.yaml` is the aggregate score source committed in this - repo. It is the only committed score data; do not add generated inventory - directories. -- There is no committed maturity-doc renderer or `pnpm maturity:*` script in - this repo. Do not invent generated scorecard files; update the source YAML - and current docs directly. -- `qa-evidence.json` artifacts provide per-run QA scorecard evidence. They can - enrich generated artifact docs, but they are not committed as inventory. +- `qa/maturity-scores.yaml` is the committed aggregate source for Quality, + Completeness, and LTS review state. +- `extensions/qa-lab/src/scorecard-taxonomy.ts` exports + `qaMaturityScoresSchema` and `readValidatedQaMaturityScoreSources`; use those + QA Lab utilities to validate score output. +- Generated public docs are `docs/maturity/scorecard.md` and + `docs/maturity/taxonomy.md`; both come from `pnpm maturity:render`. Do not + hand-edit generated Markdown to change score results. +- `qa-evidence.json` artifacts provide per-run QA scorecard evidence. Release + profile artifacts are the source of truth for Coverage. They can enrich + generated artifact docs, but they are not committed as inventory. ## Commands Run from the openclaw repo root. -Validate YAML structure after source edits: +Validate taxonomy YAML structure and the maturity score schema after source +edits: ```bash -node <<'NODE' -const fs = require("node:fs"); -const YAML = require("yaml"); -for (const file of ["taxonomy.yaml", "docs/maturity-scores.yaml", "qa/scenarios/index.yaml"]) { +node --import tsx --input-type=module <<'NODE' +import fs from "node:fs"; +import YAML from "yaml"; +import { readValidatedQaMaturityScoreSources } from "./extensions/qa-lab/src/scorecard-taxonomy.ts"; + +for (const file of ["taxonomy.yaml", "qa/scenarios/index.yaml"]) { YAML.parse(fs.readFileSync(file, "utf8")); } +readValidatedQaMaturityScoreSources(); NODE ``` @@ -83,17 +90,17 @@ When asked to score or refresh a surface: `.agents/skills/claw-score/references/completeness/`. 3. Gather public repo evidence from docs, source, tests, and QA scenario metadata. -4. Prefer existing `qa-evidence.json` artifacts for executed proof. Do not use - discrawl or unredacted private archives. -5. Update `docs/maturity-scores.yaml` only when the score change is backed by - public or redacted artifact evidence. -6. Run the YAML validation command from this skill. +4. Prefer existing release profile `qa-evidence.json` artifacts for executed + proof. +5. Update `qa/maturity-scores.yaml` only for Quality, Completeness, and LTS + review state backed by public or redacted artifact evidence. +6. Run the schema validation command from this skill. 7. Run `pnpm check:docs` if docs prose changed, and focused QA coverage checks if coverage IDs or profile membership changed. For subjective score changes, make the smallest defensible edit and leave the evidence path in the PR or task summary. Keep manual prose in current docs and -keep score data in `docs/maturity-scores.yaml`. +keep score data in `qa/maturity-scores.yaml`. ## Default Completeness Process @@ -152,15 +159,16 @@ Default Completeness bands: ## Score Semantics -- Coverage: public or redacted proof that the feature is exercised by docs, - tests, QA scenarios, live lanes, or release evidence. +- Coverage: deterministic release validation coverage derived from the release + profile `qa-evidence.json.scorecard` feature fulfillment data. - Quality: reliability, maintainability, operator safety, and regression confidence for the category. - Completeness: how much of the intended operator-visible workflow exists for the category. Use the default completeness process plus any surface-specific variation before changing this score. -- LTS: derived from score thresholds and `human_lts_override`; do not hand-edit - generated Markdown to change LTS status. +- LTS: derived from Quality, release-evidence Coverage, and + `human_lts_override`; do not hand-edit generated Markdown to change LTS + status. Bands: diff --git a/.github/workflows/maturity-scorecard.yml b/.github/workflows/maturity-scorecard.yml new file mode 100644 index 000000000000..01907aa13063 --- /dev/null +++ b/.github/workflows/maturity-scorecard.yml @@ -0,0 +1,242 @@ +name: Maturity scorecard + +on: + workflow_dispatch: + inputs: + source_run_id: + description: Optional workflow run id containing qa-evidence.json artifacts + required: false + type: string + artifact_pattern: + description: Artifact name pattern to download from source_run_id + required: false + default: "*qa*" + type: string + strict_inputs: + description: Fail when score or QA evidence inputs have non-fatal drift + required: false + default: false + type: boolean + +permissions: + actions: read + contents: read + +concurrency: + group: ${{ format('{0}-{1}', github.workflow, github.ref) }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NODE_VERSION: "24.x" + +jobs: + validate: + name: Validate maturity score sources + # Disabled until the initial generated docs and refreshed score snapshot land together. + if: ${{ false }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + install-bun: "false" + + - name: Validate maturity score sources + run: | + node --import tsx --input-type=module <<'NODE' + import { readValidatedQaMaturityScoreSources } from "./extensions/qa-lab/src/scorecard-taxonomy.ts"; + + const { warnings } = readValidatedQaMaturityScoreSources({ + scoresPath: "qa/maturity-scores.yaml", + taxonomyPath: "taxonomy.yaml", + }); + for (const warning of warnings) { + console.error(`warning: ${warning}`); + } + NODE + + publish: + name: Publish maturity docs PR + # Disabled until the initial generated docs and refreshed score snapshot land together. + if: ${{ false }} + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + install-bun: "false" + + - name: Download QA evidence artifacts + if: ${{ inputs.source_run_id != '' }} + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + ARTIFACT_PATTERN: ${{ inputs.artifact_pattern }} + run: | + set -euo pipefail + mkdir -p .artifacts/maturity-evidence + gh run download "$SOURCE_RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --pattern "$ARTIFACT_PATTERN" \ + --dir .artifacts/maturity-evidence + find .artifacts/maturity-evidence -name qa-evidence.json -print + + - name: Check QA evidence artifacts + id: evidence + run: | + set -euo pipefail + if find .artifacts/maturity-evidence -name qa-evidence.json -print -quit 2>/dev/null | grep -q .; then + echo "has_evidence=true" >> "$GITHUB_OUTPUT" + else + echo "has_evidence=false" >> "$GITHUB_OUTPUT" + fi + + - name: Require QA evidence for manual scorecard render + if: ${{ github.event_name == 'workflow_dispatch' && steps.evidence.outputs.has_evidence != 'true' }} + run: | + echo "Maturity scorecard rendering requires release QA evidence artifacts." >&2 + exit 1 + + - name: Render artifact docs + if: ${{ steps.evidence.outputs.has_evidence == 'true' }} + env: + STRICT_INPUTS: ${{ github.event_name == 'workflow_dispatch' && inputs.strict_inputs }} + run: | + set -euo pipefail + args=(--output-dir .artifacts/maturity-docs --static-assets-dir .artifacts/maturity-docs/assets/maturity --evidence-dir .artifacts/maturity-evidence) + if [[ "$STRICT_INPUTS" == "true" ]]; then + args+=(--strict-inputs) + fi + pnpm maturity:render -- "${args[@]}" + { + echo "### Maturity scorecard docs" + echo + echo "- Source validation: passed" + echo "- Artifact docs: \`.artifacts/maturity-docs\`" + echo "- Strict inputs: \`${STRICT_INPUTS:-false}\`" + echo "- QA evidence: included" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Render committed docs preview + if: ${{ steps.evidence.outputs.has_evidence == 'true' }} + run: | + set -euo pipefail + pnpm maturity:render -- \ + --output-dir docs \ + --evidence-dir .artifacts/maturity-evidence + + - name: Create generated docs PR app token + if: ${{ steps.evidence.outputs.has_evidence == 'true' }} + id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: "2729701" + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Create generated docs PR fallback app token + if: ${{ steps.evidence.outputs.has_evidence == 'true' && steps.app-token.outcome == 'failure' }} + id: app-token-fallback + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: "2971289" + private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }} + permission-contents: write + permission-pull-requests: write + + - name: Open generated docs PR + if: ${{ steps.evidence.outputs.has_evidence == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }} + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + run: | + set -euo pipefail + if [[ -z "${GH_TOKEN:-}" ]]; then + echo "Maturity scorecard PR creation requires the OpenClaw GitHub App token secrets." >&2 + exit 1 + fi + + if [[ -z "$(git status --porcelain -- docs/maturity/scorecard.md docs/maturity/taxonomy.md)" ]]; then + { + echo + echo "- Pull request: skipped; generated docs match current branch" + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + branch="automation/maturity-scorecard-${SOURCE_RUN_ID:-$GITHUB_RUN_ID}" + base_branch="${GITHUB_REF_NAME:-main}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + gh auth setup-git + git fetch --no-tags --depth=1 origin "refs/heads/${branch}:refs/remotes/origin/${branch}" || true + git switch -C "$branch" + git add docs/maturity/scorecard.md docs/maturity/taxonomy.md + git commit -m "docs: update maturity scorecard" + git push --force-with-lease origin "$branch" + + body_file=".artifacts/maturity-scorecard-pr-body.md" + mkdir -p "$(dirname "$body_file")" + cat > "$body_file" <> "$GITHUB_STEP_SUMMARY" + + - name: Upload maturity docs artifact + if: ${{ steps.evidence.outputs.has_evidence == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: maturity-scorecard-docs-${{ github.run_id }}-${{ github.run_attempt }} + path: .artifacts/maturity-docs/ + retention-days: 30 + if-no-files-found: error diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index dfc7bc9b679d..15e63b78746b 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -7,7 +7,7 @@ import { renderQaScenarioMatchesMarkdownReport, } from "./coverage-report.js"; import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; -import { buildQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; +import { buildQaScorecardTaxonomyReport, type QaMaturityTaxonomy } from "./scorecard-taxonomy.js"; const TEST_EXECUTABLE_CATEGORY_ID = "agent-runtime-and-provider-execution.agent-turn-execution"; const TEST_EXECUTABLE_COVERAGE_ID = "channels.dm"; @@ -22,14 +22,15 @@ function testMaturityTaxonomy(params?: { featureCoverageIds?: readonly (readonly string[])[]; includeAllCategories?: boolean; profileCategoryIds?: readonly string[]; -}) { +}): QaMaturityTaxonomy { const categoryId = params?.categoryId ?? TEST_EXECUTABLE_CATEGORY_ID; const firstDot = categoryId.indexOf("."); const surfaceId = firstDot === -1 ? categoryId : categoryId.slice(0, firstDot); const categoryLocalId = firstDot === -1 ? categoryId : categoryId.slice(firstDot + 1); return { - version: 1, + version: 1 as const, title: "Test taxonomy", + levels: [], profiles: [ { id: "smoke-ci", @@ -52,10 +53,15 @@ function testMaturityTaxonomy(params?: { { id: surfaceId, name: "Test surface", + family: "test", + level: "experimental", categories: [ { id: categoryLocalId, name: "Test category", + category_note: "test-category.md", + docs: [], + search_anchors: [], features: ( params?.featureCoverageIds ?? (params?.coverageIds ?? [TEST_EXECUTABLE_COVERAGE_ID]).map((coverageId) => [ diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts index bd73c0eb6234..45bb9b432e98 100644 --- a/extensions/qa-lab/src/scorecard-taxonomy.ts +++ b/extensions/qa-lab/src/scorecard-taxonomy.ts @@ -7,6 +7,22 @@ import { resolveQaRepoPath, type QaRepoPathKind } from "./repo-path.js"; import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; export const QA_MATURITY_TAXONOMY_PATH = "taxonomy.yaml"; +export const QA_MATURITY_SCORES_PATH = "qa/maturity-scores.yaml"; +export const QA_MATURITY_SCORE_KEYS = ["quality", "completeness"] as const; +export const QA_MATURITY_SCORE_LABELS = [ + "Lovable", + "Stable", + "Beta", + "Alpha", + "Experimental", +] as const; +export const QA_MATURITY_SCORE_LABEL_BANDS = [ + [QA_MATURITY_SCORE_LABELS[0], 95, 100], + [QA_MATURITY_SCORE_LABELS[1], 80, 95], + [QA_MATURITY_SCORE_LABELS[2], 70, 80], + [QA_MATURITY_SCORE_LABELS[3], 50, 70], + [QA_MATURITY_SCORE_LABELS[4], 0, 50], +] as const; const qaScorecardIdSchema = z .string() @@ -39,6 +55,132 @@ const qaScorecardProfileSchema = z.object({ categoryIds: z.array(qaScorecardIdSchema).default([]), }); +function maturityScoreLabelForScore(score: number) { + for (const [label, low, high] of QA_MATURITY_SCORE_LABEL_BANDS) { + if (score >= low && score <= high) { + return label; + } + } + throw new Error(`score outside 0-100: ${score}`); +} + +const qaMaturityScoreObjectSchema = z + .object({ + score: z.number().int().min(0).max(100), + label: z.enum(QA_MATURITY_SCORE_LABELS), + }) + .strict() + .superRefine((value, ctx) => { + const expectedLabel = maturityScoreLabelForScore(value.score); + if (value.label !== expectedLabel) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["label"], + message: `must be ${expectedLabel} for score ${value.score}`, + }); + } + }); + +export function qaMaturityScoreObjectForScore(score: number): QaMaturityScoreObject { + return qaMaturityScoreObjectSchema.parse({ + score, + label: maturityScoreLabelForScore(score), + }); +} + +const qaMaturityScoreBundleShape = { + quality: qaMaturityScoreObjectSchema, + completeness: qaMaturityScoreObjectSchema, +} satisfies z.ZodRawShape; + +const qaMaturityLegacyCoverageShape = { + coverage: qaMaturityScoreObjectSchema.optional(), +} satisfies z.ZodRawShape; + +const qaMaturityScoreBundleSchema = z + .object({ + ...qaMaturityLegacyCoverageShape, + ...qaMaturityScoreBundleShape, + }) + .strict(); + +const qaMaturityScoreLastRunSchema = z + .object({ + status: z.string().trim().min(1).optional(), + completed_at: z.string().trim().min(1).optional(), + by: z.string().trim().min(1).optional(), + source_ref: z.string().trim().min(1).nullable().optional(), + process_version: z.number().int().positive().optional(), + }) + .strict(); + +const qaMaturityScoreCategoryLtsSchema = z + .object({ + supported: z.boolean(), + reason: z.string().trim().min(1).optional(), + human_override: z.boolean(), + }) + .strict(); + +const qaMaturityScoreSurfaceLtsSchema = z + .object({ + supported_categories: z.number().int().nonnegative(), + total_categories: z.number().int().nonnegative(), + status: z.string().trim().min(1), + }) + .strict(); + +const qaMaturityScoreCategorySchema = z + .object({ + name: z.string().trim().min(1), + ...qaMaturityLegacyCoverageShape, + ...qaMaturityScoreBundleShape, + lts: qaMaturityScoreCategoryLtsSchema, + }) + .strict(); + +const qaMaturityScoreSurfaceSchema = z + .object({ + id: qaScorecardIdSchema, + name: z.string().trim().min(1), + family: z.string().trim().min(1).optional(), + level: z.union([ + z.string().trim().min(1), + z + .object({ + id: z.string().trim().min(1).optional(), + code: z.string().trim().min(1).optional(), + label: z.string().trim().min(1).optional(), + }) + .strict(), + ]), + scores: qaMaturityScoreBundleSchema, + categories: z.array(qaMaturityScoreCategorySchema), + lts: qaMaturityScoreSurfaceLtsSchema, + last_score_run: qaMaturityScoreLastRunSchema.optional(), + }) + .strict(); + +export const qaMaturityScoresSchema = z + .object({ + version: z.literal(1), + process_version: z.number().int().positive(), + counts: z + .object({ + active_surfaces: z.number().int().nonnegative(), + category_scores: z.number().int().nonnegative(), + }) + .strict(), + rollups: z + .object({ + surface_average: qaMaturityScoreBundleSchema, + category_average: qaMaturityScoreBundleSchema, + }) + .strict(), + surfaces: z.array(qaMaturityScoreSurfaceSchema), + }) + .strict(); + const qaMaturityFeatureSchema = z.object({ name: z.string().trim().min(1), coverageIds: z.array(qaCoverageIdSchema).default([]), @@ -48,22 +190,49 @@ const qaMaturityFeatureSchema = z.object({ const qaMaturityCategorySchema = z.object({ id: qaScorecardIdSchema, name: z.string().trim().min(1), + category_note: z.string().trim().min(1), features: z.array(qaMaturityFeatureSchema).default([]), + docs: z.array(z.string().trim().min(1)).default([]), + search_anchors: z.array(z.string().trim().min(1)).default([]), + human_lts_override: z.boolean().optional(), }); const qaMaturitySurfaceSchema = z.object({ id: qaScorecardIdSchema, name: z.string().trim().min(1), - level: z.string().trim().min(1).optional(), + family: z.string().trim().min(1), + level: z.string().trim().min(1), level_code: z.string().trim().min(1).optional(), + archived: z.boolean().optional(), + rationale: z.string().trim().min(1).optional(), + completeness_instructions: z.string().trim().min(1).optional(), + last_score_run: qaMaturityScoreLastRunSchema.optional(), categories: z.array(qaMaturityCategorySchema).default([]), }); -const qaMaturityTaxonomySchema = z +const qaMaturityLevelSchema = z.object({ + id: z.string().trim().min(1), + code: z.string().trim().min(1).optional(), + label: z.string().trim().min(1).optional(), + meaning: z.string().trim().min(1).optional(), + promotion_bar: z.string().trim().min(1).optional(), +}); + +export const qaMaturityTaxonomySchema = z .object({ - version: z.number(), + version: z.literal(1), + process_version: z.number().int().positive().optional(), title: z.string().trim().min(1), + summary: z.string().trim().min(1).optional(), + snapshot: z + .object({ + date: z.string().trim().min(1).optional(), + source_ref: z.string().trim().min(1).optional(), + }) + .strict() + .optional(), profiles: z.array(qaScorecardProfileSchema).default([]), + levels: z.array(qaMaturityLevelSchema).default([]), surfaces: z.array(qaMaturitySurfaceSchema).default([]), }) .superRefine((taxonomy, ctx) => { @@ -112,14 +281,69 @@ const qaMaturityTaxonomySchema = z seenProfileCategoryIds.add(categoryId); } } + + const categoryIds = new Set(); + const surfaceIds = new Set(); + for (const [surfaceIndex, surface] of taxonomy.surfaces.entries()) { + if (surfaceIds.has(surface.id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["surfaces", surfaceIndex, "id"], + message: `duplicate surface id: ${surface.id}`, + }); + } + surfaceIds.add(surface.id); + + const localCategoryIds = new Set(); + for (const [categoryIndex, category] of surface.categories.entries()) { + if (localCategoryIds.has(category.id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["surfaces", surfaceIndex, "categories", categoryIndex, "id"], + message: `duplicate category id in surface ${surface.id}: ${category.id}`, + }); + } + localCategoryIds.add(category.id); + categoryIds.add(`${surface.id}.${category.id}`); + } + } + + for (const [profileIndex, profile] of taxonomy.profiles.entries()) { + for (const [categoryIndex, categoryId] of profile.categoryIds.entries()) { + if (!categoryIds.has(categoryId)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["profiles", profileIndex, "categoryIds", categoryIndex], + message: `profile ${profile.id} references missing category ${categoryId}`, + }); + } + } + } }); export type QaNativeCoverageEvidenceKind = "script" | "vitest" | "playwright"; export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario"; export type QaScorecardEvidenceMode = z.infer; export type QaScorecardChannelDriver = z.infer; +export type QaMaturityScoreKey = (typeof QA_MATURITY_SCORE_KEYS)[number]; +export type QaMaturityScoreObject = z.infer; +export type QaMaturityScoreBundle = z.infer; +export type QaMaturityScoreLastRun = z.infer; +export type QaMaturityScoreSurfaceLts = z.infer; +export type QaMaturityScoreCategory = z.infer; +export type QaMaturityScoreSurface = z.infer; +export type QaMaturityScores = z.infer; +export type QaMaturityTaxonomyLevel = z.infer; +export type QaMaturityTaxonomyFeature = z.infer; +export type QaMaturityTaxonomyCategory = z.infer; +export type QaMaturityTaxonomySurface = z.infer; +export type QaMaturityTaxonomyProfile = z.infer; +export type QaMaturityTaxonomy = z.infer; type QaCoverageEvidenceRole = z.infer; -type QaMaturityTaxonomy = z.infer; + +export type QaMaturityCoverageScores = { + categories: Map; +}; export type QaScorecardValidationIssueCode = | "coverage-id-missing-primary-evidence" @@ -191,6 +415,14 @@ export type QaScorecardTaxonomyReport = { categories: QaScorecardCategoryCoverageReport[]; }; +export type QaMaturityTaxonomyCategoryIndex = { + active: QaMaturityTaxonomySurface[]; + surfaces: Map< + string, + { surface: QaMaturityTaxonomySurface; categories: Map } + >; +}; + type MaturityCategoryRef = { id: string; surfaceId: string; @@ -222,7 +454,7 @@ function formatZodIssuePath(pathLocal: PropertyKey[]) { return pathLocal.length ? pathLocal.map(String).join(".") : ""; } -function parseQaMaturityTaxonomy(value: unknown, label = QA_MATURITY_TAXONOMY_PATH) { +export function parseQaMaturityTaxonomy(value: unknown, label = QA_MATURITY_TAXONOMY_PATH) { const parsed = qaMaturityTaxonomySchema.safeParse(value); if (parsed.success) { return parsed.data; @@ -233,6 +465,40 @@ function parseQaMaturityTaxonomy(value: unknown, label = QA_MATURITY_TAXONOMY_PA throw new Error(`${label}: ${issues}`); } +export function parseQaMaturityScores(value: unknown, label = QA_MATURITY_SCORES_PATH) { + const parsed = qaMaturityScoresSchema.safeParse(value); + if (parsed.success) { + return parsed.data; + } + const issues = parsed.error.issues + .map((issue) => `${formatZodIssuePath(issue.path)}: ${issue.message}`) + .join("; "); + throw new Error(`${label}: ${issues}`); +} + +export function readQaMaturityTaxonomySource(taxonomyPath = QA_MATURITY_TAXONOMY_PATH) { + return parseQaMaturityTaxonomy(YAML.parse(fs.readFileSync(taxonomyPath, "utf8")), taxonomyPath); +} + +export function readValidatedQaMaturityScoreSources(params?: { + coverageScores?: QaMaturityCoverageScores; + scoresPath?: string; + taxonomy?: QaMaturityTaxonomy; + taxonomyPath?: string; +}) { + const taxonomyPath = params?.taxonomyPath ?? QA_MATURITY_TAXONOMY_PATH; + const scoresPath = params?.scoresPath ?? QA_MATURITY_SCORES_PATH; + const taxonomy = params?.taxonomy ?? readQaMaturityTaxonomySource(taxonomyPath); + const scores = parseQaMaturityScores(YAML.parse(fs.readFileSync(scoresPath, "utf8")), scoresPath); + const warnings = validateQaMaturityScoresAgainstTaxonomy({ + coverageScores: params?.coverageScores, + scores, + taxonomy, + scoresPath, + }); + return { scores, taxonomy, warnings }; +} + function readQaMaturityTaxonomy(repoRoot: string | undefined) { const taxonomyPath = repoRoot ? path.join(repoRoot, QA_MATURITY_TAXONOMY_PATH) @@ -302,6 +568,233 @@ function percent(part: number, total: number) { return total === 0 ? 0 : Number(((part / total) * 100).toFixed(1)); } +export function activeQaMaturityTaxonomySurfaces(taxonomy: QaMaturityTaxonomy) { + return taxonomy.surfaces.filter((surface) => !surface.archived); +} + +export function buildQaMaturityTaxonomyCategoryIndex( + taxonomy: QaMaturityTaxonomy, +): QaMaturityTaxonomyCategoryIndex { + const active = activeQaMaturityTaxonomySurfaces(taxonomy); + const surfaces = new Map< + string, + { surface: QaMaturityTaxonomySurface; categories: Map } + >(); + for (const surface of active) { + const categories = new Map(); + for (const category of surface.categories) { + if (categories.has(category.name)) { + throw new Error(`taxonomy.yaml: ${surface.id}: duplicate category name ${category.name}`); + } + categories.set(category.name, category); + } + surfaces.set(surface.id, { surface, categories }); + } + return { active, surfaces }; +} + +export function qaMaturityTaxonomyLevelMap(taxonomy: QaMaturityTaxonomy) { + return new Map(taxonomy.levels.map((level) => [level.id, level])); +} + +export function qaMaturityCategoryProfiles(taxonomy: QaMaturityTaxonomy): Map { + const profilesByCategory = new Map(); + for (const profile of taxonomy.profiles) { + const categoryIds = profile.includeAllCategories + ? activeQaMaturityTaxonomySurfaces(taxonomy).flatMap((surface) => + surface.categories.map((category) => `${surface.id}.${category.id}`), + ) + : profile.categoryIds; + for (const categoryId of categoryIds) { + const profiles = profilesByCategory.get(categoryId) ?? []; + profiles.push(profile.id); + profilesByCategory.set(categoryId, profiles); + } + } + return profilesByCategory; +} + +export function qaMaturityFamilyOrder(surfaces: readonly QaMaturityTaxonomySurface[]): string[] { + const seen: string[] = []; + for (const surface of surfaces) { + if (!seen.includes(surface.family)) { + seen.push(surface.family); + } + } + return seen; +} + +function averageSurfaceScore(rows: readonly QaMaturityScoreSurface[], key: QaMaturityScoreKey) { + return Math.round(rows.reduce((sum, row) => sum + row.scores[key].score, 0) / rows.length); +} + +function averageCategoryScore(rows: readonly QaMaturityScoreCategory[], key: QaMaturityScoreKey) { + return Math.round(rows.reduce((sum, row) => sum + row[key].score, 0) / rows.length); +} + +export function qaMaturityCoverageCategoryKey(surfaceId: string, categoryName: string) { + return `${surfaceId}\u0000${categoryName}`; +} + +function expectedMaturityLtsSupported(params: { + coverage?: QaMaturityScoreObject; + scoreCategory: QaMaturityScoreCategory; + taxonomyCategory: QaMaturityTaxonomyCategory; +}) { + return ( + (params.scoreCategory.quality.score > 80 && (params.coverage?.score ?? -1) > 90) || + params.taxonomyCategory.human_lts_override === true + ); +} + +function expectedMaturitySurfaceLtsStatus(supportedCategories: number, totalCategories: number) { + if (supportedCategories === 0) { + return "none"; + } + return supportedCategories === totalCategories ? "full" : "partial"; +} + +export function validateQaMaturityScoresAgainstTaxonomy(params: { + coverageScores?: QaMaturityCoverageScores; + scores: QaMaturityScores; + taxonomy: QaMaturityTaxonomy; + scoresPath?: string; +}) { + const scoresPath = params.scoresPath ?? QA_MATURITY_SCORES_PATH; + const warnings: string[] = []; + const scoreSurfaces = params.scores.surfaces; + const taxonomyIndex = buildQaMaturityTaxonomyCategoryIndex(params.taxonomy); + if (params.scores.counts.active_surfaces !== scoreSurfaces.length) { + throw new Error( + `${scoresPath}.counts.active_surfaces must match score surface count (${scoreSurfaces.length})`, + ); + } + if (params.scores.counts.active_surfaces !== taxonomyIndex.active.length) { + throw new Error( + `${scoresPath}.counts.active_surfaces must match active taxonomy surfaces (${taxonomyIndex.active.length})`, + ); + } + + const taxonomyCategoryCount = taxonomyIndex.active.reduce( + (count, surface) => count + surface.categories.length, + 0, + ); + if (params.scores.counts.category_scores !== taxonomyCategoryCount) { + throw new Error( + `${scoresPath}.counts.category_scores must match active taxonomy categories (${taxonomyCategoryCount})`, + ); + } + + const seenSurfaceIds = new Set(); + const allScoreCategories: QaMaturityScoreCategory[] = []; + for (const scoreSurface of scoreSurfaces) { + const surfaceId = scoreSurface.id; + if (seenSurfaceIds.has(surfaceId)) { + throw new Error(`${scoresPath}: duplicate surface id ${surfaceId}`); + } + seenSurfaceIds.add(surfaceId); + + const taxonomySurface = taxonomyIndex.surfaces.get(surfaceId); + if (!taxonomySurface) { + throw new Error(`${scoresPath}: surface ${surfaceId} is not an active taxonomy surface`); + } + const categories = scoreSurface.categories; + if (taxonomySurface && categories.length !== taxonomySurface.categories.size) { + throw new Error( + `${scoresPath}.${surfaceId}.categories must match taxonomy category count (${taxonomySurface.categories.size})`, + ); + } + + const seenCategoryNames = new Set(); + let supportedCategories = 0; + for (const scoreCategory of categories) { + const categoryName = scoreCategory.name; + if (seenCategoryNames.has(categoryName)) { + throw new Error(`${scoresPath}.${surfaceId}: duplicate category name ${categoryName}`); + } + seenCategoryNames.add(categoryName); + const lts = scoreCategory.lts; + + const taxonomyCategory = taxonomySurface?.categories.get(categoryName); + if (taxonomySurface && !taxonomyCategory) { + throw new Error( + `${scoresPath}.${surfaceId}: score category ${categoryName} is not in taxonomy`, + ); + } + if (taxonomyCategory) { + if (lts.human_override !== Boolean(taxonomyCategory.human_lts_override)) { + throw new Error( + `${scoresPath}.${surfaceId}.${categoryName}.lts.human_override must match taxonomy human_lts_override`, + ); + } + const coverage = params.coverageScores?.categories.get( + qaMaturityCoverageCategoryKey(surfaceId, categoryName), + ); + if (coverage || taxonomyCategory.human_lts_override === true) { + const expectedSupported = expectedMaturityLtsSupported({ + coverage, + scoreCategory, + taxonomyCategory, + }); + if (lts.supported !== expectedSupported) { + throw new Error( + `${scoresPath}.${surfaceId}.${categoryName}.lts.supported must match quality, release evidence coverage, or taxonomy human_lts_override`, + ); + } + } + } + if (lts.supported) { + supportedCategories += 1; + } + allScoreCategories.push(scoreCategory); + } + + const surfaceLts = scoreSurface.lts; + if (surfaceLts.supported_categories !== supportedCategories) { + throw new Error( + `${scoresPath}.${surfaceId}.lts.supported_categories must equal supported category count (${supportedCategories})`, + ); + } + if (surfaceLts.total_categories !== categories.length) { + throw new Error( + `${scoresPath}.${surfaceId}.lts.total_categories must equal score category count (${categories.length})`, + ); + } + const expectedStatus = expectedMaturitySurfaceLtsStatus(supportedCategories, categories.length); + if (surfaceLts.status !== expectedStatus) { + throw new Error(`${scoresPath}.${surfaceId}.lts.status must be ${expectedStatus}`); + } + } + + for (const surfaceId of taxonomyIndex.surfaces.keys()) { + if (!seenSurfaceIds.has(surfaceId)) { + throw new Error(`${scoresPath}: missing active taxonomy surface ${surfaceId}`); + } + } + if (params.scores.counts.category_scores !== allScoreCategories.length) { + throw new Error( + `${scoresPath}.counts.category_scores must match score category count (${allScoreCategories.length})`, + ); + } + + const rollups = params.scores.rollups; + for (const key of QA_MATURITY_SCORE_KEYS) { + const expectedSurfaceAverage = averageSurfaceScore(scoreSurfaces, key); + if (rollups.surface_average[key].score !== expectedSurfaceAverage) { + throw new Error( + `${scoresPath}.rollups.surface_average.${key}.score must be ${expectedSurfaceAverage}`, + ); + } + const expectedCategoryAverage = averageCategoryScore(allScoreCategories, key); + if (rollups.category_average[key].score !== expectedCategoryAverage) { + throw new Error( + `${scoresPath}.rollups.category_average.${key}.score must be ${expectedCategoryAverage}`, + ); + } + } + return warnings; +} + function buildMaturityRefs(taxonomy: QaMaturityTaxonomy | null) { const categories = new Map(); const coverageIds = new Map(); diff --git a/package.json b/package.json index 73b1c2857778..142616bb2803 100644 --- a/package.json +++ b/package.json @@ -1591,6 +1591,8 @@ "docs:list": "node scripts/docs-list.js", "docs:spellcheck": "bash scripts/docs-spellcheck.sh", "docs:spellcheck:fix": "bash scripts/docs-spellcheck.sh --write", + "maturity:check": "node --import tsx scripts/qa/render-maturity-docs.ts --check", + "maturity:render": "node --import tsx scripts/qa/render-maturity-docs.ts", "dup:check": "node scripts/check-duplicates.mjs", "dup:check:coverage": "node scripts/check-duplicates.mjs --coverage", "dup:check:json": "node scripts/check-duplicates.mjs --json", diff --git a/scripts/qa/render-maturity-docs.ts b/scripts/qa/render-maturity-docs.ts new file mode 100644 index 000000000000..7a8fbd4ce879 --- /dev/null +++ b/scripts/qa/render-maturity-docs.ts @@ -0,0 +1,927 @@ +#!/usr/bin/env node +// Renders public maturity scorecard docs from the root taxonomy and score aggregate. +import fs from "node:fs"; +import path from "node:path"; +import { + validateQaEvidenceSummaryJson, + type QaEvidenceScorecardJson, + type QaEvidenceStatus, + type QaEvidenceSummaryJson, +} from "../../extensions/qa-lab/src/evidence-summary.js"; +import { + QA_MATURITY_SCORE_LABEL_BANDS, + activeQaMaturityTaxonomySurfaces, + qaMaturityFamilyOrder, + qaMaturityCoverageCategoryKey, + qaMaturityScoreObjectForScore, + qaMaturityTaxonomyLevelMap, + readQaMaturityTaxonomySource, + readValidatedQaMaturityScoreSources, + type QaMaturityCoverageScores, + type QaMaturityScoreObject, + type QaMaturityScoreSurface, + type QaMaturityScoreSurfaceLts, + type QaMaturityScores, + type QaMaturityTaxonomy, + type QaMaturityTaxonomyLevel, + type QaMaturityTaxonomySurface, +} from "../../extensions/qa-lab/src/scorecard-taxonomy.js"; + +const DEFAULT_TAXONOMY_PATH = "taxonomy.yaml"; +const DEFAULT_SCORES_PATH = "qa/maturity-scores.yaml"; +const DEFAULT_OUTPUT_DIR = "docs"; + +type Args = { + taxonomy: string; + scores: string; + docsRoot: string; + outputDir: string; + staticAssetsDir?: string; + evidenceDir?: string; + check: boolean; + strictInputs: boolean; +}; + +type EvidenceSummary = { + sourcePath: string; + path: string; + generatedAt: string; + profile: string; + entryCount: number; + statuses: StatusCounts; + scorecard?: QaEvidenceScorecardJson; +}; + +type StatusCounts = Record; + +const EMPTY_STATUS_COUNTS: StatusCounts = { + pass: 0, + fail: 0, + blocked: 0, + skipped: 0, +}; + +type RenderInputs = { + taxonomy: QaMaturityTaxonomy; + scores: QaMaturityScores; + coverage: DerivedCoverageScores; +}; + +type DocsRouteIndex = { + routes: Set; + redirects: Map; +}; + +type RenderMaturityScorecardInputs = Pick & { + evidenceSummaries: EvidenceSummary[]; +}; + +type DerivedCoverageScores = QaMaturityCoverageScores & { + surfaces: Map; + rollups: { + surface_average?: QaMaturityScoreObject; + category_average?: QaMaturityScoreObject; + }; + warnings: string[]; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + taxonomy: DEFAULT_TAXONOMY_PATH, + scores: DEFAULT_SCORES_PATH, + docsRoot: DEFAULT_OUTPUT_DIR, + outputDir: DEFAULT_OUTPUT_DIR, + staticAssetsDir: undefined, + evidenceDir: undefined, + check: false, + strictInputs: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + continue; + } + if (arg === "--check") { + args.check = true; + continue; + } + if (arg === "--strict-inputs") { + args.strictInputs = true; + continue; + } + const next = (): string => { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`${arg} requires a value`); + } + index += 1; + return value; + }; + if (arg === "--taxonomy") { + args.taxonomy = next(); + } else if (arg === "--scores") { + args.scores = next(); + } else if (arg === "--docs-root") { + args.docsRoot = next(); + } else if (arg === "--output-dir") { + args.outputDir = next(); + } else if (arg === "--static-assets-dir") { + args.staticAssetsDir = next(); + } else if (arg === "--evidence-dir") { + args.evidenceDir = next(); + } else if (arg === "--help" || arg === "-h") { + process.stdout.write(`Usage: node --import tsx scripts/qa/render-maturity-docs.ts [options] + +Options: + --taxonomy Taxonomy YAML path (default: taxonomy.yaml) + --scores Aggregate score YAML path (default: qa/maturity-scores.yaml) + --docs-root Public docs source root for route validation (default: docs) + --output-dir Directory for maturity/scorecard.md and maturity/taxonomy.md + --static-assets-dir + Copy source YAML and QA evidence JSON for docs components + --evidence-dir Optional directory containing qa-evidence.json artifacts + --check Fail when output files are stale + --strict-inputs Fail on score or evidence input warnings + -h, --help Show this help +`); + process.exit(0); + } else { + throw new Error(`Unknown maturity docs option: ${arg}`); + } + } + return args; +} + +function familyTitle(value: string): string { + const titles: Record = { + "platform-app": "Platform", + "provider-tool": "Provider and tool", + }; + return ( + titles[value] ?? + value + .replaceAll("-", " ") + .replaceAll("_", " ") + .replace(/\b\w/g, (char) => char.toUpperCase()) + ); +} + +type RenderScalar = string | number | boolean | null | undefined; + +function markdownEscape(value: RenderScalar): string { + return String(value ?? "").replaceAll("|", "\\|"); +} + +function yamlCode(value: RenderScalar): string { + return `\`${markdownEscape(value)}\``; +} + +function markdownSlug(value: string): string { + return value + .trim() + .toLowerCase() + .replaceAll("&", "and") + .replace(/[/:]/g, " ") + .replace(/[^a-z0-9\s-]/g, "") + .replace(/\s+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, ""); +} + +function normalizeRoutePath(route: string): string { + return route.replace(/^\/+/, "").replace(/\/+$/, ""); +} + +function collectDocsRouteIndex(docsRoot: string): DocsRouteIndex { + const routes = new Set(); + const redirects = new Map(); + if (!fs.existsSync(docsRoot)) { + return { routes, redirects }; + } + const visit = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === "internal" && path.relative(docsRoot, fullPath) === "internal") { + continue; + } + visit(fullPath); + } else if (entry.isFile() && /\.(md|mdx)$/i.test(entry.name)) { + routes.add( + path + .relative(docsRoot, fullPath) + .replaceAll(path.sep, "/") + .replace(/\.(md|mdx)$/i, ""), + ); + } + } + }; + visit(docsRoot); + + const docsJsonPath = path.join(docsRoot, "docs.json"); + if (fs.existsSync(docsJsonPath)) { + const docsJson = JSON.parse(fs.readFileSync(docsJsonPath, "utf8")) as { + redirects?: Array<{ source?: string; destination?: string }>; + }; + for (const redirect of docsJson.redirects ?? []) { + if (!redirect.source || !redirect.destination || redirect.destination.startsWith("http")) { + continue; + } + redirects.set(normalizeRoutePath(redirect.source), normalizeRoutePath(redirect.destination)); + } + } + return { routes, redirects }; +} + +function docsLink(docPath: string, docsRouteIndex: DocsRouteIndex): string | undefined { + const docsPrefix = "docs/"; + const trimmedPath = docPath.trim(); + const publicPath = trimmedPath.startsWith(docsPrefix) + ? trimmedPath.slice(docsPrefix.length) + : trimmedPath; + const [pagePath = "", anchor] = publicPath.split("#", 2); + const withoutExtension = pagePath.replace(/\.(md|mdx)$/i, ""); + const lastSegment = withoutExtension.split("/").at(-1) ?? withoutExtension; + const title = familyTitle(anchor ?? lastSegment); + const publicRoute = docsRouteIndex.routes.has(withoutExtension) + ? withoutExtension + : docsRouteIndex.redirects.get(withoutExtension); + if (!publicRoute || !docsRouteIndex.routes.has(publicRoute)) { + return undefined; + } + const publicHref = anchor ? `${publicRoute}#${anchor}` : publicRoute; + return `[${markdownEscape(title)}](/${markdownEscape(publicHref)})`; +} + +function markdownTable(rows: RenderScalar[][]): string[] { + if (rows.length === 0) { + return []; + } + const columnCount = Math.max(...rows.map((row) => row.length)); + const normalizedRows = rows.map((row) => + Array.from({ length: columnCount }, (_, index) => String(row[index] ?? "")), + ); + const widths = Array.from({ length: columnCount }, (_, index) => + Math.max(3, ...normalizedRows.map((row) => row[index]?.length ?? 0)), + ); + const formatRow = (row: string[]) => + `| ${row.map((cell, index) => cell.padEnd(widths[index] ?? 3)).join(" | ")} |`; + return [ + formatRow(normalizedRows[0] ?? []), + formatRow(widths.map((width) => "-".repeat(width))), + ...normalizedRows.slice(1).map(formatRow), + ]; +} + +function scoreText(value?: QaMaturityScoreObject): string { + if (!value || typeof value !== "object") { + return "`Unscored`"; + } + return `\`${markdownEscape(value.label ?? "")} (${markdownEscape(value.score ?? "")}%)\``; +} + +function levelText( + surface: QaMaturityScoreSurface | QaMaturityTaxonomySurface, + taxonomyLevels: Map, +): string { + const scoreLevel = surface.level; + if (scoreLevel && typeof scoreLevel === "object") { + return [scoreLevel.code, scoreLevel.label].filter(Boolean).join(" "); + } + const levelId = typeof scoreLevel === "string" ? scoreLevel : ""; + const level = taxonomyLevels.get(levelId); + return [level?.code, level?.label ?? levelId].filter(Boolean).join(" "); +} + +function ltsText(lts?: QaMaturityScoreSurfaceLts): string { + if (!lts || typeof lts !== "object") { + return "unscored"; + } + const supportedCategories = lts.supported_categories ?? 0; + if (lts.status === "full") { + return `full (${supportedCategories})`; + } + if (lts.status === "partial") { + return `partial (${supportedCategories})`; + } + if (lts.status === "none") { + return "none"; + } + return lts.status ?? "unknown"; +} + +function renderScoreBands(): string[] { + return [ + "## Score bands", + "", + ...markdownTable([ + ["Label", "Score range"], + ...QA_MATURITY_SCORE_LABEL_BANDS.map(([label, low, high]) => [label, `${low}-${high}%`]), + ]), + "", + ]; +} + +function latestScoreRunDate(scores: QaMaturityScores): string | undefined { + const dates = scores.surfaces + .map((surface) => surface.last_score_run?.completed_at) + .filter((date): date is string => Boolean(date)) + .toSorted((left, right) => left.localeCompare(right)); + return dates.at(-1); +} + +function frontmatter(title: string, summary: string): string[] { + return ["---", `title: "${title}"`, `summary: "${summary}"`, "---", ""]; +} + +function surfaceScoreMap(scores: QaMaturityScores): Map { + return new Map(scores.surfaces.map((surface) => [surface.id, surface])); +} + +function categoryScoreMap( + scoreSurface?: QaMaturityScoreSurface, +): Map { + return new Map((scoreSurface?.categories ?? []).map((category) => [category.name, category])); +} + +function collectQaEvidenceFiles(root?: string): string[] { + if (!root || !fs.existsSync(root)) { + return []; + } + const files: string[] = []; + const visit = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + } else if (entry.isFile() && entry.name === "qa-evidence.json") { + files.push(fullPath); + } + } + }; + visit(root); + return files.toSorted((left, right) => left.localeCompare(right)); +} + +function countStatuses(entries: QaEvidenceSummaryJson["entries"]): StatusCounts { + const counts: StatusCounts = { ...EMPTY_STATUS_COUNTS }; + for (const entry of entries) { + counts[entry.result.status] += 1; + } + return counts; +} + +function numberText(value: unknown): string { + return Number.isFinite(value) ? String(value) : ""; +} + +function countText(counts?: QaEvidenceScorecardJson["categories"]): string { + if (!counts || typeof counts !== "object") { + return ""; + } + return `${counts.fulfilled ?? 0} of ${counts.total ?? 0} (${numberText(counts.fulfillmentPercent)}%)`; +} + +function averageScores( + scores: readonly QaMaturityScoreObject[], +): QaMaturityScoreObject | undefined { + if (scores.length === 0) { + return undefined; + } + const average = Math.round(scores.reduce((sum, score) => sum + score.score, 0) / scores.length); + return qaMaturityScoreObjectForScore(average); +} + +function checkSetTitle(profile: string): string { + const normalized = profile.trim(); + if (!normalized || normalized === "release") { + return "Release validation"; + } + return familyTitle(normalized); +} + +function resultCountsText(statuses: StatusCounts): string { + return [ + `${statuses.pass} passed`, + `${statuses.fail} failed`, + `${statuses.blocked} blocked`, + `${statuses.skipped} skipped`, + ].join(", "); +} + +function readinessStatusText(status: string): string { + if (status === "fulfilled") { + return "Ready"; + } + if (status === "partial") { + return "Partially reviewed"; + } + if (status === "missing") { + return "Needs review"; + } + return status; +} + +function followUpText(missingCoverageIds: readonly string[]): string { + if (missingCoverageIds.length === 0) { + return "None"; + } + return `${missingCoverageIds.length} capability ${missingCoverageIds.length === 1 ? "gap" : "gaps"}`; +} + +function readEvidenceSummaries(evidenceDir?: string): EvidenceSummary[] { + return collectQaEvidenceFiles(evidenceDir).map((filePath) => { + const payload = validateQaEvidenceSummaryJson(JSON.parse(fs.readFileSync(filePath, "utf8"))); + return { + sourcePath: filePath, + path: path.relative(process.cwd(), filePath), + generatedAt: payload.generatedAt, + profile: payload.profile ?? "", + entryCount: payload.entries.length, + statuses: countStatuses(payload.entries), + scorecard: payload.scorecard, + }; + }); +} + +function latestReleaseScorecard(evidenceSummaries: EvidenceSummary[]): EvidenceSummary | undefined { + return evidenceSummaries + .filter((item) => item.profile === "release" && item.scorecard) + .toSorted((left, right) => left.generatedAt.localeCompare(right.generatedAt)) + .at(-1); +} + +function deriveCoverageScores( + taxonomy: QaMaturityTaxonomy, + evidenceSummaries: EvidenceSummary[], +): DerivedCoverageScores { + const warnings: string[] = []; + const releaseSummary = latestReleaseScorecard(evidenceSummaries); + const releaseScorecardSummaries = evidenceSummaries.filter( + (item) => item.profile === "release" && item.scorecard, + ); + if (!releaseSummary) { + throw new Error( + "maturity scorecard rendering requires release profile qa-evidence.json with a scorecard field; pass --evidence-dir with release QA evidence artifacts", + ); + } + if (releaseScorecardSummaries.length > 1) { + warnings.push( + `multiple release profile evidence scorecards found; using latest from ${releaseSummary.path}`, + ); + } + + const categories = new Map(); + for (const report of releaseSummary.scorecard?.categoryReports ?? []) { + categories.set( + qaMaturityCoverageCategoryKey(report.surfaceId, report.name), + qaMaturityScoreObjectForScore(Math.round(report.features.fulfillmentPercent)), + ); + } + + const surfaces = new Map(); + const missingCoverage: string[] = []; + for (const surface of activeQaMaturityTaxonomySurfaces(taxonomy)) { + const categoryScores = surface.categories + .map((category) => { + const key = qaMaturityCoverageCategoryKey(surface.id, category.name); + const score = categories.get(key); + if (!score) { + missingCoverage.push( + `${releaseSummary.path}: release evidence is missing scorecard coverage for ${surface.name} / ${category.name}`, + ); + } + return score; + }) + .filter((score): score is QaMaturityScoreObject => Boolean(score)); + if (categoryScores.length === surface.categories.length) { + const surfaceScore = averageScores(categoryScores); + if (surfaceScore) { + surfaces.set(surface.id, surfaceScore); + } + } + } + if (missingCoverage.length > 0) { + throw new Error( + `maturity scorecard rendering requires complete release evidence coverage:\n${missingCoverage + .map((item) => `- ${item}`) + .join("\n")}`, + ); + } + + const activeSurfaces = activeQaMaturityTaxonomySurfaces(taxonomy); + const expectedCategoryCount = activeSurfaces.reduce( + (count, surface) => count + surface.categories.length, + 0, + ); + const categoryScores = Array.from(categories.values()); + const surfaceScores = Array.from(surfaces.values()); + return { + categories, + surfaces, + rollups: { + category_average: + categoryScores.length === expectedCategoryCount ? averageScores(categoryScores) : undefined, + surface_average: + surfaceScores.length === activeSurfaces.length ? averageScores(surfaceScores) : undefined, + }, + warnings, + }; +} + +function evidenceScorecardWarnings( + evidenceSummaries: EvidenceSummary[], + coverage: DerivedCoverageScores, +): string[] { + return [ + ...evidenceSummaries + .filter((item) => item.profile === "release" && !item.scorecard) + .map( + (item) => + `${item.path}: release profile qa-evidence.json does not include a scorecard field; run pnpm openclaw qa run --qa-profile release to produce deterministic scorecard rows`, + ), + ...coverage.warnings, + ]; +} + +function writeInputWarnings(warnings: string[]): void { + for (const warning of warnings) { + process.stderr.write(`warning: ${warning}\n`); + } +} + +function enforceStrictInputs(warnings: string[]): void { + if (warnings.length === 0) { + return; + } + throw new Error( + `strict input validation failed:\n${warnings.map((warning) => `- ${warning}`).join("\n")}`, + ); +} + +function copyStaticSourceAssets({ + evidenceSummaries, + scoresPath, + staticAssetsDir, + taxonomyPath, +}: { + evidenceSummaries: EvidenceSummary[]; + scoresPath: string; + staticAssetsDir: string; + taxonomyPath: string; +}): string[] { + fs.mkdirSync(staticAssetsDir, { recursive: true }); + const copied = [ + [taxonomyPath, path.join(staticAssetsDir, "taxonomy.yaml")], + [scoresPath, path.join(staticAssetsDir, "maturity-scores.yaml")], + ]; + const evidenceDir = path.join(staticAssetsDir, "evidence"); + fs.rmSync(evidenceDir, { recursive: true, force: true }); + if (evidenceSummaries.length > 0) { + fs.mkdirSync(evidenceDir, { recursive: true }); + } + for (const [index, evidence] of evidenceSummaries.entries()) { + copied.push([ + evidence.sourcePath, + path.join(evidenceDir, `qa-evidence-${String(index + 1).padStart(2, "0")}.json`), + ]); + } + for (const [source, target] of copied) { + fs.copyFileSync(source, target); + } + return copied.map(([, target]) => target); +} + +function surfaceNameMap(surfaces: QaMaturityTaxonomySurface[]): Map { + return new Map(surfaces.map((surface) => [surface.id, surface.name])); +} + +function renderEvidenceSection( + evidenceSummaries: EvidenceSummary[], + surfaceNames: Map, +): string[] { + const scorecardSummaries = evidenceSummaries.filter((item) => item.scorecard); + if (scorecardSummaries.length === 0) { + return []; + } + const lines = [ + "## Release check summary", + "", + "The checks below show which scorecard areas were exercised during release validation.", + "", + ]; + + const summaryRows: RenderScalar[][] = [ + ["Check set", "Completed", "Checks run", "Results", "Areas reviewed", "Capabilities reviewed"], + ]; + for (const item of scorecardSummaries) { + const scorecard = item.scorecard; + summaryRows.push([ + markdownEscape(checkSetTitle(item.profile)), + markdownEscape(item.generatedAt), + item.entryCount, + markdownEscape(resultCountsText(item.statuses)), + markdownEscape(countText(scorecard?.categories)), + markdownEscape(countText(scorecard?.features)), + ]); + } + lines.push(...markdownTable(summaryRows), ""); + + const categoryRows = scorecardSummaries.flatMap((item) => + (item.scorecard?.categoryReports ?? []).map((category) => ({ item, category })), + ); + if (categoryRows.length > 0) { + const readinessRows: RenderScalar[][] = [ + ["Check set", "Surface", "Area", "Status", "Capabilities reviewed", "Follow-up"], + ]; + for (const { item, category } of categoryRows) { + const features = countText(category.features); + readinessRows.push([ + markdownEscape(checkSetTitle(item.profile)), + markdownEscape(surfaceNames.get(category.surfaceId) ?? familyTitle(category.surfaceId)), + markdownEscape(category.name), + markdownEscape(readinessStatusText(category.status)), + markdownEscape(features), + markdownEscape(followUpText(category.missingCoverageIds)), + ]); + } + lines.push("### Readiness by area", "", ...markdownTable(readinessRows), ""); + } + return lines; +} + +function renderMaturityScorecard({ + coverage, + taxonomy, + scores, + evidenceSummaries, +}: RenderMaturityScorecardInputs): string { + const levels = qaMaturityTaxonomyLevelMap(taxonomy); + const scoreSurfaces = surfaceScoreMap(scores); + const surfaces = activeQaMaturityTaxonomySurfaces(taxonomy); + const surfaceNames = surfaceNameMap(surfaces); + const updatedDate = latestScoreRunDate(scores); + const lines = [ + ...frontmatter( + "Maturity scorecard", + "OpenClaw release readiness scores for product areas, integrations, and supported workflows.", + ), + "# Maturity scorecard", + "", + "These scores summarize release readiness across OpenClaw product areas, integrations, and supported workflows.", + "", + `The current scorecard covers ${scores.counts.active_surfaces} surfaces and ${scores.counts.category_scores} capability areas.`, + "", + "## Overall scores", + "", + ...markdownTable([ + ["Basis", "Coverage", "Quality", "Completeness"], + [ + "Surface average", + scoreText(coverage.rollups.surface_average), + scoreText(scores.rollups.surface_average.quality), + scoreText(scores.rollups.surface_average.completeness), + ], + [ + "Category average", + scoreText(coverage.rollups.category_average), + scoreText(scores.rollups.category_average.quality), + scoreText(scores.rollups.category_average.completeness), + ], + ]), + "", + "- Coverage is derived from release validation results.", + "- Quality measures reliability and operational confidence.", + "- Completeness measures how much of the expected user workflow is available.", + "", + ...renderScoreBands(), + ]; + + const surfaceRows: RenderScalar[][] = [ + [ + "Surface", + "Family", + "Level", + "Coverage", + "Quality", + "Completeness", + "Long-term support", + "Areas", + ], + ]; + for (const surface of surfaces) { + const scoreSurface = scoreSurfaces.get(surface.id); + const surfaceName = surface.name; + surfaceRows.push([ + `[${markdownEscape(surfaceName)}](/maturity/taxonomy#${markdownSlug(surfaceName)})`, + markdownEscape(familyTitle(surface.family)), + markdownEscape(levelText(surface, levels)), + scoreText(coverage.surfaces.get(surface.id)), + scoreText(scoreSurface?.scores?.quality), + scoreText(scoreSurface?.scores?.completeness), + markdownEscape(ltsText(scoreSurface?.lts)), + surface.categories.length, + ]); + } + lines.push( + "## Surface scorecard", + "", + ...markdownTable(surfaceRows), + "", + ...renderEvidenceSection(evidenceSummaries, surfaceNames), + ); + if (updatedDate) { + lines.push(`> Last updated: ${updatedDate}`, ""); + } + return `${lines.join("\n").trimEnd()}\n`; +} + +function renderTaxonomy({ + coverage, + docsRouteIndex, + scores, + taxonomy, +}: RenderInputs & { docsRouteIndex: DocsRouteIndex }): string { + const levels = qaMaturityTaxonomyLevelMap(taxonomy); + const scoreSurfaces = surfaceScoreMap(scores); + const surfaces = activeQaMaturityTaxonomySurfaces(taxonomy); + const lines = [ + ...frontmatter( + "Maturity taxonomy", + "Detailed reference for the product areas and checks behind the OpenClaw maturity scorecard.", + ), + "# Maturity taxonomy", + "", + "This page explains the product areas and capability groups behind the maturity scorecard.", + "", + "## Maturity levels", + "", + ...markdownTable([ + ["Level", "Label", "Meaning", "Promotion bar"], + ...taxonomy.levels.map((level) => [ + yamlCode(level.code ?? level.id), + markdownEscape(level.label ?? level.id), + markdownEscape(level.meaning ?? ""), + markdownEscape(level.promotion_bar ?? ""), + ]), + ]), + "", + "## Product areas", + "", + ]; + + for (const family of qaMaturityFamilyOrder(surfaces)) { + lines.push(`### ${familyTitle(family)}`, ""); + for (const surface of surfaces.filter((candidate) => candidate.family === family)) { + const surfaceName = surface.name; + lines.push(`- [${markdownEscape(surfaceName)}](#${markdownSlug(surfaceName)})`); + } + lines.push(""); + } + + lines.push("## Details", ""); + for (const family of qaMaturityFamilyOrder(surfaces)) { + lines.push(`### ${familyTitle(family)}`, ""); + for (const surface of surfaces.filter((candidate) => candidate.family === family)) { + const surfaceName = surface.name; + const scoreSurface = scoreSurfaces.get(surface.id); + const categoryScores = categoryScoreMap(scoreSurface); + const categoryRows: RenderScalar[][] = [ + [ + "Area", + "Capabilities", + "Docs", + "Coverage", + "Quality", + "Completeness", + "Long-term support", + ], + ]; + for (const category of surface.categories) { + const docs = (category.docs ?? []) + .map((doc) => docsLink(doc, docsRouteIndex)) + .filter((doc): doc is string => Boolean(doc)) + .join(", "); + const scoreCategory = categoryScores.get(category.name); + const coverageScore = coverage.categories.get( + qaMaturityCoverageCategoryKey(surface.id, category.name), + ); + categoryRows.push([ + markdownEscape(category.name), + category.features.length, + docs, + scoreText(coverageScore), + scoreText(scoreCategory?.quality), + scoreText(scoreCategory?.completeness), + markdownEscape(scoreCategory?.lts?.supported ? "Yes" : "No"), + ]); + } + lines.push( + `#### ${surfaceName}`, + "", + `- Level: ${markdownEscape(levelText(surface, levels))}`, + `- Rationale: ${surface.rationale ?? ""}`, + "", + ...markdownTable(categoryRows), + ); + lines.push(""); + } + } + return `${lines.join("\n").trimEnd()}\n`; +} + +function writeOrCheck(outputPath: string, content: string, check: boolean): boolean { + const oldContent = fs.existsSync(outputPath) ? fs.readFileSync(outputPath, "utf8") : ""; + if (check) { + if (oldContent !== content) { + throw new Error(`${outputPath} is stale; run pnpm maturity:render`); + } + return false; + } + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + if (oldContent !== content) { + fs.writeFileSync(outputPath, content); + return true; + } + return false; +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const taxonomyPath = path.normalize(args.taxonomy); + const scoresPath = path.normalize(args.scores); + const docsRoot = path.normalize(args.docsRoot); + const outputDir = path.normalize(args.outputDir); + const evidenceSummaries = readEvidenceSummaries(args.evidenceDir); + const taxonomy = readQaMaturityTaxonomySource(taxonomyPath); + const coverage = deriveCoverageScores(taxonomy, evidenceSummaries); + const { scores, warnings: scoreWarnings } = readValidatedQaMaturityScoreSources({ + coverageScores: coverage, + scoresPath, + taxonomy, + taxonomyPath, + }); + const evidenceWarnings = evidenceScorecardWarnings(evidenceSummaries, coverage); + const inputWarnings = [...scoreWarnings, ...evidenceWarnings]; + writeInputWarnings(inputWarnings); + if (args.strictInputs) { + enforceStrictInputs(inputWarnings); + } + const copiedStaticAssets = + !args.check && args.staticAssetsDir + ? copyStaticSourceAssets({ + evidenceSummaries, + scoresPath, + staticAssetsDir: args.staticAssetsDir, + taxonomyPath, + }) + : []; + const outputs = new Map([ + [ + "maturity/scorecard.md", + renderMaturityScorecard({ + coverage, + taxonomy, + scores, + evidenceSummaries, + }), + ], + [ + "maturity/taxonomy.md", + renderTaxonomy({ + coverage, + docsRouteIndex: collectDocsRouteIndex(docsRoot), + taxonomy, + scores, + }), + ], + ]); + const changed: string[] = []; + for (const [fileName, content] of outputs) { + const outputPath = path.join(outputDir, fileName); + if (writeOrCheck(outputPath, content, args.check)) { + changed.push(outputPath); + } + } + if (args.check) { + process.stdout.write(`maturity docs are up to date in ${outputDir}\n`); + } else if (changed.length > 0) { + process.stdout.write( + `rendered maturity docs:\n${changed.map((file) => `- ${file}`).join("\n")}\n`, + ); + } else { + process.stdout.write(`maturity docs already up to date in ${outputDir}\n`); + } + if (copiedStaticAssets.length > 0) { + process.stdout.write( + `copied maturity static assets:\n${copiedStaticAssets.map((file) => `- ${file}`).join("\n")}\n`, + ); + } +} + +try { + main(); +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +} diff --git a/taxonomy.yaml b/taxonomy.yaml index 841bd1b75249..e762bea7e093 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -6112,7 +6112,7 @@ surfaces: source_ref: null process_version: 3 - id: raspberry-pi-small-linux-devices - name: Raspberry Pi / small Linux devices + name: Raspberry Pi and small Linux devices family: platform-app level: beta level_code: M3 @@ -6335,7 +6335,7 @@ surfaces: source_ref: null process_version: 3 - id: docker-podman-hosting - name: Docker / Podman hosting + name: Docker and Podman hosting family: platform-app level: beta level_code: M3 @@ -7534,7 +7534,7 @@ surfaces: source_ref: null process_version: 3 - id: imessage-bluebubbles - name: iMessage / BlueBubbles + name: iMessage and BlueBubbles family: channel level: beta level_code: M3 @@ -8768,7 +8768,7 @@ surfaces: source_ref: null process_version: 3 - id: openai-codex-provider-path - name: OpenAI / Codex provider path + name: OpenAI and Codex provider path family: provider-tool level: beta level_code: M3 @@ -10249,7 +10249,7 @@ surfaces: source_ref: null process_version: 3 - id: browser-automation-and-exec-sandbox-tools - name: Browser automation and exec/sandbox tools + name: Browser automation, exec, and sandbox tools family: provider-tool level: beta level_code: M3 @@ -10400,7 +10400,7 @@ surfaces: source_ref: null process_version: 3 - id: image-video-music-generation-tools - name: Image/video/music generation tools + name: Image, video, and music generation tools family: provider-tool level: alpha level_code: M2