dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-08 02:52:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(qa): accept Sonnet max thinking support

Browse Source
This commit is contained in:
Vincent Koc
2026-06-23 01:41:59 +02:00
parent 5e342c774d
commit befe04f465
2 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
extensions/qa-lab/src/scenario-catalog.test.ts
View File

@@ -557,6 +557,9 @@ describe("qa scenario catalog", () => {
expect(config?.anthropicModelRef).toBe("anthropic/claude-sonnet-4-6");
expect(config?.openAiXhighModelRef).toBe("openai/gpt-5.5");
expect(config?.noXhighModelRef).toBe("anthropic/claude-sonnet-4-6");
const flowText = JSON.stringify(scenario.execution.flow);
expect(flowText).toContain("include max and omit xhigh");
expect(flowText).not.toContain("omit xhigh/max");
expect(scenario.execution.flow?.steps.map((step) => step.name)).toEqual([
"selects Anthropic and verifies adaptive options",
"maps adaptive to medium when switching to OpenAI",

12
qa/scenarios/models/thinking-slash-model-remap.yaml
View File

@@ -19,7 +19,7 @@ scenario:
anthropic/claude-sonnet-4-6:
params: {}
successCriteria:
- Anthropic Claude Sonnet 4.6 advertises adaptive but not OpenAI-only xhigh or Opus max.
- Anthropic Claude Sonnet 4.6 advertises adaptive and native max, but not OpenAI-only xhigh.
- A stored adaptive level remaps to medium when switching to OpenAI GPT-5.5.
- OpenAI GPT-5.5 advertises xhigh but not adaptive or max.
- A stored xhigh level remaps to high when switching to an Anthropic model without xhigh support.
@@ -36,7 +36,7 @@ scenario:
- extensions/openai/openai-provider.ts
execution:
kind: flow
summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a non-xhigh model.
summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a model without xhigh support.
config:
requiredProviderMode: live-frontier
requiredProvider: openai
@@ -90,9 +90,9 @@ flow:
message:
expr: "`expected Anthropic /think options to include adaptive, got ${anthropicThinkStatus.text}`"
- assert:
expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)"
expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && /Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)"
message:
expr: "`expected Sonnet /think options to omit xhigh/max, got ${anthropicThinkStatus.text}`"
expr: "`expected Sonnet /think options to include max and omit xhigh, got ${anthropicThinkStatus.text}`"
detailsExpr: "`model=${JSON.stringify(anthropicModelAck.resolved)}; think=${anthropicThinkStatus.text}`"
- name: maps adaptive to medium when switching to OpenAI
actions:
@@ -190,7 +190,7 @@ flow:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: high/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text)"
expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && /Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text)"
message:
expr: "`expected non-xhigh model /think options to include adaptive and omit xhigh/max, got ${noXhighThinkStatus.text}`"
expr: "`expected non-xhigh model /think options to include adaptive/max and omit xhigh, got ${noXhighThinkStatus.text}`"
detailsExpr: "`xhigh=${xhighAck.text}; switch=${JSON.stringify(noXhighModelAck.resolved)}; think=${noXhighThinkStatus.text}`"