mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-13 17:07:40 +00:00
fix(media): record per-attachment media dispositions and render bounded skip markers (#122098)
Inbound image, audio, and video attachments could vanish with no recorded outcome and no model-visible explanation: attachments dropped by first-only selection, disabled or unconfigured capabilities, stage failures, URL-only images, and ACP-undelivered images all left the model unaware the media existed. Fixes #122044; completes the outcome custody started for documents in #122025. Media capabilities now record one closed disposition per pre-truncation candidate at the decision site that owns the reason, with a memoized lazy native-vision probe (fires only when a marker could render; failure yields unknown and never alters outcomes; explicit image models never pay a catalog lookup). One late reader renders bounded, actionable markers from recorded facts only: native-vision and ACP-delivered images suppress per verified index, recorded failures always render, and document plus media markers share one five-marker budget with a reason-neutral overflow summary. The shipped SDK decision type stays additive (optional dispositions map). Proof: 906 + 103 focused tests including nine adversarial-review regressions, ClawSweeper local review clean (round 9, zero findings, security cleared), live Telegram image drive showing ref preservation with no false marker. Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
Ayaan Zaidi
@@ -1 +1 @@
|
||||
{"contentHash":"016fc07ce752591ba33863e49816aede24a0e90d240205dd87c882b64c611c19","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
{"contentHash":"6b0a039cf6246df1ec52695441900203343ddc3862ef0c9c100f08cab004bfe7","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"065fb0dd1605fb70dd75335c193f378afa8230538ce051197068e38fd24ec56d","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
{"contentHash":"817cf688ac6ab0bcde36b59ba9d20e6511fdc82f58f727557cd33097c49b1940","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"96db2cbf307a1fb22e9374a85a41da49ac538b0affe7ac40d9fcc99bcd646277","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"}
|
||||
{"contentHash":"f38505150fe39946aa11b2963190d3c4838783086be6c76bbca9194447afaa63","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"75935c1d837265d191436d43bbfe24c6328b35a45a68cad250c157923badf857","entrypoint":"channel-contract","importSpecifier":"openclaw/plugin-sdk/channel-contract"}
|
||||
{"contentHash":"d5872fb0a904acd1d4ed830b2597fe578daebaa855f1949bea76db923c35c373","entrypoint":"channel-contract","importSpecifier":"openclaw/plugin-sdk/channel-contract"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"59fe18c5c65144cadbfc61d22526e1475f7afb19f56303d0a007a688242e3629","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
{"contentHash":"3d6ce8758b33237aad8417ec2731c2677cabc092f3b71671b4df28663f21c2f7","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"96431e828c380418be9f996dcf41cdebd4c8a7c6052476dbbce519ef21dd2c9c","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
{"contentHash":"c7f13d09cb45ae35ce5855ed7576905ca2ab4cca11b589647e8bbf84ce8d8ded","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"99cee44d09759fdbb01bc1b50e430933c61c745ec3c7880ea7e2aa63b382c673","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
{"contentHash":"d4b4c9c5f145aa60c35f928f090f74959c6c3f1745fe164bb8146b6d341e8d88","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"13304005d532baf3d6edccf81a594e8a874ce4ebc80174b52562a3cbf2e7a12c","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
{"contentHash":"d2e3782705a71a08616f0c59ac1be8d8e03b1ec004c604bcc80a17567613dd71","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"a44717708258e2b8639874737bbc01a5ca55b0237229bf2921d7806767482156","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
{"contentHash":"7a6122d34d21e433c7a1a0996879c0964b3a22f508a793a53ffe6404ddf4b5ef","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"fee97710da33e100a8987aed897be17ab3ec4ece9fa09b7eb871a90461c8fca3","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"}
|
||||
{"contentHash":"16fa9e31deb0b1f53caedf3ec5ab54f4c1f0ab62c2e6816cec9f2abf31782b65","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"e974ac587d75b3a499215151dc2874d469168268a18cc024bc5048ecadf2de3b","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
{"contentHash":"f4a7174aefe70f8bf04bfb293ca92ab9f0f99b6073a8149eee45ce902b99b5ee","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"257ad1c723f513e2346193c87c6f886fae528f048cf2d3ef1bb1dd41b74ea1bb","entrypoint":"command-auth-native","importSpecifier":"openclaw/plugin-sdk/command-auth-native"}
|
||||
{"contentHash":"e17ce7a27cdf887381927cc2e39300105c26f72fccd34da5fead687237f022be","entrypoint":"command-auth-native","importSpecifier":"openclaw/plugin-sdk/command-auth-native"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"da6ecbd54c2a221465aa717896006354000ca530d89fa772aacd60f88b7c5d6d","entrypoint":"command-auth","importSpecifier":"openclaw/plugin-sdk/command-auth"}
|
||||
{"contentHash":"0e7eb50f296f755cbde5e14e06b754af5e6305202dce138a2d1835ce06ea2509","entrypoint":"command-auth","importSpecifier":"openclaw/plugin-sdk/command-auth"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"40df7e972484ae28718fef1095b2b44cb2cc9eadd35464bf28d4ae0d0f0f66b5","entrypoint":"config-runtime","importSpecifier":"openclaw/plugin-sdk/config-runtime"}
|
||||
{"contentHash":"0032a0767e4ddf214eb517cfe7352cd3c682a936d8c33e5602b0a9813052b11d","entrypoint":"config-runtime","importSpecifier":"openclaw/plugin-sdk/config-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"7a5a2f4d3e5e18d211480286d125419ecbf8175bb6f8eeee9a6ee105df03cb3f","entrypoint":"conversation-runtime","importSpecifier":"openclaw/plugin-sdk/conversation-runtime"}
|
||||
{"contentHash":"3bc769bb7d54a8d369e4922d24b1f7dd48b8a6df296457ee7c5bb241ab330e8b","entrypoint":"conversation-runtime","importSpecifier":"openclaw/plugin-sdk/conversation-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"6a56259993260770aa4d78eed69f1115f218c46a12a599d429e92304df4ddd0a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
{"contentHash":"c5756c96150425246ed16313fdfc11e6aa780634018a57e920fa5d7d43fcd469","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"a7ac16bfa85588ced0e46de7f75bf694a059c2af2995de3cd3f34b907dffca25","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
{"contentHash":"d01d8e0dd8da33679bf50faa64244e5c2c95f2c133ffc73f587e0b43176c43dd","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"273ebd4fcac3041e3a136410fa396b9704c73eedb66414723996127ed80db8f3","entrypoint":"hook-runtime","importSpecifier":"openclaw/plugin-sdk/hook-runtime"}
|
||||
{"contentHash":"909f37120c3e4834b772c5abf720aed63a2fc541ca4e9814360065533e925059","entrypoint":"hook-runtime","importSpecifier":"openclaw/plugin-sdk/hook-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"8bb9ea4976ff3657f3207838e2de5f3ce92d8c32c10bfbe8a3508e53edd16f3e","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
{"contentHash":"be64b90e0781977ab81a78be2238569ea7d51555b8784a2be600cac60f8422e0","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"2590b051b0e68c2e560aea3301e603ef3546de3862757f68a60b44ece3584933","entrypoint":"media-runtime","importSpecifier":"openclaw/plugin-sdk/media-runtime"}
|
||||
{"contentHash":"0c1524e093eb80f9f2bf71e2ae6814b5e521b546d2f8bb381fe278190d0a2932","entrypoint":"media-runtime","importSpecifier":"openclaw/plugin-sdk/media-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"f48e34067ab9b25a232af71d8ce17f9cbf9165ebce0832deaae3c1d9c9aeaab7","entrypoint":"media-understanding-runtime","importSpecifier":"openclaw/plugin-sdk/media-understanding-runtime"}
|
||||
{"contentHash":"24ed026c51d147a2959964e13fd0ef3e43cc44cc808d581f877f818e66ac4c04","entrypoint":"media-understanding-runtime","importSpecifier":"openclaw/plugin-sdk/media-understanding-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"e8a8cbcfc3bf20a51c93896e6add73d0c6d3677a7bac87ed9537afc7880c66f9","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
{"contentHash":"d2fd18514981c62a3f5e2937b8fc42f9679d7cec9c133198754dfacf54bdfd33","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"e43fcfe8f88cf8d338c3e3496f3bdd6ec14afc1fcf341e171f7758f5c0190d5b","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"}
|
||||
{"contentHash":"3d8b13b477b5468df65ba2e282d73a99bf6e4422fe71a3811be8456318e79688","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"d90b89f845a5df44d023e9455ff4d1fcbe51d4780a1223d624177d0ed639b0ff","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
{"contentHash":"4474179d9e5549947f7ef57a950be9ad426193c8ce9f23187d5d7c535f3d4af5","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"e7d2b6aa537db4452e8034bb57ae31bae15e471821fff539770a681d0d80b452","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
{"contentHash":"91fce99cea0d4ed66d7971f246ab66e0ec2c139caeb78c89b21a434908e2ca9f","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"7095620f46028ce52847f310ccac8583d69aed6223eb222b961f6012fa626ea7","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
{"contentHash":"927ae41bc0eeb5a07685238f4732c0caa639e28e10358555eba1096d22121172","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"2ffe953c6f47f15c3357af51a34385942165158724ccf4d0a272a7c060f9a2d9","entrypoint":"reply-dispatch-runtime","importSpecifier":"openclaw/plugin-sdk/reply-dispatch-runtime"}
|
||||
{"contentHash":"6b712448d5e6af99cee968aea960f12f9e6b385bf3bbb34d925720ebcd252d5f","entrypoint":"reply-dispatch-runtime","importSpecifier":"openclaw/plugin-sdk/reply-dispatch-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"ca4c1916f393a407fb4f5ad3a0fedc4bc66a8f87f1f38806ffb22928f2b57a2e","entrypoint":"reply-runtime","importSpecifier":"openclaw/plugin-sdk/reply-runtime"}
|
||||
{"contentHash":"cb54b3d858974c81cc1bd07e0a72b543e7f0a47f3751bd0a287c2f97bbb2b9ac","entrypoint":"reply-runtime","importSpecifier":"openclaw/plugin-sdk/reply-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"64e6bc07bb893d607cc4a194f85cbeedee23ec448edd40eb6e2757a9249ec4e0","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"}
|
||||
{"contentHash":"17e1939c2b38eba30dcd117a8448a54760b29cb13bd3e0cc9beb2795b0dee438","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"c6fb826551fd871ce2a2227ab734aa19296a590666f4a5a52d6ce2761a06fcbe","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"}
|
||||
{"contentHash":"838421524189909bb222d8249ede8f53dadfdf9c73a32258c5287b35a2db21d5","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"4626133e56c25b82d2e014f3f0e6fa2862162d603cfe57fd208f0b8d4ce89a3e","entrypoint":"session-store-runtime","importSpecifier":"openclaw/plugin-sdk/session-store-runtime"}
|
||||
{"contentHash":"c9b4b25729f5edb282348b1509c2fe943ba2a9aeacccf5392678033528329ad3","entrypoint":"session-store-runtime","importSpecifier":"openclaw/plugin-sdk/session-store-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"6738f9212ab1ca6c481cc9f3cea782baab29d63753770db25e39d83233d4f0f7","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
{"contentHash":"7b9e970e85cb5ac7d407bd5bf70e85b38407ba9d26b66a5bb8d830623cf806d7","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"ac368b3e61d2b5d15aeaf726b9e3816aa9bc44558a4ab7e738b4094276fb1fe5","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
{"contentHash":"5cfa7051b82c9d49810529f1522f584f28bc45b184416b8a898017423ebba415","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
|
||||
@@ -265,8 +265,10 @@ When `mode: "all"`, outputs are labeled `[Image 1/2]`, `[Audio 2/2]`, etc.
|
||||
- Read failures get `[Attachment could not be read]`.
|
||||
- URL attachments get `[Attachment skipped: URL file sources are disabled]` when URL file sources are disabled.
|
||||
- A file with no extractable text gets `[No extractable text]`.
|
||||
- At most five skip markers render per message; further skipped attachments collapse into one reason-neutral `[<n> more attachments skipped]` summary so junk attachments cannot grow the prompt without bound.
|
||||
- At most five skip markers render per message; further skipped attachments collapse into one reason-neutral `[<n> more attachments skipped]` summary so junk attachments cannot grow the prompt without bound. File and image, audio, or video markers share this five-marker budget.
|
||||
- If a PDF falls back to rendered page images, OpenClaw forwards those images to vision-capable reply models and keeps the placeholder `[PDF content rendered to images]` in the file block.
|
||||
- Image, audio, and video decisions record one closed disposition for every attachment candidate: handled, handed to native vision, not selected after the attachment limit, disabled, missing a model, denied by chat scope, or failed.
|
||||
- Unhandled media gets a bounded model-visible marker. Images handed to native vision and media turns owned by another harness do not add markers.
|
||||
|
||||
## Config examples
|
||||
|
||||
|
||||
@@ -108,6 +108,8 @@ describe("buildInboundMediaNote", () => {
|
||||
{
|
||||
capability: "image",
|
||||
outcome: "skipped",
|
||||
attachmentDispositions: { 0: { kind: "failed" } },
|
||||
nativeVisionActive: false,
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 0,
|
||||
@@ -249,6 +251,7 @@ describe("buildInboundMediaNote", () => {
|
||||
{
|
||||
capability: "audio",
|
||||
outcome: "success",
|
||||
attachmentDispositions: { 99: { kind: "handled" } },
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 99,
|
||||
|
||||
@@ -7,6 +7,8 @@ function createSuccessfulMediaDecision(
|
||||
return {
|
||||
capability,
|
||||
outcome: "success",
|
||||
attachmentDispositions: { 0: { kind: "handled" } },
|
||||
...(capability === "image" ? { nativeVisionActive: false } : {}),
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 0,
|
||||
|
||||
@@ -1212,6 +1212,49 @@ describe("tryDispatchAcpReplyCore", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("passes exactly the resolved attachment indexes as delivered images", async () => {
|
||||
setReadyAcpResolution();
|
||||
mockVisibleTextTurn("image turn");
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-"));
|
||||
const imagePath = path.join(tempDir, "delivered.png");
|
||||
try {
|
||||
// Real PNG bytes: the turn-attachment resolver byte-sniffs image MIME
|
||||
// through the harness buffer map keyed by local path.
|
||||
await fs.writeFile(imagePath, ACP_PNG_IMAGE_BYTES);
|
||||
acpAttachmentBuffers.set(imagePath, ACP_PNG_IMAGE_BYTES);
|
||||
|
||||
await runDispatch({
|
||||
bodyForAgent: "describe both images",
|
||||
cfg: createAcpTestConfig({
|
||||
channels: {
|
||||
imessage: {
|
||||
attachmentRoots: [tempDir],
|
||||
},
|
||||
},
|
||||
}),
|
||||
ctxOverrides: {
|
||||
Provider: "imessage",
|
||||
Surface: "imessage",
|
||||
media: [
|
||||
{ path: imagePath, contentType: "image/png", kind: "image" },
|
||||
{ url: "https://cdn.example.test/photos/remote.png", contentType: "image/png" },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// The delivered set must mirror the resolver: local image in, remote-url
|
||||
// image out — an empty or over-broad set reintroduces false skip claims.
|
||||
const delivered = requireRecord(
|
||||
mockArg(mediaUnderstandingMocks.applyMediaUnderstanding, 0, 0, "media understanding"),
|
||||
"media understanding",
|
||||
).deliveredImageIndexes as ReadonlySet<number>;
|
||||
expect(delivered.has(0)).toBe(true);
|
||||
expect(delivered.has(1)).toBe(false);
|
||||
} finally {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("selects bounded recent local history images", () => {
|
||||
const now = 1_700_000_000_000;
|
||||
const ctx = buildTestCtx({
|
||||
|
||||
@@ -707,6 +707,13 @@ export async function tryDispatchAcpReplyCore(params: {
|
||||
auditTerminalOutcome = "blocked";
|
||||
throw agentPolicyError;
|
||||
}
|
||||
// Resolve turn attachments before media understanding so marker rendering
|
||||
// suppresses exactly the image indexes ACP will deliver with the turn.
|
||||
const resolvedTurnAttachments = await resolveAgentTurnAttachments({
|
||||
ctx: params.ctx,
|
||||
cfg: params.cfg,
|
||||
includeAttachmentIndexes: true,
|
||||
});
|
||||
let extractedFileImages = params.extractedFileImages ?? [];
|
||||
if (hasInboundMediaForUnderstanding(params.ctx) && !params.ctx.MediaUnderstanding?.length) {
|
||||
try {
|
||||
@@ -714,6 +721,7 @@ export async function tryDispatchAcpReplyCore(params: {
|
||||
const mediaResult = await applyMediaUnderstanding({
|
||||
ctx: params.ctx,
|
||||
cfg: params.cfg,
|
||||
deliveredImageIndexes: new Set(resolvedTurnAttachments.attachmentIndexes ?? []),
|
||||
agentId: acpAgentId,
|
||||
agentDir: resolveAgentDir(params.cfg, acpAgentId),
|
||||
workspaceDir: resolveAgentWorkspaceDir(params.cfg, acpAgentId),
|
||||
@@ -729,11 +737,6 @@ export async function tryDispatchAcpReplyCore(params: {
|
||||
}
|
||||
|
||||
const promptText = resolveAcpPromptText(params.ctx);
|
||||
const resolvedTurnAttachments = await resolveAgentTurnAttachments({
|
||||
ctx: params.ctx,
|
||||
cfg: params.cfg,
|
||||
includeAttachmentIndexes: true,
|
||||
});
|
||||
const mediaAttachments = resolvedTurnAttachments.attachments;
|
||||
const inlineAttachments = resolveInlineAgentImageAttachments(params.images);
|
||||
const extractedAttachments = resolveInlineAgentImageAttachments(
|
||||
|
||||
@@ -1330,6 +1330,7 @@ describe("buildStatusMessage", () => {
|
||||
{
|
||||
capability: "audio",
|
||||
outcome: "skipped",
|
||||
attachmentDispositions: { 1: { kind: "failed" } },
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 1,
|
||||
@@ -1360,6 +1361,7 @@ describe("buildStatusMessage", () => {
|
||||
{
|
||||
capability: "audio",
|
||||
outcome: "success",
|
||||
attachmentDispositions: { 0: { kind: "handled" } },
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 0,
|
||||
@@ -1391,6 +1393,7 @@ describe("buildStatusMessage", () => {
|
||||
{
|
||||
capability: "audio",
|
||||
outcome: "failed",
|
||||
attachmentDispositions: { 0: { kind: "failed" } },
|
||||
attachments: [
|
||||
{
|
||||
attachmentIndex: 0,
|
||||
@@ -1425,9 +1428,25 @@ describe("buildStatusMessage", () => {
|
||||
sessionKey: "agent:main:main",
|
||||
queue: { mode: "none" },
|
||||
mediaDecisions: [
|
||||
{ capability: "image", outcome: "no-attachment", attachments: [] },
|
||||
{ capability: "audio", outcome: "no-attachment", attachments: [] },
|
||||
{ capability: "video", outcome: "no-attachment", attachments: [] },
|
||||
{
|
||||
capability: "image",
|
||||
outcome: "no-attachment",
|
||||
attachments: [],
|
||||
attachmentDispositions: {},
|
||||
nativeVisionActive: false,
|
||||
},
|
||||
{
|
||||
capability: "audio",
|
||||
outcome: "no-attachment",
|
||||
attachments: [],
|
||||
attachmentDispositions: {},
|
||||
},
|
||||
{
|
||||
capability: "video",
|
||||
outcome: "no-attachment",
|
||||
attachments: [],
|
||||
attachmentDispositions: {},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
|
||||
@@ -1,16 +1,37 @@
|
||||
// Keeps one provider failure from blocking the remaining media capabilities.
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { selectAttachments } from "./attachments.js";
|
||||
import { runCapability } from "./runner.js";
|
||||
|
||||
export async function runMediaCapability(
|
||||
params: Parameters<typeof runCapability>[0],
|
||||
): Promise<Awaited<ReturnType<typeof runCapability>> | undefined> {
|
||||
): Promise<Awaited<ReturnType<typeof runCapability>>> {
|
||||
try {
|
||||
return await runCapability(params);
|
||||
} catch (err) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`Media understanding task failed: ${String(err)}`);
|
||||
}
|
||||
return undefined;
|
||||
const selection = selectAttachments({
|
||||
capability: params.capability,
|
||||
attachments: params.media,
|
||||
policy: params.config?.attachments,
|
||||
});
|
||||
return {
|
||||
outputs: [],
|
||||
decision: {
|
||||
capability: params.capability,
|
||||
outcome: "failed",
|
||||
attachments: [],
|
||||
// Dropped attachments were never attempted; only selected ones failed.
|
||||
attachmentDispositions: Object.fromEntries([
|
||||
...selection.selected.map(({ index }) => [index, { kind: "failed" as const }] as const),
|
||||
...selection.droppedAttachmentIndexes.map(
|
||||
(index) => [index, { kind: "not-selected" as const }] as const,
|
||||
),
|
||||
]),
|
||||
...(params.capability === "image" ? { nativeVisionActive: false } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,16 +282,6 @@ async function applyWithDisabledMedia(params: {
|
||||
return { ctx, result };
|
||||
}
|
||||
|
||||
function expectFileNotApplied(params: {
|
||||
ctx: MsgContext;
|
||||
result: { appliedFile: boolean };
|
||||
body: string;
|
||||
}) {
|
||||
expect(params.result.appliedFile).toBe(false);
|
||||
expect(params.ctx.Body).toBe(params.body);
|
||||
expect(params.ctx.Body).not.toContain("<file");
|
||||
}
|
||||
|
||||
function expectUnsupportedFileApplied(params: {
|
||||
ctx: MsgContext;
|
||||
result: { appliedFile: boolean };
|
||||
@@ -687,7 +677,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
|
||||
expect(result.appliedAudio).toBe(false);
|
||||
expect(transcribeAudio).not.toHaveBeenCalled();
|
||||
expect(ctx.Body).toBe("");
|
||||
expect(ctx.Body).toBe("[Audio attachment could not be analyzed]");
|
||||
});
|
||||
|
||||
it("falls back to CLI model when provider fails", async () => {
|
||||
@@ -874,7 +864,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
);
|
||||
|
||||
expect(ctx.Transcript).toBeUndefined();
|
||||
expect(ctx.Body).toBe("");
|
||||
expect(ctx.Body).toBe("[Audio attachment could not be analyzed]");
|
||||
const [command] = getRunExecCall();
|
||||
expect(command).toBe("sherpa-onnx-offline");
|
||||
});
|
||||
@@ -1016,7 +1006,9 @@ describe("applyMediaUnderstanding", () => {
|
||||
);
|
||||
|
||||
expect(ctx.Transcript).toBeUndefined();
|
||||
expect(ctx.Body).toBe("");
|
||||
expect(ctx.Body).toBe(
|
||||
"[Audio attachment not analyzed: no audio-understanding model is configured]",
|
||||
);
|
||||
expect(mockedRunExec).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1048,7 +1040,9 @@ describe("applyMediaUnderstanding", () => {
|
||||
);
|
||||
|
||||
expect(ctx.Transcript).toBeUndefined();
|
||||
expect(ctx.Body).toBe("");
|
||||
expect(ctx.Body).toBe(
|
||||
"[Audio attachment not analyzed: no audio-understanding model is configured]",
|
||||
);
|
||||
expect(mockedRunExec).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1075,10 +1069,51 @@ describe("applyMediaUnderstanding", () => {
|
||||
expect(result.appliedImage).toBe(false);
|
||||
});
|
||||
|
||||
expect(ctx.Body).toBe("");
|
||||
expect(ctx.Body).toBe(
|
||||
"[Image attachment not analyzed: no image-understanding model is configured]",
|
||||
);
|
||||
expect(mockedRunExec).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("suppresses markers only for images the ACP caller actually delivers", async () => {
|
||||
clearMediaUnderstandingBinaryCacheForTests();
|
||||
const binDir = await createTempMediaDir();
|
||||
await createMockExecutable(binDir, "agy");
|
||||
const deliveredPath = await createTempMediaFile({
|
||||
fileName: "delivered.jpg",
|
||||
content: "image-bytes",
|
||||
});
|
||||
const undeliveredPath = await createTempMediaFile({
|
||||
fileName: "undelivered.jpg",
|
||||
content: "image-bytes",
|
||||
});
|
||||
const ctx: MsgContext = {
|
||||
Body: "",
|
||||
media: [
|
||||
{ path: deliveredPath, contentType: "image/jpeg" },
|
||||
{ path: undeliveredPath, contentType: "image/jpeg" },
|
||||
],
|
||||
};
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: { media: { image: { attachments: { mode: "all", maxAttachments: 4 } } } },
|
||||
};
|
||||
mockedResolveApiKey.mockResolvedValue({ source: "none", mode: "api-key" });
|
||||
|
||||
await withMediaAutoDetectEnv({ PATH: binDir }, async () => {
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx,
|
||||
cfg,
|
||||
deliveredImageIndexes: new Set([0]),
|
||||
});
|
||||
expect(result.appliedImage).toBe(false);
|
||||
});
|
||||
|
||||
// Index 0 rides with the ACP turn (no marker); index 1 was not resolved
|
||||
// into an attachment, so its non-delivery stays model-visible.
|
||||
const markerCount = ctx.Body?.split("[Image attachment not analyzed").length ?? 0;
|
||||
expect(markerCount - 1).toBe(1);
|
||||
});
|
||||
|
||||
it("uses CLI image understanding and preserves caption for commands", async () => {
|
||||
const imagePath = await createTempMediaFile({
|
||||
fileName: "photo.jpg",
|
||||
@@ -1270,6 +1305,56 @@ describe("applyMediaUnderstanding", () => {
|
||||
expect(ctx.Body).toBe("[Image]\nDescription:\nnormalized image");
|
||||
});
|
||||
|
||||
it("renders recorded outcomes for every image candidate when no model is configured", async () => {
|
||||
const ctx: MsgContext = {
|
||||
Body: "",
|
||||
media: Array.from({ length: 4 }, (_, index) => ({
|
||||
path: `/tmp/photo-${index}.jpg`,
|
||||
contentType: "image/jpeg",
|
||||
})),
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx,
|
||||
cfg: { tools: { media: { image: { enabled: true } } } },
|
||||
});
|
||||
|
||||
const imageDecision = result.decisions.find((decision) => decision.capability === "image");
|
||||
expect(imageDecision).toMatchObject({
|
||||
attachmentDispositions: {
|
||||
0: { kind: "no-model" },
|
||||
1: { kind: "not-selected" },
|
||||
2: { kind: "not-selected" },
|
||||
3: { kind: "not-selected" },
|
||||
},
|
||||
});
|
||||
expect(ctx.Body).toBe(
|
||||
[
|
||||
"[Image attachment not analyzed: no image-understanding model is configured]",
|
||||
"[Image attachment not processed: attachment limit reached]",
|
||||
"[Image attachment not processed: attachment limit reached]",
|
||||
"[Image attachment not processed: attachment limit reached]",
|
||||
].join("\n\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("caps markers for disabled image understanding", async () => {
|
||||
const ctx: MsgContext = {
|
||||
Body: "",
|
||||
media: Array.from({ length: 7 }, (_, index) => ({
|
||||
path: `/tmp/disabled-photo-${index}.jpg`,
|
||||
contentType: "image/jpeg",
|
||||
})),
|
||||
};
|
||||
|
||||
await applyMediaUnderstanding({ ctx, cfg: createMediaDisabledConfig() });
|
||||
|
||||
expect(
|
||||
ctx.Body?.split("[Image attachment not analyzed: image understanding is disabled]"),
|
||||
).toHaveLength(6);
|
||||
expect(ctx.Body).toContain("[2 more attachments skipped]");
|
||||
});
|
||||
|
||||
it("uses active model when enabled and models are missing", async () => {
|
||||
const audioPath = await createTempMediaFile({
|
||||
fileName: "fallback.ogg",
|
||||
@@ -1394,6 +1479,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
capability: "audio",
|
||||
outcome: "no-attachment",
|
||||
attachments: [],
|
||||
attachmentDispositions: {},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1751,7 +1837,10 @@ describe("applyMediaUnderstanding", () => {
|
||||
mediaType: "audio/mpeg",
|
||||
});
|
||||
|
||||
expectFileNotApplied({ ctx, result, body: "<media:audio>" });
|
||||
expect(result.appliedFile).toBe(false);
|
||||
expect(ctx.Body).toBe(
|
||||
"<media:audio>\n\n[Audio attachment not analyzed: audio understanding is disabled]",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports archive container attachments with +zip MIME types as unsupported", async () => {
|
||||
@@ -2176,31 +2265,32 @@ describe("applyMediaUnderstanding", () => {
|
||||
expect(ctx.Body).toContain("[2 more attachments skipped]");
|
||||
});
|
||||
|
||||
it("keeps the overflow summary reason-neutral when skipped kinds are mixed", async () => {
|
||||
it("shares one reason-neutral overflow budget across document and media markers", async () => {
|
||||
const olePayload = Buffer.from("Root Entry WordDocument legacy preview", "utf8");
|
||||
const media: { path: string; contentType: string }[] = [];
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
const filePath = await createTempMediaFile({
|
||||
fileName: `mixed-legacy-${i}.doc`,
|
||||
content: olePayload,
|
||||
});
|
||||
media.push({ path: filePath, contentType: "application/msword" });
|
||||
}
|
||||
const pdfPath = await createTempMediaFile({
|
||||
fileName: "report.pdf",
|
||||
content: Buffer.from("%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", "utf8"),
|
||||
});
|
||||
media.push({ path: pdfPath, contentType: "application/pdf" });
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
media.push({ path: `/tmp/junk-image-${i}.jpg`, contentType: "image/jpeg" });
|
||||
}
|
||||
|
||||
const ctx: MsgContext = { Body: "<media:file>", media };
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx,
|
||||
cfg: createMediaDisabledConfigWithAllowedMimes(["text/plain"]),
|
||||
cfg: createMediaDisabledConfig(),
|
||||
});
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
expect(ctx.Body).toContain("[1 more attachment skipped]");
|
||||
expect(ctx.Body).not.toContain("[Attachment type not allowed");
|
||||
expect(ctx.Body?.split("[Unsupported document format")).toHaveLength(5);
|
||||
expect(
|
||||
ctx.Body?.split("[Image attachment not analyzed: image understanding is disabled]"),
|
||||
).toHaveLength(2);
|
||||
expect(ctx.Body).toContain("[2 more attachments skipped]");
|
||||
});
|
||||
|
||||
it("keeps vendor +json attachments eligible for text extraction", async () => {
|
||||
|
||||
@@ -32,15 +32,18 @@ import type { ExtractedFileImage } from "./extracted-file-images.js";
|
||||
import {
|
||||
type FileAttachmentOutcome,
|
||||
isSkippedFileOutcome,
|
||||
MAX_SKIPPED_FILE_MARKERS,
|
||||
renderFileAttachmentOutcome,
|
||||
renderSkippedFileOverflowSummary,
|
||||
sanitizeMimeType,
|
||||
} from "./file-attachment-outcomes.js";
|
||||
import {
|
||||
type FileExtractionLimits,
|
||||
resolveFileExtractionLimits,
|
||||
} from "./file-extraction-limits.js";
|
||||
import {
|
||||
MAX_SKIPPED_FILE_MARKERS,
|
||||
renderMediaAttachmentDisposition,
|
||||
renderSkippedFileOverflowSummary,
|
||||
} from "./media-attachment-outcomes.js";
|
||||
import { resolveConcurrency } from "./resolve.js";
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
@@ -184,6 +187,8 @@ type ClassifiedFileAttachment = {
|
||||
mimeType?: string;
|
||||
};
|
||||
|
||||
type AttachmentContextBlock = { text: string; consumesMarkerBudget: boolean };
|
||||
|
||||
// URL attachments may carry signed query credentials; only the pathname
|
||||
// basename is safe to surface as a model-visible display name.
|
||||
function attachmentUrlDisplayName(url: string): string | undefined {
|
||||
@@ -350,10 +355,8 @@ async function extractFileContext(params: {
|
||||
if (!attachments || attachments.length === 0) {
|
||||
return { blocks: [], images: [] };
|
||||
}
|
||||
const blocks: string[] = [];
|
||||
const blocks: AttachmentContextBlock[] = [];
|
||||
const images: ExtractedFileImage[] = [];
|
||||
let skippedMarkers = 0;
|
||||
let skippedOverflow = 0;
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment) {
|
||||
continue;
|
||||
@@ -377,28 +380,74 @@ async function extractFileContext(params: {
|
||||
if (blockText === null) {
|
||||
continue;
|
||||
}
|
||||
if (isSkippedFileOutcome(outcome)) {
|
||||
if (skippedMarkers >= MAX_SKIPPED_FILE_MARKERS) {
|
||||
skippedOverflow += 1;
|
||||
continue;
|
||||
}
|
||||
skippedMarkers += 1;
|
||||
}
|
||||
blocks.push(
|
||||
renderFileContextBlock({
|
||||
blocks.push({
|
||||
text: renderFileContextBlock({
|
||||
filename,
|
||||
fallbackName: `file-${attachment.index + 1}`,
|
||||
mimeType,
|
||||
content: blockText,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (skippedOverflow > 0) {
|
||||
blocks.push(renderSkippedFileOverflowSummary(skippedOverflow));
|
||||
consumesMarkerBudget: isSkippedFileOutcome(outcome),
|
||||
});
|
||||
}
|
||||
return { blocks, images };
|
||||
}
|
||||
|
||||
function renderMediaAttachmentMarkers(params: {
|
||||
attachments: MediaAttachment[];
|
||||
decisions: MediaUnderstandingDecision[];
|
||||
outputs: MediaUnderstandingOutput[];
|
||||
deliveredImageIndexes?: ReadonlySet<number>;
|
||||
}): AttachmentContextBlock[] {
|
||||
const handledIndexes = new Set(params.outputs.map((output) => output.attachmentIndex));
|
||||
const decisions = new Map(params.decisions.map((decision) => [decision.capability, decision]));
|
||||
return params.attachments.flatMap((attachment) => {
|
||||
const capability = resolveAttachmentKind(attachment);
|
||||
if (capability !== "image" && capability !== "audio" && capability !== "video") {
|
||||
return [];
|
||||
}
|
||||
// The ACP caller resolved these exact indexes into native turn attachments;
|
||||
// a marker would falsely claim non-delivery. Unresolved images keep theirs.
|
||||
if (capability === "image" && params.deliveredImageIndexes?.has(attachment.index)) {
|
||||
return [];
|
||||
}
|
||||
const decision = decisions.get(capability);
|
||||
if (!decision || handledIndexes.has(attachment.index)) {
|
||||
return [];
|
||||
}
|
||||
const disposition = decision.attachmentDispositions?.[attachment.index];
|
||||
// Vision-capable model → the reply runtime hydrates images natively; an
|
||||
// absence-of-processing marker would contradict what the model sees.
|
||||
// Recorded per-attachment failures stay visible — they are authoritative
|
||||
// regardless of native delivery. Partial/failed native hydration remains
|
||||
// unexplainable at this frozen-prompt stage (#122101).
|
||||
if (
|
||||
capability === "image" &&
|
||||
decision.nativeVisionActive !== false &&
|
||||
disposition?.kind !== "failed"
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const text = disposition ? renderMediaAttachmentDisposition(capability, disposition) : null;
|
||||
return text ? [{ text, consumesMarkerBudget: true }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function applyAttachmentMarkerBudget(blocks: AttachmentContextBlock[]): string[] {
|
||||
const rendered: string[] = [];
|
||||
let markers = 0;
|
||||
let overflow = 0;
|
||||
for (const block of blocks) {
|
||||
if (block.consumesMarkerBudget && markers >= MAX_SKIPPED_FILE_MARKERS) {
|
||||
overflow += 1;
|
||||
continue;
|
||||
}
|
||||
markers += Number(block.consumesMarkerBudget);
|
||||
rendered.push(block.text);
|
||||
}
|
||||
return overflow > 0 ? [...rendered, renderSkippedFileOverflowSummary(overflow)] : rendered;
|
||||
}
|
||||
|
||||
export async function applyMediaUnderstanding(params: {
|
||||
ctx: MsgContext;
|
||||
cfg: OpenClawConfig;
|
||||
@@ -409,6 +458,8 @@ export async function applyMediaUnderstanding(params: {
|
||||
activeModel?: ActiveMediaModel;
|
||||
/** Preserve native-harness ownership of image, video, and file inputs while applying STT. */
|
||||
processingMode?: "audio-only";
|
||||
/** Attachment indexes the caller (ACP) has already resolved into native turn attachments. */
|
||||
deliveredImageIndexes?: ReadonlySet<number>;
|
||||
}): Promise<ApplyMediaUnderstandingResult> {
|
||||
const { ctx, cfg } = params;
|
||||
const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body];
|
||||
@@ -451,9 +502,6 @@ export async function applyMediaUnderstanding(params: {
|
||||
const outputs: MediaUnderstandingOutput[] = [];
|
||||
const decisions: MediaUnderstandingDecision[] = [];
|
||||
for (const entry of results) {
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
for (const output of entry.outputs) {
|
||||
outputs.push(output);
|
||||
}
|
||||
@@ -567,13 +615,23 @@ export async function applyMediaUnderstanding(params: {
|
||||
skipAttachmentIndexes:
|
||||
audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
|
||||
});
|
||||
if (fileContext.blocks.length > 0) {
|
||||
ctx.Body = appendFileBlocks(ctx.Body, fileContext.blocks);
|
||||
const mediaMarkers =
|
||||
params.processingMode === "audio-only"
|
||||
? []
|
||||
: renderMediaAttachmentMarkers({
|
||||
attachments,
|
||||
decisions,
|
||||
outputs,
|
||||
deliveredImageIndexes: params.deliveredImageIndexes,
|
||||
});
|
||||
const contextBlocks = applyAttachmentMarkerBudget([...fileContext.blocks, ...mediaMarkers]);
|
||||
if (contextBlocks.length > 0) {
|
||||
ctx.Body = appendFileBlocks(ctx.Body, contextBlocks);
|
||||
}
|
||||
if (outputs.length > 0 || fileContext.blocks.length > 0) {
|
||||
if (outputs.length > 0 || contextBlocks.length > 0) {
|
||||
finalizeInboundContext(ctx, {
|
||||
forceBodyForAgent: true,
|
||||
forceBodyForCommands: outputs.length > 0 || fileContext.blocks.length > 0,
|
||||
forceBodyForCommands: true,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ describe("media-understanding selectAttachments guards", () => {
|
||||
attachments: undefined as unknown as MediaAttachment[],
|
||||
policy: { prefer: "path" },
|
||||
}),
|
||||
).toStrictEqual([]);
|
||||
).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] });
|
||||
});
|
||||
|
||||
it("returns no selections when attachments is not an array", () => {
|
||||
@@ -22,7 +22,7 @@ describe("media-understanding selectAttachments guards", () => {
|
||||
attachments: { malformed: true } as unknown as MediaAttachment[],
|
||||
policy: { prefer: "url" },
|
||||
}),
|
||||
).toStrictEqual([]);
|
||||
).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] });
|
||||
});
|
||||
|
||||
it("returns no selections for malformed attachment entries", () => {
|
||||
@@ -37,6 +37,23 @@ describe("media-understanding selectAttachments guards", () => {
|
||||
] as unknown as MediaAttachment[],
|
||||
policy: { prefer: "path" },
|
||||
}),
|
||||
).toStrictEqual([]);
|
||||
).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] });
|
||||
});
|
||||
|
||||
it("reports only same-capability attachments dropped by truncation", () => {
|
||||
expect(
|
||||
selectAttachments({
|
||||
capability: "image",
|
||||
attachments: [
|
||||
{ index: 0, path: "/tmp/first.jpg", mime: "image/jpeg" },
|
||||
{ index: 1, path: "/tmp/note.ogg", mime: "audio/ogg" },
|
||||
{ index: 2, path: "/tmp/second.jpg", mime: "image/jpeg" },
|
||||
{ index: 3, path: "/tmp/third.jpg", mime: "image/jpeg" },
|
||||
],
|
||||
}),
|
||||
).toStrictEqual({
|
||||
selected: [{ index: 0, path: "/tmp/first.jpg", mime: "image/jpeg" }],
|
||||
droppedAttachmentIndexes: [2, 3],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,7 +64,7 @@ export function selectAttachments(params: {
|
||||
capability: MediaUnderstandingCapability;
|
||||
attachments: MediaAttachment[];
|
||||
policy?: MediaUnderstandingAttachmentsConfig;
|
||||
}): MediaAttachment[] {
|
||||
}): { selected: MediaAttachment[]; droppedAttachmentIndexes: number[] } {
|
||||
const { capability, attachments, policy } = params;
|
||||
const input = Array.isArray(attachments) ? attachments.filter(isAttachmentRecord) : [];
|
||||
const matches = input.filter((item) => {
|
||||
@@ -81,14 +81,15 @@ export function selectAttachments(params: {
|
||||
return isVideoAttachment(item);
|
||||
});
|
||||
if (matches.length === 0) {
|
||||
return [];
|
||||
return { selected: [], droppedAttachmentIndexes: [] };
|
||||
}
|
||||
|
||||
const ordered = orderAttachments(matches, policy?.prefer);
|
||||
const mode = policy?.mode ?? "first";
|
||||
const maxAttachments = policy?.maxAttachments ?? DEFAULT_MAX_ATTACHMENTS;
|
||||
if (mode === "all") {
|
||||
return ordered.slice(0, Math.max(1, maxAttachments));
|
||||
}
|
||||
return ordered.slice(0, 1);
|
||||
const limit = mode === "all" ? Math.max(1, maxAttachments) : 1;
|
||||
return {
|
||||
selected: ordered.slice(0, limit),
|
||||
droppedAttachmentIndexes: ordered.slice(limit).map((attachment) => attachment.index),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -53,10 +53,6 @@ function wrapUntrustedAttachmentContent(content: string): string {
|
||||
return wrapExternalContent(content, { source: "unknown", includeWarning: false });
|
||||
}
|
||||
|
||||
// Cap cumulative skip markers so a burst of rejected attachments cannot grow
|
||||
// model-visible context without bound; overflow collapses into one summary line.
|
||||
export const MAX_SKIPPED_FILE_MARKERS = 5;
|
||||
|
||||
const SKIPPED_FILE_OUTCOME_KINDS = new Set<FileAttachmentOutcome["kind"]>([
|
||||
"unsupported-format",
|
||||
"policy-rejected",
|
||||
@@ -68,12 +64,6 @@ export function isSkippedFileOutcome(outcome: FileAttachmentOutcome): boolean {
|
||||
return SKIPPED_FILE_OUTCOME_KINDS.has(outcome.kind);
|
||||
}
|
||||
|
||||
// Reason-neutral on purpose: overflow can mix unsupported, policy-rejected,
|
||||
// unreadable, and url-disabled kinds; naming one reason would misdirect recovery.
|
||||
export function renderSkippedFileOverflowSummary(count: number): string {
|
||||
return `[${count} more attachment${count === 1 ? "" : "s"} skipped]`;
|
||||
}
|
||||
|
||||
export function renderFileAttachmentOutcome(outcome: FileAttachmentOutcome): string | null {
|
||||
switch (outcome.kind) {
|
||||
case "extracted":
|
||||
|
||||
32
src/media-understanding/media-attachment-outcomes.ts
Normal file
32
src/media-understanding/media-attachment-outcomes.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import type { MediaAttachmentDisposition, MediaUnderstandingCapability } from "./types.js";
|
||||
|
||||
export const MAX_SKIPPED_FILE_MARKERS = 5;
|
||||
|
||||
// Reason-neutral because the shared overflow can mix file and media outcomes.
|
||||
export function renderSkippedFileOverflowSummary(count: number): string {
|
||||
return `[${count} more attachment${count === 1 ? "" : "s"} skipped]`;
|
||||
}
|
||||
|
||||
export function renderMediaAttachmentDisposition(
|
||||
capability: MediaUnderstandingCapability,
|
||||
disposition: MediaAttachmentDisposition,
|
||||
): string | null {
|
||||
const label = `${capability[0]?.toUpperCase()}${capability.slice(1)}`;
|
||||
switch (disposition.kind) {
|
||||
case "handled":
|
||||
case "handed-to-native-vision":
|
||||
return null;
|
||||
case "not-selected":
|
||||
return `[${label} attachment not processed: attachment limit reached]`;
|
||||
case "capability-disabled":
|
||||
return `[${label} attachment not analyzed: ${capability} understanding is disabled]`;
|
||||
case "no-model":
|
||||
return `[${label} attachment not analyzed: no ${capability}-understanding model is configured]`;
|
||||
case "scope-denied":
|
||||
return `[${label} attachment not analyzed in this chat]`;
|
||||
case "failed":
|
||||
return `[${label} attachment could not be analyzed]`;
|
||||
default:
|
||||
return disposition satisfies never;
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,8 @@ describe("media-understanding formatDecisionSummary guards", () => {
|
||||
capability: "image",
|
||||
outcome: "skipped",
|
||||
attachments: undefined as unknown as MediaUnderstandingDecision["attachments"],
|
||||
attachmentDispositions: {},
|
||||
nativeVisionActive: false,
|
||||
}),
|
||||
).toBe("image: skipped");
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@ import { logWarn } from "../logger.js";
|
||||
import { resolveChannelInboundAttachmentRoots } from "../media/channel-inbound-roots.js";
|
||||
import { getDefaultMediaLocalRoots } from "../media/local-roots.js";
|
||||
import { normalizeMediaFacts } from "../media/media-facts.js";
|
||||
import { classifyMediaReferenceSource } from "../media/media-reference.js";
|
||||
import { createLazyRuntimeModule, createLazyRuntimeNamedExport } from "../shared/lazy-runtime.js";
|
||||
import { MediaAttachmentCache, selectAttachments } from "./attachments.js";
|
||||
import { matchesMediaEntryCapability } from "./entry-capabilities.js";
|
||||
@@ -60,6 +61,7 @@ import {
|
||||
} from "./runner.entries.js";
|
||||
import type {
|
||||
MediaAttachment,
|
||||
MediaAttachmentDisposition,
|
||||
MediaUnderstandingCapability,
|
||||
MediaUnderstandingDecision,
|
||||
MediaUnderstandingModelDecision,
|
||||
@@ -565,23 +567,15 @@ async function resolveAutoEntries(params: {
|
||||
providerRegistry: ProviderRegistry;
|
||||
capability: MediaUnderstandingCapability;
|
||||
activeModel?: ActiveMediaModel;
|
||||
nativeVisionActive: boolean;
|
||||
}): Promise<MediaUnderstandingModelConfig[]> {
|
||||
if (params.capability === "image") {
|
||||
const activeSupportsVision = await activeModelSupportsNativeVision({
|
||||
if (params.capability === "image" && !params.nativeVisionActive) {
|
||||
const imageModelEntries = resolveImageModelFromAgentDefaults({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
activeModel: params.activeModel,
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
});
|
||||
if (!activeSupportsVision) {
|
||||
const imageModelEntries = resolveImageModelFromAgentDefaults({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
if (imageModelEntries.length > 0) {
|
||||
return imageModelEntries;
|
||||
}
|
||||
if (imageModelEntries.length > 0) {
|
||||
return imageModelEntries;
|
||||
}
|
||||
}
|
||||
const activeEntry = await resolveActiveModelEntry(params);
|
||||
@@ -840,6 +834,13 @@ function hasFailedMediaAttempt(attachments: MediaUnderstandingDecision["attachme
|
||||
);
|
||||
}
|
||||
|
||||
function createAttachmentDispositions(
|
||||
indexes: readonly number[],
|
||||
disposition: MediaAttachmentDisposition,
|
||||
): Record<number, MediaAttachmentDisposition> {
|
||||
return Object.fromEntries(indexes.map((index) => [index, disposition]));
|
||||
}
|
||||
|
||||
export async function runCapability(params: {
|
||||
capability: MediaUnderstandingCapability;
|
||||
cfg: OpenClawConfig;
|
||||
@@ -855,23 +856,81 @@ export async function runCapability(params: {
|
||||
}): Promise<RunCapabilityResult> {
|
||||
const { capability, cfg, ctx } = params;
|
||||
const config: MediaUnderstandingConfig = params.config ?? cfg.tools?.media?.[capability] ?? {};
|
||||
const selection = selectAttachments({
|
||||
capability,
|
||||
attachments: params.media,
|
||||
policy: config.attachments,
|
||||
});
|
||||
const selectedAttachmentIndexes = selection.selected.map((attachment) => attachment.index);
|
||||
const activeProvider = params.activeModel?.provider?.trim();
|
||||
// One memoized owner for the native-vision fact. Probed lazily — only when
|
||||
// the skip branch must decide, or an image decision carries a renderable
|
||||
// disposition — so explicit image models never pay a catalog lookup. A probe
|
||||
// failure yields "unknown" and never alters a decision outcome; unknown
|
||||
// suppresses image markers because a false skip claim beside a natively
|
||||
// delivered image is worse than silence (#122101).
|
||||
let nativeVisionProbe: Promise<boolean | undefined> | undefined;
|
||||
const resolveNativeVisionFlag = (): Promise<boolean | undefined> => {
|
||||
nativeVisionProbe ??= activeModelSupportsNativeVision({
|
||||
cfg,
|
||||
agentId: params.agentId,
|
||||
activeModel: params.activeModel,
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
}).catch((err: unknown) => {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`native vision support probe failed: ${String(err)}`);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
return nativeVisionProbe;
|
||||
};
|
||||
const buildDispositions = (
|
||||
selectedDisposition: MediaAttachmentDisposition,
|
||||
droppedDisposition = selectedDisposition,
|
||||
) => ({
|
||||
...createAttachmentDispositions(selectedAttachmentIndexes, selectedDisposition),
|
||||
...createAttachmentDispositions(selection.droppedAttachmentIndexes, droppedDisposition),
|
||||
});
|
||||
const rendersMarker = (dispositions: Record<number, MediaAttachmentDisposition>) =>
|
||||
Object.values(dispositions).some(
|
||||
(d) => d.kind !== "handled" && d.kind !== "handed-to-native-vision",
|
||||
);
|
||||
const buildDecision = async (
|
||||
outcome: MediaUnderstandingDecision["outcome"],
|
||||
attachments: MediaUnderstandingDecision["attachments"],
|
||||
attachmentDispositions: Record<number, MediaAttachmentDisposition>,
|
||||
): Promise<MediaUnderstandingDecision> => {
|
||||
// Record the fact whenever it is known (probe already ran) or needed
|
||||
// (a marker could render); never fire the probe for marker-free decisions.
|
||||
const nativeVisionActive =
|
||||
capability === "image" &&
|
||||
(nativeVisionProbe !== undefined || rendersMarker(attachmentDispositions))
|
||||
? await resolveNativeVisionFlag()
|
||||
: undefined;
|
||||
return {
|
||||
capability,
|
||||
outcome,
|
||||
attachments,
|
||||
attachmentDispositions,
|
||||
...(nativeVisionActive !== undefined ? { nativeVisionActive } : {}),
|
||||
};
|
||||
};
|
||||
if (config?.enabled === false) {
|
||||
return {
|
||||
outputs: [],
|
||||
decision: { capability, outcome: "disabled", attachments: [] },
|
||||
decision: await buildDecision(
|
||||
"disabled",
|
||||
[],
|
||||
buildDispositions({ kind: "capability-disabled" }),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const attachmentPolicy = config?.attachments;
|
||||
const selected = selectAttachments({
|
||||
capability,
|
||||
attachments: params.media,
|
||||
policy: attachmentPolicy,
|
||||
});
|
||||
if (selected.length === 0) {
|
||||
if (selection.selected.length === 0) {
|
||||
return {
|
||||
outputs: [],
|
||||
decision: { capability, outcome: "no-attachment", attachments: [] },
|
||||
decision: await buildDecision("no-attachment", [], {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -882,61 +941,66 @@ export async function runCapability(params: {
|
||||
}
|
||||
return {
|
||||
outputs: [],
|
||||
decision: {
|
||||
capability,
|
||||
outcome: "scope-deny",
|
||||
attachments: selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })),
|
||||
},
|
||||
decision: await buildDecision(
|
||||
"scope-deny",
|
||||
selection.selected.map((item) => ({
|
||||
attachmentIndex: item.index,
|
||||
attempts: [],
|
||||
})),
|
||||
buildDispositions({ kind: "scope-denied" }),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// Skip image understanding when the primary model supports vision natively.
|
||||
// The image will be injected directly into the model context instead.
|
||||
const activeProvider = params.activeModel?.provider?.trim();
|
||||
if (
|
||||
capability === "image" &&
|
||||
activeProvider &&
|
||||
!hasExplicitImageUnderstandingConfig({
|
||||
cfg,
|
||||
providerRegistry: params.providerRegistry,
|
||||
})
|
||||
!hasExplicitImageUnderstandingConfig({ cfg, providerRegistry: params.providerRegistry }) &&
|
||||
(await resolveNativeVisionFlag()) === true
|
||||
) {
|
||||
if (
|
||||
await activeModelSupportsNativeVision({
|
||||
cfg,
|
||||
agentId: params.agentId,
|
||||
activeModel: params.activeModel,
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
})
|
||||
) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose("Skipping image understanding: primary model supports vision natively");
|
||||
}
|
||||
const model = params.activeModel?.model?.trim();
|
||||
const reason = "primary model supports vision natively";
|
||||
return {
|
||||
outputs: [],
|
||||
decision: {
|
||||
capability,
|
||||
outcome: "skipped",
|
||||
attachments: selected.map((item) => {
|
||||
const attempt = {
|
||||
type: "provider" as const,
|
||||
provider: activeProvider,
|
||||
model: model || undefined,
|
||||
outcome: "skipped" as const,
|
||||
reason,
|
||||
};
|
||||
return {
|
||||
attachmentIndex: item.index,
|
||||
attempts: [attempt],
|
||||
chosen: attempt,
|
||||
};
|
||||
}),
|
||||
},
|
||||
};
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose("Skipping image understanding: primary model supports vision natively");
|
||||
}
|
||||
const model = params.activeModel?.model?.trim();
|
||||
const reason = "primary model supports vision natively";
|
||||
// Native hydration resolves local paths and media-store refs only; a
|
||||
// remote-URL-only image is never delivered that way, so claiming the
|
||||
// handoff would suppress its marker while it silently vanishes.
|
||||
const nativeDeliverable = (item: MediaAttachment) =>
|
||||
Boolean(item.path) ||
|
||||
(Boolean(item.url) && classifyMediaReferenceSource(item.url ?? "").isMediaStoreUrl);
|
||||
return {
|
||||
outputs: [],
|
||||
decision: await buildDecision(
|
||||
"skipped",
|
||||
selection.selected.map((item) => {
|
||||
if (!nativeDeliverable(item)) {
|
||||
return { attachmentIndex: item.index, attempts: [] };
|
||||
}
|
||||
const attempt = {
|
||||
type: "provider" as const,
|
||||
provider: activeProvider,
|
||||
model: model || undefined,
|
||||
outcome: "skipped" as const,
|
||||
reason,
|
||||
};
|
||||
return {
|
||||
attachmentIndex: item.index,
|
||||
attempts: [attempt],
|
||||
chosen: attempt,
|
||||
};
|
||||
}),
|
||||
{
|
||||
...buildDispositions({ kind: "handed-to-native-vision" }),
|
||||
...createAttachmentDispositions(
|
||||
selection.selected.filter((item) => !nativeDeliverable(item)).map((item) => item.index),
|
||||
{ kind: "failed", reason: "remote-url image is not natively deliverable" },
|
||||
),
|
||||
},
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const entries = resolveModelEntries({
|
||||
@@ -956,23 +1020,28 @@ export async function runCapability(params: {
|
||||
providerRegistry: params.providerRegistry,
|
||||
capability,
|
||||
activeModel: params.activeModel,
|
||||
nativeVisionActive: capability === "image" && (await resolveNativeVisionFlag()) === true,
|
||||
})
|
||||
).map((entry) => ({ entry }));
|
||||
}
|
||||
if (resolvedEntries.length === 0) {
|
||||
return {
|
||||
outputs: [],
|
||||
decision: {
|
||||
capability,
|
||||
outcome: "skipped",
|
||||
attachments: selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })),
|
||||
},
|
||||
decision: await buildDecision(
|
||||
"skipped",
|
||||
selection.selected.map((item) => ({
|
||||
attachmentIndex: item.index,
|
||||
attempts: [],
|
||||
})),
|
||||
buildDispositions({ kind: "no-model" }, { kind: "not-selected" }),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const outputs: MediaUnderstandingOutput[] = [];
|
||||
const attachmentDecisions: MediaUnderstandingDecision["attachments"] = [];
|
||||
for (const attachment of selected) {
|
||||
const attachmentDispositions = buildDispositions({ kind: "failed" }, { kind: "not-selected" });
|
||||
for (const attachment of selection.selected) {
|
||||
const { output, attempts } = await runAttachmentEntries({
|
||||
capability,
|
||||
cfg,
|
||||
@@ -989,22 +1058,22 @@ export async function runCapability(params: {
|
||||
if (output) {
|
||||
outputs.push(output);
|
||||
}
|
||||
attachmentDispositions[attachment.index] = output ? { kind: "handled" } : { kind: "failed" };
|
||||
attachmentDecisions.push({
|
||||
attachmentIndex: attachment.index,
|
||||
attempts,
|
||||
chosen: attempts.find((attempt) => attempt.outcome === "success"),
|
||||
});
|
||||
}
|
||||
const decision: MediaUnderstandingDecision = {
|
||||
capability,
|
||||
outcome:
|
||||
outputs.length > 0
|
||||
? "success"
|
||||
: hasFailedMediaAttempt(attachmentDecisions)
|
||||
? "failed"
|
||||
: "skipped",
|
||||
attachments: attachmentDecisions,
|
||||
};
|
||||
const decision = await buildDecision(
|
||||
outputs.length > 0
|
||||
? "success"
|
||||
: hasFailedMediaAttempt(attachmentDecisions)
|
||||
? "failed"
|
||||
: "skipped",
|
||||
attachmentDecisions,
|
||||
attachmentDispositions,
|
||||
);
|
||||
if (decision.outcome === "failed") {
|
||||
logWarn(`media-understanding: ${formatDecisionSummary(decision)}`);
|
||||
} else if (shouldLogVerbose()) {
|
||||
|
||||
@@ -187,6 +187,10 @@ describe("runCapability image skip", () => {
|
||||
let describeCalls = 0;
|
||||
const msgCtx = ctx as MsgContext;
|
||||
msgCtx.Body = "please inspect this image";
|
||||
msgCtx.media = Array.from({ length: 4 }, () => ({
|
||||
path: mediaPath,
|
||||
contentType: "image/png",
|
||||
}));
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
@@ -217,14 +221,171 @@ describe("runCapability image skip", () => {
|
||||
const attempt = imageDecision?.attachments[0]?.attempts[0];
|
||||
expect(result.appliedImage).toBe(false);
|
||||
expect(imageDecision?.outcome).toBe("skipped");
|
||||
expect(imageDecision).toMatchObject({ nativeVisionActive: true });
|
||||
expect(attempt?.outcome).toBe("skipped");
|
||||
expect(attempt?.reason).toBe("primary model supports vision natively");
|
||||
expect(describeCalls).toBe(0);
|
||||
expect(msgCtx.Body).not.toContain(plantedVisionSentinel);
|
||||
expect(msgCtx.Body).not.toContain("Image attachment not");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("markers remote-url-only images instead of claiming native handoff", async () => {
|
||||
await withMediaFixture(
|
||||
{
|
||||
filePrefix: "openclaw-image-url-only-no-handoff",
|
||||
extension: "png",
|
||||
mediaType: "image/png",
|
||||
fileContents: Buffer.from("image"),
|
||||
},
|
||||
async ({ ctx, mediaPath }) => {
|
||||
const msgCtx = ctx as MsgContext;
|
||||
msgCtx.Body = "please inspect both images";
|
||||
msgCtx.media = [
|
||||
{ path: mediaPath, contentType: "image/png" },
|
||||
{ url: "https://cdn.example.test/photos/second.png", contentType: "image/png" },
|
||||
];
|
||||
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx: msgCtx,
|
||||
cfg: {
|
||||
tools: { media: { image: { attachments: { mode: "all", maxAttachments: 4 } } } },
|
||||
} as unknown as OpenClawConfig,
|
||||
agentDir: "/tmp",
|
||||
workspaceDir: path.dirname(mediaPath),
|
||||
activeModel: { provider: "openai", model: "gpt-4.1" },
|
||||
});
|
||||
|
||||
const imageDecision = result.decisions.find((decision) => decision.capability === "image");
|
||||
expect(imageDecision?.outcome).toBe("skipped");
|
||||
expect(imageDecision?.attachmentDispositions).toMatchObject({
|
||||
0: { kind: "handed-to-native-vision" },
|
||||
1: { kind: "failed" },
|
||||
});
|
||||
// Local image stays suppressed (native hydration owns it); the
|
||||
// remote-url image renders its failure despite nativeVisionActive.
|
||||
expect(msgCtx.Body).toContain("[Image attachment could not be analyzed]");
|
||||
expect(msgCtx.Body).not.toContain("not processed");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("runs explicit image models untouched by native-vision probe failure", async () => {
|
||||
await withMediaFixture(
|
||||
{
|
||||
filePrefix: "openclaw-image-explicit-model-probe-immune",
|
||||
extension: "png",
|
||||
mediaType: "image/png",
|
||||
fileContents: Buffer.from("image"),
|
||||
},
|
||||
async ({ ctx, mediaPath }) => {
|
||||
const msgCtx = ctx as MsgContext;
|
||||
msgCtx.Body = "inspect this image";
|
||||
msgCtx.media = [{ path: mediaPath, contentType: "image/png" }];
|
||||
const cfg = {
|
||||
tools: {
|
||||
media: {
|
||||
models: [
|
||||
{
|
||||
provider: "openrouter",
|
||||
model: "google/gemini-2.5-flash",
|
||||
capabilities: ["image"],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
await loadModelCatalog.withImplementation(
|
||||
async () => {
|
||||
throw new Error("catalog unavailable");
|
||||
},
|
||||
async () => {
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx: msgCtx,
|
||||
cfg,
|
||||
agentDir: "/tmp",
|
||||
workspaceDir: path.dirname(mediaPath),
|
||||
providers: {
|
||||
openrouter: {
|
||||
id: "openrouter",
|
||||
capabilities: ["image"],
|
||||
describeImage: async (req) => ({
|
||||
text: plantedVisionSentinel,
|
||||
model: req.model,
|
||||
}),
|
||||
},
|
||||
},
|
||||
activeModel: { provider: "openai", model: "gpt-4.1" },
|
||||
});
|
||||
|
||||
const imageDecision = result.decisions.find(
|
||||
(decision) => decision.capability === "image",
|
||||
);
|
||||
// The lone selected attachment leaves nothing to marker, so the
|
||||
// probe never fires and catalog failure cannot reach this path.
|
||||
expect(result.appliedImage).toBe(true);
|
||||
expect(imageDecision?.outcome).toBe("success");
|
||||
expect(imageDecision?.attachmentDispositions).toEqual({ 0: { kind: "handled" } });
|
||||
expect(imageDecision).not.toHaveProperty("nativeVisionActive");
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps disabled outcomes precise and suppresses markers when the vision probe fails", async () => {
|
||||
const ctx: MsgContext = {
|
||||
Body: "inspect this image",
|
||||
media: [{ path: "/tmp/image.png", contentType: "image/png" }],
|
||||
};
|
||||
|
||||
await loadModelCatalog.withImplementation(
|
||||
async () => {
|
||||
throw new Error("catalog unavailable");
|
||||
},
|
||||
async () => {
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx,
|
||||
cfg: { tools: { media: { image: { enabled: false } } } },
|
||||
activeModel: { provider: "openai", model: "gpt-4.1" },
|
||||
});
|
||||
|
||||
const imageDecision = result.decisions.find((d) => d.capability === "image");
|
||||
expect(imageDecision).toMatchObject({
|
||||
outcome: "disabled",
|
||||
attachmentDispositions: { 0: { kind: "capability-disabled" } },
|
||||
});
|
||||
// Probe failure leaves the flag unknown: no false delivery claim, no marker.
|
||||
expect(imageDecision).not.toHaveProperty("nativeVisionActive");
|
||||
expect(ctx.Body).not.toContain("not analyzed");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("renders disabled markers when the active model has no native vision", async () => {
|
||||
const ctx: MsgContext = {
|
||||
Body: "inspect this image",
|
||||
media: [{ path: "/tmp/image.png", contentType: "image/png" }],
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({
|
||||
ctx,
|
||||
cfg: { tools: { media: { image: { enabled: false } } } },
|
||||
});
|
||||
|
||||
expect(result.decisions).toContainEqual(
|
||||
expect.objectContaining({
|
||||
capability: "image",
|
||||
outcome: "disabled",
|
||||
nativeVisionActive: false,
|
||||
attachmentDispositions: { 0: { kind: "capability-disabled" } },
|
||||
}),
|
||||
);
|
||||
expect(ctx.Body).toContain("[Image attachment not analyzed: image understanding is disabled]");
|
||||
});
|
||||
|
||||
it("skips agents.defaults.imageModel fallback when MiniMax M3 supports vision", async () => {
|
||||
catalog = [
|
||||
...baseCatalog,
|
||||
@@ -297,6 +458,10 @@ describe("runCapability image skip", () => {
|
||||
let describeCalls = 0;
|
||||
const msgCtx = ctx as MsgContext;
|
||||
msgCtx.Body = "please inspect this explicit image";
|
||||
msgCtx.media = Array.from({ length: 4 }, () => ({
|
||||
path: mediaPath,
|
||||
contentType: "image/png",
|
||||
}));
|
||||
const cfg = {
|
||||
tools: {
|
||||
media: {
|
||||
@@ -332,8 +497,17 @@ describe("runCapability image skip", () => {
|
||||
const imageDecision = result.decisions.find((decision) => decision.capability === "image");
|
||||
expect(result.appliedImage).toBe(true);
|
||||
expect(imageDecision?.outcome).toBe("success");
|
||||
expect(imageDecision).toMatchObject({
|
||||
nativeVisionActive: true,
|
||||
attachmentDispositions: {
|
||||
1: { kind: "not-selected" },
|
||||
2: { kind: "not-selected" },
|
||||
3: { kind: "not-selected" },
|
||||
},
|
||||
});
|
||||
expect(describeCalls).toBe(1);
|
||||
expect(msgCtx.Body).toContain(plantedVisionSentinel);
|
||||
expect(msgCtx.Body).not.toContain("attachment limit reached");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -135,7 +135,13 @@ describe("media-understanding runtime", () => {
|
||||
provider: undefined,
|
||||
model: undefined,
|
||||
output: undefined,
|
||||
decision: { capability: "image", outcome: "disabled", attachments: [] },
|
||||
decision: {
|
||||
capability: "image",
|
||||
outcome: "disabled",
|
||||
attachments: [],
|
||||
attachmentDispositions: { 0: { kind: "capability-disabled" } },
|
||||
nativeVisionActive: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(mocks.buildProviderRegistry).not.toHaveBeenCalled();
|
||||
|
||||
@@ -177,10 +177,19 @@ export async function runMediaUnderstandingFile(
|
||||
scopeContext: params.scopeContext,
|
||||
});
|
||||
const attachments = normalizeMediaAttachments(ctx);
|
||||
const decisionBase = {
|
||||
capability: params.capability,
|
||||
attachments: [],
|
||||
...(params.capability === "image" ? { nativeVisionActive: false } : {}),
|
||||
};
|
||||
if (attachments.length === 0) {
|
||||
return {
|
||||
text: undefined,
|
||||
decision: { capability: params.capability, outcome: "no-attachment", attachments: [] },
|
||||
decision: {
|
||||
...decisionBase,
|
||||
outcome: "no-attachment",
|
||||
attachmentDispositions: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
const config = cfg.tools?.media?.[params.capability];
|
||||
@@ -190,10 +199,18 @@ export async function runMediaUnderstandingFile(
|
||||
provider: undefined,
|
||||
model: undefined,
|
||||
output: undefined,
|
||||
decision: { capability: params.capability, outcome: "disabled", attachments: [] },
|
||||
decision: {
|
||||
...decisionBase,
|
||||
outcome: "disabled",
|
||||
attachmentDispositions: Object.fromEntries(
|
||||
attachments.map((attachment) => [
|
||||
attachment.index,
|
||||
{ kind: "capability-disabled" as const },
|
||||
]),
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const providerRegistry = buildProviderRegistry(undefined, cfg);
|
||||
const agentDir =
|
||||
params.agentDir ?? (params.agentId ? resolveAgentDir(cfg, params.agentId) : undefined);
|
||||
|
||||
@@ -44,10 +44,25 @@ type MediaUnderstandingAttachmentDecision = {
|
||||
chosen?: MediaUnderstandingModelDecision;
|
||||
};
|
||||
|
||||
export type MediaAttachmentDisposition =
|
||||
| { kind: "handled" }
|
||||
| { kind: "handed-to-native-vision" }
|
||||
| { kind: "not-selected" }
|
||||
| { kind: "capability-disabled" }
|
||||
| { kind: "no-model" }
|
||||
| { kind: "scope-denied" }
|
||||
| { kind: "failed"; reason?: string };
|
||||
|
||||
export type MediaUnderstandingDecision = {
|
||||
capability: MediaUnderstandingCapability;
|
||||
outcome: MediaUnderstandingDecisionOutcome;
|
||||
attachments: MediaUnderstandingAttachmentDecision[];
|
||||
// Optional on the shipped SDK contract: plugins pass FinalizedMsgContext into
|
||||
// inbound-reply dispatch and may hold legacy decision literals. Core producers
|
||||
// (runner, apply-capability, runtime) always populate it; absence renders no
|
||||
// markers rather than breaking plugin compilation.
|
||||
attachmentDispositions?: Record<number, MediaAttachmentDisposition>;
|
||||
nativeVisionActive?: boolean;
|
||||
};
|
||||
|
||||
type MediaUnderstandingProviderRequestAuthOverride =
|
||||
|
||||
Reference in New Issue
Block a user