dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-13 17:07:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): transcode renditions and waveform audio cards (#116115)

Browse Source 
* feat(ui): add chat media renditions and waveforms

* fix(ui): tighten media playback guards

* fix(ui): bound waveform decode memory

* fix(ui): harden media playback retries

* fix(ui): preserve valid media state

* fix(ui): stop detached video playback

* fix(ui): bound media artifact resolution

* fix(ui): reject failed media fallback sources

* refactor(ui): split attachment availability helpers

* fix(ui): unload media across auth resets

* fix(ui): retain the longest-lived media ticket
This commit is contained in:
Peter Steinberger
2026-07-29 16:44:05 -07:00
committed by GitHub
parent cdab176484
commit d60436a4ee
24 changed files with 3144 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ui/src/i18n/locales/en.ts
View File

@@ -4844,6 +4844,7 @@ export const en: TranslationMap = {
pause: "Pause",
seek: "Seek media",
download: "Download {filename}",
preparing: "Preparing playback…",
videoUnavailable: "Can't play this format — download instead.",
},
modelControls: {

4
ui/src/lib/chat/chat-types.ts
View File

@@ -119,6 +119,10 @@ export type MessageContentItem =
label: string;
mimeType?: string;
isVoiceNote?: boolean;
artifactId?: string;
playback?: "native" | "transcode";
sizeBytes?: number;
durationMs?: number;
width?: number;
height?: number;
};

36
ui/src/lib/chat/message-normalizer.test.ts
View File

@@ -242,6 +242,42 @@ describe("message-normalizer", () => {
]);
});
it("preserves managed media playback and artifact metadata", () => {
const result = normalizeMessage({
role: "assistant",
content: [
{
type: "audio",
artifactId: "artifact_managed_media_audio",
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full",
fileName: "voice.caf",
mimeType: "audio/x-caf",
playback: "transcode",
sizeBytes: 4096,
durationMs: 2_345,
isVoiceNote: true,
},
],
});
expect(result.content).toEqual([
{
type: "attachment",
attachment: {
artifactId: "artifact_managed_media_audio",
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full",
kind: "audio",
label: "voice.caf",
mimeType: "audio/x-caf",
playback: "transcode",
sizeBytes: 4096,
durationMs: 2_345,
isVoiceNote: true,
},
},
]);
});
it("does not normalize non-assistant structured audio blocks as attachments", () => {
const result = normalizeMessage({
role: "user",

66
ui/src/lib/chat/message-normalizer.ts
View File

@@ -294,6 +294,52 @@ function coerceAudioContentBlock(
return null;
}
function coerceManagedMediaContentBlock(
item: Record<string, unknown>,
): Extract<MessageContentItem, { type: "attachment" }> | null {
if ((item.type !== "audio" && item.type !== "video") || typeof item.url !== "string") {
return null;
}
const url = item.url.trim();
if (!url) {
return null;
}
const kind = item.type;
const fallbackLabel = kind === "audio" ? "Audio" : "Video";
const label =
typeof item.fileName === "string" && item.fileName.trim()
? item.fileName.trim()
: typeof item.label === "string" && item.label.trim()
? item.label.trim()
: fallbackLabel;
return {
type: "attachment",
attachment: {
url,
kind,
label,
...(typeof item.mimeType === "string" ? { mimeType: item.mimeType } : {}),
...(typeof item.artifactId === "string" ? { artifactId: item.artifactId } : {}),
...(kind === "audio" && item.isVoiceNote === true ? { isVoiceNote: true } : {}),
...(item.playback === "native" || item.playback === "transcode"
? { playback: item.playback }
: {}),
...(typeof item.sizeBytes === "number" && item.sizeBytes >= 0
? { sizeBytes: item.sizeBytes }
: {}),
...(typeof item.durationMs === "number" && item.durationMs >= 0
? { durationMs: item.durationMs }
: {}),
...(kind === "video" && typeof item.width === "number" && item.width > 0
? { width: item.width }
: {}),
...(kind === "video" && typeof item.height === "number" && item.height > 0
? { height: item.height }
: {}),
},
};
}
function mergeAdjacentTextItems(items: MessageContentItem[]): MessageContentItem[] {
const merged: MessageContentItem[] = [];
for (const item of items) {
@@ -444,6 +490,10 @@ export function normalizeMessage(message: unknown): NormalizedMessage {
} else if (contentItems) {
content = contentItems.flatMap((item) => {
if (isAssistantMessage) {
const managedMediaAttachment = coerceManagedMediaContentBlock(item);
if (managedMediaAttachment) {
return [managedMediaAttachment];
}
const audioAttachment = coerceAudioContentBlock(item);
if (audioAttachment) {
return [audioAttachment];
@@ -463,6 +513,10 @@ export function normalizeMessage(message: unknown): NormalizedMessage {
label?: unknown;
mimeType?: unknown;
isVoiceNote?: unknown;
artifactId?: unknown;
playback?: unknown;
sizeBytes?: unknown;
durationMs?: unknown;
width?: unknown;
height?: unknown;
};
@@ -485,6 +539,18 @@ export function normalizeMessage(message: unknown): NormalizedMessage {
label: attachment.label,
...(typeof attachment.mimeType === "string" ? { mimeType: attachment.mimeType } : {}),
...(attachment.isVoiceNote === true ? { isVoiceNote: true } : {}),
...(typeof attachment.artifactId === "string"
? { artifactId: attachment.artifactId }
: {}),
...(attachment.playback === "native" || attachment.playback === "transcode"
? { playback: attachment.playback }
: {}),
...(typeof attachment.sizeBytes === "number" && attachment.sizeBytes >= 0
? { sizeBytes: attachment.sizeBytes }
: {}),
...(typeof attachment.durationMs === "number" && attachment.durationMs >= 0
? { durationMs: attachment.durationMs }
: {}),
...(typeof attachment.width === "number" && attachment.width > 0
? { width: attachment.width }
: {}),

7
ui/src/pages/chat/chat-pane-state.test.ts
View File

@@ -86,13 +86,13 @@ describe("applySelectedSessionProjection", () => {
describe("resolveChatArtifactDownload", () => {
it("returns a trimmed ticket without exposing a gateway bearer credential", async () => {
const requests: Array<{ method: string; params: unknown }> = [];
const requests: Array<{ method: string; params: unknown; options: unknown }> = [];
const result = await resolveChatArtifactDownload(
{
connected: true,
client: {
request: async (method: string, params: unknown) => {
requests.push({ method, params });
request: async (method: string, params: unknown, options: unknown) => {
requests.push({ method, params, options });
return {
artifact: {
id: "artifact-1",
@@ -113,6 +113,7 @@ describe("resolveChatArtifactDownload", () => {
{
method: "artifacts.download",
params: { sessionKey: "agent:main:main", artifactId: "artifact-1" },
options: { timeoutMs: 30_000 },
},
]);
expect(result).toEqual({

2
ui/src/pages/chat/chat-pane-state.ts
View File

@@ -22,6 +22,7 @@ export function applySelectedSessionProjection(
}
const MAX_TRACKED_SESSION_ROWS = 256;
const CHAT_ARTIFACT_DOWNLOAD_TIMEOUT_MS = 30_000;
export class SessionParticipationTracker {
private readonly lastBlocked = new Map<string, boolean>();
@@ -94,6 +95,7 @@ export async function resolveChatArtifactDownload(
const result = await state.client.request<ArtifactDownloadResult | null>(
"artifacts.download",
params,
{ timeoutMs: CHAT_ARTIFACT_DOWNLOAD_TIMEOUT_MS },
);
const url = typeof result?.url === "string" ? result.url.trim() : "";
if (!url) {

459
ui/src/pages/chat/components/chat-audio-player.test.ts
View File

@@ -2,6 +2,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import "./chat-audio-player.ts";
import { CHAT_AUDIO_WAVEFORM_MAX_BYTES } from "./chat-audio-waveform.ts";
type ChatAudioPlayer = HTMLElementTagNameMap["openclaw-chat-audio-player"];
@@ -25,6 +26,8 @@ async function createPlayer(label: string): Promise<ChatAudioPlayer> {
afterEach(() => {
document.body.replaceChildren();
vi.useRealTimers();
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
@@ -71,10 +74,11 @@ describe("ChatAudioPlayer", () => {
expect(player.querySelector(".chat-audio-player__time")?.textContent).toContain("2:05");
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await player.updateComplete;
expect(play).toHaveBeenCalledOnce();
expect(player.querySelector(".chat-audio-player__toggle")?.getAttribute("aria-label")).toBe(
"Pause",
await vi.waitFor(() => expect(play).toHaveBeenCalledOnce());
await vi.waitFor(() =>
expect(player.querySelector(".chat-audio-player__toggle")?.getAttribute("aria-label")).toBe(
"Pause",
),
);
expect(
player
@@ -153,6 +157,453 @@ describe("ChatAudioPlayer", () => {
expect(pauseFirst).not.toHaveBeenCalled();
});
it("shows preparing, retries a 202 rendition, and then plays", async () => {
vi.useFakeTimers();
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 202 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket";
player.sourceIdentity = "/tmp/voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await player.updateComplete;
await Promise.resolve();
await vi.advanceTimersByTimeAsync(0);
await player.updateComplete;
expect(player.textContent).toContain("Preparing playback…");
await vi.advanceTimersByTimeAsync(2_000);
await player.updateComplete;
const media = player.querySelector("audio")!;
let paused = true;
Object.defineProperty(media, "paused", { configurable: true, get: () => paused });
const play = vi.spyOn(media, "play").mockImplementation(async () => {
paused = false;
media.dispatchEvent(new Event("play"));
});
expect(media.getAttribute("src")).toContain("mediaTicket=ticket&playback=1");
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.advanceTimersByTimeAsync(0);
await player.updateComplete;
expect(play).toHaveBeenCalledOnce();
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("falls back to download after rendition retries are exhausted", async () => {
vi.useFakeTimers();
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(async () => new Response(null, { status: 202 })),
);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket";
player.sourceIdentity = "/tmp/voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await player.updateComplete;
await vi.runAllTimersAsync();
await player.updateComplete;
expect(player.querySelector(".chat-assistant-attachment-card__reason")?.textContent).toContain(
"Can't play this format — download instead.",
);
expect(
player
.querySelector<HTMLAnchorElement>(".chat-assistant-attachment-card__reason a")
?.getAttribute("href"),
).not.toContain("playback=1");
});
it("does not retain an errored source when a refreshed rendition is unavailable", async () => {
let resolveFirstRefresh: ((response: Response) => void) | undefined;
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockImplementationOnce(
async () =>
await new Promise<Response>((resolve) => {
resolveFirstRefresh = resolve;
}),
)
.mockResolvedValueOnce(new Response(null, { status: 500 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=old";
player.sourceIdentity = "/tmp/voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() =>
expect(player.querySelector("audio")?.getAttribute("src")).toContain("mediaTicket=old"),
);
player.querySelector("audio")?.dispatchEvent(new Event("error"));
await player.updateComplete;
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=refresh-1";
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=refresh-2";
await vi.waitFor(() =>
expect(
player.querySelector(".chat-assistant-attachment-card__reason")?.textContent,
).toContain("Can't play this format — download instead."),
);
expect(fetchMock).toHaveBeenCalledTimes(3);
resolveFirstRefresh?.(new Response(null, { status: 200 }));
});
it("reuses the waveform fetch as the audio element Blob source", async () => {
const samples = new Float32Array([0, 0.5, -1, 0.25]);
const decodeAudioData = vi.fn(async () => ({
duration: 4,
length: samples.length,
numberOfChannels: 1,
getChannelData: () => samples,
}));
const close = vi.fn(async () => undefined);
vi.stubGlobal(
"AudioContext",
class {
decodeAudioData = decodeAudioData;
close = close;
},
);
let resolveFetch: ((response: Response) => void) | undefined;
const fetchMock = vi.fn<typeof fetch>(
async () =>
await new Promise<Response>((resolve) => {
resolveFetch = resolve;
}),
);
vi.stubGlobal("fetch", fetchMock);
vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:waveform-audio");
vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => undefined);
const player = await createPlayer("waveform-reuse");
player.serverDurationMs = 4_000;
const media = player.querySelector("audio")!;
let paused = true;
Object.defineProperty(media, "paused", { configurable: true, get: () => paused });
const play = vi.spyOn(media, "play").mockImplementation(async () => {
paused = false;
media.dispatchEvent(new Event("play"));
});
vi.spyOn(media, "pause").mockImplementation(() => {
paused = true;
media.dispatchEvent(new Event("pause"));
});
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
expect(play).toHaveBeenCalledOnce();
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce());
resolveFetch?.(
new Response(new Uint8Array([1, 2, 3, 4]), {
status: 200,
headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" },
}),
);
await vi.waitFor(() =>
expect(player.querySelectorAll(".chat-audio-player__waveform rect")).toHaveLength(96),
);
await player.updateComplete;
expect(fetchMock).toHaveBeenCalledOnce();
expect(decodeAudioData).toHaveBeenCalledOnce();
expect(media.getAttribute("src")).toBe("https://example.com/waveform-reuse.mp3");
media.pause();
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.waitFor(() => expect(play).toHaveBeenCalledTimes(2));
expect(media.getAttribute("src")).toBe("blob:waveform-audio");
expect(fetchMock).toHaveBeenCalledOnce();
player.authToken = "new-principal";
await player.updateComplete;
expect(media.getAttribute("src")).toBe("https://example.com/waveform-reuse.mp3");
player.remove();
const refreshed = document.createElement("openclaw-chat-audio-player");
refreshed.src = "https://example.com/waveform-reuse.mp3?mediaTicket=fresh";
refreshed.sourceIdentity = "media://waveform-reuse";
refreshed.authToken = "different-principal";
refreshed.label = "waveform-reuse.mp3";
refreshed.serverDurationMs = 4_000;
document.body.append(refreshed);
await refreshed.updateComplete;
const refreshedMedia = refreshed.querySelector("audio")!;
Object.defineProperty(refreshedMedia, "paused", { configurable: true, value: true });
vi.spyOn(refreshedMedia, "play").mockResolvedValue(undefined);
refreshed.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
resolveFetch?.(new Response(null, { status: 500 }));
});
it("shows preparation while the initial rendition HEAD is pending", async () => {
let resolveFetch: ((response: Response) => void) | undefined;
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(
async () =>
await new Promise<Response>((resolve) => {
resolveFetch = resolve;
}),
),
);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket";
player.sourceIdentity = "/tmp/voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…"));
resolveFetch?.(new Response(null, { status: 500 }));
});
it("skips waveform work without a server-probed duration", async () => {
const decodeAudioData = vi.fn();
vi.stubGlobal(
"AudioContext",
class {
decodeAudioData = decodeAudioData;
close = vi.fn(async () => undefined);
},
);
const fetchMock = vi.fn<typeof fetch>();
vi.stubGlobal("fetch", fetchMock);
const player = await createPlayer("unknown-duration");
const media = player.querySelector("audio")!;
Object.defineProperty(media, "paused", { configurable: true, value: true });
vi.spyOn(media, "play").mockResolvedValue(undefined);
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.waitFor(() =>
expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(),
);
expect(fetchMock).not.toHaveBeenCalled();
expect(decodeAudioData).not.toHaveBeenCalled();
expect(player.querySelector(".chat-audio-player__waveform")).toBeNull();
});
it("discards waveform peaks when decoded duration exceeds the server gate by 20 percent", async () => {
const samples = new Float32Array([0, 0.5, -1, 0.25]);
const decodeAudioData = vi.fn(async () => ({
duration: 121,
length: samples.length,
numberOfChannels: 1,
getChannelData: () => samples,
}));
const AudioContextMock = vi.fn(function (options?: AudioContextOptions) {
expect(options?.sampleRate).toBe(16_000);
return { decodeAudioData, close: vi.fn(async () => undefined) };
});
vi.stubGlobal("AudioContext", AudioContextMock);
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(
async () =>
new Response(new Uint8Array([1, 2, 3, 4]), {
status: 200,
headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" },
}),
),
);
vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:duration-mismatch");
const player = await createPlayer("duration-mismatch");
player.serverDurationMs = 100_000;
const media = player.querySelector("audio")!;
Object.defineProperty(media, "paused", { configurable: true, value: true });
setMediaNumber(media, "duration", 100);
media.dispatchEvent(new Event("loadedmetadata"));
await player.updateComplete;
vi.spyOn(media, "play").mockResolvedValue(undefined);
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.waitFor(() =>
expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(),
);
expect(decodeAudioData).toHaveBeenCalledOnce();
expect(player.querySelector(".chat-audio-player__waveform")).toBeNull();
expect(
Array.from(player.querySelectorAll(".chat-audio-player__time span"), (item) =>
item.textContent?.trim(),
),
).toEqual(["0:00", "1:40"]);
});
it("does not buffer or cache a chunked waveform response above 8 MiB", async () => {
const decodeAudioData = vi.fn();
vi.stubGlobal(
"AudioContext",
class {
decodeAudioData = decodeAudioData;
close = vi.fn(async () => undefined);
},
);
const fetchMock = vi.fn<typeof fetch>(
async () =>
new Response(
new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(CHAT_AUDIO_WAVEFORM_MAX_BYTES + 1));
controller.close();
},
}),
{ status: 200, headers: { "Content-Type": "audio/mpeg" } },
),
);
vi.stubGlobal("fetch", fetchMock);
const createObjectURL = vi.spyOn(URL, "createObjectURL");
const player = await createPlayer("oversized-waveform");
player.serverDurationMs = 4_000;
const media = player.querySelector("audio")!;
Object.defineProperty(media, "paused", { configurable: true, value: true });
const play = vi.spyOn(media, "play").mockResolvedValue(undefined);
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
expect(play).toHaveBeenCalledOnce();
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce());
await vi.waitFor(() =>
expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(),
);
expect(createObjectURL).not.toHaveBeenCalled();
expect(decodeAudioData).not.toHaveBeenCalled();
expect(player.querySelector(".chat-audio-player__waveform")).toBeNull();
expect(media.getAttribute("src")).toBe("https://example.com/oversized-waveform.mp3");
});
it("transfers Blob ownership when AudioContext construction fails", async () => {
const FailingAudioContext = vi.fn(function () {
throw new Error("audio context limit reached");
});
vi.stubGlobal("AudioContext", FailingAudioContext);
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(
async () =>
new Response(new Uint8Array([1, 2, 3, 4]), {
status: 200,
headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" },
}),
),
);
vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:context-fallback");
const player = await createPlayer("context-fallback");
player.serverDurationMs = 4_000;
const media = player.querySelector("audio")!;
let paused = true;
Object.defineProperty(media, "paused", { configurable: true, get: () => paused });
const play = vi.spyOn(media, "play").mockImplementation(async () => {
paused = false;
media.dispatchEvent(new Event("play"));
});
vi.spyOn(media, "pause").mockImplementation(() => {
paused = true;
media.dispatchEvent(new Event("pause"));
});
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
await vi.waitFor(() =>
expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(),
);
media.pause();
player.querySelector<HTMLButtonElement>(".chat-audio-player__toggle")!.click();
expect(play).toHaveBeenCalledTimes(2);
expect(media.getAttribute("src")).toBe("blob:context-fallback");
});
it("restarts transcode readiness after disconnecting during preparation", async () => {
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 202 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket";
player.sourceIdentity = "/tmp/voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce());
player.remove();
document.body.append(player);
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
await vi.waitFor(() =>
expect(player.querySelector("audio")?.getAttribute("src")).toContain("playback=1"),
);
});
it("cancels the old source when identity changes during rendition preparation", async () => {
let resolveFetch: ((response: Response) => void) | undefined;
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(
async () =>
await new Promise<Response>((resolve) => {
resolveFetch = resolve;
}),
),
);
const player = await createPlayer("identity-before");
const media = player.querySelector("audio")!;
let paused = false;
Object.defineProperty(media, "paused", { configurable: true, get: () => paused });
setMediaNumber(media, "currentTime", 20);
setMediaNumber(media, "duration", 80);
const play = vi.spyOn(media, "play").mockResolvedValue(undefined);
const pause = vi.spyOn(media, "pause").mockImplementation(() => {
paused = true;
});
player.src = "/__openclaw__/assistant-media?source=after.caf&mediaTicket=ticket";
player.sourceIdentity = "media://identity-after";
player.label = "after.caf";
player.playback = "transcode";
await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…"));
expect(pause).toHaveBeenCalledOnce();
expect(media.hasAttribute("src")).toBe(false);
resolveFetch?.(new Response(null, { status: 200 }));
await vi.waitFor(() => expect(media.getAttribute("src")).toContain("playback=1"));
media.dispatchEvent(new Event("loadedmetadata"));
expect(play).not.toHaveBeenCalled();
});
it("clears a failed rendition after reconnect succeeds", async () => {
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 500 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-audio-player");
player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket";
player.sourceIdentity = "/tmp/retry-voice.caf";
player.label = "voice.caf";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() =>
expect(
player.querySelector(".chat-assistant-attachment-card__reason")?.textContent,
).toContain("Can't play this format"),
);
player.remove();
document.body.append(player);
await vi.waitFor(() =>
expect(player.querySelector("audio")?.getAttribute("src")).toContain("playback=1"),
);
expect(player.textContent).not.toContain("Can't play this format — download instead.");
});
it("does not auto-resume a refreshed source after disconnecting before metadata", async () => {
const player = await createPlayer("disconnect-resume");
const media = player.querySelector("audio")!;

480
ui/src/pages/chat/components/chat-audio-player.ts
View File

@@ -1,4 +1,4 @@
import { html, type PropertyValues } from "lit";
import { html, svg, type PropertyValues } from "lit";
import { property, state } from "lit/decorators.js";
import { ref } from "lit/directives/ref.js";
import { styleMap } from "lit/directives/style-map.js";
@@ -11,9 +11,69 @@ import {
claimChatAudioPlayback,
releaseChatAudioPlayback,
} from "./chat-audio-coordinator.ts";
import {
cacheAndRetainChatAudioBlob,
canDecodeChatAudioWaveform,
CHAT_AUDIO_WAVEFORM_MAX_BYTES,
CHAT_AUDIO_WAVEFORM_SAMPLE_RATE,
computeChatAudioWaveformPeaks,
retainCachedChatAudioBlob,
shouldFetchChatAudioWaveform,
type CachedChatAudioBlob,
} from "./chat-audio-waveform.ts";
import {
appendChatMediaPlaybackParam,
buildChatMediaFetchHeaders,
waitForChatMediaPlayback,
type ChatMediaPlaybackMode,
} from "./chat-media-playback.ts";
import { ChatMediaSourceController } from "./chat-media-source.ts";
const SEEK_STEP_SECONDS = 5;
const WAVEFORM_FETCH_TIMEOUT_MS = 30_000;
const WAVEFORM_DECODE_DURATION_TOLERANCE = 1.2;
async function readResponseBytesWithinLimit(
response: Response,
maxBytes: number,
): Promise<ArrayBuffer | null> {
const contentLengthHeader = response.headers.get("Content-Length");
const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader);
if (contentLength !== undefined && Number.isFinite(contentLength) && contentLength > maxBytes) {
await response.body?.cancel().catch(() => undefined);
return null;
}
if (!response.body) {
const bytes = await response.arrayBuffer();
return bytes.byteLength <= maxBytes ? bytes : null;
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let totalBytes = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
totalBytes += value.byteLength;
if (totalBytes > maxBytes) {
await reader.cancel().catch(() => undefined);
return null;
}
chunks.push(value);
}
} finally {
reader.releaseLock();
}
const combined = new Uint8Array(totalBytes);
let offset = 0;
for (const chunk of chunks) {
combined.set(chunk, offset);
offset += chunk.byteLength;
}
return combined.buffer;
}
function formatChatMediaTime(seconds: number): string {
if (!Number.isFinite(seconds) || seconds < 0) {
@@ -29,6 +89,10 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
@property() src = "";
@property() sourceIdentity = "";
@property() label = "";
@property() playback: ChatMediaPlaybackMode = "native";
@property() authToken: string | null = null;
@property({ type: Number }) sizeBytes: number | undefined;
@property({ type: Number }) serverDurationMs: number | undefined;
@property({ type: Boolean }) voiceNote = false;
@property({ attribute: false }) onMediaLoaded: (() => void) | undefined;
@@ -37,12 +101,40 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
@state() private buffered = 0;
@state() private playing = false;
@state() private failed = false;
@state() private preparing = false;
@state() private playbackReady = true;
@state() private waveformPeaks: readonly number[] | null = null;
private media: HTMLAudioElement | null = null;
private readonly sourceController = new ChatMediaSourceController();
private currentSourceFailed = false;
private readonly cancelPendingResume = () => this.sourceController.cancelPendingResume();
private readinessController: AbortController | null = null;
private readinessKey = "";
private readinessPromise: Promise<boolean> | null = null;
private readySource = "";
private playRequest: Promise<void> | null = null;
private releaseWaveformBlob: (() => void) | undefined;
private waveformController: AbortController | null = null;
private waveformAttempted = false;
override connectedCallback(): void {
super.connectedCallback();
queueMicrotask(() => this.syncSource());
}
override disconnectedCallback(): void {
this.readinessController?.abort();
this.readinessController = null;
this.readinessKey = "";
this.readinessPromise = null;
this.readySource = "";
this.playbackReady = this.playback !== "transcode";
this.releaseWaveformBlob?.();
this.releaseWaveformBlob = undefined;
this.waveformController?.abort();
this.waveformController = null;
this.waveformAttempted = false;
if (this.media) {
this.sourceController.cancelPendingResume();
if (!this.media.paused) {
@@ -54,29 +146,294 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
}
override updated(changedProperties: PropertyValues<this>): void {
if (changedProperties.has("src")) {
this.failed = false;
}
if (this.media) {
this.sourceController.updateSource(this.media, this.src, this.sourceIdentity);
if (
changedProperties.has("src") ||
changedProperties.has("sourceIdentity") ||
changedProperties.has("playback") ||
changedProperties.has("authToken")
) {
const sourceIdentityChanged =
changedProperties.has("sourceIdentity") &&
Boolean(this.sourceController.currentIdentity) &&
this.sourceController.currentIdentity !== this.sourceIdentity.trim();
if (
sourceIdentityChanged ||
changedProperties.has("playback") ||
changedProperties.has("authToken")
) {
this.waveformController?.abort();
this.waveformController = null;
this.releaseWaveformBlob?.();
this.releaseWaveformBlob = undefined;
this.waveformPeaks = null;
this.waveformAttempted = false;
this.currentTime = 0;
this.duration = 0;
this.buffered = 0;
if (this.media && !this.media.paused) {
this.media.pause();
releaseChatAudioPlayback(this.media);
}
if ((sourceIdentityChanged || changedProperties.has("authToken")) && this.media) {
this.sourceController.reset(this.media);
this.currentSourceFailed = false;
}
}
this.syncSource();
}
}
private setMedia = (element: Element | undefined) => {
this.media = element instanceof HTMLAudioElement ? element : null;
this.syncSource();
};
private togglePlayback(): void {
private syncSource(): void {
const media = this.media;
const source = this.src.trim();
if (!media || !source || !this.sourceIdentity.trim() || !this.isConnected) {
return;
}
if (this.releaseWaveformBlob) {
return;
}
const playbackSource =
this.playback === "transcode" ? appendChatMediaPlaybackParam(source) : source;
const hasCurrentAttachmentSource =
this.sourceController.currentIdentity === this.sourceIdentity.trim();
const hasUsableCurrentAttachmentSource =
hasCurrentAttachmentSource && !this.currentSourceFailed;
if (this.playback !== "transcode") {
this.readinessController?.abort();
this.readinessController = null;
this.readinessKey = "";
this.readinessPromise = null;
this.preparing = false;
this.playbackReady = true;
this.readySource = playbackSource;
this.failed = false;
this.currentSourceFailed = false;
this.sourceController.updateSource(media, playbackSource, this.sourceIdentity);
return;
}
const readinessKey = `${playbackSource}\0${this.authToken?.trim() ?? ""}`;
if (readinessKey === this.readinessKey && this.readinessController && this.readinessPromise) {
return;
}
this.readinessController?.abort();
const controller = new AbortController();
this.readinessController = controller;
this.readinessKey = readinessKey;
this.readySource = "";
this.preparing = !hasUsableCurrentAttachmentSource;
this.playbackReady = hasUsableCurrentAttachmentSource;
this.failed = false;
const pending = waitForChatMediaPlayback({
source: playbackSource,
authToken: this.authToken,
signal: controller.signal,
onPreparing: () => {
if (this.readinessController === controller && !hasUsableCurrentAttachmentSource) {
this.preparing = true;
}
},
}).then((result) => {
if (this.readinessController !== controller || result === "aborted") {
return false;
}
this.preparing = false;
if (result !== "ready") {
if (hasUsableCurrentAttachmentSource) {
this.readySource = this.sourceController.currentSource;
this.playbackReady = true;
return true;
}
this.failed = true;
this.playbackReady = false;
return false;
}
this.readySource = playbackSource;
this.playbackReady = true;
this.failed = false;
this.currentSourceFailed = false;
this.sourceController.updateSource(media, playbackSource, this.sourceIdentity);
return true;
});
this.readinessPromise = pending;
}
private resolveWaveformCacheKey(): string {
return [
this.sourceIdentity.trim(),
this.playback,
this.src.trim(),
this.authToken?.trim() ?? "",
].join("\0");
}
private applyPreparedAudio(
cacheKey: string,
prepared: { value: CachedChatAudioBlob; release: () => void },
): void {
const media = this.media;
if (!media || cacheKey !== this.resolveWaveformCacheKey()) {
prepared.release();
return;
}
this.releaseWaveformBlob?.();
this.releaseWaveformBlob = prepared.release;
this.waveformPeaks = prepared.value.peaks ?? null;
if (prepared.value.durationSeconds !== undefined) {
this.duration = prepared.value.durationSeconds;
}
this.sourceController.updateSource(media, prepared.value.blobUrl, this.sourceIdentity);
}
private adoptPreparedAudioForPlayback(): void {
const media = this.media;
if (!media) {
return;
}
if (media.paused) {
claimChatAudioPlayback(media, this.cancelPendingResume);
void media.play().catch(() => {
releaseChatAudioPlayback(media);
this.playing = false;
if (!this.releaseWaveformBlob) {
const cached = retainCachedChatAudioBlob(this.resolveWaveformCacheKey());
if (cached) {
this.applyPreparedAudio(this.resolveWaveformCacheKey(), cached);
}
}
this.sourceController.applyPendingSource(media);
}
private async prepareWaveformAudio(): Promise<void> {
const media = this.media;
const source = this.readySource;
if (!media || !source || this.releaseWaveformBlob || this.waveformAttempted) {
return;
}
const cacheKey = this.resolveWaveformCacheKey();
const cached = retainCachedChatAudioBlob(cacheKey);
if (cached) {
this.applyPreparedAudio(cacheKey, cached);
return;
}
const durationSeconds =
this.serverDurationMs !== undefined ? this.serverDurationMs / 1_000 : undefined;
if (
durationSeconds === undefined ||
!shouldFetchChatAudioWaveform({ sizeBytes: this.sizeBytes, durationSeconds })
) {
return;
}
const AudioContextConstructor = globalThis.AudioContext;
if (!AudioContextConstructor) {
return;
}
this.waveformAttempted = true;
const headers = buildChatMediaFetchHeaders(this.authToken);
headers.set("Accept", "audio/*");
const controller = new AbortController();
this.waveformController = controller;
const timeout = setTimeout(
() => controller.abort(new DOMException("waveform fetch timed out", "TimeoutError")),
WAVEFORM_FETCH_TIMEOUT_MS,
);
let response: Response;
let bytes: ArrayBuffer;
try {
response = await fetch(source, {
method: "GET",
headers,
credentials: "same-origin",
signal: controller.signal,
});
if (!response.ok) {
return;
}
const boundedBytes = await readResponseBytesWithinLimit(
response,
CHAT_AUDIO_WAVEFORM_MAX_BYTES,
);
if (!boundedBytes) {
return;
}
bytes = boundedBytes;
} finally {
clearTimeout(timeout);
if (this.waveformController === controller) {
this.waveformController = null;
}
}
const blob = new Blob([bytes], {
type: response.headers.get("Content-Type")?.split(";", 1)[0]?.trim() || "audio/mpeg",
});
const blobUrl = URL.createObjectURL(blob);
let peaks: readonly number[] | undefined;
let acceptedDecodedDuration: number | undefined;
if (canDecodeChatAudioWaveform({ sizeBytes: bytes.byteLength, durationSeconds })) {
let context: AudioContext | null = null;
try {
// Duration is trusted only from the server-side ffprobe metadata.
// A 16 kHz decode bounds PCM; >20% duration mismatches are discarded.
context = new AudioContextConstructor({ sampleRate: CHAT_AUDIO_WAVEFORM_SAMPLE_RATE });
const decoded = await context.decodeAudioData(bytes.slice(0));
const decodedDuration = Number.isFinite(decoded.duration) ? decoded.duration : undefined;
if (
decodedDuration !== undefined &&
decodedDuration <= durationSeconds * WAVEFORM_DECODE_DURATION_TOLERANCE
) {
peaks = computeChatAudioWaveformPeaks(decoded);
acceptedDecodedDuration = decodedDuration;
}
} catch {
// A playable browser source can still use the fetched Blob when Web Audio cannot decode it.
} finally {
await context?.close().catch(() => undefined);
}
}
if (!this.isConnected || cacheKey !== this.resolveWaveformCacheKey()) {
URL.revokeObjectURL(blobUrl);
return;
}
const retained = cacheAndRetainChatAudioBlob(cacheKey, {
blobUrl,
sizeBytes: bytes.byteLength,
...(peaks ? { peaks } : {}),
...(acceptedDecodedDuration !== undefined
? { durationSeconds: acceptedDecodedDuration }
: {}),
});
if (retained) {
this.applyPreparedAudio(cacheKey, retained);
}
}
private togglePlayback(): void {
const media = this.media;
if (!media || !this.playbackReady) {
return;
}
if (media.paused) {
this.adoptPreparedAudioForPlayback();
claimChatAudioPlayback(media, this.cancelPendingResume);
const playback = media.play();
if (!this.playRequest) {
// Invoke play in the click task so strict browser media policies retain user activation.
this.playRequest = playback
.then(() => this.prepareWaveformAudio().catch(() => undefined))
.catch(() => {
releaseChatAudioPlayback(media);
this.playing = false;
})
.finally(() => {
this.playRequest = null;
});
} else {
void playback.catch(() => {
releaseChatAudioPlayback(media);
this.playing = false;
});
}
} else {
media.pause();
}
@@ -117,6 +474,46 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
this.buffered = Math.min(1, media.buffered.end(media.buffered.length - 1) / this.duration);
}
private renderSeek(progress: number) {
const seek = html`<input
class=${this.waveformPeaks
? "chat-audio-player__seek chat-audio-player__seek--waveform"
: "chat-audio-player__seek"}
type="range"
min="0"
max=${String(this.duration || 0)}
step=${String(SEEK_STEP_SECONDS)}
.value=${String(Math.min(this.currentTime, this.duration || this.currentTime))}
aria-label=${t("chat.mediaPlayer.seek")}
style=${styleMap({
"--chat-audio-progress": `${progress * 100}%`,
"--chat-audio-buffered": `${Math.max(progress, this.buffered) * 100}%`,
})}
@input=${(event: Event) =>
this.seekTo(Number((event.currentTarget as HTMLInputElement).value))}
/>`;
if (!this.waveformPeaks) {
return seek;
}
const count = this.waveformPeaks.length;
return html`<div class="chat-audio-player__waveform">
<svg viewBox="0 0 ${count} 24" preserveAspectRatio="none" aria-hidden="true">
${this.waveformPeaks.map((peak, index) => {
const height = Math.max(2, peak * 22);
return svg`<rect
class=${index / count < progress ? "is-played" : ""}
x=${String(index + 0.18)}
y=${String((24 - height) / 2)}
width="0.64"
height=${String(height)}
rx="0.3"
></rect>`;
})}
</svg>
${seek}
</div>`;
}
override render() {
const progress = this.duration > 0 ? Math.min(1, this.currentTime / this.duration) : 0;
const downloadHref = safeAttachmentHref(this.src);
@@ -160,41 +557,32 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
>`
: null}
</div> `
: html`<div
class="chat-audio-player"
tabindex="0"
@keydown=${(event: KeyboardEvent) => this.handlePlayerKeydown(event)}
>
<button
type="button"
class="chat-audio-player__toggle"
aria-label=${t(this.playing ? "chat.mediaPlayer.pause" : "chat.mediaPlayer.play")}
@click=${() => this.togglePlayback()}
: this.preparing
? html`<div class="chat-assistant-attachment-card__reason chat-media-preparing">
${t("chat.mediaPlayer.preparing")}
</div>`
: html`<div
class="chat-audio-player"
tabindex="0"
@keydown=${(event: KeyboardEvent) => this.handlePlayerKeydown(event)}
>
${this.playing ? icons.pause : icons.play}
</button>
<div class="chat-audio-player__timeline">
<input
class="chat-audio-player__seek"
type="range"
min="0"
max=${String(this.duration || 0)}
step=${String(SEEK_STEP_SECONDS)}
.value=${String(Math.min(this.currentTime, this.duration || this.currentTime))}
aria-label=${t("chat.mediaPlayer.seek")}
style=${styleMap({
"--chat-audio-progress": `${progress * 100}%`,
"--chat-audio-buffered": `${Math.max(progress, this.buffered) * 100}%`,
})}
@input=${(event: Event) =>
this.seekTo(Number((event.currentTarget as HTMLInputElement).value))}
/>
<div class="chat-audio-player__time" aria-live="off">
<span>${formatChatMediaTime(this.currentTime)}</span>
<span>${formatChatMediaTime(this.duration)}</span>
<button
type="button"
class="chat-audio-player__toggle"
?disabled=${!this.playbackReady}
aria-label=${t(this.playing ? "chat.mediaPlayer.pause" : "chat.mediaPlayer.play")}
@click=${() => this.togglePlayback()}
>
${this.playing ? icons.pause : icons.play}
</button>
<div class="chat-audio-player__timeline">
${this.renderSeek(progress)}
<div class="chat-audio-player__time" aria-live="off">
<span>${formatChatMediaTime(this.currentTime)}</span>
<span>${formatChatMediaTime(this.duration)}</span>
</div>
</div>
</div>
</div>`}
</div>`}
<audio
class="chat-audio-player__media"
preload="metadata"
@@ -209,6 +597,7 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
this.duration = Number.isFinite(this.media.duration) ? this.media.duration : 0;
this.currentTime = this.media.currentTime;
this.failed = false;
this.currentSourceFailed = false;
this.updateBuffered();
this.onMediaLoaded?.();
}}
@@ -245,6 +634,7 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement {
releaseChatAudioPlayback(this.media);
this.playing = false;
this.failed = true;
this.currentSourceFailed = true;
}
}}
></audio>

113
ui/src/pages/chat/components/chat-audio-waveform.test.ts Normal file
View File

@@ -0,0 +1,113 @@
import { describe, expect, it } from "vitest";
import {
cacheAndRetainChatAudioBlob,
canDecodeChatAudioWaveform,
CHAT_AUDIO_WAVEFORM_MAX_BYTES,
computeChatAudioWaveformPeaks,
shouldFetchChatAudioWaveform,
} from "./chat-audio-waveform.ts";
describe("chat audio waveform", () => {
it("caps waveform decoding by byte size and duration", () => {
expect(
canDecodeChatAudioWaveform({
sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES,
durationSeconds: 300,
}),
).toBe(true);
expect(canDecodeChatAudioWaveform({ sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES + 1 })).toBe(
false,
);
expect(
canDecodeChatAudioWaveform({
sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES,
durationSeconds: 301,
}),
).toBe(false);
expect(shouldFetchChatAudioWaveform({})).toBe(false);
expect(canDecodeChatAudioWaveform({ sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES })).toBe(false);
expect(
canDecodeChatAudioWaveform({
sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES,
durationSeconds: Number.NaN,
}),
).toBe(false);
});
it("computes normalized peak buckets", () => {
const channels = [new Float32Array([0.1, -0.5, 0.2, 0.4, -0.25, 0.75, 0, -1])];
const peaks = computeChatAudioWaveformPeaks(
{
length: channels[0]!.length,
numberOfChannels: channels.length,
getChannelData: (channel) => channels[channel]!,
},
4,
);
expect(peaks).toHaveLength(4);
expect(peaks[0]).toBeCloseTo(0.5);
expect(peaks[1]).toBeCloseTo(0.4);
expect(peaks[2]).toBeCloseTo(0.75);
expect(peaks[3]).toBe(1);
});
it("includes peaks carried only by a non-first channel", () => {
const channels = [new Float32Array([0, 0, 0, 0]), new Float32Array([0, 0.25, 0, 1])];
const peaks = computeChatAudioWaveformPeaks(
{
length: channels[0]!.length,
numberOfChannels: channels.length,
getChannelData: (channel) => channels[channel]!,
},
2,
);
expect(peaks).toEqual([0.25, 1]);
});
it("declines a new Blob when all 32 cache entries are retained", () => {
const releases: Array<() => void> = [];
for (let index = 0; index < 32; index += 1) {
const retained = cacheAndRetainChatAudioBlob(`retained-${index}`, {
blobUrl: `blob:retained-${index}`,
sizeBytes: 1,
});
expect(retained).not.toBeNull();
if (!retained) {
throw new Error("expected retained audio blob");
}
expect(retained.value.blobUrl).toBe(`blob:retained-${index}`);
releases.push(retained.release);
}
expect(
cacheAndRetainChatAudioBlob("retained-overflow", {
blobUrl: "blob:retained-overflow",
sizeBytes: 1,
}),
).toBeNull();
for (const release of releases) {
release();
}
});
it("caps retained audio bytes at 48 MiB", () => {
const releases = Array.from({ length: 6 }, (_, index) =>
cacheAndRetainChatAudioBlob(`large-${index}`, {
blobUrl: `blob:large-${index}`,
sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES,
}),
);
expect(releases.every((retained) => retained !== null)).toBe(true);
expect(
cacheAndRetainChatAudioBlob("large-overflow", {
blobUrl: "blob:large-overflow",
sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES,
}),
).toBeNull();
for (const retained of releases) {
retained?.release();
}
});
});

180
ui/src/pages/chat/components/chat-audio-waveform.ts Normal file
View File

@@ -0,0 +1,180 @@
export const CHAT_AUDIO_WAVEFORM_MAX_BYTES = 8 * 1024 * 1024;
export const CHAT_AUDIO_WAVEFORM_SAMPLE_RATE = 16_000;
const CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS = 5 * 60;
const CHAT_AUDIO_WAVEFORM_BUCKET_COUNT = 96;
type ChatAudioBufferLike = {
length: number;
numberOfChannels: number;
getChannelData(channel: number): Float32Array;
};
export type CachedChatAudioBlob = {
blobUrl: string;
sizeBytes: number;
peaks?: readonly number[];
durationSeconds?: number;
};
type ChatAudioBlobCacheEntry = CachedChatAudioBlob & { retainCount: number };
const chatAudioBlobCache = new Map<string, ChatAudioBlobCacheEntry>();
const CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES = 32;
const CHAT_AUDIO_BLOB_CACHE_MAX_BYTES = 48 * 1024 * 1024;
export function shouldFetchChatAudioWaveform(params: {
sizeBytes?: number;
durationSeconds?: number;
}): boolean {
return (
params.durationSeconds !== undefined &&
Number.isFinite(params.durationSeconds) &&
params.durationSeconds >= 0 &&
params.durationSeconds <= CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS &&
(params.sizeBytes === undefined ||
(Number.isFinite(params.sizeBytes) &&
params.sizeBytes >= 0 &&
params.sizeBytes <= CHAT_AUDIO_WAVEFORM_MAX_BYTES))
);
}
export function canDecodeChatAudioWaveform(params: {
sizeBytes: number;
durationSeconds?: number;
}): boolean {
return (
Number.isFinite(params.sizeBytes) &&
params.sizeBytes >= 0 &&
params.sizeBytes <= CHAT_AUDIO_WAVEFORM_MAX_BYTES &&
params.durationSeconds !== undefined &&
Number.isFinite(params.durationSeconds) &&
params.durationSeconds >= 0 &&
params.durationSeconds <= CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS
);
}
export function computeChatAudioWaveformPeaks(
buffer: ChatAudioBufferLike,
bucketCount = CHAT_AUDIO_WAVEFORM_BUCKET_COUNT,
): number[] {
const count = Math.max(1, Math.floor(bucketCount));
if (buffer.length <= 0 || buffer.numberOfChannels <= 0) {
return Array.from({ length: count }, () => 0);
}
const channels = Array.from({ length: buffer.numberOfChannels }, (_, channel) =>
buffer.getChannelData(channel),
);
const peaks = Array.from({ length: count }, () => 0);
for (let bucket = 0; bucket < count; bucket += 1) {
const start = Math.floor((bucket * buffer.length) / count);
const end = Math.max(start + 1, Math.floor(((bucket + 1) * buffer.length) / count));
let peak = 0;
for (const samples of channels) {
for (let index = start; index < Math.min(end, samples.length); index += 1) {
peak = Math.max(peak, Math.abs(samples[index] ?? 0));
}
}
peaks[bucket] = peak;
}
const maximum = Math.max(...peaks);
return maximum > 0 ? peaks.map((peak) => peak / maximum) : peaks;
}
function trimChatAudioBlobCache(): void {
while (
chatAudioBlobCache.size > CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES ||
[...chatAudioBlobCache.values()].reduce((total, entry) => total + entry.sizeBytes, 0) >
CHAT_AUDIO_BLOB_CACHE_MAX_BYTES
) {
const evictable = [...chatAudioBlobCache].find(([, entry]) => entry.retainCount === 0);
if (!evictable) {
return;
}
const [cacheKey, entry] = evictable;
chatAudioBlobCache.delete(cacheKey);
URL.revokeObjectURL(entry.blobUrl);
}
}
function makeRoomForChatAudioBlob(sizeBytes: number): boolean {
if (sizeBytes > CHAT_AUDIO_BLOB_CACHE_MAX_BYTES) {
return false;
}
const cachedBytes = () =>
[...chatAudioBlobCache.values()].reduce((total, entry) => total + entry.sizeBytes, 0);
while (
chatAudioBlobCache.size + 1 > CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES ||
cachedBytes() + sizeBytes > CHAT_AUDIO_BLOB_CACHE_MAX_BYTES
) {
const evictable = [...chatAudioBlobCache].find(([, entry]) => entry.retainCount === 0);
if (!evictable) {
return false;
}
const [cacheKey, entry] = evictable;
chatAudioBlobCache.delete(cacheKey);
URL.revokeObjectURL(entry.blobUrl);
}
return true;
}
export function retainCachedChatAudioBlob(
cacheKey: string,
): { value: CachedChatAudioBlob; release: () => void } | null {
const entry = chatAudioBlobCache.get(cacheKey);
if (!entry) {
return null;
}
chatAudioBlobCache.delete(cacheKey);
chatAudioBlobCache.set(cacheKey, entry);
entry.retainCount += 1;
let released = false;
return {
value: entry,
release: () => {
if (released) {
return;
}
released = true;
const current = chatAudioBlobCache.get(cacheKey);
if (current && current.retainCount > 0) {
current.retainCount -= 1;
}
trimChatAudioBlobCache();
},
};
}
export function cacheAndRetainChatAudioBlob(
cacheKey: string,
value: CachedChatAudioBlob,
): { value: CachedChatAudioBlob; release: () => void } | null {
const previous = chatAudioBlobCache.get(cacheKey);
if (previous) {
if (previous.blobUrl !== value.blobUrl) {
URL.revokeObjectURL(value.blobUrl);
}
return retainCachedChatAudioBlob(cacheKey)!;
}
if (!makeRoomForChatAudioBlob(value.sizeBytes)) {
URL.revokeObjectURL(value.blobUrl);
return null;
}
const entry = { ...value, retainCount: 1 };
chatAudioBlobCache.set(cacheKey, entry);
trimChatAudioBlobCache();
let released = false;
return {
value: entry,
release: () => {
if (released) {
return;
}
released = true;
const current = chatAudioBlobCache.get(cacheKey);
if (current && current.retainCount > 0) {
current.retainCount -= 1;
}
trimChatAudioBlobCache();
},
};
}

149
ui/src/pages/chat/components/chat-media-playback.test.ts Normal file
View File

@@ -0,0 +1,149 @@
import { describe, expect, it, vi } from "vitest";
import { appendChatMediaPlaybackParam, waitForChatMediaPlayback } from "./chat-media-playback.ts";
const EXPECTED_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 20_000];
describe("chat media playback renditions", () => {
it("appends playback=1 without dropping assistant or managed media tickets", () => {
expect(
appendChatMediaPlaybackParam(
"/__openclaw__/assistant-media?source=%2Ftmp%2Fvoice.caf&mediaTicket=assistant",
),
).toBe(
"/__openclaw__/assistant-media?source=%2Ftmp%2Fvoice.caf&mediaTicket=assistant&playback=1",
);
expect(
appendChatMediaPlaybackParam(
"/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full?mediaTicket=managed",
),
).toBe(
"/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full?mediaTicket=managed&playback=1",
);
expect(appendChatMediaPlaybackParam("media/clip.avi?mediaTicket=relative#preview")).toBe(
"media/clip.avi?mediaTicket=relative&playback=1#preview",
);
expect(appendChatMediaPlaybackParam("//cdn.example/clip.avi?mediaTicket=cdn")).toBe(
"//cdn.example/clip.avi?mediaTicket=cdn&playback=1",
);
});
it("reports preparing and retries until the rendition is ready", async () => {
const fetchImpl = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 202 }))
.mockResolvedValueOnce(new Response(null, { status: 202 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise<void>>(
async () => undefined,
);
const onPreparing = vi.fn();
await expect(
waitForChatMediaPlayback({
source: "/media?playback=1",
authToken: "secret-token",
signal: new AbortController().signal,
fetchImpl,
waitImpl,
onPreparing,
}),
).resolves.toBe("ready");
expect(onPreparing).toHaveBeenCalledTimes(2);
expect(waitImpl.mock.calls.map(([delay]) => delay)).toEqual([2_000, 4_000]);
expect(fetchImpl).toHaveBeenCalledTimes(3);
const firstRequest = fetchImpl.mock.calls[0];
expect(firstRequest?.[0]).toBe("/media?playback=1");
expect(firstRequest?.[1]?.method).toBe("HEAD");
expect(new Headers(firstRequest?.[1]?.headers).get("Authorization")).toBe(
"Bearer secret-token",
);
});
it("stops after the bounded two-minute retry schedule", async () => {
const fetchImpl = vi.fn<typeof fetch>(async () => new Response(null, { status: 202 }));
const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise<void>>(
async () => undefined,
);
await expect(
waitForChatMediaPlayback({
source: "/media?playback=1",
signal: new AbortController().signal,
fetchImpl,
waitImpl,
}),
).resolves.toBe("unavailable");
expect(waitImpl.mock.calls.map(([delay]) => delay)).toEqual(EXPECTED_RETRY_DELAYS_MS);
expect(fetchImpl).toHaveBeenCalledTimes(EXPECTED_RETRY_DELAYS_MS.length + 1);
expect(EXPECTED_RETRY_DELAYS_MS.reduce((sum, delay) => sum + delay, 0)).toBe(110_000);
});
it("performs a definitive final HEAD after the last backoff", async () => {
vi.useFakeTimers();
try {
const fetchImpl = vi.fn<typeof fetch>();
for (let attempt = 0; attempt < 7; attempt += 1) {
fetchImpl.mockResolvedValueOnce(new Response(null, { status: 202 }));
}
fetchImpl.mockResolvedValueOnce(new Response(null, { status: 200 }));
const pending = waitForChatMediaPlayback({
source: "/media?playback=1",
signal: new AbortController().signal,
fetchImpl,
});
await vi.runAllTimersAsync();
await expect(pending).resolves.toBe("ready");
expect(fetchImpl).toHaveBeenCalledTimes(8);
} finally {
vi.useRealTimers();
}
});
it("fails a stalled readiness request at its request deadline", async () => {
vi.useFakeTimers();
try {
const pending = waitForChatMediaPlayback({
source: "/media?playback=1",
signal: new AbortController().signal,
requestTimeoutMs: 10,
fetchImpl: vi.fn<typeof fetch>(async () => await new Promise<Response>(() => {})),
});
await vi.advanceTimersByTimeAsync(10);
await expect(pending).resolves.toBe("unavailable");
} finally {
vi.useRealTimers();
}
});
it("clamps a retry sleep to the remaining overall deadline", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date(0));
try {
const fetchImpl = vi
.fn<typeof fetch>()
.mockImplementationOnce(async () => {
vi.setSystemTime(new Date(119_000));
return new Response(null, { status: 202 });
})
.mockResolvedValueOnce(new Response(null, { status: 500 }));
const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise<void>>(
async () => undefined,
);
await expect(
waitForChatMediaPlayback({
source: "/media?playback=1",
signal: new AbortController().signal,
fetchImpl,
waitImpl,
}),
).resolves.toBe("unavailable");
expect(waitImpl).toHaveBeenCalledWith(1_000, expect.any(AbortSignal));
} finally {
vi.useRealTimers();
}
});
});

144
ui/src/pages/chat/components/chat-media-playback.ts Normal file
View File

@@ -0,0 +1,144 @@
import { appendAttachmentUrlSearchParam } from "./chat-message-local-media.ts";
export type ChatMediaPlaybackMode = "native" | "transcode";
const CHAT_MEDIA_PLAYBACK_RETRY_DELAYS_MS = [
2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 20_000,
] as const;
const CHAT_MEDIA_PLAYBACK_REQUEST_TIMEOUT_MS = 30_000;
const CHAT_MEDIA_PLAYBACK_MAX_WAIT_MS = 120_000;
type ChatMediaPlaybackReadiness = "ready" | "unavailable" | "aborted";
function playbackAbortError(signal: AbortSignal): Error {
return signal.reason instanceof Error
? signal.reason
: new DOMException("playback preparation aborted", "AbortError");
}
export function appendChatMediaPlaybackParam(source: string): string {
return appendAttachmentUrlSearchParam(source, "playback", "1");
}
export function buildChatMediaFetchHeaders(authToken: string | null | undefined): Headers {
const headers = new Headers();
const token = authToken?.trim();
if (token) {
headers.set("Authorization", `Bearer ${token}`);
}
return headers;
}
function waitForRetry(delayMs: number, signal: AbortSignal): Promise<void> {
if (signal.aborted) {
return Promise.reject(playbackAbortError(signal));
}
return new Promise((resolve, reject) => {
const onAbort = () => {
clearTimeout(timer);
reject(playbackAbortError(signal));
};
const timer = setTimeout(() => {
signal.removeEventListener("abort", onAbort);
resolve();
}, delayMs);
signal.addEventListener("abort", onAbort, { once: true });
});
}
async function fetchPlaybackHead(params: {
source: string;
headers: Headers;
signal: AbortSignal;
timeoutMs: number;
fetchImpl: typeof fetch;
}): Promise<Response> {
const controller = new AbortController();
let rejectDeadline: ((error: Error) => void) | undefined;
const deadline = new Promise<never>((_resolve, reject) => {
rejectDeadline = reject;
});
const onAbort = () => {
const error = playbackAbortError(params.signal);
controller.abort(error);
rejectDeadline?.(error);
};
params.signal.addEventListener("abort", onAbort, { once: true });
if (params.signal.aborted) {
onAbort();
}
const timer = setTimeout(() => {
const error = new DOMException("playback readiness request timed out", "TimeoutError");
controller.abort(error);
rejectDeadline?.(error);
}, params.timeoutMs);
try {
return await Promise.race([
params.fetchImpl(params.source, {
method: "HEAD",
headers: params.headers,
credentials: "same-origin",
signal: controller.signal,
}),
deadline,
]);
} finally {
clearTimeout(timer);
params.signal.removeEventListener("abort", onAbort);
}
}
export async function waitForChatMediaPlayback(params: {
source: string;
authToken?: string | null;
signal: AbortSignal;
onPreparing?: () => void;
fetchImpl?: typeof fetch;
retryDelaysMs?: readonly number[];
waitImpl?: (delayMs: number, signal: AbortSignal) => Promise<void>;
requestTimeoutMs?: number;
}): Promise<ChatMediaPlaybackReadiness> {
const fetchImpl = params.fetchImpl ?? fetch;
const retryDelaysMs = params.retryDelaysMs ?? CHAT_MEDIA_PLAYBACK_RETRY_DELAYS_MS;
const waitImpl = params.waitImpl ?? waitForRetry;
const headers = buildChatMediaFetchHeaders(params.authToken);
headers.set("Accept", "audio/*, video/*");
const deadline = Date.now() + CHAT_MEDIA_PLAYBACK_MAX_WAIT_MS;
for (let attempt = 0; ; attempt += 1) {
if (params.signal.aborted) {
return "aborted";
}
try {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) {
return "unavailable";
}
const response = await fetchPlaybackHead({
source: params.source,
headers,
signal: params.signal,
timeoutMs: Math.min(
params.requestTimeoutMs ?? CHAT_MEDIA_PLAYBACK_REQUEST_TIMEOUT_MS,
remainingMs,
),
fetchImpl,
});
if (response.status !== 202) {
return response.ok ? "ready" : "unavailable";
}
params.onPreparing?.();
const retryDelay = retryDelaysMs[attempt];
if (retryDelay === undefined) {
return "unavailable";
}
const remainingAfterResponseMs = deadline - Date.now();
if (remainingAfterResponseMs <= 0) {
return "unavailable";
}
await waitImpl(Math.min(retryDelay, remainingAfterResponseMs), params.signal);
} catch {
return params.signal.aborted ? "aborted" : "unavailable";
}
}
}

30
ui/src/pages/chat/components/chat-media-source.test.ts
View File

@@ -105,6 +105,36 @@ describe("ChatMediaSourceController", () => {
expect(media.getAttribute("src")).toBe("/media?mediaTicket=fresh");
});
it("applies a queued Blob before a paused player resumes", () => {
const media = document.createElement("audio");
const state = { currentTime: 18, duration: 80, paused: false };
mockMediaState(media, state);
const controller = new ChatMediaSourceController();
controller.updateSource(media, "/media?mediaTicket=old", "/tmp/audio.mp3");
controller.updateSource(media, "blob:waveform", "/tmp/audio.mp3");
state.paused = true;
expect(controller.applyPendingSource(media)).toBe(true);
expect(media.getAttribute("src")).toBe("blob:waveform");
expect(controller.currentIdentity).toBe("/tmp/audio.mp3");
});
it("resets an applied source across an authentication boundary", () => {
const media = document.createElement("audio");
const state = { currentTime: 18, duration: 80, paused: true };
mockMediaState(media, state);
const load = vi.spyOn(media, "load").mockImplementation(() => undefined);
const controller = new ChatMediaSourceController();
controller.updateSource(media, "blob:protected-audio", "/tmp/audio.mp3");
controller.reset(media);
expect(media.hasAttribute("src")).toBe(false);
expect(load).toHaveBeenCalledOnce();
expect(controller.currentSource).toBe("");
expect(controller.currentIdentity).toBe("");
});
it("applies a fresh ticket that arrives after the old source has already failed", () => {
const media = document.createElement("audio");
const state = { currentTime: 0, duration: 80, paused: true, error: null as MediaError | null };

36
ui/src/pages/chat/components/chat-media-source.ts
View File

@@ -19,6 +19,10 @@ export class ChatMediaSourceController {
return this.appliedSource;
}
get currentIdentity(): string {
return this.appliedIdentity;
}
get queuedSource(): string {
return this.pendingSource;
}
@@ -82,6 +86,17 @@ export class ChatMediaSourceController {
return true;
}
applyPendingSource(media: HTMLMediaElement): boolean {
if (!this.pendingSource) {
return false;
}
this.applySource(media, this.pendingSource, this.pendingIdentity, {
currentTime: finiteMediaTime(media.currentTime),
paused: media.paused,
});
return true;
}
seek(media: HTMLMediaElement, nextTime: number): boolean {
const targetTime = Math.max(0, finiteMediaTime(nextTime));
try {
@@ -105,6 +120,16 @@ export class ChatMediaSourceController {
}
}
reset(media: HTMLMediaElement): void {
this.appliedSource = "";
this.appliedIdentity = "";
this.pendingSource = "";
this.pendingIdentity = "";
this.restore = null;
media.removeAttribute("src");
media.load();
}
handleLoadedMetadata(media: HTMLMediaElement, canResume = () => true): void {
const restore = this.restore;
if (!restore) {
@@ -132,14 +157,3 @@ export class ChatMediaSourceController {
media.src = source;
}
}
const sourceControllers = new WeakMap<HTMLMediaElement, ChatMediaSourceController>();
export function getChatMediaSourceController(media: HTMLMediaElement): ChatMediaSourceController {
let controller = sourceControllers.get(media);
if (!controller) {
controller = new ChatMediaSourceController();
sourceControllers.set(media, controller);
}
return controller;
}

145
ui/src/pages/chat/components/chat-message-attachment-availability.ts Normal file
View File

@@ -0,0 +1,145 @@
import { buildAssistantAttachmentUrl } from "./chat-message-local-media.ts";
import {
isChatMediaResourceCurrent,
notifyChatMediaResourceSubscribers,
scheduleChatMediaResourceRefresh,
type ChatMediaResource,
} from "./chat-message-media.ts";
export type AssistantAttachmentAvailability =
| { status: "checking" }
| {
status: "available";
mediaTicket?: string;
mediaTicketExpiresAt?: number;
refreshAfter?: number;
refreshAttempts?: number;
playback?: "native" | "transcode";
sizeBytes?: number;
durationMs?: number;
width?: number;
height?: number;
}
| { status: "unavailable"; reason: string; checkedAt: number; retryAttempted?: true };
export type ManagedAttachmentAvailability =
| { status: "checking"; refreshAfter?: number; refreshAttempts?: number }
| {
status: "available";
url: string;
expiresAt?: number;
refreshAfter?: number;
refreshAttempts?: number;
}
| { status: "unavailable"; reason: string; checkedAt: number };
export const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000;
export const ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS = 30_000;
export const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000;
export const ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES = 2;
let assistantAttachmentAvailabilityRenderVersion = 0;
export function createUnavailableAssistantAttachment(
reason: string,
retryAttempted: boolean,
): Extract<AssistantAttachmentAvailability, { status: "unavailable" }> {
return {
status: "unavailable",
reason,
checkedAt: Date.now(),
...(retryAttempted ? { retryAttempted: true } : {}),
};
}
export function getAssistantAttachmentAvailabilityRenderVersion(): number {
return assistantAttachmentAvailabilityRenderVersion;
}
export function bumpAssistantAttachmentAvailabilityRenderVersion(): void {
assistantAttachmentAvailabilityRenderVersion =
(assistantAttachmentAvailabilityRenderVersion + 1) % Number.MAX_SAFE_INTEGER;
}
export function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string {
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath);
return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`;
}
export function setAssistantAttachmentAvailability(
resource: ChatMediaResource<AssistantAttachmentAvailability>,
availability: AssistantAttachmentAvailability,
): void {
if (!isChatMediaResourceCurrent(resource)) {
return;
}
resource.value = availability;
bumpAssistantAttachmentAvailabilityRenderVersion();
scheduleAssistantAttachmentRefresh(resource, availability);
}
export function scheduleAssistantAttachmentRefresh(
resource: ChatMediaResource<AssistantAttachmentAvailability>,
availability: AssistantAttachmentAvailability,
): void {
const refreshAt =
availability.status === "unavailable" && !availability.retryAttempted
? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
: availability.status === "available" &&
availability.mediaTicket &&
availability.mediaTicketExpiresAt
? (availability.refreshAfter ??
availability.mediaTicketExpiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS)
: undefined;
scheduleChatMediaResourceRefresh(resource, refreshAt, () => {
if (resource.value !== availability) {
return;
}
// Keep the failed generation until its retry can inherit the one-attempt
// budget. A ticket refresh keeps the playable generation mounted while
// its replacement is minted, otherwise the checking card resets playback.
if (availability.status === "available") {
// Virtual rows use this version as their media invalidation key. Notify
// alone updates the host but can leave the attachment row memoized.
bumpAssistantAttachmentAvailabilityRenderVersion();
} else if (availability.status !== "unavailable") {
resource.value = undefined;
bumpAssistantAttachmentAvailabilityRenderVersion();
}
notifyChatMediaResourceSubscribers(resource);
});
}
export function managedAttachmentRefreshDelayMs(refreshAttempts: number): number {
return ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS * 2 ** Math.max(0, refreshAttempts - 1);
}
export function selectLaterExpiringManagedAttachment(
current: Extract<ManagedAttachmentAvailability, { status: "available" }> | null,
incoming: Extract<ManagedAttachmentAvailability, { status: "available" }>,
): Extract<ManagedAttachmentAvailability, { status: "available" }> {
return current?.expiresAt !== undefined && current.expiresAt >= (incoming.expiresAt ?? 0)
? current
: incoming;
}
export function isManagedOutgoingMediaSource(source: string): boolean {
try {
const parsed = new URL(source, window.location.origin);
return (
parsed.origin === window.location.origin &&
parsed.pathname.startsWith("/api/chat/media/outgoing/")
);
} catch {
return false;
}
}
export function resolveManagedOutgoingMediaSessionKey(source: string): string | null {
try {
const encodedSessionKey = new URL(source, window.location.origin).pathname.split("/")[5];
return encodedSessionKey ? decodeURIComponent(encodedSessionKey) : null;
} catch {
return null;
}
}

33
ui/src/pages/chat/components/chat-message-attachment-status.ts Normal file
View File

@@ -0,0 +1,33 @@
import { html, nothing } from "lit";
import { icons } from "../../../components/icons.ts";
import type { AttachmentItem } from "./chat-message-media.ts";
export function renderAssistantAttachmentStatusCard(params: {
kind: AttachmentItem["attachment"]["kind"];
label: string;
badge: string;
reason?: string;
}) {
const icon =
params.kind === "image"
? icons.image
: params.kind === "audio"
? icons.mic
: params.kind === "video"
? icons.monitor
: icons.paperclip;
return html`
<div class="chat-assistant-attachment-card chat-assistant-attachment-card--blocked">
<div class="chat-assistant-attachment-card__header">
<span class="chat-assistant-attachment-card__icon">${icon}</span>
<span class="chat-assistant-attachment-card__title">${params.label}</span>
<span class="chat-assistant-attachment-badge chat-assistant-attachment-badge--muted"
>${params.badge}</span
>
</div>
${params.reason
? html`<div class="chat-assistant-attachment-card__reason">${params.reason}</div>`
: nothing}
</div>
`;
}

595
ui/src/pages/chat/components/chat-message-attachments.ts
View File

@@ -1,12 +1,29 @@
import { html, nothing } from "lit";
import { ref } from "lit/directives/ref.js";
import { styleMap } from "lit/directives/style-map.js";
import { icons } from "../../../components/icons.ts";
import type { ImageLightboxItem } from "../../../components/image-lightbox.ts";
import { t } from "../../../i18n/index.ts";
import "./chat-audio-player.ts";
import "./chat-video-player.ts";
import { safeAttachmentHref } from "./chat-attachment-href.ts";
import { getChatMediaSourceController } from "./chat-media-source.ts";
import {
ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES,
ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS,
ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS,
ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS,
buildAssistantAttachmentMetaUrl,
bumpAssistantAttachmentAvailabilityRenderVersion,
createUnavailableAssistantAttachment,
getAssistantAttachmentAvailabilityRenderVersion,
isManagedOutgoingMediaSource,
managedAttachmentRefreshDelayMs,
resolveManagedOutgoingMediaSessionKey,
scheduleAssistantAttachmentRefresh,
selectLaterExpiringManagedAttachment,
setAssistantAttachmentAvailability,
type AssistantAttachmentAvailability,
type ManagedAttachmentAvailability,
} from "./chat-message-attachment-availability.ts";
import { renderAssistantAttachmentStatusCard } from "./chat-message-attachment-status.ts";
import { openResolvedImage } from "./chat-message-image-open.ts";
import {
buildAssistantAttachmentUrl,
@@ -19,95 +36,11 @@ import {
observeChatMediaResource,
scheduleChatMediaResourceRefresh,
type AttachmentItem,
type ArtifactDownloadResolver,
type ChatMediaResource,
} from "./chat-message-media.ts";
type AssistantAttachmentAvailability =
| { status: "checking" }
| {
status: "available";
mediaTicket?: string;
mediaTicketExpiresAt?: number;
refreshAfter?: number;
refreshAttempts?: number;
}
| { status: "unavailable"; reason: string; checkedAt: number; retryAttempted?: true };
const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000;
const ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS = 30_000;
const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000;
const ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES = 2;
let assistantAttachmentAvailabilityRenderVersion = 0;
function createUnavailableAssistantAttachment(
reason: string,
retryAttempted: boolean,
): Extract<AssistantAttachmentAvailability, { status: "unavailable" }> {
return {
status: "unavailable",
reason,
checkedAt: Date.now(),
...(retryAttempted ? { retryAttempted: true } : {}),
};
}
export function getAssistantAttachmentAvailabilityRenderVersion(): number {
return assistantAttachmentAvailabilityRenderVersion;
}
function bumpAssistantAttachmentAvailabilityRenderVersion() {
assistantAttachmentAvailabilityRenderVersion =
(assistantAttachmentAvailabilityRenderVersion + 1) % Number.MAX_SAFE_INTEGER;
}
function setAssistantAttachmentAvailability(
resource: ChatMediaResource<AssistantAttachmentAvailability>,
availability: AssistantAttachmentAvailability,
) {
if (!isChatMediaResourceCurrent(resource)) {
return;
}
resource.value = availability;
bumpAssistantAttachmentAvailabilityRenderVersion();
scheduleAssistantAttachmentRefresh(resource, availability);
}
function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string {
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath);
return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`;
}
function scheduleAssistantAttachmentRefresh(
resource: ChatMediaResource<AssistantAttachmentAvailability>,
availability: AssistantAttachmentAvailability,
) {
const refreshAt =
availability.status === "unavailable" && !availability.retryAttempted
? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
: availability.status === "available" &&
availability.mediaTicket &&
availability.mediaTicketExpiresAt
? (availability.refreshAfter ??
availability.mediaTicketExpiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS)
: undefined;
scheduleChatMediaResourceRefresh(resource, refreshAt, () => {
if (resource.value !== availability) {
return;
}
// Keep the failed generation until its retry can inherit the one-attempt
// budget. A ticket refresh keeps the playable generation mounted while
// its replacement is minted, otherwise the checking card resets playback.
if (availability.status === "available") {
// Virtual rows use this version as their media invalidation key. Notify
// alone updates the host but can leave the attachment row memoized.
bumpAssistantAttachmentAvailabilityRenderVersion();
} else if (availability.status !== "unavailable") {
resource.value = undefined;
bumpAssistantAttachmentAvailabilityRenderVersion();
}
notifyChatMediaResourceSubscribers(resource);
});
}
export { getAssistantAttachmentAvailabilityRenderVersion };
export function resolveAssistantAttachmentAvailability(
source: string,
@@ -232,6 +165,11 @@ export function resolveAssistantAttachmentAvailability(
available?: boolean;
mediaTicket?: string;
mediaTicketExpiresAt?: string;
playback?: "native" | "transcode";
sizeBytes?: number;
durationMs?: number;
width?: number;
height?: number;
reason?: string;
} | null;
if (payload?.available === true) {
@@ -252,6 +190,13 @@ export function resolveAssistantAttachmentAvailability(
const availability: AssistantAttachmentAvailability = {
status: "available",
...(mediaTicket ? { mediaTicket, mediaTicketExpiresAt } : {}),
...(payload.playback === "native" || payload.playback === "transcode"
? { playback: payload.playback }
: {}),
...(typeof payload.sizeBytes === "number" ? { sizeBytes: payload.sizeBytes } : {}),
...(typeof payload.durationMs === "number" ? { durationMs: payload.durationMs } : {}),
...(typeof payload.width === "number" ? { width: payload.width } : {}),
...(typeof payload.height === "number" ? { height: payload.height } : {}),
};
resource.retryAttempted = false;
setAssistantAttachmentAvailability(resource, availability);
@@ -291,59 +236,283 @@ export function resolveAssistantAttachmentAvailability(
return refreshingAvailability ?? { status: "checking" };
}
function renderAssistantAttachmentStatusCard(params: {
kind: AttachmentItem["attachment"]["kind"];
label: string;
badge: string;
reason?: string;
}) {
const icon =
params.kind === "image"
? icons.image
: params.kind === "audio"
? icons.mic
: params.kind === "video"
? icons.monitor
: icons.paperclip;
return html`
<div class="chat-assistant-attachment-card chat-assistant-attachment-card--blocked">
<div class="chat-assistant-attachment-card__header">
<span class="chat-assistant-attachment-card__icon">${icon}</span>
<span class="chat-assistant-attachment-card__title">${params.label}</span>
<span class="chat-assistant-attachment-badge chat-assistant-attachment-badge--muted"
>${params.badge}</span
>
</div>
${params.reason
? html`<div class="chat-assistant-attachment-card__reason">${params.reason}</div>`
: nothing}
</div>
`;
}
function videoCardFor(media: HTMLVideoElement): HTMLElement | null {
return media.closest<HTMLElement>(".chat-assistant-attachment-card--video");
}
function markVideoMetadataLoaded(media: HTMLVideoElement, loaded: boolean): void {
videoCardFor(media)?.toggleAttribute("data-metadata-loaded", loaded);
}
function markVideoUnplayable(media: HTMLVideoElement, unplayable: boolean): void {
videoCardFor(media)?.toggleAttribute("data-unplayable", unplayable);
}
function syncVideoSource(media: HTMLVideoElement, source: string, sourceIdentity: string): void {
getChatMediaSourceController(media).updateSource(media, source, sourceIdentity);
}
function recoverVideoSource(media: HTMLVideoElement): boolean {
const recovered = getChatMediaSourceController(media).handleError(media);
if (recovered) {
markVideoMetadataLoaded(media, false);
markVideoUnplayable(media, false);
function retainManagedAttachmentUntilExpiry(
resource: ChatMediaResource<ManagedAttachmentAvailability>,
availability: Extract<ManagedAttachmentAvailability, { status: "available" }> | null,
refreshAttempts: number,
): Extract<ManagedAttachmentAvailability, { status: "available" }> | null {
if (!availability?.expiresAt || availability.expiresAt <= Date.now()) {
return null;
}
return recovered;
const retained = {
...availability,
refreshAfter: availability.expiresAt,
refreshAttempts,
};
setManagedAttachmentAvailability(resource, retained);
return retained;
}
function setManagedAttachmentAvailability(
resource: ChatMediaResource<ManagedAttachmentAvailability>,
availability: ManagedAttachmentAvailability,
scheduleExpiryOnly = false,
): void {
if (!isChatMediaResourceCurrent(resource)) {
return;
}
resource.value = availability;
bumpAssistantAttachmentAvailabilityRenderVersion();
const refreshAt =
availability.status === "checking"
? availability.refreshAfter
: availability.status === "available" && availability.expiresAt !== undefined
? scheduleExpiryOnly
? availability.expiresAt
: Math.min(
availability.refreshAfter ??
availability.expiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS,
availability.expiresAt,
)
: availability.status === "unavailable" && !resource.retryAttempted
? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
: undefined;
scheduleChatMediaResourceRefresh(resource, refreshAt, () => {
if (resource.value?.status === "unavailable") {
resource.retryAttempted = true;
resource.value = undefined;
}
bumpAssistantAttachmentAvailabilityRenderVersion();
notifyChatMediaResourceSubscribers(resource);
});
}
function resolveManagedAttachmentAvailability(
attachment: AttachmentItem["attachment"],
resolveArtifactDownload: ArtifactDownloadResolver | undefined,
onRequestUpdate: (() => void) | undefined,
): ManagedAttachmentAvailability {
if (!isManagedOutgoingMediaSource(attachment.url)) {
return { status: "available", url: attachment.url };
}
if (!attachment.artifactId || !resolveArtifactDownload) {
if (new URL(attachment.url, window.location.origin).searchParams.get("mediaTicket")?.trim()) {
return { status: "available", url: attachment.url };
}
return {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: Date.now(),
};
}
const sessionKey = resolveManagedOutgoingMediaSessionKey(attachment.url);
if (!sessionKey) {
return {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: Date.now(),
};
}
const cacheKey = `${attachment.url}::${attachment.artifactId}`;
const resource = observeChatMediaResource<ManagedAttachmentAvailability>(
"managed-media",
cacheKey,
onRequestUpdate,
attachment.url,
);
const cached = resource.value;
const now = Date.now();
if (cached?.status === "unavailable") {
setManagedAttachmentAvailability(resource, cached);
return cached;
}
if (
cached?.status === "checking" &&
cached.refreshAfter !== undefined &&
cached.refreshAfter > now
) {
setManagedAttachmentAvailability(resource, cached);
return cached;
}
if (cached?.status === "available") {
if (
cached.expiresAt !== undefined &&
cached.expiresAt <= now &&
(cached.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES
) {
resource.retryAttempted = true;
const unavailable: ManagedAttachmentAvailability = {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: now,
};
setManagedAttachmentAvailability(resource, unavailable);
return unavailable;
}
if (
cached.expiresAt !== undefined &&
cached.expiresAt <= now &&
(resource.pending || (cached.refreshAfter !== undefined && cached.refreshAfter > now))
) {
const checking: ManagedAttachmentAvailability = {
status: "checking",
...(!resource.pending && cached.refreshAfter !== undefined
? { refreshAfter: cached.refreshAfter }
: {}),
refreshAttempts: cached.refreshAttempts,
};
setManagedAttachmentAvailability(resource, checking);
return checking;
}
const refreshAt =
cached.refreshAfter ??
(cached.expiresAt === undefined
? undefined
: cached.expiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS);
if (refreshAt === undefined || refreshAt > now) {
setManagedAttachmentAvailability(resource, cached);
return cached;
}
}
if (resource.pending) {
return cached?.status === "available" ? cached : { status: "checking" };
}
const current =
cached?.status === "available" && (cached.expiresAt === undefined || cached.expiresAt > now)
? cached
: null;
const keepCurrentForRetry = () => {
if (!current && cached?.status !== "checking") {
return null;
}
const refreshAttempts = current?.refreshAttempts ?? cached?.refreshAttempts ?? 0;
if (refreshAttempts >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES) {
return retainManagedAttachmentUntilExpiry(resource, current, refreshAttempts);
}
const nextRefreshAttempts = refreshAttempts + 1;
const refreshAfter = Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts);
const retryAvailability: ManagedAttachmentAvailability =
!current || (current.expiresAt !== undefined && current.expiresAt <= Date.now())
? { status: "checking", refreshAfter, refreshAttempts: nextRefreshAttempts }
: { ...current, refreshAfter, refreshAttempts: nextRefreshAttempts };
setManagedAttachmentAvailability(resource, retryAvailability);
return retryAvailability;
};
if (!current) {
setManagedAttachmentAvailability(resource, { status: "checking" });
}
const pending = Promise.resolve()
.then(() => resolveArtifactDownload({ sessionKey, artifactId: attachment.artifactId! }))
.then((result) => {
if (!isChatMediaResourceCurrent(resource)) {
return null;
}
const url = result?.url.trim();
if (!url) {
const retryAvailability = keepCurrentForRetry();
if (retryAvailability) {
return retryAvailability;
}
if (
(cached?.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES
) {
resource.retryAttempted = true;
}
const unavailable: ManagedAttachmentAvailability = {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: Date.now(),
};
setManagedAttachmentAvailability(resource, unavailable);
return unavailable;
}
const parsedExpiresAt = Date.parse(result?.expiresAt ?? "");
const expiresAt = Number.isFinite(parsedExpiresAt)
? parsedExpiresAt
: Date.now() + 5 * 60_000;
const refreshAttempts = cached?.refreshAttempts ?? 0;
if (
expiresAt - Date.now() <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS &&
refreshAttempts >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES
) {
const incoming: Extract<ManagedAttachmentAvailability, { status: "available" }> = {
status: "available",
url,
expiresAt,
};
const retained = retainManagedAttachmentUntilExpiry(
resource,
selectLaterExpiringManagedAttachment(current, incoming),
refreshAttempts,
);
if (retained) {
return retained;
}
resource.retryAttempted = true;
const unavailable: ManagedAttachmentAvailability = {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: Date.now(),
};
setManagedAttachmentAvailability(resource, unavailable);
return unavailable;
}
const nextRefreshAttempts = refreshAttempts + 1;
const needsEarlyRefresh =
expiresAt - Date.now() <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS;
if (expiresAt <= Date.now()) {
const retryAvailability: ManagedAttachmentAvailability = {
status: "checking",
refreshAfter: Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts),
refreshAttempts: nextRefreshAttempts,
};
setManagedAttachmentAvailability(resource, retryAvailability);
return retryAvailability;
}
const availability: ManagedAttachmentAvailability = {
status: "available",
url,
expiresAt,
...(needsEarlyRefresh
? {
refreshAfter: Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts),
refreshAttempts: nextRefreshAttempts,
}
: {}),
};
if (!needsEarlyRefresh) {
resource.retryAttempted = false;
}
setManagedAttachmentAvailability(resource, availability);
return availability;
})
.catch(() => {
const retryAvailability = keepCurrentForRetry();
if (retryAvailability) {
return retryAvailability;
}
if ((cached?.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES) {
resource.retryAttempted = true;
}
const unavailable: ManagedAttachmentAvailability = {
status: "unavailable",
reason: t("chat.attachments.unavailable"),
checkedAt: Date.now(),
};
setManagedAttachmentAvailability(resource, unavailable);
return unavailable;
})
.finally(() => {
if (resource.pending === pending) {
resource.pending = undefined;
}
notifyChatMediaResourceSubscribers(resource);
});
resource.pending = pending;
if (current) {
setManagedAttachmentAvailability(resource, current, true);
}
return current ?? { status: "checking" };
}
export function renderAssistantAttachments(
@@ -355,6 +524,7 @@ export function renderAssistantAttachments(
onAssistantAttachmentLoaded?: () => void,
onRequestOpenImage?: () => number,
onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void,
resolveArtifactDownload?: ArtifactDownloadResolver,
) {
if (attachments.length === 0) {
return nothing;
@@ -362,17 +532,56 @@ export function renderAssistantAttachments(
return html`
<div class="chat-assistant-attachments">
${attachments.map(({ attachment }) => {
const availability = resolveAssistantAttachmentAvailability(
const assistantAvailability = resolveAssistantAttachmentAvailability(
attachment.url,
localMediaPreviewRoots,
basePath,
authToken,
onRequestUpdate,
);
const attachmentUrl =
availability.status === "available"
? buildAssistantAttachmentUrl(attachment.url, basePath, availability.mediaTicket)
const managedAvailability =
assistantAvailability.status === "available"
? resolveManagedAttachmentAvailability(
attachment,
resolveArtifactDownload,
onRequestUpdate,
)
: null;
const availability =
assistantAvailability.status !== "available"
? assistantAvailability
: managedAvailability?.status === "unavailable"
? managedAvailability
: managedAvailability?.status === "checking"
? managedAvailability
: assistantAvailability;
const attachmentUrl =
assistantAvailability.status === "available" &&
managedAvailability?.status === "available"
? isLocalAssistantAttachmentSource(attachment.url)
? buildAssistantAttachmentUrl(
attachment.url,
basePath,
assistantAvailability.mediaTicket,
)
: managedAvailability.url
: null;
const playback =
assistantAvailability.status === "available"
? (assistantAvailability.playback ?? attachment.playback ?? "native")
: (attachment.playback ?? "native");
const sizeBytes =
assistantAvailability.status === "available"
? (assistantAvailability.sizeBytes ?? attachment.sizeBytes)
: attachment.sizeBytes;
const serverDurationMs =
isLocalAssistantAttachmentSource(attachment.url) &&
assistantAvailability.status === "available"
? assistantAvailability.durationMs
: undefined;
const playbackAuthToken = isLocalAssistantAttachmentSource(attachment.url)
? (authToken ?? null)
: null;
if (attachment.kind === "image") {
if (!attachmentUrl) {
return renderAssistantAttachmentStatusCard({
@@ -421,6 +630,10 @@ export function renderAssistantAttachments(
.src=${attachmentUrl}
.sourceIdentity=${attachment.url}
.label=${attachment.label}
.playback=${playback}
.authToken=${playbackAuthToken}
.sizeBytes=${sizeBytes}
.serverDurationMs=${serverDurationMs}
.voiceNote=${attachment.isVoiceNote === true}
.onMediaLoaded=${onAssistantAttachmentLoaded}
></openclaw-chat-audio-player>
@@ -438,85 +651,21 @@ export function renderAssistantAttachments(
reason: availability.status === "unavailable" ? availability.reason : undefined,
});
}
const dimensions =
attachment.width && attachment.height
? { "aspect-ratio": `${attachment.width} / ${attachment.height}` }
: {};
const downloadHref = safeAttachmentHref(attachmentUrl);
return html`
<div class="chat-assistant-attachment-card chat-assistant-attachment-card--video">
<div class="chat-assistant-attachment-card__header">
<span class="chat-assistant-attachment-card__title">${attachment.label}</span>
${downloadHref
? html`<a
class="chat-assistant-attachment-card__download"
href=${downloadHref}
download=${attachment.label}
target="_blank"
rel="noreferrer"
aria-label=${t("chat.mediaPlayer.download", {
filename: attachment.label,
})}
title=${t("chat.mediaPlayer.download", { filename: attachment.label })}
>${icons.download}</a
>`
: nothing}
</div>
<div class="chat-assistant-video-frame" style=${styleMap(dimensions)}>
<span class="chat-assistant-video-frame__placeholder" aria-hidden="true"
>${icons.monitor}</span
>
<video
controls
preload="metadata"
${ref((element) => {
if (element instanceof HTMLVideoElement) {
syncVideoSource(element, attachmentUrl, attachment.url);
}
})}
@loadedmetadata=${(event: Event) => {
const media = event.currentTarget as HTMLVideoElement;
getChatMediaSourceController(media).handleLoadedMetadata(media);
markVideoMetadataLoaded(media, true);
markVideoUnplayable(media, false);
onAssistantAttachmentLoaded?.();
}}
@ended=${(event: Event) => {
const media = event.currentTarget as HTMLVideoElement;
if (getChatMediaSourceController(media).handleEnded(media)) {
markVideoMetadataLoaded(media, false);
}
}}
@seeking=${(event: Event) => {
const media = event.currentTarget as HTMLVideoElement;
if (media.error) {
recoverVideoSource(media);
}
}}
@error=${(event: Event) => {
const media = event.currentTarget as HTMLVideoElement;
if (!recoverVideoSource(media)) {
markVideoUnplayable(media, true);
}
}}
></video>
</div>
<div class="chat-assistant-video-fallback">
<div class="chat-assistant-attachment-card__reason">
${t("chat.mediaPlayer.videoUnavailable")}
</div>
${downloadHref
? html`<a
class="chat-assistant-attachment-card__link"
href=${downloadHref}
download=${attachment.label}
target="_blank"
rel="noreferrer"
>${t("chat.mediaPlayer.download", { filename: attachment.label })}</a
>`
: nothing}
</div>
</div>
<openclaw-chat-video-player
.src=${attachmentUrl}
.sourceIdentity=${attachment.url}
.label=${attachment.label}
.playback=${playback}
.authToken=${playbackAuthToken}
.mediaWidth=${assistantAvailability.status === "available"
? (assistantAvailability.width ?? attachment.width)
: attachment.width}
.mediaHeight=${assistantAvailability.status === "available"
? (assistantAvailability.height ?? attachment.height)
: attachment.height}
.onMediaLoaded=${onAssistantAttachmentLoaded}
></openclaw-chat-video-player>
`;
}
if (!attachmentUrl) {

2
ui/src/pages/chat/components/chat-message-bubble.ts
View File

@@ -422,6 +422,7 @@ export function renderGroupedMessage(
opts.onAssistantAttachmentLoaded,
opts.onRequestOpenImage,
opts.onOpenImage,
opts.resolveArtifactDownload,
)}
${assistantViewContent}
${reasoningMarkdown
@@ -488,6 +489,7 @@ export function renderGroupedMessage(
opts.onAssistantAttachmentLoaded,
opts.onRequestOpenImage,
opts.onOpenImage,
opts.resolveArtifactDownload,
)}
${reasoningMarkdown
? html`<div class="chat-thinking">

19
ui/src/pages/chat/components/chat-message-local-media.ts
View File

@@ -129,3 +129,22 @@ export function buildAssistantAttachmentUrl(
}
return `${normalizedBasePath}/__openclaw__/assistant-media?${params.toString()}`;
}
export function appendAttachmentUrlSearchParam(
source: string,
name: string,
value: string,
): string {
const trimmed = source.trim();
if (!trimmed) {
return trimmed;
}
const hashIndex = trimmed.indexOf("#");
const hash = hashIndex === -1 ? "" : trimmed.slice(hashIndex);
const withoutHash = hashIndex === -1 ? trimmed : trimmed.slice(0, hashIndex);
const queryIndex = withoutHash.indexOf("?");
const path = queryIndex === -1 ? withoutHash : withoutHash.slice(0, queryIndex);
const params = new URLSearchParams(queryIndex === -1 ? "" : withoutHash.slice(queryIndex + 1));
params.set(name, value);
return `${path}?${params.toString()}${hash}`;
}

6
ui/src/pages/chat/components/chat-message-media.ts
View File

@@ -42,7 +42,11 @@ export type RenderableImageBlock = ImageBlock & {
export type AttachmentItem = Extract<MessageContentItem, { type: "attachment" }>;
type ChatMediaResourceKind = "assistant-attachment" | "managed-image" | "pairing-qr";
type ChatMediaResourceKind =
| "assistant-attachment"
| "managed-image"
| "managed-media"
| "pairing-qr";
export type ChatMediaResource<Value> = {
kind: ChatMediaResourceKind;

396
ui/src/pages/chat/components/chat-message.test.ts
View File

@@ -595,6 +595,16 @@ async function requireAudioPlayer(container: HTMLElement) {
return player;
}
async function requireVideoPlayer(container: HTMLElement) {
const player = expectElement(
container,
"openclaw-chat-video-player",
HTMLElement,
) as HTMLElement & { updateComplete: Promise<unknown> };
await player.updateComplete;
return player;
}
afterEach(() => {
markdownRenderMock.mockClear();
document.querySelectorAll("[data-media-player-test-fixture]").forEach((element) => {
@@ -3119,7 +3129,10 @@ describe("grouped chat rendering", () => {
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
expect(url).toContain("meta=1");
expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer session-token");
return { ok: true, json: async () => mediaTicketPayload("ticket-bootstrap-audio") };
return {
ok: true,
json: async () => ({ ...mediaTicketPayload("ticket-bootstrap-audio"), durationMs: 2_345 }),
};
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
@@ -3153,6 +3166,373 @@ describe("grouped chat rendering", () => {
expect(audio.getAttribute("src")).toBe(
`/openclaw/__openclaw__/assistant-media?source=${encodeURIComponent(source)}&mediaTicket=ticket-bootstrap-audio`,
);
expect((audioPlayer as unknown as { serverDurationMs?: number }).serverDurationMs).toBe(2_345);
});
it("resolves managed transcode audio through an artifact ticket", async () => {
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const ticketedUrl = `${source}?mediaTicket=managed-ticket`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
const resolveArtifactDownload = vi.fn(async () => ({ url: ticketedUrl }));
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
expect(url).toBe(`${ticketedUrl}&playback=1`);
expect(init?.method).toBe("HEAD");
expect(new Headers(init?.headers).get("Authorization")).toBeNull();
return new Response(null, { status: 200 });
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-transcode-audio",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.caf",
mimeType: "audio/x-caf",
playback: "transcode",
sizeBytes: 4_096,
durationMs: 2_345,
},
],
timestamp: Date.now(),
},
{
showToolCalls: false,
assistantAttachmentAuthToken: "must-not-be-forwarded",
onRequestUpdate: rerender,
resolveArtifactDownload,
},
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
const player = await requireAudioPlayer(container);
await flushAssistantAttachmentAvailabilityChecks();
await player.updateComplete;
expect(resolveArtifactDownload).toHaveBeenCalledWith({
sessionKey: "agent:main:main",
artifactId,
});
expect(fetchMock).toHaveBeenCalledOnce();
expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(
`${ticketedUrl}&playback=1`,
);
expect((player as unknown as { serverDurationMs?: number }).serverDurationMs).toBeUndefined();
});
it("keeps a valid managed media ticket while refresh retries", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z"));
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const ticketedUrl = `${source}?mediaTicket=managed-old`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
const resolveArtifactDownload = vi
.fn<() => Promise<{ url: string; expiresAt: string }>>()
.mockResolvedValueOnce({
url: ticketedUrl,
expiresAt: new Date(Date.now() + 31_000).toISOString(),
})
.mockRejectedValueOnce(new Error("refresh unavailable"));
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-ticket-refresh",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload },
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
const player = await requireAudioPlayer(container);
expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(ticketedUrl);
await vi.advanceTimersByTimeAsync(1_001);
await flushAssistantAttachmentAvailabilityChecks();
await player.updateComplete;
expect(resolveArtifactDownload).toHaveBeenCalledTimes(2);
expect(container.querySelector(".chat-assistant-attachment-card--blocked")).toBeNull();
expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(ticketedUrl);
});
it("backs off stale managed ticket refreshes and eventually marks them unavailable", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z"));
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
let finishSecondRequest: (() => void) | undefined;
const resolveArtifactDownload = vi.fn(() => {
const requestNumber = resolveArtifactDownload.mock.calls.length;
const result = {
url: `${source}?mediaTicket=stale-${requestNumber}`,
expiresAt: new Date(Date.now() + (requestNumber === 1 ? 6_000 : -1_000)).toISOString(),
};
if (requestNumber !== 2) {
return Promise.resolve(result);
}
return new Promise<typeof result>((resolve) => {
finishSecondRequest = () => resolve(result);
});
});
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-stale-ticket-refresh",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload },
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(1);
expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull();
await vi.advanceTimersByTimeAsync(4_999);
expect(resolveArtifactDownload).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(1);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(2);
expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull();
await vi.advanceTimersByTimeAsync(1_000);
expect(resolveArtifactDownload).toHaveBeenCalledTimes(2);
expect(container.querySelector("openclaw-chat-audio-player")).toBeNull();
finishSecondRequest?.();
await flushAssistantAttachmentAvailabilityChecks();
await vi.advanceTimersByTimeAsync(9_999);
expect(resolveArtifactDownload).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(1);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(3);
expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull();
await vi.advanceTimersByTimeAsync(60_000);
expect(resolveArtifactDownload).toHaveBeenCalledTimes(3);
});
it("retains the current managed ticket until expiry after refresh exhaustion", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z"));
const expiresAt = new Date(Date.now() + 20_000).toISOString();
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
const resolveArtifactDownload = vi.fn(async () => ({
url: `${source}?mediaTicket=short-${resolveArtifactDownload.mock.calls.length}`,
expiresAt,
}));
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-ticket-exhaustion",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload },
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
await vi.advanceTimersByTimeAsync(5_000);
await flushAssistantAttachmentAvailabilityChecks();
await vi.advanceTimersByTimeAsync(10_000);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(3);
expect(
container.querySelector("openclaw-chat-audio-player audio")?.getAttribute("src"),
).toContain("mediaTicket=short-2");
expect(container.querySelector(".chat-assistant-attachment-card--blocked")).toBeNull();
await vi.advanceTimersByTimeAsync(4_999);
expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull();
await vi.advanceTimersByTimeAsync(1);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(3);
expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull();
});
it("retains a longer-lived incoming managed ticket after refresh exhaustion", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z"));
const initialExpiry = Date.now() + 20_000;
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
const resolveArtifactDownload = vi.fn(async () => ({
url: `${source}?mediaTicket=short-${resolveArtifactDownload.mock.calls.length}`,
expiresAt: new Date(
resolveArtifactDownload.mock.calls.length === 3 ? Date.now() + 20_000 : initialExpiry,
).toISOString(),
}));
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-later-ticket",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload },
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
await vi.advanceTimersByTimeAsync(5_000);
await flushAssistantAttachmentAvailabilityChecks();
await vi.advanceTimersByTimeAsync(10_000);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(3);
expect(
container.querySelector("openclaw-chat-audio-player audio")?.getAttribute("src"),
).toContain("mediaTicket=short-3");
await vi.advanceTimersByTimeAsync(5_000);
await flushAssistantAttachmentAvailabilityChecks();
expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull();
await vi.advanceTimersByTimeAsync(15_000);
await flushAssistantAttachmentAvailabilityChecks();
expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull();
});
it("refreshes a managed attachment that arrives with an initial ticket", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z"));
const rawSource = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const source = `${rawSource}?mediaTicket=initial`;
const refreshedSource = `${rawSource}?mediaTicket=refreshed`;
const artifactId = `artifact_managed_media_${crypto.randomUUID()}`;
const resolveArtifactDownload = vi.fn(async () => ({ url: refreshedSource }));
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const rerender = () =>
renderAssistantMessage(
container,
{
id: "assistant-managed-initial-ticket",
role: "assistant",
content: [
{
type: "audio",
artifactId,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload },
);
rerender();
await flushAssistantAttachmentAvailabilityChecks();
const player = await requireAudioPlayer(container);
expect(resolveArtifactDownload).toHaveBeenCalledWith({
sessionKey: "agent:main:main",
artifactId,
});
expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(
refreshedSource,
);
await vi.advanceTimersByTimeAsync(4 * 60_000 + 30_001);
await flushAssistantAttachmentAvailabilityChecks();
expect(resolveArtifactDownload).toHaveBeenCalledTimes(2);
});
it("does not render an unticketed managed attachment without an artifact resolver", () => {
const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const container = document.createElement("div");
renderAssistantMessage(
container,
{
id: "assistant-managed-media-without-resolver",
role: "assistant",
content: [
{
type: "audio",
artifactId: `artifact_managed_media_${crypto.randomUUID()}`,
url: source,
fileName: "voice.mp3",
mimeType: "audio/mpeg",
playback: "native",
},
],
timestamp: Date.now(),
},
{ showToolCalls: false },
);
expect(container.querySelector("openclaw-chat-audio-player")).toBeNull();
expect(
container.querySelector(".chat-assistant-attachment-card--blocked")?.textContent,
).toContain("Unavailable");
});
it("checks local assistant images against server metadata while preview roots load", async () => {
@@ -3249,7 +3629,7 @@ describe("grouped chat rendering", () => {
},
);
it("shows the download fallback when the video element emits an error", () => {
it("shows the download fallback when the video element emits an error", async () => {
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
@@ -3274,8 +3654,10 @@ describe("grouped chat rendering", () => {
{ showToolCalls: false },
);
const card = expectElement(container, ".chat-assistant-attachment-card--video", HTMLElement);
const videoPlayer = await requireVideoPlayer(container);
const card = expectElement(videoPlayer, ".chat-assistant-attachment-card--video", HTMLElement);
expectElement(card, "video", HTMLVideoElement).dispatchEvent(new Event("error"));
await videoPlayer.updateComplete;
expect(card.hasAttribute("data-unplayable")).toBe(true);
expect(card.querySelector(".chat-assistant-video-fallback")?.textContent).toContain(
"Can't play this format — download instead.",
@@ -3756,8 +4138,9 @@ describe("grouped chat rendering", () => {
expect(container.querySelector(".chat-text")?.textContent?.trim()).toBe("Blocked\nDone");
});
it("renders transcript video URLs with encoded extensions", () => {
const container = document.createElement("div");
it("renders transcript video URLs with encoded extensions", async () => {
const container = document.body.appendChild(document.createElement("div"));
container.dataset.mediaPlayerTestFixture = "";
const mediaUrl = "https://cdn.example/clip%2Emp4?download=1";
renderGroupedMessage(
@@ -3773,7 +4156,8 @@ describe("grouped chat rendering", () => {
{ showToolCalls: false },
);
expect(expectElement(container, "video", HTMLVideoElement).src).toBe(mediaUrl);
const videoPlayer = await requireVideoPlayer(container);
expect(expectElement(videoPlayer, "video", HTMLVideoElement).src).toBe(mediaUrl);
});
it("renders transcript image variants and structured image blocks", async () => {

221
ui/src/pages/chat/components/chat-video-player.test.ts Normal file
View File

@@ -0,0 +1,221 @@
/* @vitest-environment jsdom */
import { afterEach, describe, expect, it, vi } from "vitest";
import "./chat-video-player.ts";
afterEach(() => {
document.body.replaceChildren();
vi.useRealTimers();
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
describe("ChatVideoPlayer", () => {
it("keeps one video element mounted across 202 preparation", async () => {
vi.useFakeTimers();
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 202 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-video-player");
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket";
player.sourceIdentity = "media:clip";
player.label = "clip.avi";
player.playback = "transcode";
document.body.append(player);
await player.updateComplete;
const video = player.querySelector("video");
await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…"));
expect(player.querySelector("video")).toBe(video);
await vi.advanceTimersByTimeAsync(2_000);
await player.updateComplete;
expect(player.querySelector("video")).toBe(video);
expect(video?.getAttribute("src")).toContain("mediaTicket=ticket&playback=1");
});
it("does not preserve a previous attachment when a new rendition fails", async () => {
const player = document.createElement("openclaw-chat-video-player");
player.src = "https://example.com/first.mp4";
player.sourceIdentity = "media:first";
player.label = "first.mp4";
document.body.append(player);
await player.updateComplete;
expect(player.querySelector("video")?.getAttribute("src")).toBe(
"https://example.com/first.mp4",
);
vi.stubGlobal(
"fetch",
vi.fn<typeof fetch>(async () => new Response(null, { status: 500 })),
);
player.src = "/__openclaw__/assistant-media?source=second.caf&mediaTicket=ticket";
player.sourceIdentity = "media:second";
player.label = "second.caf";
player.playback = "transcode";
await player.updateComplete;
expect(player.querySelector("video")?.hasAttribute("src")).toBe(false);
await vi.waitFor(() =>
expect(
player
.querySelector(".chat-assistant-attachment-card--video")
?.hasAttribute("data-unplayable"),
).toBe(true),
);
expect(player.querySelector(".chat-assistant-video-fallback")?.textContent).toContain(
"Can't play this format — download instead.",
);
});
it("does not retain an errored source when a refreshed rendition is unavailable", async () => {
let resolveFirstRefresh: ((response: Response) => void) | undefined;
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockImplementationOnce(
async () =>
await new Promise<Response>((resolve) => {
resolveFirstRefresh = resolve;
}),
)
.mockResolvedValueOnce(new Response(null, { status: 500 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-video-player");
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=old";
player.sourceIdentity = "media:clip";
player.label = "clip.avi";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() =>
expect(player.querySelector("video")?.getAttribute("src")).toContain("mediaTicket=old"),
);
player.querySelector("video")?.dispatchEvent(new Event("error"));
await player.updateComplete;
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=refresh-1";
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=refresh-2";
await vi.waitFor(() =>
expect(
player
.querySelector(".chat-assistant-attachment-card--video")
?.hasAttribute("data-unplayable"),
).toBe(true),
);
expect(fetchMock).toHaveBeenCalledTimes(3);
resolveFirstRefresh?.(new Response(null, { status: 200 }));
});
it("hides a previous attachment while the replacement HEAD is stalled", async () => {
const player = document.createElement("openclaw-chat-video-player");
player.src = "https://example.com/first.mp4";
player.sourceIdentity = "media:first-stalled";
player.label = "first.mp4";
document.body.append(player);
await player.updateComplete;
const fetchMock = vi.fn<typeof fetch>(async () => await new Promise<Response>(() => {}));
vi.stubGlobal("fetch", fetchMock);
player.src = "/__openclaw__/assistant-media?source=second.caf&mediaTicket=ticket";
player.sourceIdentity = "media:second-stalled";
player.label = "second.caf";
player.playback = "transcode";
await player.updateComplete;
expect(fetchMock).toHaveBeenCalledOnce();
await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…"));
expect(player.querySelector("video")?.hasAttribute("src")).toBe(false);
});
it("clears authorized video while a new principal is checked", async () => {
let resolveRefresh: ((response: Response) => void) | undefined;
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockImplementationOnce(
async () =>
await new Promise<Response>((resolve) => {
resolveRefresh = resolve;
}),
);
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-video-player");
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket";
player.sourceIdentity = "media:principal-clip";
player.authToken = "principal-a";
player.label = "clip.avi";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() =>
expect(player.querySelector("video")?.getAttribute("src")).toContain("playback=1"),
);
player.authToken = "principal-b";
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…"));
expect(player.querySelector("video")?.hasAttribute("src")).toBe(false);
resolveRefresh?.(new Response(null, { status: 500 }));
});
it("clears a failed rendition after reconnect succeeds", async () => {
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValueOnce(new Response(null, { status: 500 }))
.mockResolvedValueOnce(new Response(null, { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const player = document.createElement("openclaw-chat-video-player");
player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket";
player.sourceIdentity = "media:retry-clip";
player.label = "clip.avi";
player.playback = "transcode";
document.body.append(player);
await vi.waitFor(() =>
expect(
player
.querySelector(".chat-assistant-attachment-card--video")
?.hasAttribute("data-unplayable"),
).toBe(true),
);
player.remove();
document.body.append(player);
await vi.waitFor(() =>
expect(player.querySelector("video")?.getAttribute("src")).toContain("playback=1"),
);
expect(
player
.querySelector(".chat-assistant-attachment-card--video")
?.hasAttribute("data-unplayable"),
).toBe(false);
});
it("pauses and clears the source when disconnected while playing", async () => {
const player = document.createElement("openclaw-chat-video-player");
player.src = "https://example.com/playing.mp4";
player.sourceIdentity = "media:playing";
player.label = "playing.mp4";
document.body.append(player);
await player.updateComplete;
const video = player.querySelector("video")!;
let paused = false;
Object.defineProperty(video, "paused", { configurable: true, get: () => paused });
const pause = vi.spyOn(video, "pause").mockImplementation(() => {
paused = true;
});
player.remove();
expect(pause).toHaveBeenCalledOnce();
expect(paused).toBe(true);
expect(video.hasAttribute("src")).toBe(false);
document.body.append(player);
await vi.waitFor(() =>
expect(video.getAttribute("src")).toBe("https://example.com/playing.mp4"),
);
});
});

264
ui/src/pages/chat/components/chat-video-player.ts Normal file
View File

@@ -0,0 +1,264 @@
import { html, type PropertyValues } from "lit";
import { property, state } from "lit/decorators.js";
import { ref } from "lit/directives/ref.js";
import { styleMap } from "lit/directives/style-map.js";
import { icons } from "../../../components/icons.ts";
import { t } from "../../../i18n/index.ts";
import { OpenClawLightDomContentsElement } from "../../../lit/openclaw-element.ts";
import { safeAttachmentHref } from "./chat-attachment-href.ts";
import {
appendChatMediaPlaybackParam,
waitForChatMediaPlayback,
type ChatMediaPlaybackMode,
} from "./chat-media-playback.ts";
import { ChatMediaSourceController } from "./chat-media-source.ts";
class ChatVideoPlayer extends OpenClawLightDomContentsElement {
@property() src = "";
@property() sourceIdentity = "";
@property() label = "";
@property() playback: ChatMediaPlaybackMode = "native";
@property() authToken: string | null = null;
@property({ type: Number }) mediaWidth: number | undefined;
@property({ type: Number }) mediaHeight: number | undefined;
@property({ attribute: false }) onMediaLoaded: (() => void) | undefined;
@state() private metadataLoaded = false;
@state() private failed = false;
@state() private preparing = false;
private media: HTMLVideoElement | null = null;
private readonly sourceController = new ChatMediaSourceController();
private currentSourceFailed = false;
private readinessController: AbortController | null = null;
private readinessKey = "";
private readySource = "";
override connectedCallback(): void {
super.connectedCallback();
queueMicrotask(() => this.syncSource());
}
override disconnectedCallback(): void {
this.readinessController?.abort();
this.readinessController = null;
this.readinessKey = "";
this.readySource = "";
this.sourceController.cancelPendingResume();
if (this.media) {
if (!this.media.paused) {
this.media.pause();
}
this.sourceController.reset(this.media);
}
super.disconnectedCallback();
}
override updated(changedProperties: PropertyValues<this>): void {
if (
changedProperties.has("src") ||
changedProperties.has("sourceIdentity") ||
changedProperties.has("playback") ||
changedProperties.has("authToken")
) {
const authenticationChanged = changedProperties.has("authToken");
if (
authenticationChanged ||
(changedProperties.has("sourceIdentity") &&
this.sourceController.currentIdentity &&
this.sourceController.currentIdentity !== this.sourceIdentity.trim())
) {
this.metadataLoaded = false;
if (this.media && !this.media.paused) {
this.media.pause();
}
if (this.media) {
this.sourceController.reset(this.media);
this.currentSourceFailed = false;
}
}
this.syncSource();
}
}
private setMedia = (element: Element | undefined) => {
this.media = element instanceof HTMLVideoElement ? element : null;
this.syncSource();
};
private syncSource(): void {
const media = this.media;
const source = this.src.trim();
if (!media || !source || !this.sourceIdentity.trim() || !this.isConnected) {
return;
}
const playbackSource =
this.playback === "transcode" ? appendChatMediaPlaybackParam(source) : source;
const hasCurrentAttachmentSource =
this.sourceController.currentIdentity === this.sourceIdentity.trim();
const hasUsableCurrentAttachmentSource =
hasCurrentAttachmentSource && !this.currentSourceFailed;
if (this.playback !== "transcode") {
this.readinessController?.abort();
this.readinessController = null;
this.readinessKey = "";
this.readySource = playbackSource;
this.preparing = false;
this.failed = false;
this.currentSourceFailed = false;
this.sourceController.updateSource(media, playbackSource, this.sourceIdentity);
return;
}
const readinessKey = `${playbackSource}\0${this.authToken?.trim() ?? ""}`;
if (!hasUsableCurrentAttachmentSource) {
this.metadataLoaded = false;
this.preparing = true;
}
if (readinessKey === this.readinessKey) {
if (this.readySource === playbackSource) {
this.sourceController.updateSource(media, playbackSource, this.sourceIdentity);
return;
}
if (this.readinessController) {
return;
}
}
this.readinessController?.abort();
const controller = new AbortController();
this.readinessController = controller;
this.readinessKey = readinessKey;
this.readySource = "";
this.preparing = !hasUsableCurrentAttachmentSource;
this.failed = false;
void waitForChatMediaPlayback({
source: playbackSource,
authToken: this.authToken,
signal: controller.signal,
onPreparing: () => {
if (this.readinessController === controller && !hasUsableCurrentAttachmentSource) {
this.preparing = true;
}
},
}).then((result) => {
if (this.readinessController !== controller || result === "aborted") {
return;
}
this.preparing = false;
if (result !== "ready") {
if (hasUsableCurrentAttachmentSource) {
this.readySource = this.sourceController.currentSource;
return;
}
this.failed = true;
return;
}
this.readySource = playbackSource;
this.failed = false;
this.currentSourceFailed = false;
this.sourceController.updateSource(media, playbackSource, this.sourceIdentity);
});
}
override render() {
const downloadHref = safeAttachmentHref(this.src);
const dimensions =
this.mediaWidth && this.mediaHeight
? { "aspect-ratio": `${this.mediaWidth} / ${this.mediaHeight}` }
: {};
return html`
<div
class="chat-assistant-attachment-card chat-assistant-attachment-card--video"
?data-metadata-loaded=${this.metadataLoaded}
?data-unplayable=${this.failed}
>
<div class="chat-assistant-attachment-card__header">
<span class="chat-assistant-attachment-card__title">${this.label}</span>
${downloadHref
? html`<a
class="chat-assistant-attachment-card__download"
href=${downloadHref}
download=${this.label}
target="_blank"
rel="noreferrer"
aria-label=${t("chat.mediaPlayer.download", { filename: this.label })}
title=${t("chat.mediaPlayer.download", { filename: this.label })}
>${icons.download}</a
>`
: null}
</div>
${this.preparing
? html`<div class="chat-assistant-attachment-card__reason chat-media-preparing">
${t("chat.mediaPlayer.preparing")}
</div>`
: null}
<div
class="chat-assistant-video-frame"
style=${styleMap(dimensions)}
?hidden=${this.preparing}
>
<span class="chat-assistant-video-frame__placeholder" aria-hidden="true"
>${icons.monitor}</span
>
<video
controls
preload="metadata"
${ref(this.setMedia)}
@loadedmetadata=${() => {
if (!this.media) {
return;
}
this.sourceController.handleLoadedMetadata(this.media);
this.metadataLoaded = true;
this.failed = false;
this.currentSourceFailed = false;
this.onMediaLoaded?.();
}}
@ended=${() => {
if (this.media && this.sourceController.handleEnded(this.media)) {
this.metadataLoaded = false;
}
}}
@seeking=${() => {
if (this.media?.error && this.sourceController.handleError(this.media)) {
this.metadataLoaded = false;
this.failed = false;
}
}}
@error=${() => {
if (this.media && !this.sourceController.handleError(this.media)) {
this.failed = true;
this.currentSourceFailed = true;
}
}}
></video>
</div>
<div class="chat-assistant-video-fallback">
<div class="chat-assistant-attachment-card__reason">
${t("chat.mediaPlayer.videoUnavailable")}
</div>
${downloadHref
? html`<a
class="chat-assistant-attachment-card__link"
href=${downloadHref}
download=${this.label}
target="_blank"
rel="noreferrer"
>${t("chat.mediaPlayer.download", { filename: this.label })}</a
>`
: null}
</div>
</div>
`;
}
}
if (!customElements.get("openclaw-chat-video-player")) {
customElements.define("openclaw-chat-video-player", ChatVideoPlayer);
}
declare global {
interface HTMLElementTagNameMap {
"openclaw-chat-video-player": ChatVideoPlayer;
}
}

49
ui/src/styles/chat/layout.css
View File

@@ -1079,6 +1079,11 @@ openclaw-chat-page {
display: block;
}
openclaw-chat-audio-player,
openclaw-chat-video-player {
display: block;
}
.chat-assistant-attachment-card {
display: flex;
align-items: center;
@@ -1264,6 +1269,39 @@ openclaw-chat-page {
outline-offset: 4px;
}
.chat-audio-player__waveform {
position: relative;
height: 24px;
}
.chat-audio-player__waveform svg {
display: block;
width: 100%;
height: 24px;
overflow: visible;
}
.chat-audio-player__waveform rect {
fill: color-mix(in srgb, var(--muted) 55%, transparent);
}
.chat-audio-player__waveform rect.is-played {
fill: var(--accent);
}
.chat-audio-player__seek--waveform {
position: absolute;
inset: 0;
height: 24px;
opacity: 0;
}
.chat-audio-player__waveform:has(.chat-audio-player__seek:focus-visible) {
border-radius: 4px;
outline: 2px solid var(--focus-ring);
outline-offset: 2px;
}
.chat-audio-player__time {
display: flex;
justify-content: space-between;
@@ -1290,6 +1328,10 @@ openclaw-chat-page {
border: 1px solid color-mix(in srgb, var(--border) 82%, transparent);
}
.chat-assistant-video-frame[hidden] {
display: none;
}
.chat-assistant-video-frame video {
position: relative;
z-index: 1;
@@ -1351,6 +1393,13 @@ openclaw-chat-page {
line-height: 1.4;
}
.chat-media-preparing {
padding: 10px 12px;
border-radius: var(--radius-sm);
background: color-mix(in srgb, var(--bg) 78%, transparent);
border: 1px solid color-mix(in srgb, var(--border) 82%, transparent);
}
.chat-assistant-attachment-card__icon {
display: inline-flex;
width: 16px;