From 43e138cc66bdd8b87360fb43075c0cbecddb63ea Mon Sep 17 00:00:00 2001 From: Shubhankar Tripathy <95570942+lonexreb@users.noreply.github.com> Date: Sun, 12 Jul 2026 03:16:46 -0500 Subject: [PATCH] feat(google-meet): retain the full caption transcript per session (#103387) (#103811) * feat(google-meet): retain bounded transcripts Co-authored-by: lonexreb * chore: leave changelog to release tooling * fix(google-meet): satisfy transcript CI checks --------- Co-authored-by: Peter Steinberger --- docs/plugins/google-meet.md | 10 + extensions/google-meet/index.test.ts | 406 ++++++++++++++++++ extensions/google-meet/index.ts | 45 ++ extensions/google-meet/src/cli.test.ts | 29 ++ extensions/google-meet/src/cli.ts | 45 ++ extensions/google-meet/src/runtime.ts | 183 +++++++- .../google-meet/src/transports/chrome.ts | 335 ++++++++++++++- .../google-meet/src/transports/types.ts | 16 + 8 files changed, 1045 insertions(+), 24 deletions(-) diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index 95b32e5c77d..2a04affbfea 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -121,6 +121,15 @@ If the browser fallback hits a Google login or Meet permission blocker, the tool Set `"mode": "transcribe"` to skip the duplex realtime bridge (no BlackHole/SoX requirement, no talk-back). Transcribe-mode Chrome joins also skip OpenClaw's microphone/camera permission grant and the Meet **Use microphone** path; if Meet shows the audio-choice interstitial, automation tries **Continue without microphone** first. Managed Chrome transports in this mode install a best-effort Meet caption observer. `googlemeet status --json` and `googlemeet doctor` report `captioning`, `captionsEnabledAttempted`, `transcriptLines`, `lastCaptionAt`, `lastCaptionSpeaker`, `lastCaptionText`, and a `recentTranscript` tail. +For the bounded session transcript, read the exact tracked Meet tab: + +```bash +openclaw googlemeet transcript +openclaw googlemeet transcript --since --json +``` + +The observer keeps at most 2,000 completed caption lines in the Meet page. Visible progressive text stays in the status health tail until the caption row completes, so saving `nextIndex` cannot skip a later text expansion; leaving finalizes visible rows before the snapshot. `droppedLines` reports lines lost from the head when the cap is exceeded. The four most recently ended session transcripts remain readable until the gateway restarts. Older ended transcripts return `evicted: true`. This is intentionally runtime memory, not durable meeting-history storage: restarting the gateway, closing the tab before a snapshot, or exceeding the documented caps can lose captions. + For a yes/no listen probe: ```bash @@ -838,6 +847,7 @@ Agents use the `google_meet` tool: | `attendance` | List participants and participant sessions | | `export` | Write the artifacts/attendance/transcript/manifest bundle; set `"dryRun": true` for manifest-only | | `recover_current_tab` | Focus/inspect an existing Meet tab without opening a new one | +| `transcript` | Read the bounded caption transcript; `sinceIndex` resumes from the previous `nextIndex` | | `leave` | End a session (Chrome clicks Leave; closes only tabs it opened; Twilio hangs up) | | `end_active_conference` | End the active Google Meet conference for an API-managed space | | `speak` | Make the realtime agent speak immediately, given `sessionId` and `message` | diff --git a/extensions/google-meet/index.test.ts b/extensions/google-meet/index.test.ts index 08a1418d25f..769bd45f5fe 100644 --- a/extensions/google-meet/index.test.ts +++ b/extensions/google-meet/index.test.ts @@ -980,6 +980,7 @@ describe("google-meet plugin", () => { "join", "create", "status", + "transcript", "setup_status", "resolve_space", "preflight", @@ -2629,9 +2630,27 @@ describe("google-meet plugin", () => { tabUrlAfterJoin?: string; leaveClicked?: boolean; leaveConfirmationRequired?: boolean; + nonFinalTranscriptGate?: Promise; + onNonFinalTranscriptRead?: () => void; + transcript?: { + droppedLines?: number; + epoch?: string; + lines: Array<{ at?: string; speaker?: string; text: string }>; + }; + finalTranscript?: { + droppedLines?: number; + epoch?: string; + lines: Array<{ at?: string; speaker?: string; text: string }>; + }; + transcriptSequence?: Array<{ + droppedLines?: number; + epoch?: string; + lines: Array<{ at?: string; speaker?: string; text: string }>; + }>; }) { let joined = false; let leaveStep = 0; + let transcriptReadIndex = 0; let openedTabUrl = options?.reused ? "https://meet.google.com/abc-defg-hij?hl=en" : undefined; const callGatewayFromCli = vi.fn( async ( @@ -2676,6 +2695,30 @@ describe("google-meet plugin", () => { return { ok: true }; } if (request.path === "/act") { + if (String(request.body?.fn).includes("state = window.__openclawMeetCaptions")) { + const finalizing = String(request.body?.fn).includes("if (true &&"); + if (!finalizing) { + options?.onNonFinalTranscriptRead?.(); + await options?.nonFinalTranscriptGate; + } + const transcript = finalizing + ? (options?.finalTranscript ?? options?.transcript) + : options?.transcript; + const sequencedTranscript = + options?.transcriptSequence?.[ + Math.min(transcriptReadIndex, options.transcriptSequence.length - 1) + ]; + transcriptReadIndex += 1; + const responseTranscript = sequencedTranscript ?? transcript; + return { + result: JSON.stringify({ + urlMatched: options?.tabUrlAfterJoin?.includes("/abc-defg-hij") !== false, + droppedLines: responseTranscript?.droppedLines ?? 0, + epoch: responseTranscript?.epoch, + lines: responseTranscript?.lines ?? [], + }), + }; + } if (String(request.body?.fn).includes("leaveAction")) { const currentUrl = options?.tabUrlAfterJoin ?? openedTabUrl; const urlMatched = currentUrl?.includes("/abc-defg-hij") === true; @@ -2725,6 +2768,223 @@ describe("google-meet plugin", () => { return callGatewayFromCli; } + it("reads and snapshots the bounded transcript from the exact tracked tab", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const callGatewayFromCli = mockLocalMeetBrowserRequestWithTabState({ + transcript: { + droppedLines: 2, + lines: [ + { at: "2026-07-12T06:00:00.000Z", speaker: "Alice", text: "third line" }, + { at: "2026-07-12T06:00:01.000Z", speaker: "Bob", text: "fourth line" }, + ], + }, + }); + const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" }); + const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", { + url: "https://meet.google.com/abc-defg-hij", + })) as { session: { id: string } }; + + const beforeRead = callGatewayFromCli.mock.calls.length; + const transcript = (await invokeGoogleMeetGatewayMethodForTest( + methods, + "googlemeet.transcript", + { sessionId: joined.session.id, sinceIndex: 3 }, + )) as { + droppedLines: number; + startIndex: number; + nextIndex: number; + lines: Array<{ text: string }>; + }; + expect(transcript).toMatchObject({ droppedLines: 2, startIndex: 3, nextIndex: 4 }); + expect(transcript.lines.map((line) => line.text)).toEqual(["fourth line"]); + const readCalls = callGatewayFromCli.mock.calls.slice(beforeRead); + expect(readCalls).toHaveLength(1); + expect(requireRecord(readCalls[0]?.[2], "transcript request")).toMatchObject({ + method: "POST", + path: "/act", + body: { targetId: "local-meet-tab" }, + }); + + await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", { + sessionId: joined.session.id, + }); + const afterLeave = (await invokeGoogleMeetGatewayMethodForTest( + methods, + "googlemeet.transcript", + { sessionId: joined.session.id }, + )) as { lines: Array<{ text: string }> }; + expect(afterLeave.lines.map((line) => line.text)).toEqual(["third line", "fourth line"]); + + await expect( + invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + sinceIndex: 1.5, + }), + ).rejects.toThrow("sinceIndex must be a non-negative safe integer"); + } finally { + Object.defineProperty(process, "platform", { value: originalPlatform }); + } + }); + + it("refuses to read a tracked tab after it navigates away from the meeting", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + mockLocalMeetBrowserRequestWithTabState({ + tabUrlAfterJoin: "https://meet.google.com/lookup/unrelated", + transcript: { lines: [{ text: "must not leak" }] }, + }); + const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" }); + const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", { + url: "https://meet.google.com/abc-defg-hij", + })) as { session: { id: string } }; + + await expect( + invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + }), + ).rejects.toThrow("tracked Meet tab no longer shows this session's meeting URL"); + } finally { + Object.defineProperty(process, "platform", { value: originalPlatform }); + } + }); + + it("keeps the cursor monotonic when the Meet page transcript epoch resets", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + mockLocalMeetBrowserRequestWithTabState({ + transcriptSequence: [ + { epoch: "page-1", lines: [{ text: "before reload" }] }, + { epoch: "page-2", lines: [{ text: "after reload" }] }, + { epoch: "page-2", lines: [{ text: "after reload" }] }, + ], + }); + const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" }); + const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", { + url: "https://meet.google.com/abc-defg-hij", + })) as { session: { id: string } }; + + const first = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + })) as { nextIndex: number; lines: Array<{ text: string }> }; + const second = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + sinceIndex: first.nextIndex, + })) as { nextIndex: number; lines: Array<{ text: string }> }; + expect(first).toMatchObject({ nextIndex: 1, lines: [{ text: "before reload" }] }); + expect(second).toMatchObject({ nextIndex: 2, lines: [{ text: "after reload" }] }); + + await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", { + sessionId: joined.session.id, + }); + const ended = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + })) as { lines: Array<{ text: string }> }; + expect(ended.lines.map((line) => line.text)).toEqual(["before reload", "after reload"]); + } finally { + Object.defineProperty(process, "platform", { value: originalPlatform }); + } + }); + + it("does not let a late active read replace the finalized leave snapshot", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + let releaseRead: (() => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + let activeReads = 0; + try { + const callGatewayFromCli = mockLocalMeetBrowserRequestWithTabState({ + transcript: { lines: [{ text: "partial" }] }, + finalTranscript: { lines: [{ text: "partial" }, { text: "complete caption" }] }, + nonFinalTranscriptGate: readGate, + onNonFinalTranscriptRead: () => { + activeReads += 1; + markReadStarted?.(); + }, + }); + const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" }); + const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", { + url: "https://meet.google.com/abc-defg-hij", + })) as { session: { id: string } }; + + const lateRead = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + }); + await readStarted; + const secondRead = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + }); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + expect(activeReads).toBe(1); + const leaving = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", { + sessionId: joined.session.id, + }); + const repeatedLeave = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", { + sessionId: joined.session.id, + }); + releaseRead?.(); + await Promise.allSettled([lateRead, secondRead, leaving, repeatedLeave]); + const finalCaptures = callGatewayFromCli.mock.calls.filter((call) => { + const request = call[2] as { body?: { fn?: string } }; + const script = String(request.body?.fn); + return ( + script.includes("state = window.__openclawMeetCaptions") && script.includes("if (true &&") + ); + }); + expect(finalCaptures).toHaveLength(1); + const result = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: joined.session.id, + })) as { lines: Array<{ text: string }> }; + expect(result.lines.map((line) => line.text)).toEqual(["partial", "complete caption"]); + } finally { + Object.defineProperty(process, "platform", { value: originalPlatform }); + } + }); + + it("retains only the four most recently ended transcripts", async () => { + const originalPlatform = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + mockLocalMeetBrowserRequestWithTabState({ + transcript: { lines: [{ text: "retained line" }] }, + }); + const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" }); + const sessionIds: string[] = []; + for (let index = 0; index < 5; index += 1) { + const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", { + url: "https://meet.google.com/abc-defg-hij", + })) as { session: { id: string } }; + sessionIds.push(joined.session.id); + await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", { + sessionId: joined.session.id, + }); + } + + const oldest = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: sessionIds[0], + })) as { evicted?: boolean; lines: unknown[] }; + const next = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", { + sessionId: sessionIds[1], + })) as { evicted?: boolean; lines: unknown[] }; + expect(oldest).toMatchObject({ evicted: true, lines: [] }); + expect(next.evicted).toBeUndefined(); + expect(next.lines).toHaveLength(1); + } finally { + Object.defineProperty(process, "platform", { value: originalPlatform }); + } + }); + it("leaves the Meet call in the browser when a chrome session leaves", async () => { const originalPlatform = process.platform; Object.defineProperty(process, "platform", { value: "darwin" }); @@ -3470,6 +3730,7 @@ describe("google-meet plugin", () => { `(${chromeTransportTesting.meetStatusScriptForTest({ allowMicrophone: false, autoJoin: false, + captionSessionId: "session-1", captureCaptions: true, guestName: "OpenClaw Agent", })})`, @@ -3494,6 +3755,151 @@ describe("google-meet plugin", () => { expect(captionButton.click).toHaveBeenCalledTimes(1); }); + it("coalesces progressive captions and ignores end phrases while still in call", async () => { + const leaveButton = { + disabled: false, + innerText: "", + textContent: "", + click: vi.fn(), + getAttribute: vi.fn((name: string) => (name === "aria-label" ? "Leave call" : null)), + }; + const page = { caption: "Alice\nmeeting ended", visible: true }; + const makeRegion = () => ({ + get innerText() { + return page.caption; + }, + get textContent() { + return page.caption; + }, + }); + let region = makeRegion(); + const windowState: Record = {}; + const disconnectObserver = vi.fn(); + const clearCaptionTimer = vi.fn(); + let settleCaption: (() => void) | undefined; + const document = { + body: { innerText: "meeting ended", textContent: "meeting ended" }, + title: "Meet", + querySelector: vi.fn((selector: string) => + selector.includes("aria-live") && page.visible ? region : null, + ), + querySelectorAll: vi.fn((selector: string) => { + if (selector === "button") { + return [leaveButton]; + } + if (selector === "input") { + return []; + } + if (selector.includes("aria-live")) { + return page.visible ? [region] : []; + } + return []; + }), + }; + const context = createContext({ + Date, + JSON, + clearTimeout: clearCaptionTimer, + setTimeout: (callback: () => void) => { + settleCaption = callback; + return 1; + }, + String, + URL, + document, + location: { href: "https://meet.google.com/abc-defg-hij", hostname: "meet.google.com" }, + MutationObserver: class { + observe = vi.fn(); + disconnect = disconnectObserver; + }, + window: windowState, + }); + const inspect = new Script( + `(${chromeTransportTesting.meetStatusScriptForTest({ + allowMicrophone: false, + autoJoin: false, + captionSessionId: "session-1", + captureCaptions: true, + guestName: "OpenClaw Agent", + })})`, + ).runInContext(context) as () => string | Promise; + + const first = JSON.parse(await inspect()) as { leaveReason?: string }; + expect(first.leaveReason).toBeUndefined(); + page.caption = "Alice\nmeeting ended after the recap"; + await inspect(); + const state = windowState["__openclawMeetCaptions"] as { + lines: Array<{ text: string }>; + visible: Array<{ text: string }>; + }; + expect(state.lines).toEqual([]); + expect(state.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]); + region = makeRegion(); + await inspect(); + expect(state.visible).toHaveLength(1); + + page.visible = false; + await inspect(); + page.visible = true; + await inspect(); + expect(clearCaptionTimer).toHaveBeenCalledWith(1); + expect(state.lines).toEqual([]); + + page.visible = false; + await inspect(); + settleCaption?.(); + page.visible = true; + await inspect(); + expect(state.lines.map((line) => line.text)).toEqual(["meeting ended after the recap"]); + expect(state.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]); + const readTranscript = (sessionId: string, finalize: boolean) => + new Script( + `(${chromeTransportTesting.meetTranscriptScriptForTest( + "https://meet.google.com/abc-defg-hij", + sessionId, + finalize, + )})`, + ).runInContext(context) as () => string; + const beforeFinalize = JSON.parse(readTranscript("session-1", false)()) as { lines: unknown[] }; + expect(beforeFinalize.lines).toHaveLength(1); + const afterFinalize = JSON.parse(readTranscript("session-1", true)()) as { lines: unknown[] }; + expect(afterFinalize.lines).toHaveLength(2); + + const inspectNextSession = new Script( + `(${chromeTransportTesting.meetStatusScriptForTest({ + allowMicrophone: false, + autoJoin: false, + captionSessionId: "session-2", + captureCaptions: true, + guestName: "OpenClaw Agent", + })})`, + ).runInContext(context) as () => string | Promise; + await inspectNextSession(); + const nextState = windowState["__openclawMeetCaptions"] as { + droppedLines: number; + sessionId?: string; + lines: Array<{ text: string }>; + visible: Array<{ text: string }>; + }; + expect(nextState.sessionId).toBe("session-2"); + expect(nextState.lines).toEqual([]); + expect(nextState.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]); + expect(disconnectObserver).toHaveBeenCalledTimes(1); + expect(JSON.parse(readTranscript("session-1", false)())).toMatchObject({ + sessionMatched: false, + }); + + nextState.lines = Array.from({ length: 2_000 }, (_, index) => ({ text: `line-${index}` })); + nextState.visible = []; + page.caption = "Alice\nline-2000"; + await inspectNextSession(); + readTranscript("session-2", true)(); + expect(nextState.lines).toHaveLength(2_000); + expect(nextState.lines[0]?.text).toBe("line-1"); + expect(nextState.lines.at(-1)?.text).toBe("line-2000"); + expect(nextState.droppedLines).toBe(1); + }); + it("reports in-call Meet audio permission problems from button labels", async () => { const makeButton = (label: string) => ({ disabled: false, diff --git a/extensions/google-meet/index.ts b/extensions/google-meet/index.ts index 082042647d3..14c989282cb 100644 --- a/extensions/google-meet/index.ts +++ b/extensions/google-meet/index.ts @@ -230,6 +230,7 @@ const GoogleMeetToolSchema = Type.Object({ "join", "create", "status", + "transcript", "setup_status", "resolve_space", "preflight", @@ -288,6 +289,12 @@ const GoogleMeetToolSchema = Type.Object({ ), dtmfSequence: Type.Optional(Type.String({ description: "Explicit DTMF sequence for Twilio" })), sessionId: Type.Optional(Type.String({ description: "Meet session ID" })), + sinceIndex: Type.Optional( + Type.Integer({ + description: "For transcript, resume from the previous response's nextIndex.", + minimum: 0, + }), + ), message: Type.Optional(Type.String({ description: "Realtime instructions to speak now" })), timeoutMs: optionalPositiveIntegerSchema({ description: "Probe timeout in milliseconds" }), meeting: Type.Optional(Type.String({ description: "Meet URL, meeting code, or spaces/{id}" })), @@ -400,6 +407,7 @@ type GoogleMeetGatewayToolAction = | "join" | "create" | "status" + | "transcript" | "recover_current_tab" | "setup_status" | "leave" @@ -806,6 +814,38 @@ export default definePluginEntry({ }, ); + api.registerGatewayMethod( + "googlemeet.transcript", + async ({ params, respond }: GatewayRequestHandlerOptions) => { + try { + const sessionId = normalizeOptionalString(params?.sessionId); + if (!sessionId) { + sendError(respond, new Error("sessionId required"), ErrorCodes.INVALID_REQUEST); + return; + } + const sinceIndex = (params as { sinceIndex?: unknown } | undefined)?.sinceIndex; + if ( + sinceIndex !== undefined && + (typeof sinceIndex !== "number" || !Number.isSafeInteger(sinceIndex) || sinceIndex < 0) + ) { + sendError( + respond, + new Error("sinceIndex must be a non-negative safe integer"), + ErrorCodes.INVALID_REQUEST, + ); + return; + } + const rt = await ensureRuntime(); + respond( + true, + await rt.transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex }), + ); + } catch (err) { + sendError(respond, err); + } + }, + ); + api.registerGatewayMethod( "googlemeet.recoverCurrentTab", async ({ params, respond }: GatewayRequestHandlerOptions) => { @@ -1121,6 +1161,11 @@ export default definePluginEntry({ case "status": { return json(await callGoogleMeetGatewayFromTool({ config, action: "status", raw })); } + case "transcript": { + return json( + await callGoogleMeetGatewayFromTool({ config, action: "transcript", raw }), + ); + } case "recover_current_tab": { return json( await callGoogleMeetGatewayFromTool({ diff --git a/extensions/google-meet/src/cli.test.ts b/extensions/google-meet/src/cli.test.ts index 1ec407e1394..5a638aecce0 100644 --- a/extensions/google-meet/src/cli.test.ts +++ b/extensions/google-meet/src/cli.test.ts @@ -803,6 +803,35 @@ describe("google-meet CLI", () => { } }); + it("prints cursor-based transcripts from the gateway-owned runtime", async () => { + const callGatewayFromCli = vi.fn(async () => ({ + found: true, + sessionId: "meet_gateway", + startIndex: 3, + nextIndex: 4, + droppedLines: 2, + lines: [{ at: "2026-07-12T06:00:00.000Z", speaker: "Alice", text: "fourth line" }], + })); + const stdout = captureStdout(); + try { + await setupCli({ callGatewayFromCli }).parseAsync( + ["googlemeet", "transcript", "meet_gateway", "--since", "3"], + { from: "user" }, + ); + expect(callGatewayFromCli).toHaveBeenCalledWith( + "googlemeet.transcript", + { json: true, timeout: "5000" }, + { sessionId: "meet_gateway", sinceIndex: 3 }, + { progress: false }, + ); + expect(stdout.output()).toContain("# 2 earlier lines dropped by the transcript cap"); + expect(stdout.output()).toContain("Alice: fourth line"); + expect(stdout.output()).toContain("# nextIndex: 4"); + } finally { + stdout.restore(); + } + }); + it("delegates join to the gateway-owned runtime when available", async () => { const callGatewayFromCli = vi.fn(async () => ({ session: { diff --git a/extensions/google-meet/src/cli.ts b/extensions/google-meet/src/cli.ts index a2ab29d9740..dae18413370 100644 --- a/extensions/google-meet/src/cli.ts +++ b/extensions/google-meet/src/cli.ts @@ -161,6 +161,7 @@ type GoogleMeetGatewayMethod = | "googlemeet.leave" | "googlemeet.speak" | "googlemeet.status" + | "googlemeet.transcript" | "googlemeet.testListen" | "googlemeet.testSpeech"; @@ -2251,6 +2252,50 @@ export function registerGoogleMeetCli(params: { writeStdoutJson(await rt.status(sessionId)); }); + root + .command("transcript") + .description("Print the bounded caption transcript for a Meet session") + .argument("", "Meet session ID") + .option("--since ", "Resume from the previous response's nextIndex") + .option("--json", "Print JSON output", false) + .action(async (sessionId: string, options: { since?: string; json?: boolean }) => { + const sinceIndex = options.since === undefined ? undefined : Number(options.since); + if (sinceIndex !== undefined && (!Number.isSafeInteger(sinceIndex) || sinceIndex < 0)) { + throw new Error("--since must be a non-negative safe integer"); + } + const delegated = await callGoogleMeetGateway({ + callGateway, + method: "googlemeet.transcript", + payload: { sessionId, ...(sinceIndex === undefined ? {} : { sinceIndex }) }, + }); + const result = delegated.ok + ? (delegated.payload as Awaited>) + : await ( + await params.ensureRuntime() + ).transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex }); + if (!result.found) { + throw new Error("session not found"); + } + if (options.json) { + writeStdoutJson(result); + return; + } + if (result.evicted) { + writeStdoutLine("# transcript evicted from runtime memory"); + } else if (result.droppedLines) { + writeStdoutLine("# %d earlier lines dropped by the transcript cap", result.droppedLines); + } + for (const line of result.lines ?? []) { + writeStdoutLine( + "%s%s%s", + line.at ? `[${line.at}] ` : "", + line.speaker ? `${line.speaker}: ` : "", + line.text, + ); + } + writeStdoutLine("# nextIndex: %d", result.nextIndex ?? 0); + }); + root .command("doctor") .description("Show human-readable Meet session/browser/realtime health") diff --git a/extensions/google-meet/src/runtime.ts b/extensions/google-meet/src/runtime.ts index e24e19e132d..9d7f3d8d939 100644 --- a/extensions/google-meet/src/runtime.ts +++ b/extensions/google-meet/src/runtime.ts @@ -26,6 +26,8 @@ import { launchChromeMeetOnNode, leaveChromeMeet, leaveChromeMeetOnNode, + readChromeMeetTranscript, + readChromeMeetTranscriptOnNode, recoverCurrentMeetTab, recoverCurrentMeetTabOnNode, } from "./transports/chrome.js"; @@ -34,12 +36,15 @@ import { normalizeDialInNumber, prefixDtmfWait, } from "./transports/twilio.js"; +import { GOOGLE_MEET_TRANSCRIPT_MAX_LINES } from "./transports/types.js"; import type { GoogleMeetBrowserTab, GoogleMeetChromeHealth, GoogleMeetJoinRequest, GoogleMeetJoinResult, GoogleMeetSession, + GoogleMeetTranscriptLine, + GoogleMeetTranscriptSnapshot, } from "./transports/types.js"; import { createVoiceCallGateway, @@ -73,6 +78,13 @@ type RetainedBrowserTab = { tab: GoogleMeetBrowserTab; }; +type RetainedTranscriptSnapshot = GoogleMeetTranscriptSnapshot & { + pageEpoch?: string; + pageNextIndex: number; +}; + +const GOOGLE_MEET_ENDED_TRANSCRIPTS_MAX = 4; + function nowIso(): string { return new Date().toISOString(); } @@ -270,6 +282,10 @@ export class GoogleMeetRuntime { readonly #sessionStops = new Map Promise>(); readonly #sessionSpeakers = new Map void>(); readonly #sessionHealth = new Map GoogleMeetChromeHealth>(); + readonly #transcripts = new Map(); + readonly #transcriptCaptures = new Map>(); + readonly #transcriptFinalizing = new Set(); + readonly #retiredTranscripts = new Set(); readonly #voiceCallGateway: VoiceCallGateway; constructor( @@ -308,6 +324,45 @@ export class GoogleMeetRuntime { return session ? { found: true, session } : { found: false }; } + async transcript( + sessionId: string, + options: { sinceIndex?: number } = {}, + ): Promise<{ + found: boolean; + sessionId?: string; + startIndex?: number; + nextIndex?: number; + droppedLines?: number; + evicted?: boolean; + lines?: GoogleMeetTranscriptLine[]; + }> { + const session = this.#sessions.get(sessionId); + if (!session) { + return { found: false }; + } + if (session.mode !== "transcribe") { + throw new Error("transcript is only available for transcribe-mode sessions"); + } + const sinceIndex = options.sinceIndex ?? 0; + if (!Number.isSafeInteger(sinceIndex) || sinceIndex < 0) { + throw new Error("sinceIndex must be a non-negative safe integer"); + } + if (session.state === "active" && !this.#transcriptFinalizing.has(session.id)) { + await this.#captureTranscriptSnapshot(session); + } + const snapshot = this.#transcripts.get(sessionId) ?? { droppedLines: 0, lines: [] }; + const startIndex = Math.max(sinceIndex, snapshot.droppedLines); + return { + found: true, + sessionId, + startIndex, + nextIndex: snapshot.droppedLines + snapshot.lines.length, + droppedLines: snapshot.droppedLines, + ...(session.transcriptEvicted ? { evicted: true } : {}), + lines: snapshot.lines.slice(startIndex - snapshot.droppedLines), + }; + } + async setupStatus( options: { transport?: GoogleMeetTransport; @@ -781,6 +836,16 @@ export class GoogleMeetRuntime { session: GoogleMeetSession, opts?: { keepBrowserTab?: boolean }, ): Promise { + // #leaveUnlocked publishes this promise in #sessionLeaves before another + // event can re-enter, so the final snapshot remains inside one teardown. + if (session.mode === "transcribe") { + this.#transcriptFinalizing.add(session.id); + await this.#captureTranscriptSnapshot(session, { finalize: true }).catch((error: unknown) => { + this.params.logger.debug?.( + `[google-meet] final transcript snapshot ignored: ${formatErrorMessage(error)}`, + ); + }); + } session.state = "ended"; session.updatedAt = nowIso(); const stop = this.#sessionStops.get(session.id); @@ -791,14 +856,126 @@ export class GoogleMeetRuntime { try { await stop?.(); } finally { - // A bridge teardown failure must not leave the browser participant behind. - if (!opts?.keepBrowserTab) { - browserLeft = await this.#releaseBrowserTab(session); + try { + // A bridge teardown failure must not leave the browser participant behind. + if (!opts?.keepBrowserTab) { + browserLeft = await this.#releaseBrowserTab(session); + } + } finally { + this.#retireTranscript(session.id); + this.#transcriptFinalizing.delete(session.id); } } return { found: true, session, ...(browserLeft === undefined ? {} : { browserLeft }) }; } + async #captureTranscriptSnapshot( + session: GoogleMeetSession, + options: { finalize?: boolean } = {}, + ): Promise { + // Preserve browser-read order so an older response cannot regress the cursor + // or overwrite the final snapshot queued by leave. + const previous = this.#transcriptCaptures.get(session.id) ?? Promise.resolve(); + const capture = previous + .catch(() => {}) + .then(async () => { + if (!isBrowserTransport(session.transport) || session.mode !== "transcribe") { + return; + } + const tab = session.chrome?.browserTab; + if (!tab) { + return; + } + const snapshot = + session.transport === "chrome-node" + ? await readChromeMeetTranscriptOnNode({ + runtime: this.params.runtime, + nodeId: session.chrome?.nodeId, + config: this.params.config, + ...(options.finalize === undefined ? {} : { finalize: options.finalize }), + meetingUrl: session.url, + meetingSessionId: session.id, + tab, + }) + : await readChromeMeetTranscript({ + runtime: this.params.runtime, + config: this.params.config, + ...(options.finalize === undefined ? {} : { finalize: options.finalize }), + meetingUrl: session.url, + meetingSessionId: session.id, + tab, + }); + this.#mergeTranscriptSnapshot(session.id, snapshot); + }); + this.#transcriptCaptures.set(session.id, capture); + try { + await capture; + } finally { + if (this.#transcriptCaptures.get(session.id) === capture) { + this.#transcriptCaptures.delete(session.id); + } + } + } + + #mergeTranscriptSnapshot(sessionId: string, snapshot: GoogleMeetTranscriptSnapshot): void { + const pageNextIndex = snapshot.droppedLines + snapshot.lines.length; + const retained = this.#transcripts.get(sessionId); + if (!retained) { + this.#transcripts.set(sessionId, { + droppedLines: snapshot.droppedLines, + lines: snapshot.lines, + pageEpoch: snapshot.epoch, + pageNextIndex, + }); + return; + } + + // A page reload starts a new epoch, but the runtime cursor stays absolute + // so already delivered captions remain addressable. + if (retained.pageEpoch !== snapshot.epoch) { + retained.droppedLines += snapshot.droppedLines; + retained.lines.push(...snapshot.lines); + retained.pageEpoch = snapshot.epoch; + retained.pageNextIndex = pageNextIndex; + } else if (pageNextIndex > retained.pageNextIndex) { + const appendFrom = Math.max(retained.pageNextIndex, snapshot.droppedLines); + retained.droppedLines += Math.max(0, snapshot.droppedLines - retained.pageNextIndex); + retained.lines.push(...snapshot.lines.slice(appendFrom - snapshot.droppedLines)); + retained.pageNextIndex = pageNextIndex; + } + + const excess = retained.lines.length - GOOGLE_MEET_TRANSCRIPT_MAX_LINES; + if (excess > 0) { + retained.lines.splice(0, excess); + retained.droppedLines += excess; + } + } + + #retireTranscript(sessionId: string): void { + const snapshot = this.#transcripts.get(sessionId); + if (snapshot) { + this.#transcripts.delete(sessionId); + this.#transcripts.set(sessionId, snapshot); + this.#retiredTranscripts.delete(sessionId); + this.#retiredTranscripts.add(sessionId); + } + const retainedIds = [...this.#retiredTranscripts] + .filter((id) => this.#transcripts.has(id)) + .toSorted((a, b) => + (this.#sessions.get(a)?.updatedAt ?? "").localeCompare( + this.#sessions.get(b)?.updatedAt ?? "", + ), + ); + for (const id of retainedIds.slice(0, -GOOGLE_MEET_ENDED_TRANSCRIPTS_MAX)) { + this.#transcripts.delete(id); + this.#retiredTranscripts.delete(id); + const session = this.#sessions.get(id); + if (session) { + session.transcriptEvicted = true; + } + } + } + #inheritBrowserTabOwnership(params: { transport: GoogleMeetTransport; nodeId?: string; diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index 4d3423fbc2b..8540b1039f3 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -31,7 +31,12 @@ import { resolveChromeNode, type BrowserTab, } from "./chrome-browser-proxy.js"; -import type { GoogleMeetBrowserTab, GoogleMeetChromeHealth } from "./types.js"; +import { GOOGLE_MEET_TRANSCRIPT_MAX_LINES } from "./types.js"; +import type { + GoogleMeetBrowserTab, + GoogleMeetChromeHealth, + GoogleMeetTranscriptSnapshot, +} from "./types.js"; type BrowserRequestParams = { method: "GET" | "POST" | "DELETE"; @@ -48,11 +53,14 @@ const chromeTransportDeps: { callGatewayFromCli, }; +const GOOGLE_MEET_CAPTION_SETTLE_MS = 1_000; + export const testing = { setDepsForTest(deps: { callGatewayFromCli?: typeof callGatewayFromCli } | null) { chromeTransportDeps.callGatewayFromCli = deps?.callGatewayFromCli ?? callGatewayFromCli; }, meetStatusScriptForTest: meetStatusScript, + meetTranscriptScriptForTest: meetTranscriptScript, meetLeaveScriptForTest: meetLeaveScript, parseMeetBrowserStatusForTest: parseMeetBrowserStatus, resolveBrowserGatewayTimeoutMsForTest: resolveBrowserGatewayTimeoutMs, @@ -213,6 +221,7 @@ export async function launchChromeMeet(params: { callBrowser: await resolveLocalBrowserRequest(params.runtime), config: params.config, mode: params.mode, + meetingSessionId: params.meetingSessionId, url: params.url, }); const shouldStartRealtimeBridge = @@ -408,6 +417,7 @@ async function grantMeetMediaPermissions(params: { function meetStatusScript(params: { allowMicrophone: boolean; autoJoin: boolean; + captionSessionId?: string; captureCaptions: boolean; guestName: string; readOnly?: boolean; @@ -415,6 +425,7 @@ function meetStatusScript(params: { return `async () => { const text = (node) => (node?.innerText || node?.textContent || "").trim(); const allowMicrophone = ${JSON.stringify(params.allowMicrophone)}; + const captionSessionId = ${JSON.stringify(params.captionSessionId)}; const captureCaptions = ${JSON.stringify(params.captureCaptions)}; const readOnly = ${JSON.stringify(Boolean(params.readOnly))}; const buttons = [...document.querySelectorAll('button')]; @@ -546,42 +557,125 @@ function meetStatusScript(params: { if (!captureCaptions) return undefined; const w = window; if (!inCall && !w.__openclawMeetCaptions) return undefined; - if (!w.__openclawMeetCaptions) { + // A reused tab starts a fresh logical transcript for each OpenClaw session. + // Status refreshes omit the id, so they preserve the active page-owned buffer. + if (!w.__openclawMeetCaptions || (captionSessionId && w.__openclawMeetCaptions.sessionId !== captionSessionId)) { + if (w.__openclawMeetCaptions?.settleTimer !== undefined) { + clearTimeout(w.__openclawMeetCaptions.settleTimer); + } + w.__openclawMeetCaptions?.observer?.disconnect?.(); w.__openclawMeetCaptions = { + sessionId: captionSessionId, + epoch: Date.now() + "-" + Math.random().toString(36).slice(2), enabledAttempted: false, observerInstalled: false, + observer: undefined, + droppedLines: 0, lines: [], - seen: {} + settleTimer: undefined, + visible: [] }; } return w.__openclawMeetCaptions; })(); - const recordCaption = (speaker, captionText) => { + const normalizeCaption = (speaker, captionText) => { if (!captionState) return; const clean = String(captionText || "").replace(/\\s+/g, " ").trim(); const cleanSpeaker = String(speaker || "").replace(/\\s+/g, " ").trim(); - if (!clean || clean.length < 2) return; - if (/^(turn on captions|turn off captions|captions)$/i.test(clean)) return; - const key = (cleanSpeaker + "\\n" + clean).toLowerCase(); - if (captionState.seen[key]) return; - captionState.seen[key] = true; - const entry = { at: new Date().toISOString(), speaker: cleanSpeaker || undefined, text: clean }; - captionState.lines.push(entry); - if (captionState.lines.length > 50) captionState.lines.splice(0, captionState.lines.length - 50); + if (!clean || clean.length < 2) return undefined; + if (/^(turn on captions|turn off captions|captions)$/i.test(clean)) return undefined; + return { speaker: cleanSpeaker || undefined, text: clean }; + }; + const commitLines = (state, entries) => { + state.lines.push(...entries.map((entry) => ({ + at: entry.at, + speaker: entry.speaker, + text: entry.text + }))); + const excess = state.lines.length - ${GOOGLE_MEET_TRANSCRIPT_MAX_LINES}; + if (excess > 0) { + state.lines.splice(0, excess); + state.droppedLines = (state.droppedLines || 0) + excess; + } }; const scrapeCaptions = () => { if (!captionState) return; const regions = [...document.querySelectorAll(captionSelector)]; + const rows = []; for (const region of regions) { const raw = text(region); if (!raw) continue; const pieces = raw.split(/\\n+/).map((part) => part.trim()).filter(Boolean); - if (pieces.length >= 2) { - recordCaption(pieces[0], pieces.slice(1).join(" ")); - } else { - recordCaption("", pieces[0] || raw); - } + const row = pieces.length >= 2 + ? normalizeCaption(pieces[0], pieces.slice(1).join(" ")) + : normalizeCaption("", pieces[0] || raw); + if (row) rows.push({ ...row, node: region }); } + if (rows.length === 0) { + // Meet briefly removes caption rows while rerendering. Keep them mutable + // for one settle window so a DOM gap cannot fabricate a repeated line. + if (captionState.visible.length > 0 && captionState.settleTimer === undefined) { + const pendingState = captionState; + pendingState.settleTimer = setTimeout(() => { + if (window.__openclawMeetCaptions !== pendingState) return; + commitLines(pendingState, pendingState.visible); + pendingState.visible = []; + pendingState.settleTimer = undefined; + }, ${GOOGLE_MEET_CAPTION_SETTLE_MS}); + } + return; + } + if (captionState.settleTimer !== undefined) { + clearTimeout(captionState.settleTimer); + captionState.settleTimer = undefined; + } + const previous = Array.isArray(captionState.visible) ? captionState.visible : []; + const unmatchedPrevious = [...previous]; + const nextVisible = []; + const now = Date.now(); + for (let index = 0; index < rows.length; index += 1) { + const row = rows[index]; + const priorIndex = unmatchedPrevious.findIndex((candidate) => { + const sameTextLifecycle = + candidate.text === row.text || + row.text.startsWith(candidate.text) || + candidate.text.startsWith(row.text); + const sameDomLifecycle = + candidate.node === row.node || now - candidate.seenAt <= ${GOOGLE_MEET_CAPTION_SETTLE_MS}; + return candidate.speaker === row.speaker && sameTextLifecycle && sameDomLifecycle; + }); + const prior = priorIndex >= 0 ? unmatchedPrevious.splice(priorIndex, 1)[0] : undefined; + const sameSpeaker = Boolean(prior) && prior.speaker === row.speaker; + if (sameSpeaker && prior.text === row.text) { + prior.node = row.node; + prior.seenAt = now; + nextVisible.push(prior); + continue; + } + if (sameSpeaker && row.text.startsWith(prior.text)) { + prior.text = row.text; + prior.node = row.node; + prior.seenAt = now; + nextVisible.push(prior); + continue; + } + if (sameSpeaker && prior.text.startsWith(row.text)) { + prior.node = row.node; + prior.seenAt = now; + nextVisible.push(prior); + continue; + } + const entry = { + at: new Date().toISOString(), + node: row.node, + seenAt: now, + speaker: row.speaker, + text: row.text + }; + nextVisible.push(entry); + } + commitLines(captionState, unmatchedPrevious); + captionState.visible = nextVisible; }; if (captionState) { if (!readOnly && inCall && !captionState.enabledAttempted) { @@ -600,7 +694,8 @@ function meetStatusScript(params: { } if (inCall && !captionState.observerInstalled) { captionState.observerInstalled = true; - new MutationObserver(scrapeCaptions).observe(document.body, { + captionState.observer = new MutationObserver(scrapeCaptions); + captionState.observer.observe(document.body, { childList: true, subtree: true, characterData: true @@ -610,17 +705,19 @@ function meetStatusScript(params: { if (inCall) { scrapeCaptions(); } - const lines = Array.isArray(captionState.lines) ? captionState.lines : []; + const committedLines = Array.isArray(captionState.lines) ? captionState.lines : []; + const visibleLines = Array.isArray(captionState.visible) ? captionState.visible : []; + const lines = [...committedLines, ...visibleLines]; const last = lines[lines.length - 1]; captioning = document.querySelector(captionSelector) !== null || lines.length > 0; - transcriptLines = lines.length; + transcriptLines = (captionState.droppedLines || 0) + lines.length; lastCaptionAt = last?.at; lastCaptionSpeaker = last?.speaker; lastCaptionText = last?.text; recentTranscript = lines.slice(-5); } const lobbyWaiting = !inCall && /asking to be let in|you.?ll join when someone lets you in|waiting to be let in|ask to join/i.test(pageText); - const leaveReason = /you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i.test(pageText) + const leaveReason = !inCall && /you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i.test(pageText) ? pageText.match(/you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i)?.[0] : undefined; let manualActionReason; @@ -672,6 +769,145 @@ function meetStatusScript(params: { }`; } +function meetTranscriptScript(meetingUrl: string, meetingSessionId: string, finalize: boolean) { + const expectedMeetingUrl = normalizeMeetUrlForReuse(meetingUrl); + return `() => { + const expectedMeetingUrl = ${JSON.stringify(expectedMeetingUrl)}; + const expectedSessionId = ${JSON.stringify(meetingSessionId)}; + let currentMeetingUrl; + try { + const currentUrl = new URL(location.href); + currentMeetingUrl = currentUrl.origin + currentUrl.pathname.toLowerCase().replace(/\\/$/, ""); + } catch { + return JSON.stringify({ urlMatched: false }); + } + if (!expectedMeetingUrl || currentMeetingUrl !== expectedMeetingUrl) { + return JSON.stringify({ urlMatched: false }); + } + const state = window.__openclawMeetCaptions; + if (state?.sessionId && state.sessionId !== expectedSessionId) { + return JSON.stringify({ urlMatched: true, sessionMatched: false }); + } + if (${JSON.stringify(finalize)} && Array.isArray(state?.visible) && state.visible.length > 0) { + if (state.settleTimer !== undefined) clearTimeout(state.settleTimer); + state.settleTimer = undefined; + state.lines = Array.isArray(state.lines) ? state.lines : []; + state.lines.push(...state.visible.map((entry) => ({ + at: entry.at, + speaker: entry.speaker, + text: entry.text + }))); + state.visible = []; + const excess = state.lines.length - ${GOOGLE_MEET_TRANSCRIPT_MAX_LINES}; + if (excess > 0) { + state.lines.splice(0, excess); + state.droppedLines = (state.droppedLines || 0) + excess; + } + } + const lines = Array.isArray(state?.lines) ? state.lines : []; + return JSON.stringify({ + urlMatched: true, + sessionMatched: true, + epoch: typeof state?.epoch === "string" ? state.epoch : undefined, + droppedLines: Number.isFinite(state?.droppedLines) ? Math.max(0, Math.trunc(state.droppedLines)) : 0, + lines: lines.map((line) => ({ + at: typeof line?.at === "string" ? line.at : undefined, + speaker: typeof line?.speaker === "string" ? line.speaker : undefined, + text: typeof line?.text === "string" ? line.text : "" + })).filter((line) => line.text) + }); +}`; +} + +function parseMeetTranscriptSnapshot( + result: unknown, +): GoogleMeetTranscriptSnapshot & { sessionMatched?: boolean; urlMatched?: boolean } { + const record = result && typeof result === "object" ? (result as Record) : {}; + const raw = record.result; + if (typeof raw !== "string" || !raw.trim()) { + return { droppedLines: 0, lines: [] }; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("Google Meet transcript JSON is malformed."); + } + if (!parsed || typeof parsed !== "object") { + throw new Error("Google Meet transcript payload is invalid."); + } + const payload = parsed as { + droppedLines?: unknown; + epoch?: unknown; + lines?: unknown; + sessionMatched?: unknown; + urlMatched?: unknown; + }; + const droppedLines = + typeof payload.droppedLines === "number" && Number.isSafeInteger(payload.droppedLines) + ? Math.max(0, payload.droppedLines) + : 0; + const lines = Array.isArray(payload.lines) + ? payload.lines.flatMap((value) => { + if (!value || typeof value !== "object") { + return []; + } + const line = value as { at?: unknown; speaker?: unknown; text?: unknown }; + if (typeof line.text !== "string" || !line.text.trim()) { + return []; + } + return [ + { + ...(typeof line.at === "string" ? { at: line.at } : {}), + ...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}), + text: line.text, + }, + ]; + }) + : []; + return { + droppedLines, + ...(typeof payload.epoch === "string" ? { epoch: payload.epoch } : {}), + lines, + ...(typeof payload.urlMatched === "boolean" ? { urlMatched: payload.urlMatched } : {}), + ...(typeof payload.sessionMatched === "boolean" + ? { sessionMatched: payload.sessionMatched } + : {}), + }; +} + +async function readMeetTranscriptWithBrowserRequest(params: { + callBrowser: BrowserRequestCaller; + finalize: boolean; + meetingUrl: string; + meetingSessionId: string; + tab: GoogleMeetBrowserTab; + timeoutMs: number; +}): Promise { + const result = await params.callBrowser({ + method: "POST", + path: "/act", + body: { + kind: "evaluate", + targetId: params.tab.targetId, + fn: meetTranscriptScript(params.meetingUrl, params.meetingSessionId, params.finalize), + }, + timeoutMs: params.timeoutMs, + }); + const snapshot = parseMeetTranscriptSnapshot(result); + if (snapshot.urlMatched === false) { + throw new Error("The tracked Meet tab no longer shows this session's meeting URL."); + } + if (snapshot.sessionMatched === false) { + throw new Error("The tracked Meet tab now belongs to another OpenClaw meeting session."); + } + return { + droppedLines: snapshot.droppedLines, + ...(snapshot.epoch ? { epoch: snapshot.epoch } : {}), + lines: snapshot.lines, + }; +} + function meetLeaveScript(meetingUrl: string) { const expectedMeetingUrl = normalizeMeetUrlForReuse(meetingUrl); return `() => { @@ -909,6 +1145,58 @@ export async function leaveChromeMeet(params: { }); } +export async function readChromeMeetTranscript(params: { + runtime: PluginRuntime; + config: GoogleMeetConfig; + finalize?: boolean; + meetingUrl: string; + meetingSessionId: string; + tab: GoogleMeetBrowserTab; +}): Promise { + return await readMeetTranscriptWithBrowserRequest({ + callBrowser: await resolveLocalBrowserRequest(params.runtime), + finalize: params.finalize === true, + meetingUrl: params.meetingUrl, + meetingSessionId: params.meetingSessionId, + tab: params.tab, + timeoutMs: Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000), + }); +} + +export async function readChromeMeetTranscriptOnNode(params: { + runtime: PluginRuntime; + nodeId?: string; + config: GoogleMeetConfig; + finalize?: boolean; + meetingUrl: string; + meetingSessionId: string; + tab: GoogleMeetBrowserTab; +}): Promise { + const nodeId = + params.nodeId ?? + (await resolveChromeNode({ + runtime: params.runtime, + requestedNode: params.config.chromeNode.node, + })); + const timeoutMs = Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000); + return await readMeetTranscriptWithBrowserRequest({ + callBrowser: async (request) => + await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId, + method: request.method, + path: request.path, + body: request.body, + timeoutMs: request.timeoutMs, + }), + finalize: params.finalize === true, + meetingUrl: params.meetingUrl, + meetingSessionId: params.meetingSessionId, + tab: params.tab, + timeoutMs, + }); +} + export async function leaveChromeMeetOnNode(params: { runtime: PluginRuntime; nodeId?: string; @@ -943,6 +1231,7 @@ async function openMeetWithBrowserProxy(params: { nodeId: string; config: GoogleMeetConfig; mode: GoogleMeetMode; + meetingSessionId: string; url: string; }): Promise<{ launched: boolean; browser?: GoogleMeetChromeHealth; tab?: GoogleMeetBrowserTab }> { return await openMeetWithBrowserRequest({ @@ -957,6 +1246,7 @@ async function openMeetWithBrowserProxy(params: { }), config: params.config, mode: params.mode, + meetingSessionId: params.meetingSessionId, url: params.url, }); } @@ -965,6 +1255,7 @@ async function openMeetWithBrowserRequest(params: { callBrowser: BrowserRequestCaller; config: GoogleMeetConfig; mode: GoogleMeetMode; + meetingSessionId: string; url: string; }): Promise<{ launched: boolean; browser?: GoogleMeetChromeHealth; tab?: GoogleMeetBrowserTab }> { if (!params.config.chrome.launch) { @@ -1055,6 +1346,7 @@ async function openMeetWithBrowserRequest(params: { targetId, fn: meetStatusScript({ allowMicrophone: isGoogleMeetTalkBackMode(params.mode), + captionSessionId: params.meetingSessionId, captureCaptions: params.mode === "transcribe", guestName: params.config.chrome.guestName, autoJoin: params.config.chrome.autoJoin, @@ -1366,6 +1658,7 @@ export async function launchChromeMeetOnNode(params: { nodeId, config: params.config, mode: params.mode, + meetingSessionId: params.meetingSessionId, url: params.url, }); const raw = await params.runtime.nodes.invoke({ diff --git a/extensions/google-meet/src/transports/types.ts b/extensions/google-meet/src/transports/types.ts index 683d4c1315c..5bc5185ff2d 100644 --- a/extensions/google-meet/src/transports/types.ts +++ b/extensions/google-meet/src/transports/types.ts @@ -3,6 +3,20 @@ import type { GoogleMeetMode, GoogleMeetModeInput, GoogleMeetTransport } from ". type GoogleMeetSessionState = "active" | "ended"; +export const GOOGLE_MEET_TRANSCRIPT_MAX_LINES = 2_000; + +export type GoogleMeetTranscriptLine = { + at?: string; + speaker?: string; + text: string; +}; + +export type GoogleMeetTranscriptSnapshot = { + droppedLines: number; + epoch?: string; + lines: GoogleMeetTranscriptLine[]; +}; + export type GoogleMeetJoinRequest = { url: string; transport?: GoogleMeetTransport; @@ -120,6 +134,8 @@ export type GoogleMeetSession = { /** Canonical agent owner for every later consult and bridge restart. */ agentId: string; state: GoogleMeetSessionState; + /** Set when this ended session's ephemeral transcript aged out of memory. */ + transcriptEvicted?: boolean; /** Terminal browser-departure result, retained so repeated leave calls stay idempotent. */ browserLeft?: boolean; createdAt: string;