mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-10 17:16:55 +00:00
* feat(google-meet): retain bounded transcripts Co-authored-by: lonexreb <reach2shubhankar@gmail.com> * chore: leave changelog to release tooling * fix(google-meet): satisfy transcript CI checks --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
committed by
GitHub
parent
648d742ee7
commit
43e138cc66
@@ -121,6 +121,15 @@ If the browser fallback hits a Google login or Meet permission blocker, the tool
|
||||
|
||||
Set `"mode": "transcribe"` to skip the duplex realtime bridge (no BlackHole/SoX requirement, no talk-back). Transcribe-mode Chrome joins also skip OpenClaw's microphone/camera permission grant and the Meet **Use microphone** path; if Meet shows the audio-choice interstitial, automation tries **Continue without microphone** first. Managed Chrome transports in this mode install a best-effort Meet caption observer. `googlemeet status --json` and `googlemeet doctor` report `captioning`, `captionsEnabledAttempted`, `transcriptLines`, `lastCaptionAt`, `lastCaptionSpeaker`, `lastCaptionText`, and a `recentTranscript` tail.
|
||||
|
||||
For the bounded session transcript, read the exact tracked Meet tab:
|
||||
|
||||
```bash
|
||||
openclaw googlemeet transcript <session-id>
|
||||
openclaw googlemeet transcript <session-id> --since <next-index> --json
|
||||
```
|
||||
|
||||
The observer keeps at most 2,000 completed caption lines in the Meet page. Visible progressive text stays in the status health tail until the caption row completes, so saving `nextIndex` cannot skip a later text expansion; leaving finalizes visible rows before the snapshot. `droppedLines` reports lines lost from the head when the cap is exceeded. The four most recently ended session transcripts remain readable until the gateway restarts. Older ended transcripts return `evicted: true`. This is intentionally runtime memory, not durable meeting-history storage: restarting the gateway, closing the tab before a snapshot, or exceeding the documented caps can lose captions.
|
||||
|
||||
For a yes/no listen probe:
|
||||
|
||||
```bash
|
||||
@@ -838,6 +847,7 @@ Agents use the `google_meet` tool:
|
||||
| `attendance` | List participants and participant sessions |
|
||||
| `export` | Write the artifacts/attendance/transcript/manifest bundle; set `"dryRun": true` for manifest-only |
|
||||
| `recover_current_tab` | Focus/inspect an existing Meet tab without opening a new one |
|
||||
| `transcript` | Read the bounded caption transcript; `sinceIndex` resumes from the previous `nextIndex` |
|
||||
| `leave` | End a session (Chrome clicks Leave; closes only tabs it opened; Twilio hangs up) |
|
||||
| `end_active_conference` | End the active Google Meet conference for an API-managed space |
|
||||
| `speak` | Make the realtime agent speak immediately, given `sessionId` and `message` |
|
||||
|
||||
@@ -980,6 +980,7 @@ describe("google-meet plugin", () => {
|
||||
"join",
|
||||
"create",
|
||||
"status",
|
||||
"transcript",
|
||||
"setup_status",
|
||||
"resolve_space",
|
||||
"preflight",
|
||||
@@ -2629,9 +2630,27 @@ describe("google-meet plugin", () => {
|
||||
tabUrlAfterJoin?: string;
|
||||
leaveClicked?: boolean;
|
||||
leaveConfirmationRequired?: boolean;
|
||||
nonFinalTranscriptGate?: Promise<void>;
|
||||
onNonFinalTranscriptRead?: () => void;
|
||||
transcript?: {
|
||||
droppedLines?: number;
|
||||
epoch?: string;
|
||||
lines: Array<{ at?: string; speaker?: string; text: string }>;
|
||||
};
|
||||
finalTranscript?: {
|
||||
droppedLines?: number;
|
||||
epoch?: string;
|
||||
lines: Array<{ at?: string; speaker?: string; text: string }>;
|
||||
};
|
||||
transcriptSequence?: Array<{
|
||||
droppedLines?: number;
|
||||
epoch?: string;
|
||||
lines: Array<{ at?: string; speaker?: string; text: string }>;
|
||||
}>;
|
||||
}) {
|
||||
let joined = false;
|
||||
let leaveStep = 0;
|
||||
let transcriptReadIndex = 0;
|
||||
let openedTabUrl = options?.reused ? "https://meet.google.com/abc-defg-hij?hl=en" : undefined;
|
||||
const callGatewayFromCli = vi.fn(
|
||||
async (
|
||||
@@ -2676,6 +2695,30 @@ describe("google-meet plugin", () => {
|
||||
return { ok: true };
|
||||
}
|
||||
if (request.path === "/act") {
|
||||
if (String(request.body?.fn).includes("state = window.__openclawMeetCaptions")) {
|
||||
const finalizing = String(request.body?.fn).includes("if (true &&");
|
||||
if (!finalizing) {
|
||||
options?.onNonFinalTranscriptRead?.();
|
||||
await options?.nonFinalTranscriptGate;
|
||||
}
|
||||
const transcript = finalizing
|
||||
? (options?.finalTranscript ?? options?.transcript)
|
||||
: options?.transcript;
|
||||
const sequencedTranscript =
|
||||
options?.transcriptSequence?.[
|
||||
Math.min(transcriptReadIndex, options.transcriptSequence.length - 1)
|
||||
];
|
||||
transcriptReadIndex += 1;
|
||||
const responseTranscript = sequencedTranscript ?? transcript;
|
||||
return {
|
||||
result: JSON.stringify({
|
||||
urlMatched: options?.tabUrlAfterJoin?.includes("/abc-defg-hij") !== false,
|
||||
droppedLines: responseTranscript?.droppedLines ?? 0,
|
||||
epoch: responseTranscript?.epoch,
|
||||
lines: responseTranscript?.lines ?? [],
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (String(request.body?.fn).includes("leaveAction")) {
|
||||
const currentUrl = options?.tabUrlAfterJoin ?? openedTabUrl;
|
||||
const urlMatched = currentUrl?.includes("/abc-defg-hij") === true;
|
||||
@@ -2725,6 +2768,223 @@ describe("google-meet plugin", () => {
|
||||
return callGatewayFromCli;
|
||||
}
|
||||
|
||||
it("reads and snapshots the bounded transcript from the exact tracked tab", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
try {
|
||||
const callGatewayFromCli = mockLocalMeetBrowserRequestWithTabState({
|
||||
transcript: {
|
||||
droppedLines: 2,
|
||||
lines: [
|
||||
{ at: "2026-07-12T06:00:00.000Z", speaker: "Alice", text: "third line" },
|
||||
{ at: "2026-07-12T06:00:01.000Z", speaker: "Bob", text: "fourth line" },
|
||||
],
|
||||
},
|
||||
});
|
||||
const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" });
|
||||
const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", {
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
})) as { session: { id: string } };
|
||||
|
||||
const beforeRead = callGatewayFromCli.mock.calls.length;
|
||||
const transcript = (await invokeGoogleMeetGatewayMethodForTest(
|
||||
methods,
|
||||
"googlemeet.transcript",
|
||||
{ sessionId: joined.session.id, sinceIndex: 3 },
|
||||
)) as {
|
||||
droppedLines: number;
|
||||
startIndex: number;
|
||||
nextIndex: number;
|
||||
lines: Array<{ text: string }>;
|
||||
};
|
||||
expect(transcript).toMatchObject({ droppedLines: 2, startIndex: 3, nextIndex: 4 });
|
||||
expect(transcript.lines.map((line) => line.text)).toEqual(["fourth line"]);
|
||||
const readCalls = callGatewayFromCli.mock.calls.slice(beforeRead);
|
||||
expect(readCalls).toHaveLength(1);
|
||||
expect(requireRecord(readCalls[0]?.[2], "transcript request")).toMatchObject({
|
||||
method: "POST",
|
||||
path: "/act",
|
||||
body: { targetId: "local-meet-tab" },
|
||||
});
|
||||
|
||||
await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
const afterLeave = (await invokeGoogleMeetGatewayMethodForTest(
|
||||
methods,
|
||||
"googlemeet.transcript",
|
||||
{ sessionId: joined.session.id },
|
||||
)) as { lines: Array<{ text: string }> };
|
||||
expect(afterLeave.lines.map((line) => line.text)).toEqual(["third line", "fourth line"]);
|
||||
|
||||
await expect(
|
||||
invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
sinceIndex: 1.5,
|
||||
}),
|
||||
).rejects.toThrow("sinceIndex must be a non-negative safe integer");
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", { value: originalPlatform });
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses to read a tracked tab after it navigates away from the meeting", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
try {
|
||||
mockLocalMeetBrowserRequestWithTabState({
|
||||
tabUrlAfterJoin: "https://meet.google.com/lookup/unrelated",
|
||||
transcript: { lines: [{ text: "must not leak" }] },
|
||||
});
|
||||
const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" });
|
||||
const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", {
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
})) as { session: { id: string } };
|
||||
|
||||
await expect(
|
||||
invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
}),
|
||||
).rejects.toThrow("tracked Meet tab no longer shows this session's meeting URL");
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", { value: originalPlatform });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the cursor monotonic when the Meet page transcript epoch resets", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
try {
|
||||
mockLocalMeetBrowserRequestWithTabState({
|
||||
transcriptSequence: [
|
||||
{ epoch: "page-1", lines: [{ text: "before reload" }] },
|
||||
{ epoch: "page-2", lines: [{ text: "after reload" }] },
|
||||
{ epoch: "page-2", lines: [{ text: "after reload" }] },
|
||||
],
|
||||
});
|
||||
const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" });
|
||||
const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", {
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
})) as { session: { id: string } };
|
||||
|
||||
const first = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
})) as { nextIndex: number; lines: Array<{ text: string }> };
|
||||
const second = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
sinceIndex: first.nextIndex,
|
||||
})) as { nextIndex: number; lines: Array<{ text: string }> };
|
||||
expect(first).toMatchObject({ nextIndex: 1, lines: [{ text: "before reload" }] });
|
||||
expect(second).toMatchObject({ nextIndex: 2, lines: [{ text: "after reload" }] });
|
||||
|
||||
await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
const ended = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
})) as { lines: Array<{ text: string }> };
|
||||
expect(ended.lines.map((line) => line.text)).toEqual(["before reload", "after reload"]);
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", { value: originalPlatform });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not let a late active read replace the finalized leave snapshot", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
let releaseRead: (() => void) | undefined;
|
||||
let markReadStarted: (() => void) | undefined;
|
||||
const readGate = new Promise<void>((resolve) => {
|
||||
releaseRead = resolve;
|
||||
});
|
||||
const readStarted = new Promise<void>((resolve) => {
|
||||
markReadStarted = resolve;
|
||||
});
|
||||
let activeReads = 0;
|
||||
try {
|
||||
const callGatewayFromCli = mockLocalMeetBrowserRequestWithTabState({
|
||||
transcript: { lines: [{ text: "partial" }] },
|
||||
finalTranscript: { lines: [{ text: "partial" }, { text: "complete caption" }] },
|
||||
nonFinalTranscriptGate: readGate,
|
||||
onNonFinalTranscriptRead: () => {
|
||||
activeReads += 1;
|
||||
markReadStarted?.();
|
||||
},
|
||||
});
|
||||
const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" });
|
||||
const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", {
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
})) as { session: { id: string } };
|
||||
|
||||
const lateRead = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
await readStarted;
|
||||
const secondRead = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
expect(activeReads).toBe(1);
|
||||
const leaving = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
const repeatedLeave = invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
releaseRead?.();
|
||||
await Promise.allSettled([lateRead, secondRead, leaving, repeatedLeave]);
|
||||
const finalCaptures = callGatewayFromCli.mock.calls.filter((call) => {
|
||||
const request = call[2] as { body?: { fn?: string } };
|
||||
const script = String(request.body?.fn);
|
||||
return (
|
||||
script.includes("state = window.__openclawMeetCaptions") && script.includes("if (true &&")
|
||||
);
|
||||
});
|
||||
expect(finalCaptures).toHaveLength(1);
|
||||
const result = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: joined.session.id,
|
||||
})) as { lines: Array<{ text: string }> };
|
||||
expect(result.lines.map((line) => line.text)).toEqual(["partial", "complete caption"]);
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", { value: originalPlatform });
|
||||
}
|
||||
});
|
||||
|
||||
it("retains only the four most recently ended transcripts", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
try {
|
||||
mockLocalMeetBrowserRequestWithTabState({
|
||||
transcript: { lines: [{ text: "retained line" }] },
|
||||
});
|
||||
const { methods } = setup({ defaultMode: "transcribe", defaultTransport: "chrome" });
|
||||
const sessionIds: string[] = [];
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const joined = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.join", {
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
})) as { session: { id: string } };
|
||||
sessionIds.push(joined.session.id);
|
||||
await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.leave", {
|
||||
sessionId: joined.session.id,
|
||||
});
|
||||
}
|
||||
|
||||
const oldest = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: sessionIds[0],
|
||||
})) as { evicted?: boolean; lines: unknown[] };
|
||||
const next = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.transcript", {
|
||||
sessionId: sessionIds[1],
|
||||
})) as { evicted?: boolean; lines: unknown[] };
|
||||
expect(oldest).toMatchObject({ evicted: true, lines: [] });
|
||||
expect(next.evicted).toBeUndefined();
|
||||
expect(next.lines).toHaveLength(1);
|
||||
} finally {
|
||||
Object.defineProperty(process, "platform", { value: originalPlatform });
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves the Meet call in the browser when a chrome session leaves", async () => {
|
||||
const originalPlatform = process.platform;
|
||||
Object.defineProperty(process, "platform", { value: "darwin" });
|
||||
@@ -3470,6 +3730,7 @@ describe("google-meet plugin", () => {
|
||||
`(${chromeTransportTesting.meetStatusScriptForTest({
|
||||
allowMicrophone: false,
|
||||
autoJoin: false,
|
||||
captionSessionId: "session-1",
|
||||
captureCaptions: true,
|
||||
guestName: "OpenClaw Agent",
|
||||
})})`,
|
||||
@@ -3494,6 +3755,151 @@ describe("google-meet plugin", () => {
|
||||
expect(captionButton.click).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("coalesces progressive captions and ignores end phrases while still in call", async () => {
|
||||
const leaveButton = {
|
||||
disabled: false,
|
||||
innerText: "",
|
||||
textContent: "",
|
||||
click: vi.fn(),
|
||||
getAttribute: vi.fn((name: string) => (name === "aria-label" ? "Leave call" : null)),
|
||||
};
|
||||
const page = { caption: "Alice\nmeeting ended", visible: true };
|
||||
const makeRegion = () => ({
|
||||
get innerText() {
|
||||
return page.caption;
|
||||
},
|
||||
get textContent() {
|
||||
return page.caption;
|
||||
},
|
||||
});
|
||||
let region = makeRegion();
|
||||
const windowState: Record<string, unknown> = {};
|
||||
const disconnectObserver = vi.fn();
|
||||
const clearCaptionTimer = vi.fn();
|
||||
let settleCaption: (() => void) | undefined;
|
||||
const document = {
|
||||
body: { innerText: "meeting ended", textContent: "meeting ended" },
|
||||
title: "Meet",
|
||||
querySelector: vi.fn((selector: string) =>
|
||||
selector.includes("aria-live") && page.visible ? region : null,
|
||||
),
|
||||
querySelectorAll: vi.fn((selector: string) => {
|
||||
if (selector === "button") {
|
||||
return [leaveButton];
|
||||
}
|
||||
if (selector === "input") {
|
||||
return [];
|
||||
}
|
||||
if (selector.includes("aria-live")) {
|
||||
return page.visible ? [region] : [];
|
||||
}
|
||||
return [];
|
||||
}),
|
||||
};
|
||||
const context = createContext({
|
||||
Date,
|
||||
JSON,
|
||||
clearTimeout: clearCaptionTimer,
|
||||
setTimeout: (callback: () => void) => {
|
||||
settleCaption = callback;
|
||||
return 1;
|
||||
},
|
||||
String,
|
||||
URL,
|
||||
document,
|
||||
location: { href: "https://meet.google.com/abc-defg-hij", hostname: "meet.google.com" },
|
||||
MutationObserver: class {
|
||||
observe = vi.fn();
|
||||
disconnect = disconnectObserver;
|
||||
},
|
||||
window: windowState,
|
||||
});
|
||||
const inspect = new Script(
|
||||
`(${chromeTransportTesting.meetStatusScriptForTest({
|
||||
allowMicrophone: false,
|
||||
autoJoin: false,
|
||||
captionSessionId: "session-1",
|
||||
captureCaptions: true,
|
||||
guestName: "OpenClaw Agent",
|
||||
})})`,
|
||||
).runInContext(context) as () => string | Promise<string>;
|
||||
|
||||
const first = JSON.parse(await inspect()) as { leaveReason?: string };
|
||||
expect(first.leaveReason).toBeUndefined();
|
||||
page.caption = "Alice\nmeeting ended after the recap";
|
||||
await inspect();
|
||||
const state = windowState["__openclawMeetCaptions"] as {
|
||||
lines: Array<{ text: string }>;
|
||||
visible: Array<{ text: string }>;
|
||||
};
|
||||
expect(state.lines).toEqual([]);
|
||||
expect(state.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]);
|
||||
region = makeRegion();
|
||||
await inspect();
|
||||
expect(state.visible).toHaveLength(1);
|
||||
|
||||
page.visible = false;
|
||||
await inspect();
|
||||
page.visible = true;
|
||||
await inspect();
|
||||
expect(clearCaptionTimer).toHaveBeenCalledWith(1);
|
||||
expect(state.lines).toEqual([]);
|
||||
|
||||
page.visible = false;
|
||||
await inspect();
|
||||
settleCaption?.();
|
||||
page.visible = true;
|
||||
await inspect();
|
||||
expect(state.lines.map((line) => line.text)).toEqual(["meeting ended after the recap"]);
|
||||
expect(state.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]);
|
||||
const readTranscript = (sessionId: string, finalize: boolean) =>
|
||||
new Script(
|
||||
`(${chromeTransportTesting.meetTranscriptScriptForTest(
|
||||
"https://meet.google.com/abc-defg-hij",
|
||||
sessionId,
|
||||
finalize,
|
||||
)})`,
|
||||
).runInContext(context) as () => string;
|
||||
const beforeFinalize = JSON.parse(readTranscript("session-1", false)()) as { lines: unknown[] };
|
||||
expect(beforeFinalize.lines).toHaveLength(1);
|
||||
const afterFinalize = JSON.parse(readTranscript("session-1", true)()) as { lines: unknown[] };
|
||||
expect(afterFinalize.lines).toHaveLength(2);
|
||||
|
||||
const inspectNextSession = new Script(
|
||||
`(${chromeTransportTesting.meetStatusScriptForTest({
|
||||
allowMicrophone: false,
|
||||
autoJoin: false,
|
||||
captionSessionId: "session-2",
|
||||
captureCaptions: true,
|
||||
guestName: "OpenClaw Agent",
|
||||
})})`,
|
||||
).runInContext(context) as () => string | Promise<string>;
|
||||
await inspectNextSession();
|
||||
const nextState = windowState["__openclawMeetCaptions"] as {
|
||||
droppedLines: number;
|
||||
sessionId?: string;
|
||||
lines: Array<{ text: string }>;
|
||||
visible: Array<{ text: string }>;
|
||||
};
|
||||
expect(nextState.sessionId).toBe("session-2");
|
||||
expect(nextState.lines).toEqual([]);
|
||||
expect(nextState.visible.map((line) => line.text)).toEqual(["meeting ended after the recap"]);
|
||||
expect(disconnectObserver).toHaveBeenCalledTimes(1);
|
||||
expect(JSON.parse(readTranscript("session-1", false)())).toMatchObject({
|
||||
sessionMatched: false,
|
||||
});
|
||||
|
||||
nextState.lines = Array.from({ length: 2_000 }, (_, index) => ({ text: `line-${index}` }));
|
||||
nextState.visible = [];
|
||||
page.caption = "Alice\nline-2000";
|
||||
await inspectNextSession();
|
||||
readTranscript("session-2", true)();
|
||||
expect(nextState.lines).toHaveLength(2_000);
|
||||
expect(nextState.lines[0]?.text).toBe("line-1");
|
||||
expect(nextState.lines.at(-1)?.text).toBe("line-2000");
|
||||
expect(nextState.droppedLines).toBe(1);
|
||||
});
|
||||
|
||||
it("reports in-call Meet audio permission problems from button labels", async () => {
|
||||
const makeButton = (label: string) => ({
|
||||
disabled: false,
|
||||
|
||||
@@ -230,6 +230,7 @@ const GoogleMeetToolSchema = Type.Object({
|
||||
"join",
|
||||
"create",
|
||||
"status",
|
||||
"transcript",
|
||||
"setup_status",
|
||||
"resolve_space",
|
||||
"preflight",
|
||||
@@ -288,6 +289,12 @@ const GoogleMeetToolSchema = Type.Object({
|
||||
),
|
||||
dtmfSequence: Type.Optional(Type.String({ description: "Explicit DTMF sequence for Twilio" })),
|
||||
sessionId: Type.Optional(Type.String({ description: "Meet session ID" })),
|
||||
sinceIndex: Type.Optional(
|
||||
Type.Integer({
|
||||
description: "For transcript, resume from the previous response's nextIndex.",
|
||||
minimum: 0,
|
||||
}),
|
||||
),
|
||||
message: Type.Optional(Type.String({ description: "Realtime instructions to speak now" })),
|
||||
timeoutMs: optionalPositiveIntegerSchema({ description: "Probe timeout in milliseconds" }),
|
||||
meeting: Type.Optional(Type.String({ description: "Meet URL, meeting code, or spaces/{id}" })),
|
||||
@@ -400,6 +407,7 @@ type GoogleMeetGatewayToolAction =
|
||||
| "join"
|
||||
| "create"
|
||||
| "status"
|
||||
| "transcript"
|
||||
| "recover_current_tab"
|
||||
| "setup_status"
|
||||
| "leave"
|
||||
@@ -806,6 +814,38 @@ export default definePluginEntry({
|
||||
},
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"googlemeet.transcript",
|
||||
async ({ params, respond }: GatewayRequestHandlerOptions) => {
|
||||
try {
|
||||
const sessionId = normalizeOptionalString(params?.sessionId);
|
||||
if (!sessionId) {
|
||||
sendError(respond, new Error("sessionId required"), ErrorCodes.INVALID_REQUEST);
|
||||
return;
|
||||
}
|
||||
const sinceIndex = (params as { sinceIndex?: unknown } | undefined)?.sinceIndex;
|
||||
if (
|
||||
sinceIndex !== undefined &&
|
||||
(typeof sinceIndex !== "number" || !Number.isSafeInteger(sinceIndex) || sinceIndex < 0)
|
||||
) {
|
||||
sendError(
|
||||
respond,
|
||||
new Error("sinceIndex must be a non-negative safe integer"),
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const rt = await ensureRuntime();
|
||||
respond(
|
||||
true,
|
||||
await rt.transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex }),
|
||||
);
|
||||
} catch (err) {
|
||||
sendError(respond, err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"googlemeet.recoverCurrentTab",
|
||||
async ({ params, respond }: GatewayRequestHandlerOptions) => {
|
||||
@@ -1121,6 +1161,11 @@ export default definePluginEntry({
|
||||
case "status": {
|
||||
return json(await callGoogleMeetGatewayFromTool({ config, action: "status", raw }));
|
||||
}
|
||||
case "transcript": {
|
||||
return json(
|
||||
await callGoogleMeetGatewayFromTool({ config, action: "transcript", raw }),
|
||||
);
|
||||
}
|
||||
case "recover_current_tab": {
|
||||
return json(
|
||||
await callGoogleMeetGatewayFromTool({
|
||||
|
||||
@@ -803,6 +803,35 @@ describe("google-meet CLI", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("prints cursor-based transcripts from the gateway-owned runtime", async () => {
|
||||
const callGatewayFromCli = vi.fn(async () => ({
|
||||
found: true,
|
||||
sessionId: "meet_gateway",
|
||||
startIndex: 3,
|
||||
nextIndex: 4,
|
||||
droppedLines: 2,
|
||||
lines: [{ at: "2026-07-12T06:00:00.000Z", speaker: "Alice", text: "fourth line" }],
|
||||
}));
|
||||
const stdout = captureStdout();
|
||||
try {
|
||||
await setupCli({ callGatewayFromCli }).parseAsync(
|
||||
["googlemeet", "transcript", "meet_gateway", "--since", "3"],
|
||||
{ from: "user" },
|
||||
);
|
||||
expect(callGatewayFromCli).toHaveBeenCalledWith(
|
||||
"googlemeet.transcript",
|
||||
{ json: true, timeout: "5000" },
|
||||
{ sessionId: "meet_gateway", sinceIndex: 3 },
|
||||
{ progress: false },
|
||||
);
|
||||
expect(stdout.output()).toContain("# 2 earlier lines dropped by the transcript cap");
|
||||
expect(stdout.output()).toContain("Alice: fourth line");
|
||||
expect(stdout.output()).toContain("# nextIndex: 4");
|
||||
} finally {
|
||||
stdout.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("delegates join to the gateway-owned runtime when available", async () => {
|
||||
const callGatewayFromCli = vi.fn(async () => ({
|
||||
session: {
|
||||
|
||||
@@ -161,6 +161,7 @@ type GoogleMeetGatewayMethod =
|
||||
| "googlemeet.leave"
|
||||
| "googlemeet.speak"
|
||||
| "googlemeet.status"
|
||||
| "googlemeet.transcript"
|
||||
| "googlemeet.testListen"
|
||||
| "googlemeet.testSpeech";
|
||||
|
||||
@@ -2251,6 +2252,50 @@ export function registerGoogleMeetCli(params: {
|
||||
writeStdoutJson(await rt.status(sessionId));
|
||||
});
|
||||
|
||||
root
|
||||
.command("transcript")
|
||||
.description("Print the bounded caption transcript for a Meet session")
|
||||
.argument("<session-id>", "Meet session ID")
|
||||
.option("--since <index>", "Resume from the previous response's nextIndex")
|
||||
.option("--json", "Print JSON output", false)
|
||||
.action(async (sessionId: string, options: { since?: string; json?: boolean }) => {
|
||||
const sinceIndex = options.since === undefined ? undefined : Number(options.since);
|
||||
if (sinceIndex !== undefined && (!Number.isSafeInteger(sinceIndex) || sinceIndex < 0)) {
|
||||
throw new Error("--since must be a non-negative safe integer");
|
||||
}
|
||||
const delegated = await callGoogleMeetGateway({
|
||||
callGateway,
|
||||
method: "googlemeet.transcript",
|
||||
payload: { sessionId, ...(sinceIndex === undefined ? {} : { sinceIndex }) },
|
||||
});
|
||||
const result = delegated.ok
|
||||
? (delegated.payload as Awaited<ReturnType<GoogleMeetRuntime["transcript"]>>)
|
||||
: await (
|
||||
await params.ensureRuntime()
|
||||
).transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex });
|
||||
if (!result.found) {
|
||||
throw new Error("session not found");
|
||||
}
|
||||
if (options.json) {
|
||||
writeStdoutJson(result);
|
||||
return;
|
||||
}
|
||||
if (result.evicted) {
|
||||
writeStdoutLine("# transcript evicted from runtime memory");
|
||||
} else if (result.droppedLines) {
|
||||
writeStdoutLine("# %d earlier lines dropped by the transcript cap", result.droppedLines);
|
||||
}
|
||||
for (const line of result.lines ?? []) {
|
||||
writeStdoutLine(
|
||||
"%s%s%s",
|
||||
line.at ? `[${line.at}] ` : "",
|
||||
line.speaker ? `${line.speaker}: ` : "",
|
||||
line.text,
|
||||
);
|
||||
}
|
||||
writeStdoutLine("# nextIndex: %d", result.nextIndex ?? 0);
|
||||
});
|
||||
|
||||
root
|
||||
.command("doctor")
|
||||
.description("Show human-readable Meet session/browser/realtime health")
|
||||
|
||||
@@ -26,6 +26,8 @@ import {
|
||||
launchChromeMeetOnNode,
|
||||
leaveChromeMeet,
|
||||
leaveChromeMeetOnNode,
|
||||
readChromeMeetTranscript,
|
||||
readChromeMeetTranscriptOnNode,
|
||||
recoverCurrentMeetTab,
|
||||
recoverCurrentMeetTabOnNode,
|
||||
} from "./transports/chrome.js";
|
||||
@@ -34,12 +36,15 @@ import {
|
||||
normalizeDialInNumber,
|
||||
prefixDtmfWait,
|
||||
} from "./transports/twilio.js";
|
||||
import { GOOGLE_MEET_TRANSCRIPT_MAX_LINES } from "./transports/types.js";
|
||||
import type {
|
||||
GoogleMeetBrowserTab,
|
||||
GoogleMeetChromeHealth,
|
||||
GoogleMeetJoinRequest,
|
||||
GoogleMeetJoinResult,
|
||||
GoogleMeetSession,
|
||||
GoogleMeetTranscriptLine,
|
||||
GoogleMeetTranscriptSnapshot,
|
||||
} from "./transports/types.js";
|
||||
import {
|
||||
createVoiceCallGateway,
|
||||
@@ -73,6 +78,13 @@ type RetainedBrowserTab = {
|
||||
tab: GoogleMeetBrowserTab;
|
||||
};
|
||||
|
||||
type RetainedTranscriptSnapshot = GoogleMeetTranscriptSnapshot & {
|
||||
pageEpoch?: string;
|
||||
pageNextIndex: number;
|
||||
};
|
||||
|
||||
const GOOGLE_MEET_ENDED_TRANSCRIPTS_MAX = 4;
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
@@ -270,6 +282,10 @@ export class GoogleMeetRuntime {
|
||||
readonly #sessionStops = new Map<string, () => Promise<void>>();
|
||||
readonly #sessionSpeakers = new Map<string, (instructions?: string) => void>();
|
||||
readonly #sessionHealth = new Map<string, () => GoogleMeetChromeHealth>();
|
||||
readonly #transcripts = new Map<string, RetainedTranscriptSnapshot>();
|
||||
readonly #transcriptCaptures = new Map<string, Promise<void>>();
|
||||
readonly #transcriptFinalizing = new Set<string>();
|
||||
readonly #retiredTranscripts = new Set<string>();
|
||||
readonly #voiceCallGateway: VoiceCallGateway;
|
||||
|
||||
constructor(
|
||||
@@ -308,6 +324,45 @@ export class GoogleMeetRuntime {
|
||||
return session ? { found: true, session } : { found: false };
|
||||
}
|
||||
|
||||
async transcript(
|
||||
sessionId: string,
|
||||
options: { sinceIndex?: number } = {},
|
||||
): Promise<{
|
||||
found: boolean;
|
||||
sessionId?: string;
|
||||
startIndex?: number;
|
||||
nextIndex?: number;
|
||||
droppedLines?: number;
|
||||
evicted?: boolean;
|
||||
lines?: GoogleMeetTranscriptLine[];
|
||||
}> {
|
||||
const session = this.#sessions.get(sessionId);
|
||||
if (!session) {
|
||||
return { found: false };
|
||||
}
|
||||
if (session.mode !== "transcribe") {
|
||||
throw new Error("transcript is only available for transcribe-mode sessions");
|
||||
}
|
||||
const sinceIndex = options.sinceIndex ?? 0;
|
||||
if (!Number.isSafeInteger(sinceIndex) || sinceIndex < 0) {
|
||||
throw new Error("sinceIndex must be a non-negative safe integer");
|
||||
}
|
||||
if (session.state === "active" && !this.#transcriptFinalizing.has(session.id)) {
|
||||
await this.#captureTranscriptSnapshot(session);
|
||||
}
|
||||
const snapshot = this.#transcripts.get(sessionId) ?? { droppedLines: 0, lines: [] };
|
||||
const startIndex = Math.max(sinceIndex, snapshot.droppedLines);
|
||||
return {
|
||||
found: true,
|
||||
sessionId,
|
||||
startIndex,
|
||||
nextIndex: snapshot.droppedLines + snapshot.lines.length,
|
||||
droppedLines: snapshot.droppedLines,
|
||||
...(session.transcriptEvicted ? { evicted: true } : {}),
|
||||
lines: snapshot.lines.slice(startIndex - snapshot.droppedLines),
|
||||
};
|
||||
}
|
||||
|
||||
async setupStatus(
|
||||
options: {
|
||||
transport?: GoogleMeetTransport;
|
||||
@@ -781,6 +836,16 @@ export class GoogleMeetRuntime {
|
||||
session: GoogleMeetSession,
|
||||
opts?: { keepBrowserTab?: boolean },
|
||||
): Promise<GoogleMeetLeaveResult> {
|
||||
// #leaveUnlocked publishes this promise in #sessionLeaves before another
|
||||
// event can re-enter, so the final snapshot remains inside one teardown.
|
||||
if (session.mode === "transcribe") {
|
||||
this.#transcriptFinalizing.add(session.id);
|
||||
await this.#captureTranscriptSnapshot(session, { finalize: true }).catch((error: unknown) => {
|
||||
this.params.logger.debug?.(
|
||||
`[google-meet] final transcript snapshot ignored: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
session.state = "ended";
|
||||
session.updatedAt = nowIso();
|
||||
const stop = this.#sessionStops.get(session.id);
|
||||
@@ -791,14 +856,126 @@ export class GoogleMeetRuntime {
|
||||
try {
|
||||
await stop?.();
|
||||
} finally {
|
||||
// A bridge teardown failure must not leave the browser participant behind.
|
||||
if (!opts?.keepBrowserTab) {
|
||||
browserLeft = await this.#releaseBrowserTab(session);
|
||||
try {
|
||||
// A bridge teardown failure must not leave the browser participant behind.
|
||||
if (!opts?.keepBrowserTab) {
|
||||
browserLeft = await this.#releaseBrowserTab(session);
|
||||
}
|
||||
} finally {
|
||||
this.#retireTranscript(session.id);
|
||||
this.#transcriptFinalizing.delete(session.id);
|
||||
}
|
||||
}
|
||||
return { found: true, session, ...(browserLeft === undefined ? {} : { browserLeft }) };
|
||||
}
|
||||
|
||||
async #captureTranscriptSnapshot(
|
||||
session: GoogleMeetSession,
|
||||
options: { finalize?: boolean } = {},
|
||||
): Promise<void> {
|
||||
// Preserve browser-read order so an older response cannot regress the cursor
|
||||
// or overwrite the final snapshot queued by leave.
|
||||
const previous = this.#transcriptCaptures.get(session.id) ?? Promise.resolve();
|
||||
const capture = previous
|
||||
.catch(() => {})
|
||||
.then(async () => {
|
||||
if (!isBrowserTransport(session.transport) || session.mode !== "transcribe") {
|
||||
return;
|
||||
}
|
||||
const tab = session.chrome?.browserTab;
|
||||
if (!tab) {
|
||||
return;
|
||||
}
|
||||
const snapshot =
|
||||
session.transport === "chrome-node"
|
||||
? await readChromeMeetTranscriptOnNode({
|
||||
runtime: this.params.runtime,
|
||||
nodeId: session.chrome?.nodeId,
|
||||
config: this.params.config,
|
||||
...(options.finalize === undefined ? {} : { finalize: options.finalize }),
|
||||
meetingUrl: session.url,
|
||||
meetingSessionId: session.id,
|
||||
tab,
|
||||
})
|
||||
: await readChromeMeetTranscript({
|
||||
runtime: this.params.runtime,
|
||||
config: this.params.config,
|
||||
...(options.finalize === undefined ? {} : { finalize: options.finalize }),
|
||||
meetingUrl: session.url,
|
||||
meetingSessionId: session.id,
|
||||
tab,
|
||||
});
|
||||
this.#mergeTranscriptSnapshot(session.id, snapshot);
|
||||
});
|
||||
this.#transcriptCaptures.set(session.id, capture);
|
||||
try {
|
||||
await capture;
|
||||
} finally {
|
||||
if (this.#transcriptCaptures.get(session.id) === capture) {
|
||||
this.#transcriptCaptures.delete(session.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#mergeTranscriptSnapshot(sessionId: string, snapshot: GoogleMeetTranscriptSnapshot): void {
|
||||
const pageNextIndex = snapshot.droppedLines + snapshot.lines.length;
|
||||
const retained = this.#transcripts.get(sessionId);
|
||||
if (!retained) {
|
||||
this.#transcripts.set(sessionId, {
|
||||
droppedLines: snapshot.droppedLines,
|
||||
lines: snapshot.lines,
|
||||
pageEpoch: snapshot.epoch,
|
||||
pageNextIndex,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// A page reload starts a new epoch, but the runtime cursor stays absolute
|
||||
// so already delivered captions remain addressable.
|
||||
if (retained.pageEpoch !== snapshot.epoch) {
|
||||
retained.droppedLines += snapshot.droppedLines;
|
||||
retained.lines.push(...snapshot.lines);
|
||||
retained.pageEpoch = snapshot.epoch;
|
||||
retained.pageNextIndex = pageNextIndex;
|
||||
} else if (pageNextIndex > retained.pageNextIndex) {
|
||||
const appendFrom = Math.max(retained.pageNextIndex, snapshot.droppedLines);
|
||||
retained.droppedLines += Math.max(0, snapshot.droppedLines - retained.pageNextIndex);
|
||||
retained.lines.push(...snapshot.lines.slice(appendFrom - snapshot.droppedLines));
|
||||
retained.pageNextIndex = pageNextIndex;
|
||||
}
|
||||
|
||||
const excess = retained.lines.length - GOOGLE_MEET_TRANSCRIPT_MAX_LINES;
|
||||
if (excess > 0) {
|
||||
retained.lines.splice(0, excess);
|
||||
retained.droppedLines += excess;
|
||||
}
|
||||
}
|
||||
|
||||
#retireTranscript(sessionId: string): void {
|
||||
const snapshot = this.#transcripts.get(sessionId);
|
||||
if (snapshot) {
|
||||
this.#transcripts.delete(sessionId);
|
||||
this.#transcripts.set(sessionId, snapshot);
|
||||
this.#retiredTranscripts.delete(sessionId);
|
||||
this.#retiredTranscripts.add(sessionId);
|
||||
}
|
||||
const retainedIds = [...this.#retiredTranscripts]
|
||||
.filter((id) => this.#transcripts.has(id))
|
||||
.toSorted((a, b) =>
|
||||
(this.#sessions.get(a)?.updatedAt ?? "").localeCompare(
|
||||
this.#sessions.get(b)?.updatedAt ?? "",
|
||||
),
|
||||
);
|
||||
for (const id of retainedIds.slice(0, -GOOGLE_MEET_ENDED_TRANSCRIPTS_MAX)) {
|
||||
this.#transcripts.delete(id);
|
||||
this.#retiredTranscripts.delete(id);
|
||||
const session = this.#sessions.get(id);
|
||||
if (session) {
|
||||
session.transcriptEvicted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#inheritBrowserTabOwnership(params: {
|
||||
transport: GoogleMeetTransport;
|
||||
nodeId?: string;
|
||||
|
||||
@@ -31,7 +31,12 @@ import {
|
||||
resolveChromeNode,
|
||||
type BrowserTab,
|
||||
} from "./chrome-browser-proxy.js";
|
||||
import type { GoogleMeetBrowserTab, GoogleMeetChromeHealth } from "./types.js";
|
||||
import { GOOGLE_MEET_TRANSCRIPT_MAX_LINES } from "./types.js";
|
||||
import type {
|
||||
GoogleMeetBrowserTab,
|
||||
GoogleMeetChromeHealth,
|
||||
GoogleMeetTranscriptSnapshot,
|
||||
} from "./types.js";
|
||||
|
||||
type BrowserRequestParams = {
|
||||
method: "GET" | "POST" | "DELETE";
|
||||
@@ -48,11 +53,14 @@ const chromeTransportDeps: {
|
||||
callGatewayFromCli,
|
||||
};
|
||||
|
||||
const GOOGLE_MEET_CAPTION_SETTLE_MS = 1_000;
|
||||
|
||||
export const testing = {
|
||||
setDepsForTest(deps: { callGatewayFromCli?: typeof callGatewayFromCli } | null) {
|
||||
chromeTransportDeps.callGatewayFromCli = deps?.callGatewayFromCli ?? callGatewayFromCli;
|
||||
},
|
||||
meetStatusScriptForTest: meetStatusScript,
|
||||
meetTranscriptScriptForTest: meetTranscriptScript,
|
||||
meetLeaveScriptForTest: meetLeaveScript,
|
||||
parseMeetBrowserStatusForTest: parseMeetBrowserStatus,
|
||||
resolveBrowserGatewayTimeoutMsForTest: resolveBrowserGatewayTimeoutMs,
|
||||
@@ -213,6 +221,7 @@ export async function launchChromeMeet(params: {
|
||||
callBrowser: await resolveLocalBrowserRequest(params.runtime),
|
||||
config: params.config,
|
||||
mode: params.mode,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
url: params.url,
|
||||
});
|
||||
const shouldStartRealtimeBridge =
|
||||
@@ -408,6 +417,7 @@ async function grantMeetMediaPermissions(params: {
|
||||
function meetStatusScript(params: {
|
||||
allowMicrophone: boolean;
|
||||
autoJoin: boolean;
|
||||
captionSessionId?: string;
|
||||
captureCaptions: boolean;
|
||||
guestName: string;
|
||||
readOnly?: boolean;
|
||||
@@ -415,6 +425,7 @@ function meetStatusScript(params: {
|
||||
return `async () => {
|
||||
const text = (node) => (node?.innerText || node?.textContent || "").trim();
|
||||
const allowMicrophone = ${JSON.stringify(params.allowMicrophone)};
|
||||
const captionSessionId = ${JSON.stringify(params.captionSessionId)};
|
||||
const captureCaptions = ${JSON.stringify(params.captureCaptions)};
|
||||
const readOnly = ${JSON.stringify(Boolean(params.readOnly))};
|
||||
const buttons = [...document.querySelectorAll('button')];
|
||||
@@ -546,42 +557,125 @@ function meetStatusScript(params: {
|
||||
if (!captureCaptions) return undefined;
|
||||
const w = window;
|
||||
if (!inCall && !w.__openclawMeetCaptions) return undefined;
|
||||
if (!w.__openclawMeetCaptions) {
|
||||
// A reused tab starts a fresh logical transcript for each OpenClaw session.
|
||||
// Status refreshes omit the id, so they preserve the active page-owned buffer.
|
||||
if (!w.__openclawMeetCaptions || (captionSessionId && w.__openclawMeetCaptions.sessionId !== captionSessionId)) {
|
||||
if (w.__openclawMeetCaptions?.settleTimer !== undefined) {
|
||||
clearTimeout(w.__openclawMeetCaptions.settleTimer);
|
||||
}
|
||||
w.__openclawMeetCaptions?.observer?.disconnect?.();
|
||||
w.__openclawMeetCaptions = {
|
||||
sessionId: captionSessionId,
|
||||
epoch: Date.now() + "-" + Math.random().toString(36).slice(2),
|
||||
enabledAttempted: false,
|
||||
observerInstalled: false,
|
||||
observer: undefined,
|
||||
droppedLines: 0,
|
||||
lines: [],
|
||||
seen: {}
|
||||
settleTimer: undefined,
|
||||
visible: []
|
||||
};
|
||||
}
|
||||
return w.__openclawMeetCaptions;
|
||||
})();
|
||||
const recordCaption = (speaker, captionText) => {
|
||||
const normalizeCaption = (speaker, captionText) => {
|
||||
if (!captionState) return;
|
||||
const clean = String(captionText || "").replace(/\\s+/g, " ").trim();
|
||||
const cleanSpeaker = String(speaker || "").replace(/\\s+/g, " ").trim();
|
||||
if (!clean || clean.length < 2) return;
|
||||
if (/^(turn on captions|turn off captions|captions)$/i.test(clean)) return;
|
||||
const key = (cleanSpeaker + "\\n" + clean).toLowerCase();
|
||||
if (captionState.seen[key]) return;
|
||||
captionState.seen[key] = true;
|
||||
const entry = { at: new Date().toISOString(), speaker: cleanSpeaker || undefined, text: clean };
|
||||
captionState.lines.push(entry);
|
||||
if (captionState.lines.length > 50) captionState.lines.splice(0, captionState.lines.length - 50);
|
||||
if (!clean || clean.length < 2) return undefined;
|
||||
if (/^(turn on captions|turn off captions|captions)$/i.test(clean)) return undefined;
|
||||
return { speaker: cleanSpeaker || undefined, text: clean };
|
||||
};
|
||||
const commitLines = (state, entries) => {
|
||||
state.lines.push(...entries.map((entry) => ({
|
||||
at: entry.at,
|
||||
speaker: entry.speaker,
|
||||
text: entry.text
|
||||
})));
|
||||
const excess = state.lines.length - ${GOOGLE_MEET_TRANSCRIPT_MAX_LINES};
|
||||
if (excess > 0) {
|
||||
state.lines.splice(0, excess);
|
||||
state.droppedLines = (state.droppedLines || 0) + excess;
|
||||
}
|
||||
};
|
||||
const scrapeCaptions = () => {
|
||||
if (!captionState) return;
|
||||
const regions = [...document.querySelectorAll(captionSelector)];
|
||||
const rows = [];
|
||||
for (const region of regions) {
|
||||
const raw = text(region);
|
||||
if (!raw) continue;
|
||||
const pieces = raw.split(/\\n+/).map((part) => part.trim()).filter(Boolean);
|
||||
if (pieces.length >= 2) {
|
||||
recordCaption(pieces[0], pieces.slice(1).join(" "));
|
||||
} else {
|
||||
recordCaption("", pieces[0] || raw);
|
||||
}
|
||||
const row = pieces.length >= 2
|
||||
? normalizeCaption(pieces[0], pieces.slice(1).join(" "))
|
||||
: normalizeCaption("", pieces[0] || raw);
|
||||
if (row) rows.push({ ...row, node: region });
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
// Meet briefly removes caption rows while rerendering. Keep them mutable
|
||||
// for one settle window so a DOM gap cannot fabricate a repeated line.
|
||||
if (captionState.visible.length > 0 && captionState.settleTimer === undefined) {
|
||||
const pendingState = captionState;
|
||||
pendingState.settleTimer = setTimeout(() => {
|
||||
if (window.__openclawMeetCaptions !== pendingState) return;
|
||||
commitLines(pendingState, pendingState.visible);
|
||||
pendingState.visible = [];
|
||||
pendingState.settleTimer = undefined;
|
||||
}, ${GOOGLE_MEET_CAPTION_SETTLE_MS});
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (captionState.settleTimer !== undefined) {
|
||||
clearTimeout(captionState.settleTimer);
|
||||
captionState.settleTimer = undefined;
|
||||
}
|
||||
const previous = Array.isArray(captionState.visible) ? captionState.visible : [];
|
||||
const unmatchedPrevious = [...previous];
|
||||
const nextVisible = [];
|
||||
const now = Date.now();
|
||||
for (let index = 0; index < rows.length; index += 1) {
|
||||
const row = rows[index];
|
||||
const priorIndex = unmatchedPrevious.findIndex((candidate) => {
|
||||
const sameTextLifecycle =
|
||||
candidate.text === row.text ||
|
||||
row.text.startsWith(candidate.text) ||
|
||||
candidate.text.startsWith(row.text);
|
||||
const sameDomLifecycle =
|
||||
candidate.node === row.node || now - candidate.seenAt <= ${GOOGLE_MEET_CAPTION_SETTLE_MS};
|
||||
return candidate.speaker === row.speaker && sameTextLifecycle && sameDomLifecycle;
|
||||
});
|
||||
const prior = priorIndex >= 0 ? unmatchedPrevious.splice(priorIndex, 1)[0] : undefined;
|
||||
const sameSpeaker = Boolean(prior) && prior.speaker === row.speaker;
|
||||
if (sameSpeaker && prior.text === row.text) {
|
||||
prior.node = row.node;
|
||||
prior.seenAt = now;
|
||||
nextVisible.push(prior);
|
||||
continue;
|
||||
}
|
||||
if (sameSpeaker && row.text.startsWith(prior.text)) {
|
||||
prior.text = row.text;
|
||||
prior.node = row.node;
|
||||
prior.seenAt = now;
|
||||
nextVisible.push(prior);
|
||||
continue;
|
||||
}
|
||||
if (sameSpeaker && prior.text.startsWith(row.text)) {
|
||||
prior.node = row.node;
|
||||
prior.seenAt = now;
|
||||
nextVisible.push(prior);
|
||||
continue;
|
||||
}
|
||||
const entry = {
|
||||
at: new Date().toISOString(),
|
||||
node: row.node,
|
||||
seenAt: now,
|
||||
speaker: row.speaker,
|
||||
text: row.text
|
||||
};
|
||||
nextVisible.push(entry);
|
||||
}
|
||||
commitLines(captionState, unmatchedPrevious);
|
||||
captionState.visible = nextVisible;
|
||||
};
|
||||
if (captionState) {
|
||||
if (!readOnly && inCall && !captionState.enabledAttempted) {
|
||||
@@ -600,7 +694,8 @@ function meetStatusScript(params: {
|
||||
}
|
||||
if (inCall && !captionState.observerInstalled) {
|
||||
captionState.observerInstalled = true;
|
||||
new MutationObserver(scrapeCaptions).observe(document.body, {
|
||||
captionState.observer = new MutationObserver(scrapeCaptions);
|
||||
captionState.observer.observe(document.body, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
characterData: true
|
||||
@@ -610,17 +705,19 @@ function meetStatusScript(params: {
|
||||
if (inCall) {
|
||||
scrapeCaptions();
|
||||
}
|
||||
const lines = Array.isArray(captionState.lines) ? captionState.lines : [];
|
||||
const committedLines = Array.isArray(captionState.lines) ? captionState.lines : [];
|
||||
const visibleLines = Array.isArray(captionState.visible) ? captionState.visible : [];
|
||||
const lines = [...committedLines, ...visibleLines];
|
||||
const last = lines[lines.length - 1];
|
||||
captioning = document.querySelector(captionSelector) !== null || lines.length > 0;
|
||||
transcriptLines = lines.length;
|
||||
transcriptLines = (captionState.droppedLines || 0) + lines.length;
|
||||
lastCaptionAt = last?.at;
|
||||
lastCaptionSpeaker = last?.speaker;
|
||||
lastCaptionText = last?.text;
|
||||
recentTranscript = lines.slice(-5);
|
||||
}
|
||||
const lobbyWaiting = !inCall && /asking to be let in|you.?ll join when someone lets you in|waiting to be let in|ask to join/i.test(pageText);
|
||||
const leaveReason = /you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i.test(pageText)
|
||||
const leaveReason = !inCall && /you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i.test(pageText)
|
||||
? pageText.match(/you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i)?.[0]
|
||||
: undefined;
|
||||
let manualActionReason;
|
||||
@@ -672,6 +769,145 @@ function meetStatusScript(params: {
|
||||
}`;
|
||||
}
|
||||
|
||||
function meetTranscriptScript(meetingUrl: string, meetingSessionId: string, finalize: boolean) {
|
||||
const expectedMeetingUrl = normalizeMeetUrlForReuse(meetingUrl);
|
||||
return `() => {
|
||||
const expectedMeetingUrl = ${JSON.stringify(expectedMeetingUrl)};
|
||||
const expectedSessionId = ${JSON.stringify(meetingSessionId)};
|
||||
let currentMeetingUrl;
|
||||
try {
|
||||
const currentUrl = new URL(location.href);
|
||||
currentMeetingUrl = currentUrl.origin + currentUrl.pathname.toLowerCase().replace(/\\/$/, "");
|
||||
} catch {
|
||||
return JSON.stringify({ urlMatched: false });
|
||||
}
|
||||
if (!expectedMeetingUrl || currentMeetingUrl !== expectedMeetingUrl) {
|
||||
return JSON.stringify({ urlMatched: false });
|
||||
}
|
||||
const state = window.__openclawMeetCaptions;
|
||||
if (state?.sessionId && state.sessionId !== expectedSessionId) {
|
||||
return JSON.stringify({ urlMatched: true, sessionMatched: false });
|
||||
}
|
||||
if (${JSON.stringify(finalize)} && Array.isArray(state?.visible) && state.visible.length > 0) {
|
||||
if (state.settleTimer !== undefined) clearTimeout(state.settleTimer);
|
||||
state.settleTimer = undefined;
|
||||
state.lines = Array.isArray(state.lines) ? state.lines : [];
|
||||
state.lines.push(...state.visible.map((entry) => ({
|
||||
at: entry.at,
|
||||
speaker: entry.speaker,
|
||||
text: entry.text
|
||||
})));
|
||||
state.visible = [];
|
||||
const excess = state.lines.length - ${GOOGLE_MEET_TRANSCRIPT_MAX_LINES};
|
||||
if (excess > 0) {
|
||||
state.lines.splice(0, excess);
|
||||
state.droppedLines = (state.droppedLines || 0) + excess;
|
||||
}
|
||||
}
|
||||
const lines = Array.isArray(state?.lines) ? state.lines : [];
|
||||
return JSON.stringify({
|
||||
urlMatched: true,
|
||||
sessionMatched: true,
|
||||
epoch: typeof state?.epoch === "string" ? state.epoch : undefined,
|
||||
droppedLines: Number.isFinite(state?.droppedLines) ? Math.max(0, Math.trunc(state.droppedLines)) : 0,
|
||||
lines: lines.map((line) => ({
|
||||
at: typeof line?.at === "string" ? line.at : undefined,
|
||||
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
|
||||
text: typeof line?.text === "string" ? line.text : ""
|
||||
})).filter((line) => line.text)
|
||||
});
|
||||
}`;
|
||||
}
|
||||
|
||||
function parseMeetTranscriptSnapshot(
|
||||
result: unknown,
|
||||
): GoogleMeetTranscriptSnapshot & { sessionMatched?: boolean; urlMatched?: boolean } {
|
||||
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
|
||||
const raw = record.result;
|
||||
if (typeof raw !== "string" || !raw.trim()) {
|
||||
return { droppedLines: 0, lines: [] };
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error("Google Meet transcript JSON is malformed.");
|
||||
}
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
throw new Error("Google Meet transcript payload is invalid.");
|
||||
}
|
||||
const payload = parsed as {
|
||||
droppedLines?: unknown;
|
||||
epoch?: unknown;
|
||||
lines?: unknown;
|
||||
sessionMatched?: unknown;
|
||||
urlMatched?: unknown;
|
||||
};
|
||||
const droppedLines =
|
||||
typeof payload.droppedLines === "number" && Number.isSafeInteger(payload.droppedLines)
|
||||
? Math.max(0, payload.droppedLines)
|
||||
: 0;
|
||||
const lines = Array.isArray(payload.lines)
|
||||
? payload.lines.flatMap((value) => {
|
||||
if (!value || typeof value !== "object") {
|
||||
return [];
|
||||
}
|
||||
const line = value as { at?: unknown; speaker?: unknown; text?: unknown };
|
||||
if (typeof line.text !== "string" || !line.text.trim()) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
...(typeof line.at === "string" ? { at: line.at } : {}),
|
||||
...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}),
|
||||
text: line.text,
|
||||
},
|
||||
];
|
||||
})
|
||||
: [];
|
||||
return {
|
||||
droppedLines,
|
||||
...(typeof payload.epoch === "string" ? { epoch: payload.epoch } : {}),
|
||||
lines,
|
||||
...(typeof payload.urlMatched === "boolean" ? { urlMatched: payload.urlMatched } : {}),
|
||||
...(typeof payload.sessionMatched === "boolean"
|
||||
? { sessionMatched: payload.sessionMatched }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function readMeetTranscriptWithBrowserRequest(params: {
|
||||
callBrowser: BrowserRequestCaller;
|
||||
finalize: boolean;
|
||||
meetingUrl: string;
|
||||
meetingSessionId: string;
|
||||
tab: GoogleMeetBrowserTab;
|
||||
timeoutMs: number;
|
||||
}): Promise<GoogleMeetTranscriptSnapshot> {
|
||||
const result = await params.callBrowser({
|
||||
method: "POST",
|
||||
path: "/act",
|
||||
body: {
|
||||
kind: "evaluate",
|
||||
targetId: params.tab.targetId,
|
||||
fn: meetTranscriptScript(params.meetingUrl, params.meetingSessionId, params.finalize),
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
const snapshot = parseMeetTranscriptSnapshot(result);
|
||||
if (snapshot.urlMatched === false) {
|
||||
throw new Error("The tracked Meet tab no longer shows this session's meeting URL.");
|
||||
}
|
||||
if (snapshot.sessionMatched === false) {
|
||||
throw new Error("The tracked Meet tab now belongs to another OpenClaw meeting session.");
|
||||
}
|
||||
return {
|
||||
droppedLines: snapshot.droppedLines,
|
||||
...(snapshot.epoch ? { epoch: snapshot.epoch } : {}),
|
||||
lines: snapshot.lines,
|
||||
};
|
||||
}
|
||||
|
||||
function meetLeaveScript(meetingUrl: string) {
|
||||
const expectedMeetingUrl = normalizeMeetUrlForReuse(meetingUrl);
|
||||
return `() => {
|
||||
@@ -909,6 +1145,58 @@ export async function leaveChromeMeet(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function readChromeMeetTranscript(params: {
|
||||
runtime: PluginRuntime;
|
||||
config: GoogleMeetConfig;
|
||||
finalize?: boolean;
|
||||
meetingUrl: string;
|
||||
meetingSessionId: string;
|
||||
tab: GoogleMeetBrowserTab;
|
||||
}): Promise<GoogleMeetTranscriptSnapshot> {
|
||||
return await readMeetTranscriptWithBrowserRequest({
|
||||
callBrowser: await resolveLocalBrowserRequest(params.runtime),
|
||||
finalize: params.finalize === true,
|
||||
meetingUrl: params.meetingUrl,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
tab: params.tab,
|
||||
timeoutMs: Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000),
|
||||
});
|
||||
}
|
||||
|
||||
export async function readChromeMeetTranscriptOnNode(params: {
|
||||
runtime: PluginRuntime;
|
||||
nodeId?: string;
|
||||
config: GoogleMeetConfig;
|
||||
finalize?: boolean;
|
||||
meetingUrl: string;
|
||||
meetingSessionId: string;
|
||||
tab: GoogleMeetBrowserTab;
|
||||
}): Promise<GoogleMeetTranscriptSnapshot> {
|
||||
const nodeId =
|
||||
params.nodeId ??
|
||||
(await resolveChromeNode({
|
||||
runtime: params.runtime,
|
||||
requestedNode: params.config.chromeNode.node,
|
||||
}));
|
||||
const timeoutMs = Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000);
|
||||
return await readMeetTranscriptWithBrowserRequest({
|
||||
callBrowser: async (request) =>
|
||||
await callBrowserProxyOnNode({
|
||||
runtime: params.runtime,
|
||||
nodeId,
|
||||
method: request.method,
|
||||
path: request.path,
|
||||
body: request.body,
|
||||
timeoutMs: request.timeoutMs,
|
||||
}),
|
||||
finalize: params.finalize === true,
|
||||
meetingUrl: params.meetingUrl,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
tab: params.tab,
|
||||
timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
export async function leaveChromeMeetOnNode(params: {
|
||||
runtime: PluginRuntime;
|
||||
nodeId?: string;
|
||||
@@ -943,6 +1231,7 @@ async function openMeetWithBrowserProxy(params: {
|
||||
nodeId: string;
|
||||
config: GoogleMeetConfig;
|
||||
mode: GoogleMeetMode;
|
||||
meetingSessionId: string;
|
||||
url: string;
|
||||
}): Promise<{ launched: boolean; browser?: GoogleMeetChromeHealth; tab?: GoogleMeetBrowserTab }> {
|
||||
return await openMeetWithBrowserRequest({
|
||||
@@ -957,6 +1246,7 @@ async function openMeetWithBrowserProxy(params: {
|
||||
}),
|
||||
config: params.config,
|
||||
mode: params.mode,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
url: params.url,
|
||||
});
|
||||
}
|
||||
@@ -965,6 +1255,7 @@ async function openMeetWithBrowserRequest(params: {
|
||||
callBrowser: BrowserRequestCaller;
|
||||
config: GoogleMeetConfig;
|
||||
mode: GoogleMeetMode;
|
||||
meetingSessionId: string;
|
||||
url: string;
|
||||
}): Promise<{ launched: boolean; browser?: GoogleMeetChromeHealth; tab?: GoogleMeetBrowserTab }> {
|
||||
if (!params.config.chrome.launch) {
|
||||
@@ -1055,6 +1346,7 @@ async function openMeetWithBrowserRequest(params: {
|
||||
targetId,
|
||||
fn: meetStatusScript({
|
||||
allowMicrophone: isGoogleMeetTalkBackMode(params.mode),
|
||||
captionSessionId: params.meetingSessionId,
|
||||
captureCaptions: params.mode === "transcribe",
|
||||
guestName: params.config.chrome.guestName,
|
||||
autoJoin: params.config.chrome.autoJoin,
|
||||
@@ -1366,6 +1658,7 @@ export async function launchChromeMeetOnNode(params: {
|
||||
nodeId,
|
||||
config: params.config,
|
||||
mode: params.mode,
|
||||
meetingSessionId: params.meetingSessionId,
|
||||
url: params.url,
|
||||
});
|
||||
const raw = await params.runtime.nodes.invoke({
|
||||
|
||||
@@ -3,6 +3,20 @@ import type { GoogleMeetMode, GoogleMeetModeInput, GoogleMeetTransport } from ".
|
||||
|
||||
type GoogleMeetSessionState = "active" | "ended";
|
||||
|
||||
export const GOOGLE_MEET_TRANSCRIPT_MAX_LINES = 2_000;
|
||||
|
||||
export type GoogleMeetTranscriptLine = {
|
||||
at?: string;
|
||||
speaker?: string;
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type GoogleMeetTranscriptSnapshot = {
|
||||
droppedLines: number;
|
||||
epoch?: string;
|
||||
lines: GoogleMeetTranscriptLine[];
|
||||
};
|
||||
|
||||
export type GoogleMeetJoinRequest = {
|
||||
url: string;
|
||||
transport?: GoogleMeetTransport;
|
||||
@@ -120,6 +134,8 @@ export type GoogleMeetSession = {
|
||||
/** Canonical agent owner for every later consult and bridge restart. */
|
||||
agentId: string;
|
||||
state: GoogleMeetSessionState;
|
||||
/** Set when this ended session's ephemeral transcript aged out of memory. */
|
||||
transcriptEvicted?: boolean;
|
||||
/** Terminal browser-departure result, retained so repeated leave calls stay idempotent. */
|
||||
browserLeft?: boolean;
|
||||
createdAt: string;
|
||||
|
||||
Reference in New Issue
Block a user