fix(android): show Canvas outside Settings (#103330)

* test(android): reproduce detached canvas lifecycle failures

* fix(android): restore shell-owned canvas presentation

* fix(android): avoid canvas renderer crash loops

* fix(android): keep canvas within safe drawing bounds

* fix(android): cover canvas inset gutters

* chore(android): refresh native i18n inventory

* fix(android): keep canvas host lint-clean

* style(android): format canvas host documentation
This commit is contained in:
Josh Avant
2026-07-10 04:20:53 -05:00
committed by GitHub
parent a6b907825a
commit fc60a8e66b
16 changed files with 842 additions and 165 deletions

View File

@@ -155,7 +155,7 @@
},
{
"kind": "ui-state-text",
"line": 180,
"line": 182,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Searching…",
"surface": "android",
@@ -163,7 +163,7 @@
},
{
"kind": "ui-state-text",
"line": 283,
"line": 285,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Mic off",
"surface": "android",
@@ -171,7 +171,7 @@
},
{
"kind": "ui-state-text",
"line": 297,
"line": 299,
"path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt",
"source": "Off",
"surface": "android",
@@ -475,7 +475,7 @@
},
{
"kind": "conditional-branch",
"line": 47,
"line": 45,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Home canvas",
"surface": "android",
@@ -483,7 +483,7 @@
},
{
"kind": "conditional-branch",
"line": 47,
"line": 45,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Live page",
"surface": "android",
@@ -491,7 +491,7 @@
},
{
"kind": "ui-named-argument",
"line": 58,
"line": 56,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Canvas",
"surface": "android",
@@ -499,7 +499,7 @@
},
{
"kind": "ui-named-argument",
"line": 59,
"line": 57,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Current screen output and interactive app surface.",
"surface": "android",
@@ -507,7 +507,7 @@
},
{
"kind": "conditional-branch",
"line": 66,
"line": 64,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Offline",
"surface": "android",
@@ -515,7 +515,7 @@
},
{
"kind": "conditional-branch",
"line": 66,
"line": 64,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Online",
"surface": "android",
@@ -523,7 +523,7 @@
},
{
"kind": "conditional-branch",
"line": 68,
"line": 66,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Ready",
"surface": "android",
@@ -531,7 +531,7 @@
},
{
"kind": "conditional-branch",
"line": 68,
"line": 66,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Standby",
"surface": "android",
@@ -539,7 +539,7 @@
},
{
"kind": "conditional-branch",
"line": 73,
"line": 71,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Refresh Screen",
"surface": "android",
@@ -547,23 +547,31 @@
},
{
"kind": "conditional-branch",
"line": 73,
"line": 71,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Refreshing",
"surface": "android",
"id": "native.android.2a870631873082b6"
},
{
"kind": "ui-named-argument",
"line": 79,
"kind": "conditional-branch",
"line": 77,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Reconnect",
"source": "Open Screen",
"surface": "android",
"id": "native.android.ec3669686494575b"
"id": "native.android.7b8f4a7869855aea"
},
{
"kind": "conditional-branch",
"line": 130,
"line": 77,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Reconnect",
"surface": "android",
"id": "native.android.0e24c627e6384ecc"
},
{
"kind": "conditional-branch",
"line": 122,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Connect the gateway",
"surface": "android",
@@ -571,7 +579,7 @@
},
{
"kind": "conditional-branch",
"line": 130,
"line": 122,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Screen surface ready",
"surface": "android",
@@ -579,20 +587,20 @@
},
{
"kind": "conditional-branch",
"line": 136,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Canvas output appears here when OpenClaw opens an app surface.",
"surface": "android",
"id": "native.android.53dcd29a9bdd99c0"
},
{
"kind": "conditional-branch",
"line": 136,
"line": 128,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Canvas output needs an active gateway connection.",
"surface": "android",
"id": "native.android.3b37959b1d369a28"
},
{
"kind": "conditional-branch",
"line": 128,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt",
"source": "Open the current Canvas surface to inspect or interact with it.",
"surface": "android",
"id": "native.android.cd92e0961b0bf500"
},
{
"kind": "ui-named-argument",
"line": 47,
@@ -4371,7 +4379,15 @@
},
{
"kind": "ui-named-argument",
"line": 438,
"line": 345,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Close Canvas",
"surface": "android",
"id": "native.android.30d8b822fa68d6c4"
},
{
"kind": "ui-named-argument",
"line": 488,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Overview",
"surface": "android",
@@ -4379,7 +4395,7 @@
},
{
"kind": "ui-named-argument",
"line": 483,
"line": 533,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "No recent sessions",
"surface": "android",
@@ -4387,7 +4403,7 @@
},
{
"kind": "ui-named-argument",
"line": 484,
"line": 534,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Start a chat and your active OpenClaw conversations will appear here.",
"surface": "android",
@@ -4395,7 +4411,7 @@
},
{
"kind": "ui-named-argument",
"line": 485,
"line": 535,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Start Chat",
"surface": "android",
@@ -4403,7 +4419,7 @@
},
{
"kind": "ui-named-argument",
"line": 531,
"line": 581,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "OpenClaw",
"surface": "android",
@@ -4411,7 +4427,7 @@
},
{
"kind": "ui-named-argument",
"line": 539,
"line": 589,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Search",
"surface": "android",
@@ -4419,7 +4435,7 @@
},
{
"kind": "ui-named-argument",
"line": 591,
"line": 641,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "ACTIVE AGENT",
"surface": "android",
@@ -4427,7 +4443,7 @@
},
{
"kind": "ui-named-argument",
"line": 600,
"line": 650,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "View",
"surface": "android",
@@ -4435,7 +4451,7 @@
},
{
"kind": "conditional-branch",
"line": 603,
"line": 653,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$pendingRunCount active",
"surface": "android",
@@ -4443,7 +4459,7 @@
},
{
"kind": "conditional-branch",
"line": 603,
"line": 653,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Idle",
"surface": "android",
@@ -4451,7 +4467,7 @@
},
{
"kind": "ui-named-argument",
"line": 603,
"line": 653,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Runs",
"surface": "android",
@@ -4459,7 +4475,7 @@
},
{
"kind": "conditional-branch",
"line": 604,
"line": 654,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$sessionCount recent",
"surface": "android",
@@ -4467,7 +4483,7 @@
},
{
"kind": "ui-named-argument",
"line": 604,
"line": 654,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Sessions",
"surface": "android",
@@ -4475,7 +4491,7 @@
},
{
"kind": "ui-named-argument",
"line": 605,
"line": 655,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Cron",
"surface": "android",
@@ -4483,7 +4499,7 @@
},
{
"kind": "ui-named-argument",
"line": 608,
"line": 658,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Chat",
"surface": "android",
@@ -4491,7 +4507,7 @@
},
{
"kind": "ui-named-argument",
"line": 612,
"line": 662,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Reconnect gateway",
"surface": "android",
@@ -4499,7 +4515,7 @@
},
{
"kind": "ui-named-argument",
"line": 760,
"line": 810,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Open ${card.title}",
"surface": "android",
@@ -4507,7 +4523,7 @@
},
{
"kind": "ui-named-argument",
"line": 828,
"line": 878,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Talk",
"surface": "android",
@@ -4515,7 +4531,7 @@
},
{
"kind": "ui-named-argument",
"line": 829,
"line": 879,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Open Talk",
"surface": "android",
@@ -4523,7 +4539,7 @@
},
{
"kind": "ui-named-argument",
"line": 831,
"line": 881,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Talk settings",
"surface": "android",
@@ -4531,7 +4547,7 @@
},
{
"kind": "ui-named-argument",
"line": 839,
"line": 889,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Recent Sessions",
"surface": "android",
@@ -4539,7 +4555,7 @@
},
{
"kind": "ui-named-argument",
"line": 849,
"line": 899,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "View all",
"surface": "android",
@@ -4547,7 +4563,7 @@
},
{
"kind": "conditional-branch",
"line": 988,
"line": 1038,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$onlineNodes/$nodeCount",
"surface": "android",
@@ -4555,7 +4571,7 @@
},
{
"kind": "conditional-branch",
"line": 988,
"line": 1038,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "None",
"surface": "android",
@@ -4563,7 +4579,7 @@
},
{
"kind": "conditional-branch",
"line": 1020,
"line": 1070,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Recent conversations",
"surface": "android",
@@ -4571,7 +4587,7 @@
},
{
"kind": "conditional-branch",
"line": 1027,
"line": 1077,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Browse",
"surface": "android",
@@ -4579,7 +4595,7 @@
},
{
"kind": "conditional-branch",
"line": 1027,
"line": 1077,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Offline",
"surface": "android",
@@ -4587,7 +4603,7 @@
},
{
"kind": "conditional-branch",
"line": 1079,
"line": 1129,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Working · $pendingRunCount active ${pluralize(\"run\", pendingRunCount)}",
"surface": "android",
@@ -4595,7 +4611,7 @@
},
{
"kind": "ui-named-argument",
"line": 1200,
"line": 1250,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Needs attention",
"surface": "android",
@@ -4603,7 +4619,7 @@
},
{
"kind": "ui-named-argument",
"line": 1265,
"line": 1315,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Open ${row.title}",
"surface": "android",
@@ -4611,7 +4627,7 @@
},
{
"kind": "ui-named-argument",
"line": 1373,
"line": 1423,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Open session",
"surface": "android",
@@ -4619,7 +4635,7 @@
},
{
"kind": "ui-named-argument",
"line": 1488,
"line": 1538,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Back",
"surface": "android",
@@ -4627,7 +4643,7 @@
},
{
"kind": "ui-named-argument",
"line": 1491,
"line": 1541,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Settings",
"surface": "android",
@@ -4635,7 +4651,7 @@
},
{
"kind": "ui-named-argument",
"line": 1494,
"line": 1544,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Search settings",
"surface": "android",
@@ -4643,7 +4659,7 @@
},
{
"kind": "conditional-branch",
"line": 1518,
"line": 1568,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$readyProviderCount ready",
"surface": "android",
@@ -4651,7 +4667,7 @@
},
{
"kind": "conditional-branch",
"line": 1518,
"line": 1568,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Review readiness",
"surface": "android",
@@ -4659,7 +4675,7 @@
},
{
"kind": "conditional-branch",
"line": 1536,
"line": 1586,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Speaker muted",
"surface": "android",
@@ -4667,7 +4683,7 @@
},
{
"kind": "conditional-branch",
"line": 1536,
"line": 1586,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Speaker on",
"surface": "android",
@@ -4675,7 +4691,7 @@
},
{
"kind": "conditional-branch",
"line": 1538,
"line": 1588,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Off",
"surface": "android",
@@ -4683,7 +4699,7 @@
},
{
"kind": "conditional-branch",
"line": 1538,
"line": 1588,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Smart delivery",
"surface": "android",
@@ -4691,7 +4707,7 @@
},
{
"kind": "conditional-branch",
"line": 1539,
"line": 1589,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Camera enabled",
"surface": "android",
@@ -4699,7 +4715,7 @@
},
{
"kind": "conditional-branch",
"line": 1539,
"line": 1589,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Locked",
"surface": "android",
@@ -4707,7 +4723,7 @@
},
{
"kind": "ui-named-argument",
"line": 1586,
"line": 1636,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "OpenClaw ${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})",
"surface": "android",
@@ -4715,7 +4731,7 @@
},
{
"kind": "conditional-branch",
"line": 1589,
"line": 1639,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "All systems operational",
"surface": "android",
@@ -4723,7 +4739,7 @@
},
{
"kind": "conditional-branch",
"line": 1589,
"line": 1639,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Gateway not connected",
"surface": "android",
@@ -4731,7 +4747,7 @@
},
{
"kind": "conditional-branch",
"line": 1621,
"line": 1671,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "No provider usage",
"surface": "android",
@@ -4739,7 +4755,7 @@
},
{
"kind": "conditional-branch",
"line": 1622,
"line": 1672,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "1 provider",
"surface": "android",
@@ -4747,7 +4763,7 @@
},
{
"kind": "conditional-branch",
"line": 1623,
"line": 1673,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$count providers",
"surface": "android",
@@ -4755,7 +4771,7 @@
},
{
"kind": "conditional-branch",
"line": 1629,
"line": 1679,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$ready/${skills.size} ready",
"surface": "android",
@@ -4763,7 +4779,7 @@
},
{
"kind": "conditional-branch",
"line": 1629,
"line": 1679,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "No skills",
"surface": "android",
@@ -4771,7 +4787,7 @@
},
{
"kind": "conditional-branch",
"line": 1643,
"line": 1693,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$pending pending",
"surface": "android",
@@ -4779,7 +4795,7 @@
},
{
"kind": "conditional-branch",
"line": 1643,
"line": 1693,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "1 pending",
"surface": "android",
@@ -4787,7 +4803,7 @@
},
{
"kind": "conditional-branch",
"line": 1648,
"line": 1698,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$held held",
"surface": "android",
@@ -4795,7 +4811,7 @@
},
{
"kind": "conditional-branch",
"line": 1648,
"line": 1698,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "1 held",
"surface": "android",
@@ -4803,7 +4819,7 @@
},
{
"kind": "conditional-branch",
"line": 1649,
"line": 1699,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "$applied applied",
"surface": "android",
@@ -4811,7 +4827,7 @@
},
{
"kind": "conditional-branch",
"line": 1649,
"line": 1699,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "1 applied",
"surface": "android",
@@ -4819,7 +4835,7 @@
},
{
"kind": "ui-named-argument",
"line": 1830,
"line": 1880,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "OpenClaw mobile",
"surface": "android",
@@ -4827,7 +4843,7 @@
},
{
"kind": "ui-named-argument",
"line": 1834,
"line": 1884,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Open profile",
"surface": "android",
@@ -4835,7 +4851,7 @@
},
{
"kind": "conditional-branch",
"line": 1919,
"line": 1969,
"path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt",
"source": "Main session",
"surface": "android",

View File

@@ -88,6 +88,7 @@ android {
applicationId = "ai.openclaw.app"
minSdk = 31
targetSdk = 36
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
versionCode = openClawAndroidVersionCode
versionName = openClawAndroidVersionName
ndk {
@@ -289,6 +290,10 @@ dependencies {
testImplementation(libs.mockwebserver)
testImplementation(libs.robolectric)
testRuntimeOnly(libs.junit.vintage.engine)
androidTestImplementation(libs.androidx.test.ext.junit)
androidTestImplementation(libs.androidx.test.runner)
androidTestImplementation(libs.androidx.uiautomator)
}
tasks.withType<Test>().configureEach {

View File

@@ -0,0 +1,203 @@
package ai.openclaw.app.ui
import ai.openclaw.app.node.CanvasController
import android.content.pm.ActivityInfo
import android.os.SystemClock
import android.view.View
import androidx.test.core.app.ActivityScenario
import androidx.test.ext.junit.rules.ActivityScenarioRule
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.uiautomator.UiDevice
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class CanvasHostLifecycleTest {
@get:Rule
val activityRule = ActivityScenarioRule(CanvasLifecycleTestActivity::class.java)
@Before
fun resetMetrics() {
CanvasLifecycleTestMetrics.reset()
}
@Test
fun hiddenHostRetainsOneWebViewWithoutBlockingShellInput() {
activityRule.scenario.onActivity { activity ->
assertEquals(CanvasController.PresentationState.Unmounted, activity.controller.presentationState.value)
assertNull(activity.host)
}
val presentElapsedMs = activityRule.scenario.readActivity { activity -> activity.presentSlowPage() }
assertTrue(
"present waited for the remote page: ${presentElapsedMs}ms",
presentElapsedMs < canvasLifecycleSlowPageDelayMs / 2,
)
assertTrue("slow page never finished", activityRule.scenario.waitForPageFinished())
val firstWebView =
activityRule.scenario.readActivity { activity ->
val host = checkNotNull(activity.host)
assertEquals(1, host.childCount)
assertEquals(CanvasController.PresentationState.Visible, activity.controller.presentationState.value)
val webView = checkNotNull(activity.currentWebView())
activity.hideCanvas()
assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value)
webView
}
assertTrue(
"hidden host remained visible",
activityRule.scenario.waitUntilActivity { activity -> activity.host?.visibility == View.INVISIBLE },
)
val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
assertTrue(device.click(device.displayWidth / 2, device.displayHeight / 2))
device.waitForIdle()
activityRule.scenario.onActivity { activity ->
assertEquals(1, activity.underlayClickCount)
repeat(3) {
activity.presentFastPage()
activity.hideCanvas()
}
assertEquals(1, activity.host?.childCount)
assertTrue(firstWebView === activity.currentWebView())
}
}
@Test
fun stalePageCompletionCannotReshowCanvasAfterHide() {
activityRule.scenario.onActivity { activity -> activity.presentSlowPage() }
assertTrue(
"Canvas host was not attached",
activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() != null },
)
activityRule.scenario.onActivity { activity -> activity.hideCanvas() }
assertTrue(
"hidden host remained visible",
activityRule.scenario.waitUntilActivity { activity -> activity.host?.visibility == View.INVISIBLE },
)
assertTrue("slow page never finished", activityRule.scenario.waitForPageFinished())
activityRule.scenario.onActivity { activity ->
assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value)
assertEquals(View.INVISIBLE, checkNotNull(activity.host).visibility)
assertNotNull(activity.currentWebView())
}
}
@Test
fun rendererTerminationForgetsFailedPageAndNextShowRecreatesIt() {
activityRule.scenario.onActivity { activity -> activity.presentFastPage() }
assertTrue("initial page never finished", activityRule.scenario.waitForPageFinished())
val firstWebView =
activityRule.scenario.readActivity { activity ->
assertNotNull(activity.controller.currentUrl())
checkNotNull(activity.currentWebView())
}
val terminated =
activityRule.scenario.readActivity { activity ->
activity.currentWebView()?.webViewRenderProcess?.terminate() == true
}
assertTrue("WebView renderer did not terminate", terminated)
assertTrue(
"renderer loss did not clear the invalid WebView",
activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() == null },
)
activityRule.scenario.onActivity { activity ->
assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value)
assertNull(activity.controller.currentUrl())
assertEquals(0, activity.host?.childCount)
activity.showCanvas()
}
assertTrue(
"next show did not create a replacement WebView",
activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() != null },
)
assertTrue("replacement scaffold never finished", activityRule.scenario.waitForPageFinished())
activityRule.scenario.onActivity { activity ->
assertEquals(CanvasController.PresentationState.Visible, activity.controller.presentationState.value)
assertEquals(1, activity.host?.childCount)
assertNotEquals(firstWebView, activity.currentWebView())
}
}
@Test
fun configurationChangesKeepTheSameHostAndWebView() {
activityRule.scenario.onActivity { activity -> activity.presentFastPage() }
assertTrue("initial page never finished", activityRule.scenario.waitForPageFinished())
val firstHost = activityRule.scenario.readActivity { activity -> checkNotNull(activity.host) }
val firstWebView = activityRule.scenario.readActivity { activity -> checkNotNull(activity.currentWebView()) }
activityRule.scenario.onActivity { activity ->
activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE
}
UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()).waitForIdle()
activityRule.scenario.onActivity { activity ->
assertTrue(firstHost === activity.host)
assertTrue(firstWebView === activity.currentWebView())
activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT
}
}
}
@RunWith(AndroidJUnit4::class)
class CanvasHostReleaseTest {
@Before
fun resetMetrics() {
CanvasLifecycleTestMetrics.reset()
}
@Test
fun activityTeardownReleasesTheHostAndWebView() {
ActivityScenario.launch(CanvasLifecycleTestActivity::class.java).use { scenario ->
scenario.onActivity { activity -> activity.presentFastPage() }
}
assertTrue(
"AndroidView onRelease was not called",
waitUntil { CanvasLifecycleTestMetrics.hostReleaseCount.get() == 1 },
)
assertEquals(1, CanvasLifecycleTestMetrics.webViewDestroyCount.get())
}
}
private inline fun <T> ActivityScenario<CanvasLifecycleTestActivity>.readActivity(crossinline block: (CanvasLifecycleTestActivity) -> T): T {
var result: Result<T>? = null
onActivity { activity -> result = runCatching { block(activity) } }
return checkNotNull(result).getOrThrow()
}
private fun ActivityScenario<CanvasLifecycleTestActivity>.waitForPageFinished(): Boolean = waitUntilActivity { activity -> activity.currentWebView()?.progress == 100 }
private inline fun ActivityScenario<CanvasLifecycleTestActivity>.waitUntilActivity(
timeoutMs: Long = 5_000L,
crossinline predicate: (CanvasLifecycleTestActivity) -> Boolean,
): Boolean =
waitUntil(timeoutMs) {
readActivity(predicate)
}
private fun waitUntil(
timeoutMs: Long = 5_000L,
predicate: () -> Boolean,
): Boolean {
val deadline = SystemClock.elapsedRealtime() + timeoutMs
while (SystemClock.elapsedRealtime() < deadline) {
if (predicate()) return true
SystemClock.sleep(20)
}
return predicate()
}

View File

@@ -7,6 +7,11 @@
<uses-permission android:name="${applicationId}.permission.RUN_VOICE_E2E" />
<application>
<activity
android:name=".ui.CanvasLifecycleTestActivity"
android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|uiMode|density|keyboard|keyboardHidden|navigation"
android:exported="false"
android:theme="@android:style/Theme.Material.Light.NoActionBar" />
<receiver
android:name=".VoiceE2eReceiver"
android:permission="${applicationId}.permission.RUN_VOICE_E2E"

View File

@@ -0,0 +1,129 @@
package ai.openclaw.app.ui
import ai.openclaw.app.node.CanvasController
import android.os.Bundle
import android.os.SystemClock
import android.util.Base64
import android.webkit.WebView
import android.widget.Button
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.viewinterop.AndroidView
import java.util.concurrent.atomic.AtomicInteger
const val canvasLifecycleSlowPageDelayMs = 2_000L
class CanvasLifecycleTestActivity : ComponentActivity() {
val controller = CanvasController()
internal var host: CanvasHostView? = null
internal set
var underlayClickCount: Int = 0
private set
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContent {
CanvasLifecycleTestContent(
activity = this,
onUnderlayClick = { underlayClickCount += 1 },
)
}
}
fun presentSlowPage(): Long = presentHtml(slowPageHtml)
fun presentFastPage(): Long = presentHtml("<html><body>ready</body></html>")
fun hideCanvas() {
controller.hide()
}
fun showCanvas() {
controller.show()
}
fun currentWebView(): WebView? = host?.currentWebView
private fun presentHtml(html: String): Long {
val startedAt = SystemClock.elapsedRealtime()
val encoded = Base64.encodeToString(html.toByteArray(Charsets.UTF_8), Base64.NO_WRAP)
controller.navigate("data:text/html;base64,$encoded")
controller.show()
return SystemClock.elapsedRealtime() - startedAt
}
}
@Composable
private fun CanvasLifecycleTestContent(
activity: CanvasLifecycleTestActivity,
onUnderlayClick: () -> Unit,
) {
val state by activity.controller.presentationState.collectAsState()
Box(modifier = Modifier.fillMaxSize()) {
AndroidView(
factory = { context ->
Button(context).apply {
setOnClickListener { onUnderlayClick() }
}
},
modifier = Modifier.fillMaxSize(),
)
if (state != CanvasController.PresentationState.Unmounted) {
AndroidView(
factory = { context ->
CanvasHostView(
context = context,
controller = activity.controller,
isTrustedPage = { false },
onA2uiMessage = {},
).also { host ->
activity.host = host
host.updateVisible(state == CanvasController.PresentationState.Visible)
}
},
update = { host ->
host.updateVisible(state == CanvasController.PresentationState.Visible)
},
modifier = Modifier.fillMaxSize(),
onRelease = { host ->
val hadWebView = host.currentWebView != null
host.release()
if (activity.host === host) activity.host = null
CanvasLifecycleTestMetrics.hostReleaseCount.incrementAndGet()
if (hadWebView) CanvasLifecycleTestMetrics.webViewDestroyCount.incrementAndGet()
},
)
}
}
}
object CanvasLifecycleTestMetrics {
val hostReleaseCount = AtomicInteger()
val webViewDestroyCount = AtomicInteger()
fun reset() {
hostReleaseCount.set(0)
webViewDestroyCount.set(0)
}
}
private val slowPageHtml =
"""
<html>
<body>
<script>
const deadline = Date.now() + $canvasLifecycleSlowPageDelayMs;
while (Date.now() < deadline) {}
</script>
ready
</body>
</html>
""".trimIndent()

View File

@@ -172,6 +172,8 @@ class MainViewModel(
.stateIn(viewModelScope, SharingStarted.Eagerly, false)
val canvasCurrentUrl: StateFlow<String?> = runtimeState(initial = null) { it.canvas.currentUrl }
val canvasPresentationState: StateFlow<CanvasController.PresentationState> =
runtimeState(initial = CanvasController.PresentationState.Unmounted) { it.canvas.presentationState }
val canvasA2uiHydrated: StateFlow<Boolean> = runtimeState(initial = false) { it.canvasA2uiHydrated }
val canvasRehydratePending: StateFlow<Boolean> = runtimeState(initial = false) { it.canvasRehydratePending }
val canvasRehydrateErrorText: StateFlow<String?> = runtimeState(initial = null) { it.canvasRehydrateErrorText }
@@ -740,6 +742,14 @@ class MainViewModel(
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
}
fun showCanvas() {
ensureRuntime().canvas.show()
}
fun hideCanvas() {
runtimeRef.value?.canvas?.hide()
}
fun refreshHomeCanvasOverviewIfConnected() {
ensureRuntime().refreshHomeCanvasOverviewIfConnected()
}

View File

@@ -16,11 +16,15 @@ class A2UIHandler(
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean = CanvasActionTrust.isTrustedCanvasActionUrl(rawUrl)
suspend fun ensureA2uiReady(): Boolean {
if (canvas.currentUrl()?.trim() == CanvasActionTrust.localA2uiAssetUrl && isA2uiReady()) {
val alreadyOnA2uiHost = canvas.currentUrl()?.trim() == CanvasActionTrust.localA2uiAssetUrl
if (!canvas.showAndAwaitHost()) return false
if (!alreadyOnA2uiHost) {
canvas.showLocalA2ui()
}
if (alreadyOnA2uiHost && isA2uiReady()) {
return true
}
canvas.showLocalA2ui()
// The bundled A2UI host bootstraps asynchronously after navigation; poll briefly before failing the command.
repeat(50) {
if (isA2uiReady()) return true

View File

@@ -13,8 +13,10 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
@@ -27,6 +29,12 @@ import kotlin.coroutines.resume
* Owns the Android WebView canvas surface used by canvas and A2UI commands.
*/
class CanvasController {
enum class PresentationState {
Unmounted,
Hidden,
Visible,
}
enum class SnapshotFormat(
val rawValue: String,
) {
@@ -47,6 +55,9 @@ class CanvasController {
@Volatile private var homeCanvasStateJson: String? = null
private val _currentUrl = MutableStateFlow<String?>(null)
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
private val _presentationState = MutableStateFlow(PresentationState.Unmounted)
val presentationState: StateFlow<PresentationState> = _presentationState.asStateFlow()
private val hostAttachedState = MutableStateFlow(false)
private val scaffoldAssetUrl = CanvasActionTrust.scaffoldAssetUrl
private val localA2uiAssetUrl = CanvasActionTrust.localA2uiAssetUrl
@@ -67,19 +78,61 @@ class CanvasController {
/** Attaches the active WebView and replays state that may have arrived before the view existed. */
fun attach(webView: WebView) {
this.webView = webView
hostAttachedState.value = true
// Replay persisted state because WebView attachment can happen after gateway events arrive.
reload()
applyDebugStatus()
applyHomeCanvasState()
}
/** Detaches only the currently attached WebView instance. */
fun detach(webView: WebView) {
if (this.webView === webView) {
this.webView = null
/** Releases the shell-owned host when its UI owner permanently leaves composition. */
fun releaseHost() {
webView = null
hostAttachedState.value = false
_presentationState.value = PresentationState.Unmounted
}
/** Invalid renderer processes cannot be reused; retain the host but require a new child. */
fun onRenderProcessGone(webView: WebView) {
if (this.webView !== webView) return
this.webView = null
// Do not replay the page that terminated its renderer into the replacement WebView.
url = null
_currentUrl.value = null
hostAttachedState.value = false
_presentationState.value = PresentationState.Hidden
}
fun show() {
_presentationState.value = PresentationState.Visible
}
fun hide() {
if (_presentationState.value != PresentationState.Unmounted) {
_presentationState.value = PresentationState.Hidden
}
}
/**
* Requests presentation and waits only for the shell host to accept it.
* Remote page loading remains asynchronous and must not delay invoke completion.
*/
suspend fun showAndAwaitHost(): Boolean {
val previousState = _presentationState.value
show()
if (hostAttachedState.value) return true
val attached =
withTimeoutOrNull(hostAttachTimeoutMs) {
hostAttachedState.first { it }
true
} ?: hostAttachedState.value
if (!attached && _presentationState.value == PresentationState.Visible) {
// A failed foreground handoff must not leave a pending overlay for the next Activity.
_presentationState.value = previousState
}
return attached
}
/** Navigates the canvas to a remote URL or back to the bundled scaffold for blank/root input. */
fun navigate(url: String) {
this.url = CanvasNavigationPolicy.normalize(url).ifBlank { null }
@@ -243,6 +296,8 @@ class CanvasController {
}
companion object {
private const val hostAttachTimeoutMs = 5_000L
/**
* Parsed canvas.snapshot options used by invoke dispatch.
*/

View File

@@ -14,6 +14,8 @@ import ai.openclaw.app.protocol.OpenClawNotificationsCommand
import ai.openclaw.app.protocol.OpenClawSmsCommand
import ai.openclaw.app.protocol.OpenClawSystemCommand
import ai.openclaw.app.protocol.OpenClawTalkCommand
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
/** Runtime state for SMS search, split so permission prompts are not reported as hard unavailability. */
internal enum class SmsSearchAvailabilityReason {
@@ -92,6 +94,8 @@ class InvokeDispatcher(
private val motionActivityAvailable: () -> Boolean,
private val motionPedometerAvailable: () -> Boolean,
) {
private val canvasCommandMutex = Mutex()
/** Dispatches one gateway node.invoke command after foreground and availability gates pass. */
suspend fun handleInvoke(
command: String,
@@ -112,19 +116,40 @@ class InvokeDispatcher(
}
availabilityError(spec.availability)?.let { return it }
if (command.startsWith(OpenClawCanvasCommand.NamespacePrefix)) {
// GatewaySession may deliver invokes concurrently. Canvas presentation, navigation, and
// A2UI evaluation share one WebView and must observe command arrival order.
return canvasCommandMutex.withLock { dispatchInvoke(command, paramsJson) }
}
return dispatchInvoke(command, paramsJson)
}
private suspend fun dispatchInvoke(
command: String,
paramsJson: String?,
): GatewaySession.InvokeResult {
// Command strings come from OpenClawProtocolConstants; the registry above owns advertised availability.
return when (command) {
// Canvas commands
OpenClawCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
withCanvasAvailable {
check(canvas.showAndAwaitHost()) { "canvas host unavailable" }
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
}
OpenClawCanvasCommand.Hide.rawValue -> {
canvas.hide()
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
OpenClawCanvasCommand.Navigate.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
withCanvasAvailable {
check(canvas.showAndAwaitHost()) { "canvas host unavailable" }
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
}
OpenClawCanvasCommand.Eval.rawValue -> {
val js =

View File

@@ -1,12 +1,16 @@
package ai.openclaw.app.ui
import ai.openclaw.app.MainViewModel
import ai.openclaw.app.node.CanvasController
import ai.openclaw.app.node.CanvasNavigationPolicy
import android.annotation.SuppressLint
import android.content.Context
import android.net.Uri
import android.util.Log
import android.view.View
import android.view.ViewGroup
import android.webkit.ConsoleMessage
import android.webkit.RenderProcessGoneDetail
import android.webkit.WebChromeClient
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
@@ -14,11 +18,9 @@ import android.webkit.WebResourceResponse
import android.webkit.WebSettings
import android.webkit.WebView
import android.webkit.WebViewClient
import android.widget.FrameLayout
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.remember
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.viewinterop.AndroidView
import androidx.webkit.JavaScriptReplyProxy
import androidx.webkit.WebMessageCompat
@@ -37,28 +39,69 @@ fun CanvasScreen(
visible: Boolean,
modifier: Modifier = Modifier,
) {
val context = LocalContext.current
val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
val webViewRef = remember { arrayOfNulls<WebView>(1) }
val currentPageUrlRef = remember { AtomicReference<String?>(null) }
DisposableEffect(viewModel) {
onDispose {
val webView = webViewRef[0] ?: return@onDispose
viewModel.canvas.detach(webView)
if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
WebViewCompat.removeWebMessageListener(webView, CanvasA2UIActionBridge.interfaceName)
AndroidView(
modifier = modifier,
factory = { context ->
CanvasHostView(
context = context,
controller = viewModel.canvas,
isTrustedPage = viewModel::isTrustedCanvasActionUrl,
onA2uiMessage = viewModel::handleCanvasA2UIActionFromWebView,
).apply {
updateVisible(visible)
}
webView.stopLoading()
webView.destroy()
webViewRef[0] = null
},
update = { host -> host.updateVisible(visible) },
onRelease = CanvasHostView::release,
)
}
/**
* Retained shell host whose WebView child can be replaced after renderer death.
*
* Compose creates this host directly; XML inflation cannot supply its controller and callbacks.
*/
@SuppressLint("SetJavaScriptEnabled", "ViewConstructor")
@Suppress("DEPRECATION")
internal class CanvasHostView(
context: Context,
private val controller: CanvasController,
private val isTrustedPage: (String?) -> Boolean,
private val onA2uiMessage: (String) -> Unit,
) : FrameLayout(context) {
internal var currentWebView: WebView? = null
private set
private val isDebuggable =
(context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
private val currentPageUrlRef = AtomicReference<String?>(null)
init {
visibility = View.INVISIBLE
}
fun updateVisible(visible: Boolean) {
if (visible) {
val webView = currentWebView ?: createWebView()
visibility = View.VISIBLE
webView.visibility = View.VISIBLE
webView.onResume()
return
}
visibility = View.INVISIBLE
currentWebView?.let { webView ->
webView.visibility = View.INVISIBLE
webView.onPause()
}
}
AndroidView(
modifier = modifier,
factory = {
val webView = WebView(context)
fun release() {
controller.releaseHost()
currentWebView?.let(::destroyWebView)
}
private fun createWebView(): WebView =
WebView(context).also { webView ->
val webSettings = webView.settings
webSettings.setAllowContentAccess(false)
webSettings.setAllowFileAccess(false)
@@ -73,7 +116,7 @@ fun CanvasScreen(
webSettings.builtInZoomControls = false
webSettings.displayZoomControls = false
webSettings.setSupportZoom(false)
webView.visibility = if (visible) View.VISIBLE else View.INVISIBLE
webView.visibility = View.INVISIBLE
// targetSdk 33+ ignores Force Dark APIs, so only opt out through the supported
// algorithmic darkening flag when this WebView implementation exposes it.
if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) {
@@ -93,7 +136,7 @@ fun CanvasScreen(
request: WebResourceRequest,
): Boolean {
if (!request.isForMainFrame) return false
return blockUnsafeCanvasNavigation(viewModel, currentPageUrlRef, request.url.toString())
return blockUnsafeCanvasNavigation(controller, currentPageUrlRef, request.url.toString())
}
override fun shouldInterceptRequest(
@@ -109,7 +152,7 @@ fun CanvasScreen(
// shouldOverrideUrlLoading excludes POST navigations and their redirects. WebView does
// not expose those redirect targets, so non-GET main-frame loads fail closed here.
currentPageUrlRef.set(null)
view.post { viewModel.canvas.navigate("") }
view.post { controller.navigate("") }
return blockedCanvasResponse()
}
@@ -150,12 +193,12 @@ fun CanvasScreen(
if (isDebuggable) {
Log.d("OpenClawWebView", "onPageFinished: $url")
}
viewModel.canvas.onPageFinished()
controller.onPageFinished()
}
override fun onRenderProcessGone(
view: WebView,
detail: android.webkit.RenderProcessGoneDetail,
detail: RenderProcessGoneDetail,
): Boolean {
if (isDebuggable) {
Log.e(
@@ -163,6 +206,11 @@ fun CanvasScreen(
"onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}",
)
}
if (view === currentWebView) {
controller.onRenderProcessGone(view)
destroyWebView(view)
visibility = View.INVISIBLE
}
return true
}
}
@@ -183,10 +231,9 @@ fun CanvasScreen(
// dispatch still requires the live URL to be an app-owned bundled page.
val bridge =
CanvasA2UIActionBridge(
isTrustedPage = { viewModel.isTrustedCanvasActionUrl(currentPageUrlRef.get()) },
) { payload ->
viewModel.handleCanvasA2UIActionFromWebView(payload)
}
isTrustedPage = { isTrustedPage(currentPageUrlRef.get()) },
onMessage = onA2uiMessage,
)
if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
WebViewCompat.addWebMessageListener(
webView,
@@ -197,32 +244,38 @@ fun CanvasScreen(
} else if (isDebuggable) {
Log.w("OpenClawWebView", "WebMessageListener unsupported; canvas actions disabled")
}
viewModel.canvas.attach(webView)
webViewRef[0] = webView
webView
},
update = { webView ->
webView.visibility = if (visible) View.VISIBLE else View.INVISIBLE
if (visible) {
webView.resumeTimers()
webView.onResume()
} else {
webView.onPause()
webView.pauseTimers()
}
},
)
addView(
webView,
ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT,
),
)
currentWebView = webView
controller.attach(webView)
}
private fun destroyWebView(webView: WebView) {
if (currentWebView !== webView) return
if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
WebViewCompat.removeWebMessageListener(webView, CanvasA2UIActionBridge.interfaceName)
}
removeView(webView)
webView.stopLoading()
webView.destroy()
currentWebView = null
}
}
private fun blockUnsafeCanvasNavigation(
viewModel: MainViewModel,
controller: CanvasController,
currentPageUrlRef: AtomicReference<String?>,
rawUrl: String,
): Boolean {
val url = rawUrl.trim()
if (!CanvasNavigationPolicy.shouldBlock(url)) return false
currentPageUrlRef.set(null)
viewModel.canvas.navigate("")
controller.navigate("")
return true
}

View File

@@ -12,7 +12,6 @@ import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.RoundedCornerShape
@@ -31,7 +30,7 @@ import androidx.compose.ui.draw.clip
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
/** Settings detail surface for live canvas status, refresh, and embedded preview. */
/** Settings detail surface for live canvas status, refresh, and presentation. */
@Composable
internal fun CanvasSettingsScreen(
viewModel: MainViewModel,
@@ -43,12 +42,11 @@ internal fun CanvasSettingsScreen(
val rehydratePending by viewModel.canvasRehydratePending.collectAsState()
val rehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState()
val hasLivePage = currentUrl?.isNotBlank() == true
val showCanvasSurface = isConnected
val canvasLabel = if (hasLivePage) "Live page" else "Home canvas"
LaunchedEffect(isConnected) {
if (isConnected) {
// Refresh once when the gateway comes online so the settings preview is
// Refresh once when the gateway comes online so current Canvas status is
// populated before the user manually asks for a rehydrate.
viewModel.refreshHomeCanvasOverviewIfConnected()
}
@@ -76,8 +74,8 @@ internal fun CanvasSettingsScreen(
modifier = Modifier.weight(1f),
)
ClawSecondaryButton(
text = "Reconnect",
onClick = viewModel::refreshGatewayConnection,
text = if (isConnected) "Open Screen" else "Reconnect",
onClick = if (isConnected) viewModel::showCanvas else viewModel::refreshGatewayConnection,
modifier = Modifier.weight(1f),
)
}
@@ -90,18 +88,12 @@ internal fun CanvasSettingsScreen(
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
Text(text = canvasLabel, style = ClawTheme.type.section, color = ClawTheme.colors.text, maxLines = 1, overflow = TextOverflow.Ellipsis)
Surface(
modifier = Modifier.fillMaxWidth().height(520.dp).clip(RoundedCornerShape(ClawTheme.radii.panel)),
modifier = Modifier.fillMaxWidth().clip(RoundedCornerShape(ClawTheme.radii.panel)),
shape = RoundedCornerShape(ClawTheme.radii.panel),
color = ClawTheme.colors.canvas,
border = BorderStroke(1.dp, ClawTheme.colors.border),
) {
Box {
if (showCanvasSurface) {
CanvasScreen(viewModel = viewModel, visible = true, modifier = Modifier.fillMaxWidth().height(520.dp))
} else {
CanvasStandbyPanel(isConnected = isConnected)
}
}
CanvasStandbyPanel(isConnected = isConnected)
}
}
}
@@ -111,7 +103,7 @@ internal fun CanvasSettingsScreen(
@Composable
private fun CanvasStandbyPanel(isConnected: Boolean) {
Column(
modifier = Modifier.fillMaxWidth().height(520.dp).padding(horizontal = 24.dp),
modifier = Modifier.fillMaxWidth().padding(horizontal = 24.dp, vertical = 48.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.Center,
) {
@@ -133,7 +125,7 @@ private fun CanvasStandbyPanel(isConnected: Boolean) {
modifier = Modifier.padding(top = 18.dp),
)
Text(
text = if (isConnected) "Canvas output appears here when OpenClaw opens an app surface." else "Canvas output needs an active gateway connection.",
text = if (isConnected) "Open the current Canvas surface to inspect or interact with it." else "Canvas output needs an active gateway connection.",
style = ClawTheme.type.body,
color = ClawTheme.colors.textMuted,
modifier = Modifier.padding(top = 6.dp),

View File

@@ -16,10 +16,12 @@ import ai.openclaw.app.NodeRuntime
import ai.openclaw.app.R
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.currentAppLanguage
import ai.openclaw.app.node.CanvasController
import ai.openclaw.app.ui.chat.ChatScreen
import ai.openclaw.app.ui.design.ClawBottomNav
import ai.openclaw.app.ui.design.ClawDesignTheme
import ai.openclaw.app.ui.design.ClawEmptyState
import ai.openclaw.app.ui.design.ClawIconButton
import ai.openclaw.app.ui.design.ClawNavItem
import ai.openclaw.app.ui.design.ClawPanel
import ai.openclaw.app.ui.design.ClawPlainIconButton
@@ -52,6 +54,7 @@ import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
@@ -61,6 +64,7 @@ import androidx.compose.material.icons.automirrored.filled.ExitToApp
import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight
import androidx.compose.material.icons.automirrored.filled.ScreenShare
import androidx.compose.material.icons.filled.ArrowDropDown
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Cloud
import androidx.compose.material.icons.filled.Favorite
import androidx.compose.material.icons.filled.GraphicEq
@@ -156,9 +160,16 @@ fun ShellScreen(
val requestedHomeDestination by viewModel.requestedHomeDestination.collectAsState()
val pendingTrust by viewModel.pendingGatewayTrust.collectAsState()
val runtimeInitialized by viewModel.runtimeInitialized.collectAsState()
val canvasPresentationState by viewModel.canvasPresentationState.collectAsState()
val canvasVisible = canvasPresentationState == CanvasController.PresentationState.Visible
LaunchedEffect(requestedHomeDestination) {
val destination = requestedHomeDestination ?: return@LaunchedEffect
if (destination == HomeDestination.Screen) {
viewModel.showCanvas()
viewModel.clearRequestedHomeDestination()
return@LaunchedEffect
}
// HomeDestination is a one-shot command from launch intents and settings
// actions; consume it after translating to local shell state.
nav.selectTab(
@@ -166,7 +177,7 @@ fun ShellScreen(
HomeDestination.Connect -> Tab.Overview
HomeDestination.Chat -> Tab.Chat
HomeDestination.Voice -> Tab.Voice
HomeDestination.Screen -> Tab.Chat
HomeDestination.Screen -> Tab.Overview
HomeDestination.Settings -> Tab.Settings
},
)
@@ -191,7 +202,8 @@ fun ShellScreen(
val density = LocalDensity.current
val keyboardVisible = WindowInsets.ime.getBottom(density) > 0
val showBottomNav = shellBottomNavVisible(keyboardVisible = keyboardVisible, commandOpen = commandOpen)
val showBottomNav =
shellBottomNavVisible(keyboardVisible = keyboardVisible, commandOpen = commandOpen) && !canvasVisible
Scaffold(
modifier = modifier.fillMaxSize(),
@@ -290,6 +302,14 @@ fun ShellScreen(
)
}
if (canvasPresentationState != CanvasController.PresentationState.Unmounted) {
CanvasOverlay(
viewModel = viewModel,
visible = canvasVisible,
onClose = viewModel::hideCanvas,
)
}
pendingTrust?.let { prompt ->
// Gateway certificate trust is modal across the shell so navigation
// cannot hide a changed TLS identity prompt.
@@ -304,6 +324,36 @@ fun ShellScreen(
}
}
@Composable
private fun CanvasOverlay(
viewModel: MainViewModel,
visible: Boolean,
onClose: () -> Unit,
) {
BackHandler(enabled = visible, onBack = onClose)
val overlayColor = if (visible) ClawTheme.colors.canvas else Color.Transparent
Box(modifier = Modifier.fillMaxSize().background(overlayColor)) {
// The shell owns system-bar avoidance; arbitrary Canvas pages cannot know Android insets.
CanvasScreen(
viewModel = viewModel,
visible = visible,
modifier = Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.safeDrawing),
)
if (visible) {
ClawIconButton(
icon = Icons.Default.Close,
contentDescription = "Close Canvas",
onClick = onClose,
modifier =
Modifier
.align(Alignment.TopEnd)
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal))
.padding(top = 12.dp, end = 12.dp),
)
}
}
}
/** Modal trust decision for first-seen or changed gateway TLS fingerprints. */
@Composable
private fun GatewayTrustDialog(

View File

@@ -0,0 +1,38 @@
package ai.openclaw.app.node
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Test
class CanvasControllerPresentationTest {
@Test
fun presentationStateKeepsTheHostUnmountedUntilFirstShow() {
val controller = CanvasController()
controller.hide()
assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value)
controller.show()
assertEquals(CanvasController.PresentationState.Visible, controller.presentationState.value)
controller.hide()
assertEquals(CanvasController.PresentationState.Hidden, controller.presentationState.value)
controller.show()
assertEquals(CanvasController.PresentationState.Visible, controller.presentationState.value)
controller.releaseHost()
assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value)
}
@Test
fun failedHostHandoffRestoresThePreviousPresentationState() =
runTest {
val controller = CanvasController()
assertFalse(controller.showAndAwaitHost())
assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value)
}
}

View File

@@ -4,6 +4,7 @@ import ai.openclaw.app.gateway.DeviceIdentityStore
import ai.openclaw.app.gateway.GatewaySession
import ai.openclaw.app.protocol.OpenClawCallLogCommand
import ai.openclaw.app.protocol.OpenClawCameraCommand
import ai.openclaw.app.protocol.OpenClawCanvasCommand
import ai.openclaw.app.protocol.OpenClawDeviceCommand
import ai.openclaw.app.protocol.OpenClawLocationCommand
import ai.openclaw.app.protocol.OpenClawMotionCommand
@@ -12,6 +13,7 @@ import ai.openclaw.app.protocol.OpenClawSmsCommand
import ai.openclaw.app.protocol.OpenClawTalkCommand
import android.content.Context
import android.content.pm.PackageManager
import android.webkit.WebView
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
@@ -275,6 +277,58 @@ class InvokeDispatcherTest {
assertEquals(listOf("start", "stop", "cancel"), talk.calls)
}
@Test
fun handleInvoke_presentAndHideDriveTheShellOwnedCanvasState() =
runTest {
val appContext = RuntimeEnvironment.getApplication()
val canvas = CanvasController()
val webView = WebView(appContext)
canvas.attach(webView)
val dispatcher = newDispatcher(canvas = canvas)
val present =
dispatcher.handleInvoke(
OpenClawCanvasCommand.Present.rawValue,
"""{"url":"https://example.com/canvas"}""",
)
assertNull(present.error)
assertEquals("https://example.com/canvas", canvas.currentUrl())
assertEquals(CanvasController.PresentationState.Visible, canvas.presentationState.value)
val hide = dispatcher.handleInvoke(OpenClawCanvasCommand.Hide.rawValue, null)
assertNull(hide.error)
assertEquals(CanvasController.PresentationState.Hidden, canvas.presentationState.value)
canvas.releaseHost()
webView.destroy()
}
@Test
fun handleInvoke_rejectsBackgroundCanvasPresentationBeforeMountingAHost() =
runTest {
val canvas = CanvasController()
val result =
newDispatcher(isForeground = false, canvas = canvas)
.handleInvoke(OpenClawCanvasCommand.Present.rawValue, """{"url":"https://example.com"}""")
assertEquals("NODE_BACKGROUND_UNAVAILABLE", result.error?.code)
assertEquals(CanvasController.PresentationState.Unmounted, canvas.presentationState.value)
}
@Test
fun handleInvoke_doesNotCommitNavigationWhenTheShellHostCannotAttach() =
runTest {
val canvas = CanvasController()
val result =
newDispatcher(canvas = canvas)
.handleInvoke(OpenClawCanvasCommand.Present.rawValue, """{"url":"https://example.com"}""")
assertEquals("NODE_BACKGROUND_UNAVAILABLE", result.error?.code)
assertNull(canvas.currentUrl())
assertEquals(CanvasController.PresentationState.Unmounted, canvas.presentationState.value)
}
private fun newDispatcher(
isForeground: Boolean = true,
cameraEnabled: Boolean = false,
@@ -290,10 +344,10 @@ class InvokeDispatcherTest {
motionActivityAvailable: Boolean = false,
motionPedometerAvailable: Boolean = false,
talkHandler: TalkHandler = InvokeDispatcherFakeTalkHandler(),
canvas: CanvasController = CanvasController(),
): InvokeDispatcher {
val appContext = RuntimeEnvironment.getApplication()
shadowOf(appContext.packageManager).setSystemFeature(PackageManager.FEATURE_TELEPHONY, smsTelephonyAvailable)
val canvas = CanvasController()
return InvokeDispatcher(
canvas = canvas,
cameraHandler = newCameraHandler(appContext),

View File

@@ -10,6 +10,7 @@ androidx-exifinterface = "1.4.2"
androidx-lifecycle = "2.11.0"
androidx-security = "1.1.0"
androidx-test-ext = "1.3.0"
androidx-test-runner = "1.7.0"
androidx-uiautomator = "2.4.0"
androidx-webkit = "1.15.0"
bcprov = "1.84"
@@ -51,6 +52,7 @@ androidx-room-compiler = { module = "androidx.room:room-compiler", version.ref =
androidx-room-runtime = { module = "androidx.room:room-runtime", version.ref = "room" }
androidx-security-crypto = { module = "androidx.security:security-crypto", version.ref = "androidx-security" }
androidx-test-ext-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-ext" }
androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" }
androidx-uiautomator = { module = "androidx.test.uiautomator:uiautomator", version.ref = "androidx-uiautomator" }
androidx-webkit = { module = "androidx.webkit:webkit", version.ref = "androidx-webkit" }
bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bcprov" }

View File

@@ -104,6 +104,36 @@ function assertObjectPayload(command: string, payload: unknown): Record<string,
return obj;
}
const VALID_A2UI_JSONL = [
JSON.stringify({
surfaceUpdate: {
surfaceId: "main",
components: [
{
id: "root",
component: { Column: { children: { explicitList: ["text"] } } },
},
{
id: "text",
component: {
Text: {
text: { literalString: "Android Canvas live test" },
usageHint: "body",
},
},
},
],
},
}),
JSON.stringify({ beginRendering: { surfaceId: "main", root: "root" } }),
].join("\n");
function assertA2uiPushPayload(command: string, payload: unknown) {
const obj = assertObjectPayload(command, payload);
expect(obj.ok).toBe(true);
expect(readStringArray(obj.surfaces)).toContain("main");
}
const COMMAND_PROFILES: Record<string, CommandProfile> = {
"canvas.present": {
buildParams: () => ({ url: "about:blank" }),
@@ -140,14 +170,20 @@ const COMMAND_PROFILES: Record<string, CommandProfile> = {
},
},
"canvas.a2ui.push": {
buildParams: () => ({ jsonl: '{"beginRendering":{}}\n' }),
buildParams: () => ({ jsonl: VALID_A2UI_JSONL }),
timeoutMs: 30_000,
outcome: "success",
onSuccess: (payload) => {
assertA2uiPushPayload("canvas.a2ui.push", payload);
},
},
"canvas.a2ui.pushJSONL": {
buildParams: () => ({ jsonl: '{"beginRendering":{}}\n' }),
buildParams: () => ({ jsonl: VALID_A2UI_JSONL }),
timeoutMs: 30_000,
outcome: "success",
onSuccess: (payload) => {
assertA2uiPushPayload("canvas.a2ui.pushJSONL", payload);
},
},
"canvas.a2ui.reset": {
buildParams: () => ({}),