diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index a74331c68e05..cc0da61335f8 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -155,7 +155,7 @@ }, { "kind": "ui-state-text", - "line": 180, + "line": 182, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Searching…", "surface": "android", @@ -163,7 +163,7 @@ }, { "kind": "ui-state-text", - "line": 283, + "line": 285, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Mic off", "surface": "android", @@ -171,7 +171,7 @@ }, { "kind": "ui-state-text", - "line": 297, + "line": 299, "path": "apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt", "source": "Off", "surface": "android", @@ -475,7 +475,7 @@ }, { "kind": "conditional-branch", - "line": 47, + "line": 45, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Home canvas", "surface": "android", @@ -483,7 +483,7 @@ }, { "kind": "conditional-branch", - "line": 47, + "line": 45, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Live page", "surface": "android", @@ -491,7 +491,7 @@ }, { "kind": "ui-named-argument", - "line": 58, + "line": 56, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Canvas", "surface": "android", @@ -499,7 +499,7 @@ }, { "kind": "ui-named-argument", - "line": 59, + "line": 57, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Current screen output and interactive app surface.", "surface": "android", @@ -507,7 +507,7 @@ }, { "kind": "conditional-branch", - "line": 66, + "line": 64, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Offline", "surface": "android", @@ -515,7 +515,7 @@ }, { "kind": "conditional-branch", - "line": 66, + "line": 64, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Online", "surface": "android", @@ -523,7 +523,7 @@ }, { "kind": "conditional-branch", - "line": 68, + "line": 66, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Ready", "surface": "android", @@ -531,7 +531,7 @@ }, { "kind": "conditional-branch", - "line": 68, + "line": 66, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Standby", "surface": "android", @@ -539,7 +539,7 @@ }, { "kind": "conditional-branch", - "line": 73, + "line": 71, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Refresh Screen", "surface": "android", @@ -547,23 +547,31 @@ }, { "kind": "conditional-branch", - "line": 73, + "line": 71, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Refreshing", "surface": "android", "id": "native.android.2a870631873082b6" }, { - "kind": "ui-named-argument", - "line": 79, + "kind": "conditional-branch", + "line": 77, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", - "source": "Reconnect", + "source": "Open Screen", "surface": "android", - "id": "native.android.ec3669686494575b" + "id": "native.android.7b8f4a7869855aea" }, { "kind": "conditional-branch", - "line": 130, + "line": 77, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", + "source": "Reconnect", + "surface": "android", + "id": "native.android.0e24c627e6384ecc" + }, + { + "kind": "conditional-branch", + "line": 122, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Connect the gateway", "surface": "android", @@ -571,7 +579,7 @@ }, { "kind": "conditional-branch", - "line": 130, + "line": 122, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Screen surface ready", "surface": "android", @@ -579,20 +587,20 @@ }, { "kind": "conditional-branch", - "line": 136, - "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", - "source": "Canvas output appears here when OpenClaw opens an app surface.", - "surface": "android", - "id": "native.android.53dcd29a9bdd99c0" - }, - { - "kind": "conditional-branch", - "line": 136, + "line": 128, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", "source": "Canvas output needs an active gateway connection.", "surface": "android", "id": "native.android.3b37959b1d369a28" }, + { + "kind": "conditional-branch", + "line": 128, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt", + "source": "Open the current Canvas surface to inspect or interact with it.", + "surface": "android", + "id": "native.android.cd92e0961b0bf500" + }, { "kind": "ui-named-argument", "line": 47, @@ -4371,7 +4379,15 @@ }, { "kind": "ui-named-argument", - "line": 438, + "line": 345, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", + "source": "Close Canvas", + "surface": "android", + "id": "native.android.30d8b822fa68d6c4" + }, + { + "kind": "ui-named-argument", + "line": 488, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Overview", "surface": "android", @@ -4379,7 +4395,7 @@ }, { "kind": "ui-named-argument", - "line": 483, + "line": 533, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "No recent sessions", "surface": "android", @@ -4387,7 +4403,7 @@ }, { "kind": "ui-named-argument", - "line": 484, + "line": 534, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Start a chat and your active OpenClaw conversations will appear here.", "surface": "android", @@ -4395,7 +4411,7 @@ }, { "kind": "ui-named-argument", - "line": 485, + "line": 535, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Start Chat", "surface": "android", @@ -4403,7 +4419,7 @@ }, { "kind": "ui-named-argument", - "line": 531, + "line": 581, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "OpenClaw", "surface": "android", @@ -4411,7 +4427,7 @@ }, { "kind": "ui-named-argument", - "line": 539, + "line": 589, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Search", "surface": "android", @@ -4419,7 +4435,7 @@ }, { "kind": "ui-named-argument", - "line": 591, + "line": 641, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "ACTIVE AGENT", "surface": "android", @@ -4427,7 +4443,7 @@ }, { "kind": "ui-named-argument", - "line": 600, + "line": 650, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "View", "surface": "android", @@ -4435,7 +4451,7 @@ }, { "kind": "conditional-branch", - "line": 603, + "line": 653, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$pendingRunCount active", "surface": "android", @@ -4443,7 +4459,7 @@ }, { "kind": "conditional-branch", - "line": 603, + "line": 653, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Idle", "surface": "android", @@ -4451,7 +4467,7 @@ }, { "kind": "ui-named-argument", - "line": 603, + "line": 653, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Runs", "surface": "android", @@ -4459,7 +4475,7 @@ }, { "kind": "conditional-branch", - "line": 604, + "line": 654, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$sessionCount recent", "surface": "android", @@ -4467,7 +4483,7 @@ }, { "kind": "ui-named-argument", - "line": 604, + "line": 654, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Sessions", "surface": "android", @@ -4475,7 +4491,7 @@ }, { "kind": "ui-named-argument", - "line": 605, + "line": 655, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Cron", "surface": "android", @@ -4483,7 +4499,7 @@ }, { "kind": "ui-named-argument", - "line": 608, + "line": 658, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Chat", "surface": "android", @@ -4491,7 +4507,7 @@ }, { "kind": "ui-named-argument", - "line": 612, + "line": 662, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Reconnect gateway", "surface": "android", @@ -4499,7 +4515,7 @@ }, { "kind": "ui-named-argument", - "line": 760, + "line": 810, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Open ${card.title}", "surface": "android", @@ -4507,7 +4523,7 @@ }, { "kind": "ui-named-argument", - "line": 828, + "line": 878, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Talk", "surface": "android", @@ -4515,7 +4531,7 @@ }, { "kind": "ui-named-argument", - "line": 829, + "line": 879, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Open Talk", "surface": "android", @@ -4523,7 +4539,7 @@ }, { "kind": "ui-named-argument", - "line": 831, + "line": 881, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Talk settings", "surface": "android", @@ -4531,7 +4547,7 @@ }, { "kind": "ui-named-argument", - "line": 839, + "line": 889, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Recent Sessions", "surface": "android", @@ -4539,7 +4555,7 @@ }, { "kind": "ui-named-argument", - "line": 849, + "line": 899, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "View all", "surface": "android", @@ -4547,7 +4563,7 @@ }, { "kind": "conditional-branch", - "line": 988, + "line": 1038, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$onlineNodes/$nodeCount", "surface": "android", @@ -4555,7 +4571,7 @@ }, { "kind": "conditional-branch", - "line": 988, + "line": 1038, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "None", "surface": "android", @@ -4563,7 +4579,7 @@ }, { "kind": "conditional-branch", - "line": 1020, + "line": 1070, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Recent conversations", "surface": "android", @@ -4571,7 +4587,7 @@ }, { "kind": "conditional-branch", - "line": 1027, + "line": 1077, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Browse", "surface": "android", @@ -4579,7 +4595,7 @@ }, { "kind": "conditional-branch", - "line": 1027, + "line": 1077, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Offline", "surface": "android", @@ -4587,7 +4603,7 @@ }, { "kind": "conditional-branch", - "line": 1079, + "line": 1129, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Working · $pendingRunCount active ${pluralize(\"run\", pendingRunCount)}", "surface": "android", @@ -4595,7 +4611,7 @@ }, { "kind": "ui-named-argument", - "line": 1200, + "line": 1250, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Needs attention", "surface": "android", @@ -4603,7 +4619,7 @@ }, { "kind": "ui-named-argument", - "line": 1265, + "line": 1315, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Open ${row.title}", "surface": "android", @@ -4611,7 +4627,7 @@ }, { "kind": "ui-named-argument", - "line": 1373, + "line": 1423, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Open session", "surface": "android", @@ -4619,7 +4635,7 @@ }, { "kind": "ui-named-argument", - "line": 1488, + "line": 1538, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Back", "surface": "android", @@ -4627,7 +4643,7 @@ }, { "kind": "ui-named-argument", - "line": 1491, + "line": 1541, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Settings", "surface": "android", @@ -4635,7 +4651,7 @@ }, { "kind": "ui-named-argument", - "line": 1494, + "line": 1544, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Search settings", "surface": "android", @@ -4643,7 +4659,7 @@ }, { "kind": "conditional-branch", - "line": 1518, + "line": 1568, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$readyProviderCount ready", "surface": "android", @@ -4651,7 +4667,7 @@ }, { "kind": "conditional-branch", - "line": 1518, + "line": 1568, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Review readiness", "surface": "android", @@ -4659,7 +4675,7 @@ }, { "kind": "conditional-branch", - "line": 1536, + "line": 1586, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Speaker muted", "surface": "android", @@ -4667,7 +4683,7 @@ }, { "kind": "conditional-branch", - "line": 1536, + "line": 1586, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Speaker on", "surface": "android", @@ -4675,7 +4691,7 @@ }, { "kind": "conditional-branch", - "line": 1538, + "line": 1588, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Off", "surface": "android", @@ -4683,7 +4699,7 @@ }, { "kind": "conditional-branch", - "line": 1538, + "line": 1588, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Smart delivery", "surface": "android", @@ -4691,7 +4707,7 @@ }, { "kind": "conditional-branch", - "line": 1539, + "line": 1589, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Camera enabled", "surface": "android", @@ -4699,7 +4715,7 @@ }, { "kind": "conditional-branch", - "line": 1539, + "line": 1589, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Locked", "surface": "android", @@ -4707,7 +4723,7 @@ }, { "kind": "ui-named-argument", - "line": 1586, + "line": 1636, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "OpenClaw ${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})", "surface": "android", @@ -4715,7 +4731,7 @@ }, { "kind": "conditional-branch", - "line": 1589, + "line": 1639, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "All systems operational", "surface": "android", @@ -4723,7 +4739,7 @@ }, { "kind": "conditional-branch", - "line": 1589, + "line": 1639, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Gateway not connected", "surface": "android", @@ -4731,7 +4747,7 @@ }, { "kind": "conditional-branch", - "line": 1621, + "line": 1671, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "No provider usage", "surface": "android", @@ -4739,7 +4755,7 @@ }, { "kind": "conditional-branch", - "line": 1622, + "line": 1672, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "1 provider", "surface": "android", @@ -4747,7 +4763,7 @@ }, { "kind": "conditional-branch", - "line": 1623, + "line": 1673, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$count providers", "surface": "android", @@ -4755,7 +4771,7 @@ }, { "kind": "conditional-branch", - "line": 1629, + "line": 1679, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$ready/${skills.size} ready", "surface": "android", @@ -4763,7 +4779,7 @@ }, { "kind": "conditional-branch", - "line": 1629, + "line": 1679, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "No skills", "surface": "android", @@ -4771,7 +4787,7 @@ }, { "kind": "conditional-branch", - "line": 1643, + "line": 1693, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$pending pending", "surface": "android", @@ -4779,7 +4795,7 @@ }, { "kind": "conditional-branch", - "line": 1643, + "line": 1693, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "1 pending", "surface": "android", @@ -4787,7 +4803,7 @@ }, { "kind": "conditional-branch", - "line": 1648, + "line": 1698, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$held held", "surface": "android", @@ -4795,7 +4811,7 @@ }, { "kind": "conditional-branch", - "line": 1648, + "line": 1698, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "1 held", "surface": "android", @@ -4803,7 +4819,7 @@ }, { "kind": "conditional-branch", - "line": 1649, + "line": 1699, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "$applied applied", "surface": "android", @@ -4811,7 +4827,7 @@ }, { "kind": "conditional-branch", - "line": 1649, + "line": 1699, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "1 applied", "surface": "android", @@ -4819,7 +4835,7 @@ }, { "kind": "ui-named-argument", - "line": 1830, + "line": 1880, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "OpenClaw mobile", "surface": "android", @@ -4827,7 +4843,7 @@ }, { "kind": "ui-named-argument", - "line": 1834, + "line": 1884, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Open profile", "surface": "android", @@ -4835,7 +4851,7 @@ }, { "kind": "conditional-branch", - "line": 1919, + "line": 1969, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt", "source": "Main session", "surface": "android", diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts index 73be1f4ff133..72708e516057 100644 --- a/apps/android/app/build.gradle.kts +++ b/apps/android/app/build.gradle.kts @@ -88,6 +88,7 @@ android { applicationId = "ai.openclaw.app" minSdk = 31 targetSdk = 36 + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" versionCode = openClawAndroidVersionCode versionName = openClawAndroidVersionName ndk { @@ -289,6 +290,10 @@ dependencies { testImplementation(libs.mockwebserver) testImplementation(libs.robolectric) testRuntimeOnly(libs.junit.vintage.engine) + + androidTestImplementation(libs.androidx.test.ext.junit) + androidTestImplementation(libs.androidx.test.runner) + androidTestImplementation(libs.androidx.uiautomator) } tasks.withType().configureEach { diff --git a/apps/android/app/src/androidTest/java/ai/openclaw/app/ui/CanvasHostLifecycleTest.kt b/apps/android/app/src/androidTest/java/ai/openclaw/app/ui/CanvasHostLifecycleTest.kt new file mode 100644 index 000000000000..f27c8a824641 --- /dev/null +++ b/apps/android/app/src/androidTest/java/ai/openclaw/app/ui/CanvasHostLifecycleTest.kt @@ -0,0 +1,203 @@ +package ai.openclaw.app.ui + +import ai.openclaw.app.node.CanvasController +import android.content.pm.ActivityInfo +import android.os.SystemClock +import android.view.View +import androidx.test.core.app.ActivityScenario +import androidx.test.ext.junit.rules.ActivityScenarioRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.UiDevice +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class CanvasHostLifecycleTest { + @get:Rule + val activityRule = ActivityScenarioRule(CanvasLifecycleTestActivity::class.java) + + @Before + fun resetMetrics() { + CanvasLifecycleTestMetrics.reset() + } + + @Test + fun hiddenHostRetainsOneWebViewWithoutBlockingShellInput() { + activityRule.scenario.onActivity { activity -> + assertEquals(CanvasController.PresentationState.Unmounted, activity.controller.presentationState.value) + assertNull(activity.host) + } + + val presentElapsedMs = activityRule.scenario.readActivity { activity -> activity.presentSlowPage() } + + assertTrue( + "present waited for the remote page: ${presentElapsedMs}ms", + presentElapsedMs < canvasLifecycleSlowPageDelayMs / 2, + ) + assertTrue("slow page never finished", activityRule.scenario.waitForPageFinished()) + + val firstWebView = + activityRule.scenario.readActivity { activity -> + val host = checkNotNull(activity.host) + assertEquals(1, host.childCount) + assertEquals(CanvasController.PresentationState.Visible, activity.controller.presentationState.value) + val webView = checkNotNull(activity.currentWebView()) + activity.hideCanvas() + assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value) + webView + } + assertTrue( + "hidden host remained visible", + activityRule.scenario.waitUntilActivity { activity -> activity.host?.visibility == View.INVISIBLE }, + ) + + val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) + assertTrue(device.click(device.displayWidth / 2, device.displayHeight / 2)) + device.waitForIdle() + + activityRule.scenario.onActivity { activity -> + assertEquals(1, activity.underlayClickCount) + repeat(3) { + activity.presentFastPage() + activity.hideCanvas() + } + assertEquals(1, activity.host?.childCount) + assertTrue(firstWebView === activity.currentWebView()) + } + } + + @Test + fun stalePageCompletionCannotReshowCanvasAfterHide() { + activityRule.scenario.onActivity { activity -> activity.presentSlowPage() } + assertTrue( + "Canvas host was not attached", + activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() != null }, + ) + activityRule.scenario.onActivity { activity -> activity.hideCanvas() } + assertTrue( + "hidden host remained visible", + activityRule.scenario.waitUntilActivity { activity -> activity.host?.visibility == View.INVISIBLE }, + ) + + assertTrue("slow page never finished", activityRule.scenario.waitForPageFinished()) + activityRule.scenario.onActivity { activity -> + assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value) + assertEquals(View.INVISIBLE, checkNotNull(activity.host).visibility) + assertNotNull(activity.currentWebView()) + } + } + + @Test + fun rendererTerminationForgetsFailedPageAndNextShowRecreatesIt() { + activityRule.scenario.onActivity { activity -> activity.presentFastPage() } + assertTrue("initial page never finished", activityRule.scenario.waitForPageFinished()) + + val firstWebView = + activityRule.scenario.readActivity { activity -> + assertNotNull(activity.controller.currentUrl()) + checkNotNull(activity.currentWebView()) + } + val terminated = + activityRule.scenario.readActivity { activity -> + activity.currentWebView()?.webViewRenderProcess?.terminate() == true + } + assertTrue("WebView renderer did not terminate", terminated) + assertTrue( + "renderer loss did not clear the invalid WebView", + activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() == null }, + ) + + activityRule.scenario.onActivity { activity -> + assertEquals(CanvasController.PresentationState.Hidden, activity.controller.presentationState.value) + assertNull(activity.controller.currentUrl()) + assertEquals(0, activity.host?.childCount) + activity.showCanvas() + } + assertTrue( + "next show did not create a replacement WebView", + activityRule.scenario.waitUntilActivity { activity -> activity.currentWebView() != null }, + ) + assertTrue("replacement scaffold never finished", activityRule.scenario.waitForPageFinished()) + activityRule.scenario.onActivity { activity -> + assertEquals(CanvasController.PresentationState.Visible, activity.controller.presentationState.value) + assertEquals(1, activity.host?.childCount) + assertNotEquals(firstWebView, activity.currentWebView()) + } + } + + @Test + fun configurationChangesKeepTheSameHostAndWebView() { + activityRule.scenario.onActivity { activity -> activity.presentFastPage() } + assertTrue("initial page never finished", activityRule.scenario.waitForPageFinished()) + val firstHost = activityRule.scenario.readActivity { activity -> checkNotNull(activity.host) } + val firstWebView = activityRule.scenario.readActivity { activity -> checkNotNull(activity.currentWebView()) } + + activityRule.scenario.onActivity { activity -> + activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE + } + UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()).waitForIdle() + + activityRule.scenario.onActivity { activity -> + assertTrue(firstHost === activity.host) + assertTrue(firstWebView === activity.currentWebView()) + activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT + } + } +} + +@RunWith(AndroidJUnit4::class) +class CanvasHostReleaseTest { + @Before + fun resetMetrics() { + CanvasLifecycleTestMetrics.reset() + } + + @Test + fun activityTeardownReleasesTheHostAndWebView() { + ActivityScenario.launch(CanvasLifecycleTestActivity::class.java).use { scenario -> + scenario.onActivity { activity -> activity.presentFastPage() } + } + + assertTrue( + "AndroidView onRelease was not called", + waitUntil { CanvasLifecycleTestMetrics.hostReleaseCount.get() == 1 }, + ) + assertEquals(1, CanvasLifecycleTestMetrics.webViewDestroyCount.get()) + } +} + +private inline fun ActivityScenario.readActivity(crossinline block: (CanvasLifecycleTestActivity) -> T): T { + var result: Result? = null + onActivity { activity -> result = runCatching { block(activity) } } + return checkNotNull(result).getOrThrow() +} + +private fun ActivityScenario.waitForPageFinished(): Boolean = waitUntilActivity { activity -> activity.currentWebView()?.progress == 100 } + +private inline fun ActivityScenario.waitUntilActivity( + timeoutMs: Long = 5_000L, + crossinline predicate: (CanvasLifecycleTestActivity) -> Boolean, +): Boolean = + waitUntil(timeoutMs) { + readActivity(predicate) + } + +private fun waitUntil( + timeoutMs: Long = 5_000L, + predicate: () -> Boolean, +): Boolean { + val deadline = SystemClock.elapsedRealtime() + timeoutMs + while (SystemClock.elapsedRealtime() < deadline) { + if (predicate()) return true + SystemClock.sleep(20) + } + return predicate() +} diff --git a/apps/android/app/src/debug/AndroidManifest.xml b/apps/android/app/src/debug/AndroidManifest.xml index 2eccec1b6307..8398d3086ca7 100644 --- a/apps/android/app/src/debug/AndroidManifest.xml +++ b/apps/android/app/src/debug/AndroidManifest.xml @@ -7,6 +7,11 @@ + ready") + + fun hideCanvas() { + controller.hide() + } + + fun showCanvas() { + controller.show() + } + + fun currentWebView(): WebView? = host?.currentWebView + + private fun presentHtml(html: String): Long { + val startedAt = SystemClock.elapsedRealtime() + val encoded = Base64.encodeToString(html.toByteArray(Charsets.UTF_8), Base64.NO_WRAP) + controller.navigate("data:text/html;base64,$encoded") + controller.show() + return SystemClock.elapsedRealtime() - startedAt + } +} + +@Composable +private fun CanvasLifecycleTestContent( + activity: CanvasLifecycleTestActivity, + onUnderlayClick: () -> Unit, +) { + val state by activity.controller.presentationState.collectAsState() + Box(modifier = Modifier.fillMaxSize()) { + AndroidView( + factory = { context -> + Button(context).apply { + setOnClickListener { onUnderlayClick() } + } + }, + modifier = Modifier.fillMaxSize(), + ) + if (state != CanvasController.PresentationState.Unmounted) { + AndroidView( + factory = { context -> + CanvasHostView( + context = context, + controller = activity.controller, + isTrustedPage = { false }, + onA2uiMessage = {}, + ).also { host -> + activity.host = host + host.updateVisible(state == CanvasController.PresentationState.Visible) + } + }, + update = { host -> + host.updateVisible(state == CanvasController.PresentationState.Visible) + }, + modifier = Modifier.fillMaxSize(), + onRelease = { host -> + val hadWebView = host.currentWebView != null + host.release() + if (activity.host === host) activity.host = null + CanvasLifecycleTestMetrics.hostReleaseCount.incrementAndGet() + if (hadWebView) CanvasLifecycleTestMetrics.webViewDestroyCount.incrementAndGet() + }, + ) + } + } +} + +object CanvasLifecycleTestMetrics { + val hostReleaseCount = AtomicInteger() + val webViewDestroyCount = AtomicInteger() + + fun reset() { + hostReleaseCount.set(0) + webViewDestroyCount.set(0) + } +} + +private val slowPageHtml = + """ + + + + ready + + + """.trimIndent() diff --git a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt index 6eeac1e336b5..b41a7ee0ee6e 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt @@ -172,6 +172,8 @@ class MainViewModel( .stateIn(viewModelScope, SharingStarted.Eagerly, false) val canvasCurrentUrl: StateFlow = runtimeState(initial = null) { it.canvas.currentUrl } + val canvasPresentationState: StateFlow = + runtimeState(initial = CanvasController.PresentationState.Unmounted) { it.canvas.presentationState } val canvasA2uiHydrated: StateFlow = runtimeState(initial = false) { it.canvasA2uiHydrated } val canvasRehydratePending: StateFlow = runtimeState(initial = false) { it.canvasRehydratePending } val canvasRehydrateErrorText: StateFlow = runtimeState(initial = null) { it.canvasRehydrateErrorText } @@ -740,6 +742,14 @@ class MainViewModel( ensureRuntime().requestCanvasRehydrate(source = source, force = true) } + fun showCanvas() { + ensureRuntime().canvas.show() + } + + fun hideCanvas() { + runtimeRef.value?.canvas?.hide() + } + fun refreshHomeCanvasOverviewIfConnected() { ensureRuntime().refreshHomeCanvasOverviewIfConnected() } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/node/A2UIHandler.kt b/apps/android/app/src/main/java/ai/openclaw/app/node/A2UIHandler.kt index 25b73a6dd903..a818e2dab1f5 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/node/A2UIHandler.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/node/A2UIHandler.kt @@ -16,11 +16,15 @@ class A2UIHandler( fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean = CanvasActionTrust.isTrustedCanvasActionUrl(rawUrl) suspend fun ensureA2uiReady(): Boolean { - if (canvas.currentUrl()?.trim() == CanvasActionTrust.localA2uiAssetUrl && isA2uiReady()) { + val alreadyOnA2uiHost = canvas.currentUrl()?.trim() == CanvasActionTrust.localA2uiAssetUrl + if (!canvas.showAndAwaitHost()) return false + if (!alreadyOnA2uiHost) { + canvas.showLocalA2ui() + } + if (alreadyOnA2uiHost && isA2uiReady()) { return true } - canvas.showLocalA2ui() // The bundled A2UI host bootstraps asynchronously after navigation; poll briefly before failing the command. repeat(50) { if (isA2uiReady()) return true diff --git a/apps/android/app/src/main/java/ai/openclaw/app/node/CanvasController.kt b/apps/android/app/src/main/java/ai/openclaw/app/node/CanvasController.kt index 58def93ea727..d450153c2025 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/node/CanvasController.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/node/CanvasController.kt @@ -13,8 +13,10 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject @@ -27,6 +29,12 @@ import kotlin.coroutines.resume * Owns the Android WebView canvas surface used by canvas and A2UI commands. */ class CanvasController { + enum class PresentationState { + Unmounted, + Hidden, + Visible, + } + enum class SnapshotFormat( val rawValue: String, ) { @@ -47,6 +55,9 @@ class CanvasController { @Volatile private var homeCanvasStateJson: String? = null private val _currentUrl = MutableStateFlow(null) val currentUrl: StateFlow = _currentUrl.asStateFlow() + private val _presentationState = MutableStateFlow(PresentationState.Unmounted) + val presentationState: StateFlow = _presentationState.asStateFlow() + private val hostAttachedState = MutableStateFlow(false) private val scaffoldAssetUrl = CanvasActionTrust.scaffoldAssetUrl private val localA2uiAssetUrl = CanvasActionTrust.localA2uiAssetUrl @@ -67,19 +78,61 @@ class CanvasController { /** Attaches the active WebView and replays state that may have arrived before the view existed. */ fun attach(webView: WebView) { this.webView = webView + hostAttachedState.value = true // Replay persisted state because WebView attachment can happen after gateway events arrive. reload() applyDebugStatus() applyHomeCanvasState() } - /** Detaches only the currently attached WebView instance. */ - fun detach(webView: WebView) { - if (this.webView === webView) { - this.webView = null + /** Releases the shell-owned host when its UI owner permanently leaves composition. */ + fun releaseHost() { + webView = null + hostAttachedState.value = false + _presentationState.value = PresentationState.Unmounted + } + + /** Invalid renderer processes cannot be reused; retain the host but require a new child. */ + fun onRenderProcessGone(webView: WebView) { + if (this.webView !== webView) return + this.webView = null + // Do not replay the page that terminated its renderer into the replacement WebView. + url = null + _currentUrl.value = null + hostAttachedState.value = false + _presentationState.value = PresentationState.Hidden + } + + fun show() { + _presentationState.value = PresentationState.Visible + } + + fun hide() { + if (_presentationState.value != PresentationState.Unmounted) { + _presentationState.value = PresentationState.Hidden } } + /** + * Requests presentation and waits only for the shell host to accept it. + * Remote page loading remains asynchronous and must not delay invoke completion. + */ + suspend fun showAndAwaitHost(): Boolean { + val previousState = _presentationState.value + show() + if (hostAttachedState.value) return true + val attached = + withTimeoutOrNull(hostAttachTimeoutMs) { + hostAttachedState.first { it } + true + } ?: hostAttachedState.value + if (!attached && _presentationState.value == PresentationState.Visible) { + // A failed foreground handoff must not leave a pending overlay for the next Activity. + _presentationState.value = previousState + } + return attached + } + /** Navigates the canvas to a remote URL or back to the bundled scaffold for blank/root input. */ fun navigate(url: String) { this.url = CanvasNavigationPolicy.normalize(url).ifBlank { null } @@ -243,6 +296,8 @@ class CanvasController { } companion object { + private const val hostAttachTimeoutMs = 5_000L + /** * Parsed canvas.snapshot options used by invoke dispatch. */ diff --git a/apps/android/app/src/main/java/ai/openclaw/app/node/InvokeDispatcher.kt b/apps/android/app/src/main/java/ai/openclaw/app/node/InvokeDispatcher.kt index f3fae39b162a..48a22eb60103 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/node/InvokeDispatcher.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/node/InvokeDispatcher.kt @@ -14,6 +14,8 @@ import ai.openclaw.app.protocol.OpenClawNotificationsCommand import ai.openclaw.app.protocol.OpenClawSmsCommand import ai.openclaw.app.protocol.OpenClawSystemCommand import ai.openclaw.app.protocol.OpenClawTalkCommand +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock /** Runtime state for SMS search, split so permission prompts are not reported as hard unavailability. */ internal enum class SmsSearchAvailabilityReason { @@ -92,6 +94,8 @@ class InvokeDispatcher( private val motionActivityAvailable: () -> Boolean, private val motionPedometerAvailable: () -> Boolean, ) { + private val canvasCommandMutex = Mutex() + /** Dispatches one gateway node.invoke command after foreground and availability gates pass. */ suspend fun handleInvoke( command: String, @@ -112,19 +116,40 @@ class InvokeDispatcher( } availabilityError(spec.availability)?.let { return it } + if (command.startsWith(OpenClawCanvasCommand.NamespacePrefix)) { + // GatewaySession may deliver invokes concurrently. Canvas presentation, navigation, and + // A2UI evaluation share one WebView and must observe command arrival order. + return canvasCommandMutex.withLock { dispatchInvoke(command, paramsJson) } + } + return dispatchInvoke(command, paramsJson) + } + + private suspend fun dispatchInvoke( + command: String, + paramsJson: String?, + ): GatewaySession.InvokeResult { // Command strings come from OpenClawProtocolConstants; the registry above owns advertised availability. return when (command) { // Canvas commands OpenClawCanvasCommand.Present.rawValue -> { val url = CanvasController.parseNavigateUrl(paramsJson) - canvas.navigate(url) + withCanvasAvailable { + check(canvas.showAndAwaitHost()) { "canvas host unavailable" } + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } + } + OpenClawCanvasCommand.Hide.rawValue -> { + canvas.hide() GatewaySession.InvokeResult.ok(null) } - OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null) OpenClawCanvasCommand.Navigate.rawValue -> { val url = CanvasController.parseNavigateUrl(paramsJson) - canvas.navigate(url) - GatewaySession.InvokeResult.ok(null) + withCanvasAvailable { + check(canvas.showAndAwaitHost()) { "canvas host unavailable" } + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } } OpenClawCanvasCommand.Eval.rawValue -> { val js = diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasScreen.kt index 0d9cbdaf9482..23084bd7c10a 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasScreen.kt @@ -1,12 +1,16 @@ package ai.openclaw.app.ui import ai.openclaw.app.MainViewModel +import ai.openclaw.app.node.CanvasController import ai.openclaw.app.node.CanvasNavigationPolicy import android.annotation.SuppressLint +import android.content.Context import android.net.Uri import android.util.Log import android.view.View +import android.view.ViewGroup import android.webkit.ConsoleMessage +import android.webkit.RenderProcessGoneDetail import android.webkit.WebChromeClient import android.webkit.WebResourceError import android.webkit.WebResourceRequest @@ -14,11 +18,9 @@ import android.webkit.WebResourceResponse import android.webkit.WebSettings import android.webkit.WebView import android.webkit.WebViewClient +import android.widget.FrameLayout import androidx.compose.runtime.Composable -import androidx.compose.runtime.DisposableEffect -import androidx.compose.runtime.remember import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.viewinterop.AndroidView import androidx.webkit.JavaScriptReplyProxy import androidx.webkit.WebMessageCompat @@ -37,28 +39,69 @@ fun CanvasScreen( visible: Boolean, modifier: Modifier = Modifier, ) { - val context = LocalContext.current - val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0 - val webViewRef = remember { arrayOfNulls(1) } - val currentPageUrlRef = remember { AtomicReference(null) } - - DisposableEffect(viewModel) { - onDispose { - val webView = webViewRef[0] ?: return@onDispose - viewModel.canvas.detach(webView) - if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) { - WebViewCompat.removeWebMessageListener(webView, CanvasA2UIActionBridge.interfaceName) + AndroidView( + modifier = modifier, + factory = { context -> + CanvasHostView( + context = context, + controller = viewModel.canvas, + isTrustedPage = viewModel::isTrustedCanvasActionUrl, + onA2uiMessage = viewModel::handleCanvasA2UIActionFromWebView, + ).apply { + updateVisible(visible) } - webView.stopLoading() - webView.destroy() - webViewRef[0] = null + }, + update = { host -> host.updateVisible(visible) }, + onRelease = CanvasHostView::release, + ) +} + +/** + * Retained shell host whose WebView child can be replaced after renderer death. + * + * Compose creates this host directly; XML inflation cannot supply its controller and callbacks. + */ +@SuppressLint("SetJavaScriptEnabled", "ViewConstructor") +@Suppress("DEPRECATION") +internal class CanvasHostView( + context: Context, + private val controller: CanvasController, + private val isTrustedPage: (String?) -> Boolean, + private val onA2uiMessage: (String) -> Unit, +) : FrameLayout(context) { + internal var currentWebView: WebView? = null + private set + + private val isDebuggable = + (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0 + private val currentPageUrlRef = AtomicReference(null) + + init { + visibility = View.INVISIBLE + } + + fun updateVisible(visible: Boolean) { + if (visible) { + val webView = currentWebView ?: createWebView() + visibility = View.VISIBLE + webView.visibility = View.VISIBLE + webView.onResume() + return + } + visibility = View.INVISIBLE + currentWebView?.let { webView -> + webView.visibility = View.INVISIBLE + webView.onPause() } } - AndroidView( - modifier = modifier, - factory = { - val webView = WebView(context) + fun release() { + controller.releaseHost() + currentWebView?.let(::destroyWebView) + } + + private fun createWebView(): WebView = + WebView(context).also { webView -> val webSettings = webView.settings webSettings.setAllowContentAccess(false) webSettings.setAllowFileAccess(false) @@ -73,7 +116,7 @@ fun CanvasScreen( webSettings.builtInZoomControls = false webSettings.displayZoomControls = false webSettings.setSupportZoom(false) - webView.visibility = if (visible) View.VISIBLE else View.INVISIBLE + webView.visibility = View.INVISIBLE // targetSdk 33+ ignores Force Dark APIs, so only opt out through the supported // algorithmic darkening flag when this WebView implementation exposes it. if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) { @@ -93,7 +136,7 @@ fun CanvasScreen( request: WebResourceRequest, ): Boolean { if (!request.isForMainFrame) return false - return blockUnsafeCanvasNavigation(viewModel, currentPageUrlRef, request.url.toString()) + return blockUnsafeCanvasNavigation(controller, currentPageUrlRef, request.url.toString()) } override fun shouldInterceptRequest( @@ -109,7 +152,7 @@ fun CanvasScreen( // shouldOverrideUrlLoading excludes POST navigations and their redirects. WebView does // not expose those redirect targets, so non-GET main-frame loads fail closed here. currentPageUrlRef.set(null) - view.post { viewModel.canvas.navigate("") } + view.post { controller.navigate("") } return blockedCanvasResponse() } @@ -150,12 +193,12 @@ fun CanvasScreen( if (isDebuggable) { Log.d("OpenClawWebView", "onPageFinished: $url") } - viewModel.canvas.onPageFinished() + controller.onPageFinished() } override fun onRenderProcessGone( view: WebView, - detail: android.webkit.RenderProcessGoneDetail, + detail: RenderProcessGoneDetail, ): Boolean { if (isDebuggable) { Log.e( @@ -163,6 +206,11 @@ fun CanvasScreen( "onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}", ) } + if (view === currentWebView) { + controller.onRenderProcessGone(view) + destroyWebView(view) + visibility = View.INVISIBLE + } return true } } @@ -183,10 +231,9 @@ fun CanvasScreen( // dispatch still requires the live URL to be an app-owned bundled page. val bridge = CanvasA2UIActionBridge( - isTrustedPage = { viewModel.isTrustedCanvasActionUrl(currentPageUrlRef.get()) }, - ) { payload -> - viewModel.handleCanvasA2UIActionFromWebView(payload) - } + isTrustedPage = { isTrustedPage(currentPageUrlRef.get()) }, + onMessage = onA2uiMessage, + ) if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) { WebViewCompat.addWebMessageListener( webView, @@ -197,32 +244,38 @@ fun CanvasScreen( } else if (isDebuggable) { Log.w("OpenClawWebView", "WebMessageListener unsupported; canvas actions disabled") } - viewModel.canvas.attach(webView) - webViewRef[0] = webView - webView - }, - update = { webView -> - webView.visibility = if (visible) View.VISIBLE else View.INVISIBLE - if (visible) { - webView.resumeTimers() - webView.onResume() - } else { - webView.onPause() - webView.pauseTimers() - } - }, - ) + addView( + webView, + ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + ), + ) + currentWebView = webView + controller.attach(webView) + } + + private fun destroyWebView(webView: WebView) { + if (currentWebView !== webView) return + if (WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) { + WebViewCompat.removeWebMessageListener(webView, CanvasA2UIActionBridge.interfaceName) + } + removeView(webView) + webView.stopLoading() + webView.destroy() + currentWebView = null + } } private fun blockUnsafeCanvasNavigation( - viewModel: MainViewModel, + controller: CanvasController, currentPageUrlRef: AtomicReference, rawUrl: String, ): Boolean { val url = rawUrl.trim() if (!CanvasNavigationPolicy.shouldBlock(url)) return false currentPageUrlRef.set(null) - viewModel.canvas.navigate("") + controller.navigate("") return true } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt index 35b713e313dc..2ecfb96d7701 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/CanvasSettingsScreen.kt @@ -12,7 +12,6 @@ import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.shape.RoundedCornerShape @@ -31,7 +30,7 @@ import androidx.compose.ui.draw.clip import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp -/** Settings detail surface for live canvas status, refresh, and embedded preview. */ +/** Settings detail surface for live canvas status, refresh, and presentation. */ @Composable internal fun CanvasSettingsScreen( viewModel: MainViewModel, @@ -43,12 +42,11 @@ internal fun CanvasSettingsScreen( val rehydratePending by viewModel.canvasRehydratePending.collectAsState() val rehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState() val hasLivePage = currentUrl?.isNotBlank() == true - val showCanvasSurface = isConnected val canvasLabel = if (hasLivePage) "Live page" else "Home canvas" LaunchedEffect(isConnected) { if (isConnected) { - // Refresh once when the gateway comes online so the settings preview is + // Refresh once when the gateway comes online so current Canvas status is // populated before the user manually asks for a rehydrate. viewModel.refreshHomeCanvasOverviewIfConnected() } @@ -76,8 +74,8 @@ internal fun CanvasSettingsScreen( modifier = Modifier.weight(1f), ) ClawSecondaryButton( - text = "Reconnect", - onClick = viewModel::refreshGatewayConnection, + text = if (isConnected) "Open Screen" else "Reconnect", + onClick = if (isConnected) viewModel::showCanvas else viewModel::refreshGatewayConnection, modifier = Modifier.weight(1f), ) } @@ -90,18 +88,12 @@ internal fun CanvasSettingsScreen( Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { Text(text = canvasLabel, style = ClawTheme.type.section, color = ClawTheme.colors.text, maxLines = 1, overflow = TextOverflow.Ellipsis) Surface( - modifier = Modifier.fillMaxWidth().height(520.dp).clip(RoundedCornerShape(ClawTheme.radii.panel)), + modifier = Modifier.fillMaxWidth().clip(RoundedCornerShape(ClawTheme.radii.panel)), shape = RoundedCornerShape(ClawTheme.radii.panel), color = ClawTheme.colors.canvas, border = BorderStroke(1.dp, ClawTheme.colors.border), ) { - Box { - if (showCanvasSurface) { - CanvasScreen(viewModel = viewModel, visible = true, modifier = Modifier.fillMaxWidth().height(520.dp)) - } else { - CanvasStandbyPanel(isConnected = isConnected) - } - } + CanvasStandbyPanel(isConnected = isConnected) } } } @@ -111,7 +103,7 @@ internal fun CanvasSettingsScreen( @Composable private fun CanvasStandbyPanel(isConnected: Boolean) { Column( - modifier = Modifier.fillMaxWidth().height(520.dp).padding(horizontal = 24.dp), + modifier = Modifier.fillMaxWidth().padding(horizontal = 24.dp, vertical = 48.dp), horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.Center, ) { @@ -133,7 +125,7 @@ private fun CanvasStandbyPanel(isConnected: Boolean) { modifier = Modifier.padding(top = 18.dp), ) Text( - text = if (isConnected) "Canvas output appears here when OpenClaw opens an app surface." else "Canvas output needs an active gateway connection.", + text = if (isConnected) "Open the current Canvas surface to inspect or interact with it." else "Canvas output needs an active gateway connection.", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted, modifier = Modifier.padding(top = 6.dp), diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt index 888756241443..080911703278 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/ShellScreen.kt @@ -16,10 +16,12 @@ import ai.openclaw.app.NodeRuntime import ai.openclaw.app.R import ai.openclaw.app.chat.ChatSessionEntry import ai.openclaw.app.currentAppLanguage +import ai.openclaw.app.node.CanvasController import ai.openclaw.app.ui.chat.ChatScreen import ai.openclaw.app.ui.design.ClawBottomNav import ai.openclaw.app.ui.design.ClawDesignTheme import ai.openclaw.app.ui.design.ClawEmptyState +import ai.openclaw.app.ui.design.ClawIconButton import ai.openclaw.app.ui.design.ClawNavItem import ai.openclaw.app.ui.design.ClawPanel import ai.openclaw.app.ui.design.ClawPlainIconButton @@ -52,6 +54,7 @@ import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.safeDrawing import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.windowInsetsPadding import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape @@ -61,6 +64,7 @@ import androidx.compose.material.icons.automirrored.filled.ExitToApp import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight import androidx.compose.material.icons.automirrored.filled.ScreenShare import androidx.compose.material.icons.filled.ArrowDropDown +import androidx.compose.material.icons.filled.Close import androidx.compose.material.icons.filled.Cloud import androidx.compose.material.icons.filled.Favorite import androidx.compose.material.icons.filled.GraphicEq @@ -156,9 +160,16 @@ fun ShellScreen( val requestedHomeDestination by viewModel.requestedHomeDestination.collectAsState() val pendingTrust by viewModel.pendingGatewayTrust.collectAsState() val runtimeInitialized by viewModel.runtimeInitialized.collectAsState() + val canvasPresentationState by viewModel.canvasPresentationState.collectAsState() + val canvasVisible = canvasPresentationState == CanvasController.PresentationState.Visible LaunchedEffect(requestedHomeDestination) { val destination = requestedHomeDestination ?: return@LaunchedEffect + if (destination == HomeDestination.Screen) { + viewModel.showCanvas() + viewModel.clearRequestedHomeDestination() + return@LaunchedEffect + } // HomeDestination is a one-shot command from launch intents and settings // actions; consume it after translating to local shell state. nav.selectTab( @@ -166,7 +177,7 @@ fun ShellScreen( HomeDestination.Connect -> Tab.Overview HomeDestination.Chat -> Tab.Chat HomeDestination.Voice -> Tab.Voice - HomeDestination.Screen -> Tab.Chat + HomeDestination.Screen -> Tab.Overview HomeDestination.Settings -> Tab.Settings }, ) @@ -191,7 +202,8 @@ fun ShellScreen( val density = LocalDensity.current val keyboardVisible = WindowInsets.ime.getBottom(density) > 0 - val showBottomNav = shellBottomNavVisible(keyboardVisible = keyboardVisible, commandOpen = commandOpen) + val showBottomNav = + shellBottomNavVisible(keyboardVisible = keyboardVisible, commandOpen = commandOpen) && !canvasVisible Scaffold( modifier = modifier.fillMaxSize(), @@ -290,6 +302,14 @@ fun ShellScreen( ) } + if (canvasPresentationState != CanvasController.PresentationState.Unmounted) { + CanvasOverlay( + viewModel = viewModel, + visible = canvasVisible, + onClose = viewModel::hideCanvas, + ) + } + pendingTrust?.let { prompt -> // Gateway certificate trust is modal across the shell so navigation // cannot hide a changed TLS identity prompt. @@ -304,6 +324,36 @@ fun ShellScreen( } } +@Composable +private fun CanvasOverlay( + viewModel: MainViewModel, + visible: Boolean, + onClose: () -> Unit, +) { + BackHandler(enabled = visible, onBack = onClose) + val overlayColor = if (visible) ClawTheme.colors.canvas else Color.Transparent + Box(modifier = Modifier.fillMaxSize().background(overlayColor)) { + // The shell owns system-bar avoidance; arbitrary Canvas pages cannot know Android insets. + CanvasScreen( + viewModel = viewModel, + visible = visible, + modifier = Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.safeDrawing), + ) + if (visible) { + ClawIconButton( + icon = Icons.Default.Close, + contentDescription = "Close Canvas", + onClick = onClose, + modifier = + Modifier + .align(Alignment.TopEnd) + .windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)) + .padding(top = 12.dp, end = 12.dp), + ) + } + } +} + /** Modal trust decision for first-seen or changed gateway TLS fingerprints. */ @Composable private fun GatewayTrustDialog( diff --git a/apps/android/app/src/test/java/ai/openclaw/app/node/CanvasControllerPresentationTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/node/CanvasControllerPresentationTest.kt new file mode 100644 index 000000000000..2c098659243f --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/node/CanvasControllerPresentationTest.kt @@ -0,0 +1,38 @@ +package ai.openclaw.app.node + +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Test + +class CanvasControllerPresentationTest { + @Test + fun presentationStateKeepsTheHostUnmountedUntilFirstShow() { + val controller = CanvasController() + + controller.hide() + assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value) + + controller.show() + assertEquals(CanvasController.PresentationState.Visible, controller.presentationState.value) + + controller.hide() + assertEquals(CanvasController.PresentationState.Hidden, controller.presentationState.value) + + controller.show() + assertEquals(CanvasController.PresentationState.Visible, controller.presentationState.value) + + controller.releaseHost() + assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value) + } + + @Test + fun failedHostHandoffRestoresThePreviousPresentationState() = + runTest { + val controller = CanvasController() + + assertFalse(controller.showAndAwaitHost()) + + assertEquals(CanvasController.PresentationState.Unmounted, controller.presentationState.value) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/node/InvokeDispatcherTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/node/InvokeDispatcherTest.kt index d391b8600801..34432fbac236 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/node/InvokeDispatcherTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/node/InvokeDispatcherTest.kt @@ -4,6 +4,7 @@ import ai.openclaw.app.gateway.DeviceIdentityStore import ai.openclaw.app.gateway.GatewaySession import ai.openclaw.app.protocol.OpenClawCallLogCommand import ai.openclaw.app.protocol.OpenClawCameraCommand +import ai.openclaw.app.protocol.OpenClawCanvasCommand import ai.openclaw.app.protocol.OpenClawDeviceCommand import ai.openclaw.app.protocol.OpenClawLocationCommand import ai.openclaw.app.protocol.OpenClawMotionCommand @@ -12,6 +13,7 @@ import ai.openclaw.app.protocol.OpenClawSmsCommand import ai.openclaw.app.protocol.OpenClawTalkCommand import android.content.Context import android.content.pm.PackageManager +import android.webkit.WebView import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.test.runTest import kotlinx.serialization.json.Json @@ -275,6 +277,58 @@ class InvokeDispatcherTest { assertEquals(listOf("start", "stop", "cancel"), talk.calls) } + @Test + fun handleInvoke_presentAndHideDriveTheShellOwnedCanvasState() = + runTest { + val appContext = RuntimeEnvironment.getApplication() + val canvas = CanvasController() + val webView = WebView(appContext) + canvas.attach(webView) + val dispatcher = newDispatcher(canvas = canvas) + + val present = + dispatcher.handleInvoke( + OpenClawCanvasCommand.Present.rawValue, + """{"url":"https://example.com/canvas"}""", + ) + + assertNull(present.error) + assertEquals("https://example.com/canvas", canvas.currentUrl()) + assertEquals(CanvasController.PresentationState.Visible, canvas.presentationState.value) + + val hide = dispatcher.handleInvoke(OpenClawCanvasCommand.Hide.rawValue, null) + + assertNull(hide.error) + assertEquals(CanvasController.PresentationState.Hidden, canvas.presentationState.value) + canvas.releaseHost() + webView.destroy() + } + + @Test + fun handleInvoke_rejectsBackgroundCanvasPresentationBeforeMountingAHost() = + runTest { + val canvas = CanvasController() + val result = + newDispatcher(isForeground = false, canvas = canvas) + .handleInvoke(OpenClawCanvasCommand.Present.rawValue, """{"url":"https://example.com"}""") + + assertEquals("NODE_BACKGROUND_UNAVAILABLE", result.error?.code) + assertEquals(CanvasController.PresentationState.Unmounted, canvas.presentationState.value) + } + + @Test + fun handleInvoke_doesNotCommitNavigationWhenTheShellHostCannotAttach() = + runTest { + val canvas = CanvasController() + val result = + newDispatcher(canvas = canvas) + .handleInvoke(OpenClawCanvasCommand.Present.rawValue, """{"url":"https://example.com"}""") + + assertEquals("NODE_BACKGROUND_UNAVAILABLE", result.error?.code) + assertNull(canvas.currentUrl()) + assertEquals(CanvasController.PresentationState.Unmounted, canvas.presentationState.value) + } + private fun newDispatcher( isForeground: Boolean = true, cameraEnabled: Boolean = false, @@ -290,10 +344,10 @@ class InvokeDispatcherTest { motionActivityAvailable: Boolean = false, motionPedometerAvailable: Boolean = false, talkHandler: TalkHandler = InvokeDispatcherFakeTalkHandler(), + canvas: CanvasController = CanvasController(), ): InvokeDispatcher { val appContext = RuntimeEnvironment.getApplication() shadowOf(appContext.packageManager).setSystemFeature(PackageManager.FEATURE_TELEPHONY, smsTelephonyAvailable) - val canvas = CanvasController() return InvokeDispatcher( canvas = canvas, cameraHandler = newCameraHandler(appContext), diff --git a/apps/android/gradle/libs.versions.toml b/apps/android/gradle/libs.versions.toml index 7fda068b3697..93a16eec80f4 100644 --- a/apps/android/gradle/libs.versions.toml +++ b/apps/android/gradle/libs.versions.toml @@ -10,6 +10,7 @@ androidx-exifinterface = "1.4.2" androidx-lifecycle = "2.11.0" androidx-security = "1.1.0" androidx-test-ext = "1.3.0" +androidx-test-runner = "1.7.0" androidx-uiautomator = "2.4.0" androidx-webkit = "1.15.0" bcprov = "1.84" @@ -51,6 +52,7 @@ androidx-room-compiler = { module = "androidx.room:room-compiler", version.ref = androidx-room-runtime = { module = "androidx.room:room-runtime", version.ref = "room" } androidx-security-crypto = { module = "androidx.security:security-crypto", version.ref = "androidx-security" } androidx-test-ext-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-ext" } +androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } androidx-uiautomator = { module = "androidx.test.uiautomator:uiautomator", version.ref = "androidx-uiautomator" } androidx-webkit = { module = "androidx.webkit:webkit", version.ref = "androidx-webkit" } bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bcprov" } diff --git a/src/gateway/android-node.capabilities.live.test.ts b/src/gateway/android-node.capabilities.live.test.ts index 5f6c50b6c6a3..4847e5b89279 100644 --- a/src/gateway/android-node.capabilities.live.test.ts +++ b/src/gateway/android-node.capabilities.live.test.ts @@ -104,6 +104,36 @@ function assertObjectPayload(command: string, payload: unknown): Record = { "canvas.present": { buildParams: () => ({ url: "about:blank" }), @@ -140,14 +170,20 @@ const COMMAND_PROFILES: Record = { }, }, "canvas.a2ui.push": { - buildParams: () => ({ jsonl: '{"beginRendering":{}}\n' }), + buildParams: () => ({ jsonl: VALID_A2UI_JSONL }), timeoutMs: 30_000, outcome: "success", + onSuccess: (payload) => { + assertA2uiPushPayload("canvas.a2ui.push", payload); + }, }, "canvas.a2ui.pushJSONL": { - buildParams: () => ({ jsonl: '{"beginRendering":{}}\n' }), + buildParams: () => ({ jsonl: VALID_A2UI_JSONL }), timeoutMs: 30_000, outcome: "success", + onSuccess: (payload) => { + assertA2uiPushPayload("canvas.a2ui.pushJSONL", payload); + }, }, "canvas.a2ui.reset": { buildParams: () => ({}),