fix(desktop): restart a stale attached gateway instead of re-attaching it (review M7)

What: `gateway ensure --json` now publishes the owner's boot commit
(`endpoint.code_sha`, from its control-socket `identify`) and the ensure
client's own commit (`client_code_sha`, same `get_code_identity` resolver).
Desktop compares them in ensureLocalGateway: a ready owner that provably runs
other code is restarted once with a supervised `hermes --profile P gateway
restart` (P = the profile owning the process: `default` for a served
secondary), then re-ensured. The Models-page "Restart Hermes"
(hermes:backend:recycle) runs the same supervised restart for a local
descriptor before dropping it. Both reuse runGatewayEnsure (resolver, spawn
env scrub, Windows cmd.exe quoting) with a 120 s bounded client.

Why: main's 55b51fa8ad8..61b69853eec made Desktop attach only to a backend
booted from its checkout. The attach model dropped that, so after
`hermes update` the running gateway answered model calls 503 "Restart
required", and Restart only tore down processes Desktop owns (none) — the next
ensure re-attached the same stale gateway forever.

Guards: unknown identity on either side (non-git install, older gateway, older
CLI) keeps the plain attach. One restart per (gateway, client) commit pair per
process, shared by concurrent dials; an owner a restart cannot move (a unit
pinned to another checkout) is then attached as-is rather than restarted on
every dial. A failed restart is logged; the re-ensure reports what answers.

Red: local-gateway.test.ts 'a ready owner running other code than this Hermes
is restarted once, then re-ensured' (pre-fix: 1 failed | 13 passed);
backend-recycle.test.ts 'kills the owned SSH serve before the primary child…'
and 'recycles a pooled profile…' (2 failed | 3 passed);
test_gateway_runtime_home_identity.py::test_ready_endpoint_carries_the_owner_boot_commit
(AttributeError: no code_sha). Green: vitest 20/20 (local-gateway,
backend-recycle, local-gateway-python); pytest gateway_runtime* 16 passed,
3 skipped. Live (fake HOME): ensure payload carries matching code_sha /
client_code_sha; `gateway restart` on an unsupervised owner drained it and
the next ensure attached a fresh instance_id.

(cherry picked from commit d8c5356625168e582c5c2ebc64f4950ecf7e0972)

apps/desktop/electron/backend-recycle.test.ts

@@ -28,13 +28,19 @@ describe('recycleOwnedBackend', () => {
       teardownPrimary: async () => {
         events.push('primary')
       },
+      restartLocalGateway: async profile => {
+        events.push(`restart:${profile}`)
+      },
       teardownSsh: async profile => {
         events.push(`ssh:${profile}`)
       }
     })
 
     expect(target).toBe('primary')
-    expect(events).toEqual(['ssh:', 'primary', 'applied'])
+    // The attached gateway owns its own lifetime: dropping descriptors alone re-attached the
+    // same stale process (503 "Restart required" after `hermes update`), so it is restarted
+    // between the SSH teardown and the descriptor drop that forces the re-ensure.
+    expect(events).toEqual(['ssh:', 'restart:default', 'primary', 'applied'])
   })
 
   it('recycles a pooled profile without tearing down the primary', async () => {
@@ -50,13 +56,16 @@ describe('recycleOwnedBackend', () => {
       teardownPrimary: async () => {
         events.push('primary')
       },
+      restartLocalGateway: async profile => {
+        events.push(`restart:${profile}`)
+      },
       teardownSsh: async profile => {
         events.push(`ssh:${profile}`)
       }
     })
 
     expect(target).toBe('pool')
-    expect(events).toEqual(['ssh:paid-ads', 'pool:paid-ads'])
+    expect(events).toEqual(['ssh:paid-ads', 'restart:paid-ads', 'pool:paid-ads'])
   })
 
   it('awaits SSH teardown before the local child even when SSH is slow', async () => {
@@ -70,6 +79,7 @@ describe('recycleOwnedBackend', () => {
     const run = recycleOwnedBackend({
       notifyApplied: () => events.push('applied'),
       primaryProfile: 'default',
+      restartLocalGateway: async () => undefined,
       teardownPool: vi.fn(),
       teardownPrimary: async () => {
         events.push('primary')

apps/desktop/electron/backend-recycle.ts

@@ -1,11 +1,17 @@
 /**
- * Recycle a Desktop-owned backend after a code-skew 503.
+ * Recycle the backend behind a code-skew 503 ("Restart required" after `hermes update`).
  *
  * Closing the local tunnel/child is not enough for SSH: `serve --isolated`
  * detaches with setsid/nohup, so a reconnect would reuse the still-alive
  * stale process via the lockfile. Kill the owned remote serve first (while
  * the SSH channel can still exec), then tear down the local child — the
  * same order as connection apply (#97046, #91668).
+ *
+ * A LOCAL profile attaches to the per-host gateway (`hermes gateway ensure`),
+ * which Desktop never owns: dropping its descriptor only re-attached the same
+ * stale process. `restartLocalGateway` runs a supervised
+ * `hermes --profile P gateway restart` (a no-op for a remote profile) before
+ * the descriptor drop, so the next ensure dials the replacement.
  */
 
 export type RecycleOwnedBackendTarget = 'pool' | 'primary'
@@ -14,6 +20,7 @@ export interface RecycleOwnedBackendDeps {
   notifyApplied: () => void
   primaryProfile: string
   profile?: null | string
+  restartLocalGateway: (profile: string) => Promise<void>
   teardownPool: (profile: string) => Promise<void>
   teardownPrimary: () => Promise<void>
   teardownSsh: (profile: string) => Promise<void>
@@ -34,6 +41,7 @@ export async function recycleOwnedBackend(deps: RecycleOwnedBackendDeps): Promis
 
   if (target === 'primary') {
     await deps.teardownSsh('')
+    await deps.restartLocalGateway(deps.primaryProfile)
     await deps.teardownPrimary()
     deps.notifyApplied()
 
@@ -41,6 +49,7 @@ export async function recycleOwnedBackend(deps: RecycleOwnedBackendDeps): Promis
   }
 
   await deps.teardownSsh(profile)
+  await deps.restartLocalGateway(profile)
   await deps.teardownPool(profile)
 
   return target

apps/desktop/electron/local-gateway.test.ts

@@ -180,6 +180,24 @@ test('a dial against a replaced local gateway forgets the cached endpoint once a
   expect(ensures).toBe(2)
 })
 
+test('a ready owner running other code than this Hermes is restarted once, then re-ensured', async () => {
+  // After `hermes update` the live gateway still serves the old code and answers model calls
+  // 503 "Restart required"; attaching to it on every dial pinned the user to it forever.
+  const { createStaleGatewayRestarter } = await import('./local-gateway')
+  const endpoint = { profile_id: '/h/profiles/w', control_home: '/h', instance_id: 'old', authority_epoch: 1, runtime_protocol: 1, api_origin: 'http://127.0.0.1:1234', capabilities: ['session-authority-v1'], supervisor: 'none', code_sha: 'OLD' }
+  const answers = [{ ...endpoint, code_sha: 'old' }, { ...endpoint, instance_id: 'new', code_sha: 'new' }, endpoint, endpoint]
+  const restarted: string[] = []
+  const restartStale = createStaleGatewayRestarter(async owner => { restarted.push(owner) }, () => undefined)
+  const ensure = () => ensureLocalGateway(async () => ({ code: 0, stdout: JSON.stringify({ state: 'ready', endpoint: answers.shift(), client_code_sha: 'new' }) }), undefined, restartStale)
+
+  expect((await ensure()).gatewayEndpoint.instance_id).toBe('new')
+  // A served secondary is replaced through the multiplexer that owns its process, not `-p w`.
+  expect(restarted).toEqual(['default'])
+  // A restart that cannot move the owner (a unit pinned to another checkout) is not repeated per dial.
+  expect((await ensure()).gatewayEndpoint.instance_id).toBe('old')
+  expect(restarted).toEqual(['default'])
+})
+
 test('a dial that keeps failing after one re-ensure surfaces the error instead of looping', async () => {
   const { redialLocalGateway } = await import('./local-gateway')
   let ensures = 0

apps/desktop/electron/local-gateway.ts

@@ -3,8 +3,10 @@ import crypto from 'node:crypto'
 
 import { hiddenWindowsChildOptions, windowsShellCommand } from './windows-child-options'
 
+/** Run one bounded `hermes ... gateway <verb>` client (`ensure` by default; `restart` for a stale owner). */
 export function runGatewayEnsure(
-  backend, cwd: string, home: string, parentEnv: NodeJS.ProcessEnv = process.env
+  backend, cwd: string, home: string, parentEnv: NodeJS.ProcessEnv = process.env,
+  { timeoutMs = 40_000, label = 'hermes gateway ensure' }: { timeoutMs?: number; label?: string } = {}
 ): Promise<{ code: number; stdout: string; stderr: string }> {
   return new Promise((resolve, reject) => {
     // shell: true hands cmd.exe one command line it cuts at the first unquoted space,
@@ -12,15 +14,15 @@ export function runGatewayEnsure(
     const child = spawn(windowsShellCommand(backend.command, Boolean(backend.shell)), backend.args, hiddenWindowsChildOptions({ cwd, env: { ...parentEnv, HERMES_HOME: home, ...backend.env }, shell: backend.shell, stdio: ['ignore', 'pipe', 'pipe'] }))
     let stdout = ''
     let stderr = ''
-    // Only the bounded ensure client is ours. Never retain/kill its detached owner.
-    const timer = setTimeout(() => child.kill(), 40_000)
+    // Only the bounded CLI client is ours. Never retain/kill its detached owner.
+    const timer = setTimeout(() => child.kill(), timeoutMs)
     child.stdout.on('data', data => { stdout += data.toString();
 
  if (stdout.length > 65536) {child.kill()} })
     // Diagnostics only (never protocol): an older `hermes` without the subcommand, a missing
     // profile or an import crash explain themselves here while stdout stays empty.
     child.stderr.on('data', data => { stderr = (stderr + data.toString()).slice(-4096) })
-    child.on('error', () => { clearTimeout(timer); reject(new Error('Could not run hermes gateway ensure')) })
+    child.on('error', () => { clearTimeout(timer); reject(new Error(`Could not run ${label}`)) })
     child.on('close', code => { clearTimeout(timer); resolve({ code: code ?? 7, stdout, stderr }) })
   })
 }
@@ -39,6 +41,60 @@ export interface GatewayEndpoint {
   supervisor: string
   /** Multiplexer home whose control socket answers for a served secondary; null when the profile owns its own. */
   control_home?: string | null
+  /** Commit the owner booted from; null when it cannot name one (see `client_code_sha`). */
+  code_sha?: string | null
+}
+
+/** Restarts a ready owner whose code differs from this client's; true when a restart was requested. */
+export type StaleGatewayRestart = (endpoint: GatewayEndpoint, stale: { gateway: string; client: string }) => Promise<boolean>
+
+/** The profile whose `hermes --profile P gateway restart` replaces the process behind `endpoint`:
+ *  the multiplexer root (`default`) for a served secondary, else the endpoint's own profile home. */
+export function gatewayOwnerProfile(endpoint: GatewayEndpoint): string {
+  const home = endpoint.control_home || endpoint.profile_id
+
+  return path.basename(path.dirname(home)) === 'profiles' ? path.basename(home) : 'default'
+}
+
+/** One restart per stale (gateway, client) code pair per process. An owner a restart cannot move
+ *  (a service unit pinned to another checkout) is then attached as-is instead of being restarted
+ *  on every dial. */
+export function createStaleGatewayRestarter(
+  restart: (ownerProfile: string) => Promise<void>,
+  log: (message: string) => void
+): StaleGatewayRestart {
+  // Concurrent dials (primary + a served profile) that see the same stale owner share ONE restart;
+  // a dial arriving after it finished attaches as-is.
+  const restarts = new Map<string, { done: boolean; promise: Promise<void> }>()
+
+  return async (endpoint, stale) => {
+    const key = `${stale.gateway}\n${stale.client}`.toLowerCase()
+    const pending = restarts.get(key)
+
+    if (pending && !pending.done) {
+      await pending.promise
+
+      return true
+    }
+
+    if (pending) {
+      log(`[gateway] owner still runs ${stale.gateway} (this Hermes is ${stale.client}) after a restart; attaching as-is`)
+
+      return false
+    }
+
+    log(`[gateway] owner runs ${stale.gateway} but this Hermes is ${stale.client}; restarting it`)
+    const entry = { done: false, promise: restart(gatewayOwnerProfile(endpoint)) }
+    restarts.set(key, entry)
+
+    try {
+      await entry.promise
+    } finally {
+      entry.done = true
+    }
+
+    return true
+  }
 }
 
 /** The single-line JSON `hermes gateway ensure --json` prints, or a diagnosis of why there is none.
@@ -60,10 +116,34 @@ export function parseGatewayEnsureOutput(result: { code: number; stdout: string;
   throw new Error(`hermes gateway ensure produced no result (exit ${result.code})${reason ? `: ${reason}` : ''}. Update Hermes or check the profile, then retry.`)
 }
 
-export async function ensureLocalGateway(run: () => Promise<{ code: number; stdout: string; stderr?: string }>, beforeEnsure?: () => Promise<void>) {
+/** `{ gateway, client }` commits when a ready owner provably runs other code than the ensure
+ *  client (a gateway that outlived `hermes update` answers model calls 503 "Restart required");
+ *  null when they match or either side cannot name its code. */
+export function staleGatewayCode(payload: Record<string, any>): { gateway: string; client: string } | null {
+  const gateway = payload?.endpoint?.code_sha
+  const client = payload?.client_code_sha
+
+  if (typeof gateway !== 'string' || !gateway || typeof client !== 'string' || !client) {return null}
+
+  return gateway.toLowerCase() === client.toLowerCase() ? null : { gateway, client }
+}
+
+export async function ensureLocalGateway(
+  run: () => Promise<{ code: number; stdout: string; stderr?: string }>,
+  beforeEnsure?: () => Promise<void>,
+  restartStale?: StaleGatewayRestart
+) {
   await beforeEnsure?.()
-  const result = await run()
-  const payload = parseGatewayEnsureOutput(result)
+  let result = await run()
+  let payload = parseGatewayEnsureOutput(result)
+  const stale = restartStale && result.code === 0 && payload.state === 'ready' ? staleGatewayCode(payload) : null
+
+  // Attaching would pin a stale owner (503 "Restart required" forever): replace it once, then
+  // take whatever answers the re-ensure.
+  if (stale && await restartStale!(payload.endpoint, stale)) {
+    result = await run()
+    payload = parseGatewayEnsureOutput(result)
+  }
 
   if (result.code !== 0 || payload.state !== 'ready') {
     throw new Error(`Gateway ${payload.state || 'inaccessible'} (${payload.reason_code || 'ensure_failed'}). Use hermes gateway status for recovery.`)

apps/desktop/electron/main.ts

@@ -1,7 +1,7 @@
 import { type ChildProcess, execFileSync, spawn } from 'node:child_process'
 
 import type { GatewayEndpoint } from './local-gateway'
-import { configureWindowsGatewayTicketClient, createLocalGatewayDials, ensureLocalGateway, mintLocalGatewayTicket, nativeGatewayHttpHeaders, redialLocalGateway, routedGatewayEndpoint, runGatewayEnsure } from './local-gateway'
+import { configureWindowsGatewayTicketClient, createLocalGatewayDials, createStaleGatewayRestarter, ensureLocalGateway, gatewayOwnerProfile, mintLocalGatewayTicket, nativeGatewayHttpHeaders, redialLocalGateway, routedGatewayEndpoint, runGatewayEnsure } from './local-gateway'
 import { mintGatewayTicketWithPython } from './local-gateway-python'
 const localGatewayDials = createLocalGatewayDials()
 configureWindowsGatewayTicketClient(async (endpoint, purpose) => {
@@ -11796,13 +11796,58 @@ async function dialPoolBackend(profile, entry, opts: { forceLocal?: boolean; poo
     // Update waits yield: retirement or profile deletion may win in that gap.
     assertPoolEntryStillOwned(poolKey, entry)
     assertLocalProfileCanStart(profile, profileDeletionGate, key => directoryExists(path.join(HERMES_HOME, 'profiles', key)))
-  })
+  }, restartStaleLocalGateway)
 
   assertPoolEntryStillOwned(poolKey, entry)
 
   return { ...connection, profile, logs: hermesLog.slice(-80), ...getWindowState() }
 }
 
+// `hermes gateway restart` drains in-flight turns before the owner exits; the CLI client is
+// bounded so a wedged drain cannot hang Restart. The gateway's own drain is unaffected by the kill.
+const GATEWAY_RESTART_CLIENT_TIMEOUT_MS = 120_000
+
+// Supervised restart of the per-host gateway Desktop attaches to (it never owns that process):
+// the same resolver, spawn helper and Windows quoting as `gateway ensure`, against the profile
+// that owns the gateway process. A non-zero exit is logged, not thrown: the re-ensure that
+// follows reports what actually answers.
+async function restartLocalGatewayOwner(ownerProfile: string): Promise<void> {
+  const backend = await ensureRuntime(
+    await resolveHermesBackend(['--profile', ownerProfile, 'gateway', 'restart']),
+    () => undefined
+  )
+
+  const result = await runGatewayEnsure(
+    { ...backend, env: desktopBackendSpawnEnv(backend.env || {}, GUEST_ONBOARDING) },
+    resolveHermesCwd(),
+    HERMES_HOME,
+    profileBackendParentEnv({ hermesHome: HERMES_HOME, profile: ownerProfile }),
+    { timeoutMs: GATEWAY_RESTART_CLIENT_TIMEOUT_MS, label: 'hermes gateway restart' }
+  )
+
+  const reason = `${result.stdout}\n${result.stderr}`.trim().split(/\r?\n/).filter(Boolean).pop()
+  rememberLog(`[gateway] hermes --profile ${ownerProfile} gateway restart exited ${result.code}${reason ? `: ${reason}` : ''}`)
+}
+
+// A ready gateway whose boot commit differs from this Hermes (it outlived `hermes update`) is
+// restarted once before attaching instead of being re-attached on every dial.
+const restartStaleLocalGateway = createStaleGatewayRestarter(restartLocalGatewayOwner, rememberLog)
+
+// Models-page "Restart Hermes": restart the local gateway behind `profile`'s cached descriptor.
+// A remote/SSH descriptor has no gatewayEndpoint, so nothing local is touched for it.
+async function restartAttachedLocalGateway(profile: string): Promise<void> {
+  const pending = profile === primaryProfileKey()
+    ? backendConnectionState.getPromise()
+    : localProfilePoolKeys(profile).map(key => backendPool.get(key)?.connectionPromise).find(Boolean)
+
+  const connection = await pending?.catch(() => null)
+  const endpoint = connection?.gatewayEndpoint
+
+  if (endpoint) {
+    await restartLocalGatewayOwner(gatewayOwnerProfile(endpoint))
+  }
+}
+
 // Forget a cached descriptor. Nothing is killed: the gateway behind it owns its
 // own lifetime (`hermes gateway ensure` attach semantics), so the next
 // ensure*() for this key simply re-dials. Every retire path — dispatch probe
@@ -12200,7 +12245,7 @@ async function startHermes(requestedProfile?: string) {
       resolveHermesCwd(),
       HERMES_HOME,
       profileBackendParentEnv({ hermesHome: HERMES_HOME, profile: primaryProfile })
-    ))
+    ), undefined, restartStaleLocalGateway)
 
     backendConnectionState.assertCurrentAttempt(connectionAttempt)
 
@@ -14657,11 +14702,13 @@ const hudIpc = registerHudIpc({
 ipcMain.handle('hermes:backend:recycle', async (_event, profile) => {
   // Models-page recovery after a code-skew 503 (#97046): kill the owned
   // SSH serve (if any) before the local child so reconnect cannot reuse a
-  // stale lockfile. Soft primary teardown keeps the renderer shell mounted.
+  // stale lockfile, and restart the attached local gateway (Desktop does not
+  // own it). Soft primary teardown keeps the renderer shell mounted.
   await recycleOwnedBackend({
     notifyApplied: sendConnectionApplied,
     primaryProfile: primaryProfileKey(),
     profile: typeof profile === 'string' ? profile : '',
+    restartLocalGateway: restartAttachedLocalGateway,
     teardownPool: teardownPoolBackendAndWait,
     teardownPrimary: () => teardownPrimaryBackendAndWait({ soft: true }),
     teardownSsh: value => teardownSshConnection(value || null)

hermes_cli/gateway_runtime.py

@@ -32,6 +32,10 @@ class GatewayEndpoint:
     # Home whose control socket answers for this endpoint: the profile's own home, or the
     # default multiplexer's root when the profile is served by it.
     control_home: str | None = None
+    # Commit the owner BOOTED from (its ``identify`` ``code_sha``); None when it cannot name one.
+    # A gateway that outlived ``hermes update`` keeps serving old code, so an attaching client
+    # compares this with its own checkout and restarts the owner instead of re-attaching it.
+    code_sha: str | None = None
 
 
 @dataclass(frozen=True)
@@ -93,6 +97,7 @@ def _endpoint(payload: dict, home: Path, control_home: Path | None = None) -> Ga
         authority_epoch=epoch, runtime_protocol=1, api_origin=origin,
         supervisor=supervisor, capabilities=frozenset(capabilities),
         control_home=str(control_home) if control_home is not None else None,
+        code_sha=code_sha if isinstance(code_sha := payload.get("code_sha"), str) and code_sha else None,
     ))
 
 

hermes_cli/gateway_runtime_cli.py

@@ -25,6 +25,11 @@ def cmd_gateway_ensure(args) -> None:
         payload = asdict(result)
         if result.endpoint:
             payload["endpoint"]["capabilities"] = sorted(result.endpoint.capabilities)
+        # The code THIS client (and any gateway it starts) runs, resolved exactly like the owner's
+        # ``endpoint.code_sha``: a client that sees them differ is attached to a gateway that
+        # outlived ``hermes update`` and must restart it rather than re-attach.
+        from hermes_cli.version_info import get_code_identity
+        payload["client_code_sha"] = get_code_identity().get("sha")
         code = ensure_exit_code(result)
     except (ValueError, TypeError):
         payload, code = {"state": "inaccessible", "reason_code": "invalid_invocation", "endpoint": None}, 2

tests/hermes_cli/test_gateway_runtime_home_identity.py

@@ -38,3 +38,17 @@ def test_endpoint_matches_served_home_under_any_canonical_spelling(tmp_path):
     other.mkdir()
     assert _endpoint(_ready_payload(served), other).reason_code == 'profile_mismatch'
     assert _endpoint(_ready_payload(served), Path(served + '-suffix')).reason_code == 'profile_mismatch'
+
+
+def test_ready_endpoint_carries_the_owner_boot_commit(tmp_path):
+    """`gateway ensure --json` publishes the code the owner BOOTED from, so Desktop can restart a
+    gateway that outlived `hermes update` instead of re-attaching it forever (503 Restart required)."""
+    from hermes_cli.gateway_runtime import _endpoint
+    home = tmp_path / 'home'
+    home.mkdir()
+    served = str(home.resolve())
+    sha = '0123456789abcdef0123456789abcdef01234567'
+    assert _endpoint({**_ready_payload(served), 'code_sha': sha}, home).endpoint.code_sha == sha
+    # An owner that cannot name its code (non-git, unreadable stamp) or answers garbage is unknown, not a mismatch.
+    assert _endpoint(_ready_payload(served), home).endpoint.code_sha is None
+    assert _endpoint({**_ready_payload(served), 'code_sha': ['x']}, home).endpoint.code_sha is None