fix(macos): prevent remote discovery hangs during Tailscale checks (#115852)

* fix(macos): bound discovery subprocess execution

* refactor(macos): migrate tailscale discovery runner

* refactor(macos): make wide-area discovery asynchronous

* test(macos): prove discovery subprocess cleanup

* chore(macos): keep release notes in the PR
This commit is contained in:
Vincent Koc
2026-07-29 21:38:57 +08:00
committed by GitHub
parent b8dcac19bf
commit cba59e4674
7 changed files with 135 additions and 88 deletions

View File

@@ -37,6 +37,7 @@ let package = Package(
name: "OpenClawDiscovery",
dependencies: [
.product(name: "OpenClawKit", package: "OpenClawKit"),
.product(name: "Subprocess", package: "swift-subprocess"),
],
path: "Sources/OpenClawDiscovery",
swiftSettings: [

View File

@@ -0,0 +1,68 @@
import Foundation
import Subprocess
enum BoundedCommand {
private static let defaultOutputLimit = 8 * 1024 * 1024
static func run(
path: String,
arguments: [String],
environment: [String: String]? = nil,
timeout: TimeInterval,
outputLimit: Int = defaultOutputLimit) async -> String?
{
guard timeout > 0, outputLimit > 0 else { return nil }
let executable: Executable = path.contains("/")
? .path(.init(path))
: .name(path)
let subprocessEnvironment = environment.map(self.environment(from:)) ?? .inherit
do {
return try await withThrowingTaskGroup(
of: String?.self,
returning: String?.self)
{ group in
group.addTask {
let result = try await Subprocess.run(
executable,
arguments: Arguments(arguments),
environment: subprocessEnvironment,
output: .string(limit: outputLimit))
guard result.terminationStatus.isSuccess,
let output = result.standardOutput?
.trimmingCharacters(in: .whitespacesAndNewlines),
!output.isEmpty
else {
return nil
}
return output
}
group.addTask {
try await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000))
return nil
}
// Keep timeout structured so swift-subprocess kills and reaps a canceled
// child before callers can start the next discovery probe.
defer { group.cancelAll() }
guard let result = try await group.next() else {
return nil
}
return result
}
} catch {
return nil
}
}
private static func environment(from values: [String: String]) -> Environment {
var converted: [Environment.Key: String] = [:]
converted.reserveCapacity(values.count)
for (key, value) in values {
guard let environmentKey = Environment.Key(rawValue: key) else { continue }
converted[environmentKey] = value
}
return .custom(converted)
}
}

View File

@@ -126,7 +126,7 @@ public final class GatewayDiscoveryModel {
guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return }
Task.detached(priority: .utility) { [weak self] in
guard let self else { return }
let beacons = WideAreaGatewayDiscovery.discover(timeoutSeconds: timeoutSeconds)
let beacons = await WideAreaGatewayDiscovery.discover(timeoutSeconds: timeoutSeconds)
await MainActor.run { [weak self] in
guard let self else { return }
self.wideAreaFallbackGateways = self.mapWideAreaBeacons(beacons, domain: domain)
@@ -325,7 +325,7 @@ public final class GatewayDiscoveryModel {
// Wide-area discovery can be racy (Tailscale not yet up, DNS zone not
// published yet). Retry with a short backoff while onboarding is open.
let beacons = WideAreaGatewayDiscovery.discover(timeoutSeconds: 2.0)
let beacons = await WideAreaGatewayDiscovery.discover(timeoutSeconds: 2.0)
if !beacons.isEmpty {
await MainActor.run { [weak self] in
guard let self else { return }

View File

@@ -152,7 +152,12 @@ enum TailscaleServeGatewayDiscovery {
for candidate in candidates {
guard let executable = self.resolveExecutablePath(candidate) else { continue }
if let stdout = await self.run(path: executable, args: ["status", "--json"], timeout: 1.0) {
if let stdout = await BoundedCommand.run(
path: executable,
arguments: ["status", "--json"],
environment: self.commandEnvironment(),
timeout: 1.0)
{
return stdout
}
}
@@ -189,43 +194,6 @@ enum TailscaleServeGatewayDiscovery {
return nil
}
private static func run(path: String, args: [String], timeout: TimeInterval) async -> String? {
await withCheckedContinuation { continuation in
DispatchQueue.global(qos: .utility).async {
continuation.resume(returning: self.runBlocking(path: path, args: args, timeout: timeout))
}
}
}
private static func runBlocking(path: String, args: [String], timeout: TimeInterval) -> String? {
let process = Process()
process.executableURL = URL(fileURLWithPath: path)
process.arguments = args
process.environment = self.commandEnvironment()
let outPipe = Pipe()
process.standardOutput = outPipe
process.standardError = FileHandle.nullDevice
do {
try process.run()
} catch {
return nil
}
let deadline = Date().addingTimeInterval(timeout)
while process.isRunning, Date() < deadline {
Thread.sleep(forTimeInterval: 0.02)
}
if process.isRunning {
process.terminate()
}
process.waitUntilExit()
let data = (try? outPipe.fileHandleForReading.readToEnd()) ?? Data()
let output = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines)
return output?.isEmpty == false ? output : nil
}
static func commandEnvironment(
base: [String: String] = ProcessInfo.processInfo.environment) -> [String: String]
{

View File

@@ -23,28 +23,28 @@ enum WideAreaGatewayDiscovery {
private static let tailscaleDNSResolver = "100.100.100.100"
struct DiscoveryContext {
var tailscaleStatus: @Sendable () -> String?
var dig: @Sendable (_ args: [String], _ timeout: TimeInterval) -> String?
var tailscaleStatus: @Sendable () async -> String?
var dig: @Sendable (_ args: [String], _ timeout: TimeInterval) async -> String?
static let live = DiscoveryContext(
tailscaleStatus: { readTailscaleStatus() },
tailscaleStatus: { await readTailscaleStatus() },
dig: { args, timeout in
runDig(args: args, timeout: timeout)
await runDig(args: args, timeout: timeout)
})
}
static func discover(
timeoutSeconds: TimeInterval = 2.0,
context: DiscoveryContext = .live) -> [WideAreaGatewayBeacon]
context: DiscoveryContext = .live) async -> [WideAreaGatewayBeacon]
{
let startedAt = Date()
let remaining = {
timeoutSeconds - Date().timeIntervalSince(startedAt)
}
guard let statusJson = context.tailscaleStatus(),
guard let statusJson = await context.tailscaleStatus(),
!collectTailnetIPv4s(statusJson: statusJson).isEmpty,
let discovery = loadWideAreaPtrRecords(
let discovery = await loadWideAreaPtrRecords(
remaining: remaining,
dig: context.dig)
else { return [] }
@@ -64,13 +64,13 @@ enum WideAreaGatewayDiscovery {
: ptrName
let instanceName = self.decodeDnsSdEscapes(rawInstanceName)
guard let srv = context.dig(
guard let srv = await context.dig(
["+short", "+time=1", "+tries=1", "@\(nameserver)", ptrName, "SRV"],
min(defaultTimeoutSeconds, remaining()))
else { continue }
guard let (host, port) = parseSrv(srv) else { continue }
let txtRaw = context.dig(
let txtRaw = await context.dig(
["+short", "+time=1", "+tries=1", "@\(nameserver)", ptrName, "TXT"],
min(self.defaultTimeoutSeconds, remaining()))
let txtTokens = txtRaw.map(self.parseTxtTokens) ?? []
@@ -119,7 +119,7 @@ enum WideAreaGatewayDiscovery {
}
}
private static func readTailscaleStatus() -> String? {
private static func readTailscaleStatus() async -> String? {
let candidates = [
"/usr/local/bin/tailscale",
"/opt/homebrew/bin/tailscale",
@@ -129,9 +129,9 @@ enum WideAreaGatewayDiscovery {
var output: String?
for candidate in candidates {
if let result = run(
if let result = await BoundedCommand.run(
path: candidate,
args: ["status", "--json"],
arguments: ["status", "--json"],
timeout: 0.7)
{
output = result
@@ -144,8 +144,8 @@ enum WideAreaGatewayDiscovery {
private static func loadWideAreaPtrRecords(
remaining: () -> TimeInterval,
dig: @escaping @Sendable (_ args: [String], _ timeout: TimeInterval) -> String?)
-> (domainTrimmed: String, ptrLines: [Substring])?
dig: @escaping @Sendable (_ args: [String], _ timeout: TimeInterval) async -> String?)
async -> (domainTrimmed: String, ptrLines: [Substring])?
{
guard let domain = OpenClawBonjour.wideAreaGatewayServiceDomain else { return nil }
let domainTrimmed = domain.trimmingCharacters(in: CharacterSet(charactersIn: "."))
@@ -153,7 +153,7 @@ enum WideAreaGatewayDiscovery {
let budget = max(0, remaining())
if budget <= 0 { return nil }
guard let stdout = dig(
guard let stdout = await dig(
["+short", "+time=1", "+tries=1", "@\(self.tailscaleDNSResolver)", probeName, "PTR"],
min(defaultTimeoutSeconds, budget)),
let ptrLines = stdout.split(whereSeparator: \.isNewline).nonEmpty
@@ -164,37 +164,8 @@ enum WideAreaGatewayDiscovery {
return (domainTrimmed, ptrLines)
}
private static func runDig(args: [String], timeout: TimeInterval) -> String? {
self.run(path: self.digPath, args: args, timeout: timeout)
}
private static func run(path: String, args: [String], timeout: TimeInterval) -> String? {
let process = Process()
process.executableURL = URL(fileURLWithPath: path)
process.arguments = args
let outPipe = Pipe()
process.standardOutput = outPipe
// Avoid stderr pipe backpressure; we don't consume it.
process.standardError = FileHandle.nullDevice
do {
try process.run()
} catch {
return nil
}
let deadline = Date().addingTimeInterval(timeout)
while process.isRunning, Date() < deadline {
Thread.sleep(forTimeInterval: 0.02)
}
if process.isRunning {
process.terminate()
}
process.waitUntilExit()
let data = (try? outPipe.fileHandleForReading.readToEnd()) ?? Data()
let output = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines)
return output?.isEmpty == false ? output : nil
private static func runDig(args: [String], timeout: TimeInterval) async -> String? {
await BoundedCommand.run(path: self.digPath, arguments: args, timeout: timeout)
}
private static func parseSrv(_ stdout: String) -> (String, Int)? {

View File

@@ -0,0 +1,39 @@
import Darwin
import Foundation
import Testing
@testable import OpenClawDiscovery
struct BoundedCommandTests {
@Test func `drains output larger than a process pipe`() async throws {
let byteCount = 256 * 1024
let output = await BoundedCommand.run(
path: "/usr/bin/head",
arguments: ["-c", "\(byteCount)", "/dev/zero"],
timeout: 1.0)
let value = try #require(output)
#expect(value.utf8.count == byteCount)
}
@Test func `force kills and reaps a command that ignores termination`() async throws {
let pidFile = FileManager.default.temporaryDirectory
.appendingPathComponent("openclaw-bounded-command-\(UUID().uuidString).pid")
defer { try? FileManager.default.removeItem(at: pidFile) }
let clock = ContinuousClock()
let startedAt = clock.now
let output = await BoundedCommand.run(
path: "/bin/sh",
arguments: ["-c", "echo $$ > \"$PID_FILE\"; trap '' TERM; while :; do :; done"],
environment: ["PID_FILE": pidFile.path],
timeout: 0.1)
#expect(output == nil)
#expect(startedAt.duration(to: clock.now) < .seconds(1))
let pidString = try String(contentsOf: pidFile, encoding: .utf8)
.trimmingCharacters(in: .whitespacesAndNewlines)
let pid = try #require(pid_t(pidString))
#expect(kill(pid, 0) == -1)
#expect(errno == ESRCH)
}
}

View File

@@ -56,7 +56,7 @@ struct WideAreaGatewayDiscoveryTests {
let beacons = await TestIsolation.withEnvValues(
["OPENCLAW_WIDE_AREA_DOMAIN": "openclaw.internal"])
{
WideAreaGatewayDiscovery.discover(
await WideAreaGatewayDiscovery.discover(
timeoutSeconds: 2.0,
context: context)
}
@@ -107,7 +107,7 @@ struct WideAreaGatewayDiscoveryTests {
let beacons = await TestIsolation.withEnvValues(
["OPENCLAW_WIDE_AREA_DOMAIN": "openclaw.internal"])
{
WideAreaGatewayDiscovery.discover(
await WideAreaGatewayDiscovery.discover(
timeoutSeconds: 2.0,
context: context)
}