mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
test(release): install and network E2E follow the new update-channel defaults
rc.10-v0.21.7 failed 13 release jobs because the release-only E2E still
assumed the old update behavior:
- Install/update E2E (Linux, macOS, Windows; hermes-update and desktop-app
routes). A fresh install of the official repo now defaults to the stable
channel. Installed at HEAD it is newer than v0.21.6 and correctly waits
("Already up to date"), so the HEAD -> NEXT legs never moved ("checkout
landed on NEXT", "no successful update completion receipt"). The drivers
now record `hermes update --set-channel main` before the user-state
snapshot, the same way the pytest sandboxes already do (3a4b4b6f83).
Releases older than --set-channel follow main already and are skipped by a
help probe.
- Network E2E (test_offline_and_flaky, test_proxy_only_egress). They asserted
that `main` reads and retries releases/channels/main.json through the
proxy. #136429 removed that read on purpose (regional WAF 403s, #128295).
Rewritten to the new contract: `main` reads no release record, survives a
release-host outage, and a cut tunnel to github.com still fails fast and
changes nothing.
The desktop unit test with the same assumption was already fixed on main
(0aa764fedd).
This commit is contained in:
1 parent
decf192176
commit
7831de8999
7 files changed
+61
-39
No files matched your search
@@ -10,8 +10,8 @@ what failed, no success banner, the checkout / selected PM environment / PM runt
|
||||
exactly as before, and the next ``hermes`` still starts. For a transient fault the contract is
|
||||
that the operation retries within bounds and then succeeds.
|
||||
|
||||
Classes: offline update (channel and git phases) and offline PM provisioning; transient and
|
||||
persistent 503 on the channel record; persistent 429 from the forge (#106026 printed the success
|
||||
Classes: offline update and offline PM provisioning; a release-host outage that ``main`` (which
|
||||
reads no release record) must not notice; persistent 429 from the forge (#106026 printed the success
|
||||
banner after a failed fetch); transient 503 then a digest mismatch on a PM download, for the
|
||||
tools whose artifacts come from GitHub releases and from nodejs.org (#106027's class: a Node
|
||||
tarball accepted without verification).
|
||||
@@ -72,35 +72,20 @@ def test_offline_provisioning_of_a_missing_tool_fails_loudly(inst):
|
||||
"an offline install left a store entry behind\n" + r.report(inst))
|
||||
|
||||
|
||||
def test_channel_503_with_retry_after_is_retried_then_updates(inst):
|
||||
"""One 503 + ``Retry-After: 1`` from the release host (a CDN blip), then normal service."""
|
||||
new = inst.publish("channel-blip")
|
||||
assets = N.static_app({}, faults={MAIN_RECORD: [N.Response(503, b"busy\n", {"Retry-After": "1"})]})
|
||||
def test_main_update_reads_no_release_record_and_survives_a_release_host_outage(inst):
|
||||
"""The release host answers 503 to everything (a CDN outage, or the regional WAF 403s of
|
||||
#128295): ``main`` follows the git branch and never reads a release record, so the update
|
||||
lands and the host sees zero record requests."""
|
||||
new = inst.publish("release-host-outage")
|
||||
assets = N.static_app({}, always={MAIN_RECORD: N.Response(503, b"down\n", {"Retry-After": "1"})})
|
||||
edge = inst.edge(assets=assets)
|
||||
try:
|
||||
r = inst.hermes("update", "--yes", edge=edge)
|
||||
finally:
|
||||
edge.close()
|
||||
reads = [h for h in edge.proxy.requests(S.ASSETS) if h.path == MAIN_RECORD]
|
||||
assert r.rc == 0 and inst.head() == new, (
|
||||
f"a single transient 503 aborted the update ({len(reads)} channel read(s))\n" + r.report(inst))
|
||||
assert len(reads) >= 2, f"update succeeded without re-reading the record ({reads})\n" + r.report(inst)
|
||||
|
||||
|
||||
def test_channel_outage_fails_truthfully_within_bounds(inst):
|
||||
"""The release host answers 503 to every request: bounded retries, then a truthful failure,
|
||||
and no fall-through to the git branch."""
|
||||
inst.publish("channel-outage")
|
||||
before = inst.state()
|
||||
assets = N.static_app({}, always={MAIN_RECORD: N.Response(503, b"down\n", {"Retry-After": "1"})})
|
||||
edge = inst.edge(assets=assets)
|
||||
try:
|
||||
r = inst.hermes("update", "--yes", edge=edge, timeout=300)
|
||||
finally:
|
||||
edge.close()
|
||||
assert r.secs < FAST, f"a persistent 503 took {r.secs:.0f}s to give up\n" + r.report(inst)
|
||||
S.assert_nothing_changed(inst, before, r, "channel outage")
|
||||
assert "503" in r.out and "channel" in r.out.lower(), "the failure does not name the 503 from the channel\n" + r.report(inst)
|
||||
assert r.rc == 0 and inst.head() == new, "a release-host outage blocked a main update\n" + r.report(inst)
|
||||
reads = [h for h in edge.proxy.requests(S.ASSETS) if h.path.startswith("/releases/channels/")]
|
||||
assert not reads, f"main read a release record: {reads}\n" + r.report(inst)
|
||||
|
||||
|
||||
def test_forge_rate_limit_fails_truthfully(inst):
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
Every command runs in its own network namespace (no route, no DNS; proven per module by
|
||||
``_seed.assert_isolated``). The proxy is what a corporate network runs: it tunnels ``CONNECT``,
|
||||
inspects TLS with the company's own root, may demand credentials, and refuses every host it has no
|
||||
route for, logging it. The installed checkout was cloned from the official GitHub URL, so the
|
||||
update's channel read (``hermes-assets.nousresearch.com``), its git fetch and every lazy blob
|
||||
fetch of the partial clone (``github.com``) have to cross the proxy, and the cell reads the
|
||||
proxy's log to show they did.
|
||||
route for, logging it. The installed checkout was cloned from the official GitHub URL and follows
|
||||
``main``, so the update's git fetch and every lazy blob fetch of the partial clone
|
||||
(``github.com``) have to cross the proxy, and the cell reads the proxy's log to show they did and
|
||||
that no release record was read.
|
||||
|
||||
Failure classes: #124022 (updater dies with ``UNEXPECTED_EOF`` when direct TLS is cut and it
|
||||
ignores the proxy), a proxy that requires auth, and a corporate root supplied via
|
||||
@@ -48,7 +48,7 @@ def _assert_updated_through_proxy(inst: S.Installed, r: S.Result, new: str) -> N
|
||||
assert r.rc == 0, "update through the proxy failed\n" + r.report(inst)
|
||||
assert inst.head() == new, f"update exited 0 but HEAD is {inst.head()}, not the published {new}\n" + r.report(inst)
|
||||
channel = [h for h in edge.proxy.requests(S.ASSETS) if h.path.startswith("/releases/channels/")]
|
||||
assert channel, "the channel record was never read through the proxy\n" + r.report(inst)
|
||||
assert not channel, f"main read a release record (it follows the git branch): {channel}\n" + r.report(inst)
|
||||
fetches = [h for h in edge.proxy.requests("github.com") if h.path.startswith(GIT_PATH) and h.status == 200]
|
||||
assert any("git-upload-pack" in h.path for h in fetches), "no git fetch crossed the proxy\n" + r.report(inst)
|
||||
stray = sorted(edge.proxy.hosts("refused") | edge.proxy.hosts("tls-rejected"))
|
||||
@@ -96,18 +96,16 @@ def test_update_with_corporate_root_only_in_ssl_cert_file(inst):
|
||||
_assert_updated_through_proxy(inst, r, new)
|
||||
|
||||
|
||||
def test_tunnel_cut_to_channel_host_fails_fast_and_changes_nothing(inst):
|
||||
"""The #124022 symptom made deterministic: the proxy accepts the CONNECT to the release
|
||||
host and hangs up before TLS (``UNEXPECTED_EOF``). The update must stop, say the channel
|
||||
could not be read, and must not silently fall through to the git branch."""
|
||||
def test_tunnel_cut_to_the_git_host_fails_fast_and_changes_nothing(inst):
|
||||
"""The #124022 symptom made deterministic: the proxy accepts the CONNECT to github.com and
|
||||
hangs up before TLS (``UNEXPECTED_EOF``). The update must stop within bounds, change
|
||||
nothing, and print no success banner."""
|
||||
inst.publish("tunnel-cut")
|
||||
before = inst.state()
|
||||
edge = inst.edge(eof_hosts=[S.ASSETS])
|
||||
edge = inst.edge(eof_hosts=["github.com"])
|
||||
try:
|
||||
r = inst.hermes("update", "--yes", edge=edge, timeout=300)
|
||||
finally:
|
||||
edge.close()
|
||||
assert r.secs < 60, f"a cut tunnel took {r.secs:.0f}s to fail\n" + r.report(inst)
|
||||
S.assert_nothing_changed(inst, before, r, "tunnel cut to the channel host")
|
||||
assert "channel" in r.out.lower() and "unavailable" in r.out.lower(), (
|
||||
"the failure does not say the release channel was unreachable\n" + r.report(inst))
|
||||
S.assert_nothing_changed(inst, before, r, "tunnel cut to the git host")
|
||||
@@ -39,3 +39,24 @@ function Get-SourceHermesForStartup([string]$Root) {
|
||||
if (Test-Path -LiteralPath $legacy -PathType Leaf) { return $legacy }
|
||||
throw "No installed Hermes command to start under $Root"
|
||||
}
|
||||
|
||||
# Official source checkouts default to the stable channel (the latest published
|
||||
# vX.Y.Z), and these fixtures publish commits on main, not releases: a fresh
|
||||
# install at HEAD is newer than the latest release and correctly waits, so the
|
||||
# HEAD -> NEXT legs would find no update. Record `main` for the install BEFORE
|
||||
# the user-state snapshot (config.yaml is part of the baseline). Releases that
|
||||
# predate --set-channel follow main already.
|
||||
function Set-SourceMainChannel([string]$Root) {
|
||||
$exe = Get-SourceHermes $Root
|
||||
$prevEap = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$help = & $exe update --help 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -ne 0) { throw "update --help failed: $help" }
|
||||
if ($help -notmatch '--set-channel') { return }
|
||||
$out = & $exe update --set-channel main 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -ne 0) { throw "could not record the main update channel: $out" }
|
||||
Write-Host " $($out.Trim())"
|
||||
} finally {
|
||||
$ErrorActionPreference = $prevEap
|
||||
}
|
||||
}
|
||||
@@ -14,3 +14,16 @@ build_source_update_command() {
|
||||
update_cmd+=(--branch main)
|
||||
fi
|
||||
}
|
||||
|
||||
# Official source checkouts default to the stable channel (the latest published
|
||||
# vX.Y.Z), and these fixtures publish commits on main, not releases: a fresh
|
||||
# install at HEAD is newer than the latest release and correctly waits, so the
|
||||
# HEAD -> NEXT legs would find no update. Record `main` for the install BEFORE
|
||||
# the user-state snapshot (config.yaml is part of the baseline). Releases that
|
||||
# predate --set-channel follow main already.
|
||||
pin_source_main_channel() {
|
||||
local hermes="$1" help
|
||||
help="$("$hermes" update --help 2>&1)" || { printf 'update --help failed: %s\n' "$help" >&2; return 1; }
|
||||
grep -qF -- --set-channel <<< "$help" || return 0
|
||||
"$hermes" update --set-channel main < /dev/null
|
||||
}
|
||||
@@ -358,6 +358,7 @@ desktop_checkpoint old "$OLD_SHA" "$INSTALL_METHOD"
|
||||
# must survive. Done as late as possible before the update so the window
|
||||
# verify() covers contains only the upgrade.
|
||||
HERMES="$(source_hermes "$INSTALL_DIR")" || fail "no installed command to drive"
|
||||
source_build_env pin_source_main_channel "$HERMES" || fail "could not record the main update channel"
|
||||
source_build_env user_state_produce "$HERMES"
|
||||
user_state_before_upgrade
|
||||
assert_redirect_is_transport_only
|
||||
|
||||
@@ -292,6 +292,9 @@ phase_update() {
|
||||
# shellcheck disable=SC1090
|
||||
. "$STATE"
|
||||
arm_redirect
|
||||
local pin_hermes
|
||||
pin_hermes="$(source_hermes "$INSTALL_DIR")" || fail "no installed command to record the update channel"
|
||||
source_build_env pin_source_main_channel "$pin_hermes" || fail "could not record the main update channel"
|
||||
# Snapshot every plugin tree BEFORE the upgrade moves anything: fixtures
|
||||
# seeded here must survive through the verify after the update lands.
|
||||
preserve_before_upgrade
|
||||
|
||||
@@ -1572,6 +1572,7 @@ function Invoke-PhaseUpdate {
|
||||
if ($Route -in @('open-app-update', 'hermes-desktop-app-update', 'desktop-installer@latest')) {
|
||||
Start-JourneyChat
|
||||
}
|
||||
Set-SourceMainChannel $InstallDir
|
||||
# Snapshot every plugin tree BEFORE the upgrade moves anything.
|
||||
Invoke-PreserveSnapshot
|
||||
# ... and the user's own durable state, produced by the install phase
|
||||
|
||||
Reference in new issue
Block a user