diff --git a/pm/environment.py b/pm/environment.py index 83c7d42461..0a12a25ef6 100644 --- a/pm/environment.py +++ b/pm/environment.py @@ -60,6 +60,15 @@ UNDATED_INDEX_HINT = ("the package index did not report upload times, which lock "dependencies needs for its 3-day release-age check. Use https://pypi.org/simple " "or a mirror that serves upload times (pip.conf index-url / UV_INDEX_URL), then retry.") +# uv's own English diagnostic. A localized Windows tail (os error 1224) rides +# along, but the marker that means "packages may be installed, only the +# optional compile step failed" is this line. #124268 +_BYTECODE_COMPILE_MARKER = "failed to bytecode-compile" +# Streamed uv output keeps only the last 2000 characters. Resolver markers are +# stitched back if a later verbose flood evicts them; the bytecode marker has +# to ride the same path or repair/bootstrap never sees the retry signal. +_STREAM_TAIL_MARKERS = _RESOLVER_MARKERS + (_BYTECODE_COMPILE_MARKER,) + class ResolutionConflict(InstallError): """uv's resolver proved the union has no valid solution.""" @@ -69,6 +78,20 @@ class BuildFailure(InstallError): """A package's build backend ran and failed.""" +def _is_bytecode_compile_failure(result: subprocess.CompletedProcess) -> bool: + """True when uv failed in the optional bytecode-compile step, not the install. + + ``--compile-bytecode`` exists so the first user-facing import does not pay + compile cost (#100461). It is not required for a usable environment. Windows + real-time AV can lock uv's temporary compile script (os error 1224) after + the packages are already on disk (#124268). + """ + if not result.returncode: + return False + text = f"{result.stderr or ''}\n{result.stdout or ''}".lower() + return _BYTECODE_COMPILE_MARKER in text + + def classify_uv_failure(stage: str, returncode: int, output: str) -> InstallError: """Turn a failed `uv ` into the right classified error. @@ -185,7 +208,7 @@ def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], # evicts it. Scan across read boundaries, never retain the full log. if not conflict: lowered = (tail + text).lower() - conflict = next((marker for marker in _RESOLVER_MARKERS if marker in lowered), "") + conflict = next((marker for marker in _STREAM_TAIL_MARKERS if marker in lowered), "") tail = (tail + text)[-2000:] output.write(text) output.flush() @@ -328,7 +351,12 @@ class PythonEnvironment: except BaseException: tail.close(False) raise - tail.close(result.returncode == 0) + if "--compile-bytecode" in args and _is_bytecode_compile_failure(result): + # Caller retries without the flag. A failure banner here + # would report a recovered install as failed. + tail.discard() + else: + tail.close(result.returncode == 0) return result return subprocess.run(command, cwd=native(cwd), env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False) @@ -398,6 +426,17 @@ class PythonEnvironment: for group in sorted(set(groups)): command += ["--group", group] result = self._run(command, cwd=source, timeout=timeout) + if _is_bytecode_compile_failure(result): + # Compile is an optimization. Retry once without the flag so a + # locked temp script cannot fail pm repair/bootstrap. Imports + # compile lazily on first use. + if self.output is not None: + self.output.write( + "Bytecode compile failed; retrying the install without it. " + "Modules will compile on first import.\n" + ) + fallback = [arg for arg in command if arg != "--compile-bytecode"] + result = self._run(fallback, cwd=source, timeout=timeout) if result.returncode: raise classify_uv_failure("sync", result.returncode, result.stderr or result.stdout) diff --git a/pm/progress.py b/pm/progress.py index 72706826d4..0b448a0ca1 100644 --- a/pm/progress.py +++ b/pm/progress.py @@ -90,6 +90,19 @@ class LiveTail: def flush(self) -> None: return None + def discard(self) -> None: + """Clear the status line without a success or failure verdict. + + Used when a bytecode-compile failure will be retried. Closing as a + failure here would report a recovered install as failed. + """ + if self._partial: + self._line(self._partial) + self._partial = "" + if self.live: + self._draw("") + self._emit("\r") + def close(self, ok: bool) -> None: if self._partial: self._line(self._partial) diff --git a/tests/pm/test_sync_bytecode_compile_fallback.py b/tests/pm/test_sync_bytecode_compile_fallback.py new file mode 100644 index 0000000000..3705db1c3a --- /dev/null +++ b/tests/pm/test_sync_bytecode_compile_fallback.py @@ -0,0 +1,211 @@ +"""uv --compile-bytecode is an optimization. A compile-step failure must not +fail the dependency sync (#124268). + +Windows real-time AV can lock uv's temporary compile script (os error 1224) +after packages are already installed. The sync has to retry once without the +flag so pm repair/bootstrap still produce a usable environment. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from pm.environment import PythonEnvironment +from pm.package import InstallError + + +_BYTECODE_STDERR = ( + "error: Failed to bytecode-compile Python file in: " + "C:\\hermes\\cache\\uv\\venv\\Lib\\site-packages\n" + " Caused by: Failed to create temporary script file\n" + " Caused by: failed to write to file " + "`C:\\hermes\\cache\\uv\\.tmpKKlVim\\pip_compileall.py`: " + "The requested operation cannot be performed on a file with a " + "user-mapped section open. (os error 1224)\n" +) + + +def _env(tmp_path: Path) -> PythonEnvironment: + return PythonEnvironment( + uv=tmp_path / "uv", + python=tmp_path / "python", + destination=tmp_path / "venv", + cache=tmp_path / "cache", + env={}, + ) + + +def _completed(args: list[str], returncode: int, stderr: str = "") -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(args, returncode, stdout="", stderr=stderr) + + +def test_sync_retries_without_bytecode_compile_when_that_step_fails(tmp_path, monkeypatch): + """A bytecode-compile failure is not an install failure. The retry omits + the flag and a clean second sync leaves the environment usable.""" + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + if "--compile-bytecode" in args: + return _completed(args, 1, _BYTECODE_STDERR) + return _completed(args, 0) + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + _env(tmp_path).sync(tmp_path) + + assert len(calls) == 2 + assert "--compile-bytecode" in calls[0] + assert "--compile-bytecode" not in calls[1] + assert calls[1][0] == "sync" + assert "--all-packages" in calls[1] + assert "--frozen" in calls[1] + + +def test_sync_does_not_retry_an_unrelated_uv_failure(tmp_path, monkeypatch): + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + return _completed(args, 1, "error: No solution found for the requested dependencies") + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + with pytest.raises(InstallError): + _env(tmp_path).sync(tmp_path) + assert len(calls) == 1 + assert "--compile-bytecode" in calls[0] + + +def test_sync_keeps_bytecode_compile_on_the_happy_path(tmp_path, monkeypatch): + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + return _completed(args, 0) + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + _env(tmp_path).sync(tmp_path, extras=("web",), groups=("dev",)) + + assert len(calls) == 1 + assert "--compile-bytecode" in calls[0] + assert "--extra" in calls[0] and "web" in calls[0] + assert "--group" in calls[0] and "dev" in calls[0] + + +def test_sync_recognizes_bytecode_failure_reported_only_on_stdout(tmp_path, monkeypatch): + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + if "--compile-bytecode" in args: + return subprocess.CompletedProcess( + args, 1, stdout="error: Failed to bytecode-compile Python file in: venv\n", stderr="" + ) + return _completed(args, 0) + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + _env(tmp_path).sync(tmp_path) + assert len(calls) == 2 + assert "--compile-bytecode" not in calls[1] + + +def test_sync_fallback_keeps_selection_flags_and_warns(tmp_path, monkeypatch): + import io + + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + if "--compile-bytecode" in args: + return _completed(args, 1, _BYTECODE_STDERR) + return _completed(args, 0) + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + notes = io.StringIO() + PythonEnvironment( + uv=tmp_path / "uv", + python=tmp_path / "python", + destination=tmp_path / "venv", + cache=tmp_path / "cache", + env={}, + output=notes, + ).sync(tmp_path, extras=("web",), no_default_groups=True) + + assert "--extra" in calls[1] and "web" in calls[1] + assert "--no-default-groups" in calls[1] + assert "--compile-bytecode" not in calls[1] + assert "retrying the install without it" in notes.getvalue() + + +def test_bytecode_compile_failure_does_not_banner_the_status_as_failed(tmp_path, monkeypatch): + """The non-verbose repair path wraps uv in LiveTail. Closing that tail as + a failure before the retry paints a recovered sync as failed.""" + import io + + notes = io.StringIO() + + def fake_stream(command, *, cwd, env, timeout, output): + output.write("error: Failed to bytecode-compile Python file in: venv\n") + return subprocess.CompletedProcess( + command, 1, "", "error: Failed to bytecode-compile Python file in: venv\n" + ) + + monkeypatch.setattr("pm.environment._run_streaming", fake_stream) + monkeypatch.setattr("pm.environment.verbose_output", lambda: False) + PythonEnvironment( + uv=tmp_path / "uv", + python=tmp_path / "python", + destination=tmp_path / "venv", + cache=tmp_path / "cache", + env={}, + output=notes, + )._run(["sync", "--frozen", "--compile-bytecode"], cwd=tmp_path, timeout=30) + + assert "✗" not in notes.getvalue() + assert "failed" not in notes.getvalue().lower() + + +def test_streaming_tail_keeps_bytecode_marker_after_later_output(tmp_path): + """Repair and bootstrap stream uv output. The diagnostic tail is 2000 + characters, and resolver markers are stitched back if verbose output + evicts them. The bytecode marker has to survive the same way or the + retry never runs on the path that actually failed (#124268).""" + import sys + + from pm.environment import _run_streaming + + code = ( + "import sys\n" + "sys.stdout.write('error: Failed to bytecode-compile Python file in: venv\\n')\n" + "sys.stdout.write('N' * 4000 + '\\n')\n" + "sys.stdout.flush()\n" + "raise SystemExit(1)\n" + ) + notes = __import__("io").StringIO() + result = _run_streaming( + [sys.executable, "-c", code], + cwd=tmp_path, + env={}, + timeout=30, + output=notes, + ) + assert result.returncode == 1 + assert "failed to bytecode-compile" in (result.stderr or "").lower() + + +def test_sync_raises_when_the_fallback_sync_also_fails(tmp_path, monkeypatch): + calls: list[list[str]] = [] + + def fake_run(self, args, *, cwd, timeout): + calls.append(list(args)) + if "--compile-bytecode" in args: + return _completed(args, 1, _BYTECODE_STDERR) + return _completed(args, 1, "error: failed to download distribution") + + monkeypatch.setattr(PythonEnvironment, "_run", fake_run) + with pytest.raises(InstallError, match="failed to download"): + _env(tmp_path).sync(tmp_path) + assert len(calls) == 2 + assert "--compile-bytecode" not in calls[1]