fix: three correctness bugs found in review + a dropped-log

fail() blanket-marked the instance `failed` for every dead-lettered task kind.
Only provision is correct. The others each left the instance in a state the
UPDATE then corrupted:
  - deprovision: `deleting` -> `failed` stranded the instance, because
    due_for_deprovision only re-selects ready/deleting, leaking the release
    reconcile.py promises to reclaim.
  - upgrade: helm --atomic rolled back, so the instance was still `ready` and
    serving the old version; `failed` mislabelled a healthy service and dropped
    it off the upgrade work-list.
  - verify: handle_verify already halted the rollout; the instance was `ready`.
Now gated on kind == provision, with a regression test per kind (control-tested
against the blanket UPDATE, which fails all three).

The API exception handler was registered on fastapi.HTTPException, a subclass
of starlette's. Starlette matches handlers by walking type(exc).__mro__, so
framework-raised 404/405 never hit it and returned {"detail": ...} instead of
ErrorBody. Registered on the starlette parent, and added a
RequestValidationError handler so body-validation 422s share the shape too.
Tests assert the ErrorBody shape for framework 404, 405, and a forbidden field.

helm._list_releases_via_api built the httpx client with verify=<ca path>, which
loads the CA eagerly and raises OSError on a half-mounted ServiceAccount — an
error the except clause did not catch, crashing the reconciler tick as a bare
bug. Now the CA is checked for readability alongside the token, and a missing
one means "not in-cluster" and falls back to helm. Also documents the no-limit
pagination invariant and pins it with a test.

redis.py logged through stdlib logging with extra={"team": team}, which the
structlog bridge drops on the floor — the trap notify.py documents. Switched to
a bound logger with team as a kwarg.
This commit is contained in:
Nguyen Minh Phuc
2026-07-21 01:31:11 +00:00
parent d6c1b64512
commit d64c3c9f39
7 changed files with 299 additions and 31 deletions
+90 -1
View File
@@ -21,6 +21,7 @@ from svcforge_core.adapters import helm
from svcforge_core.adapters.helm import (
MANAGED_BY_LABEL,
MANAGED_BY_VALUE,
HelmError,
HelmProvisioner,
)
from svcforge_core.domain.models import CatalogEntry, SizeSpec
@@ -131,8 +132,10 @@ def _fake_api(
token = tmp_path / "token"
token.write_text("tok", encoding="utf-8")
ca = tmp_path / "ca.crt"
ca.write_text("ca", encoding="utf-8") # a complete SA has both; the readability check needs it
monkeypatch.setattr(helm, "_SA_TOKEN", token)
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
monkeypatch.setattr(helm, "_SA_CA", ca)
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
monkeypatch.setenv("KUBERNETES_SERVICE_PORT_HTTPS", "443")
@@ -267,3 +270,89 @@ async def test_api_read_handles_kubernetes_serialising_empty_as_null(
monkeypatch.setattr(httpx, "AsyncClient", _NullClient)
assert await HelmProvisioner().list_releases() == []
@pytest.mark.asyncio
async def test_api_read_passes_tls_verify_and_timeout(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""The API client must verify against the SA CA and carry the read timeout.
A refactor that dropped `verify` to the default (or None) is a TLS regression the happy
path would not reveal, so it is pinned here off the captured client kwargs.
"""
cap = _fake_api(monkeypatch, tmp_path, [])
await HelmProvisioner().list_releases()
assert cap["client_kwargs"]["verify"] == str(helm._SA_CA)
assert cap["client_kwargs"]["timeout"] == helm._API_TIMEOUT_S
@pytest.mark.asyncio
async def test_api_read_sends_no_limit_param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""No `limit`, so the apiserver returns the full set and the single read is complete.
Pins the pagination invariant: adding `limit` without consuming `metadata.continue`
would silently truncate the release list.
"""
cap = _fake_api(monkeypatch, tmp_path, [])
await HelmProvisioner().list_releases()
assert "limit" not in cap["params"]
@pytest.mark.asyncio
async def test_api_error_raises_helmerror_and_does_not_fall_back(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A reachable-but-erroring apiserver raises HelmError; it must not shell out to helm.
Falling back would swap a visible error for the 330s helm timeout this method exists to
remove. The fallback is only for a ServiceAccount that is not present at all.
"""
(tmp_path / "ca.crt").write_text("ca", encoding="utf-8")
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
(tmp_path / "token").write_text("tok", encoding="utf-8")
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
seen = _capture(monkeypatch)
class _ErrClient:
def __init__(self, **kw: object) -> None:
pass
async def __aenter__(self) -> _ErrClient:
return self
async def __aexit__(self, *exc: object) -> None:
return None
async def get(self, url: str, **kw: object) -> object:
raise httpx.ConnectError("connection reset by peer")
monkeypatch.setattr(httpx, "AsyncClient", _ErrClient)
with pytest.raises(HelmError):
await HelmProvisioner().list_releases()
assert seen == [], "an API error must not fall back to `helm list`"
@pytest.mark.asyncio
async def test_missing_ca_falls_back_to_helm(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""A token without a readable CA is a half-mounted SA: fall back rather than crash.
httpx loads the CA when the client is built, raising OSError that the API except clause
does not catch, so the readability check has to happen before the request. A missing CA
means "not in-cluster", the same as a missing token.
"""
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
(tmp_path / "token").write_text("tok", encoding="utf-8")
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "absent-ca.crt") # never created
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
seen = _capture(monkeypatch)
await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
assert seen and seen[0][:2] == ["helm", "list"]