fix: three correctness bugs found in review + a dropped-log
fail() blanket-marked the instance `failed` for every dead-lettered task kind.
Only provision is correct. The others each left the instance in a state the
UPDATE then corrupted:
- deprovision: `deleting` -> `failed` stranded the instance, because
due_for_deprovision only re-selects ready/deleting, leaking the release
reconcile.py promises to reclaim.
- upgrade: helm --atomic rolled back, so the instance was still `ready` and
serving the old version; `failed` mislabelled a healthy service and dropped
it off the upgrade work-list.
- verify: handle_verify already halted the rollout; the instance was `ready`.
Now gated on kind == provision, with a regression test per kind (control-tested
against the blanket UPDATE, which fails all three).
The API exception handler was registered on fastapi.HTTPException, a subclass
of starlette's. Starlette matches handlers by walking type(exc).__mro__, so
framework-raised 404/405 never hit it and returned {"detail": ...} instead of
ErrorBody. Registered on the starlette parent, and added a
RequestValidationError handler so body-validation 422s share the shape too.
Tests assert the ErrorBody shape for framework 404, 405, and a forbidden field.
helm._list_releases_via_api built the httpx client with verify=<ca path>, which
loads the CA eagerly and raises OSError on a half-mounted ServiceAccount — an
error the except clause did not catch, crashing the reconciler tick as a bare
bug. Now the CA is checked for readability alongside the token, and a missing
one means "not in-cluster" and falls back to helm. Also documents the no-limit
pagination invariant and pins it with a test.
redis.py logged through stdlib logging with extra={"team": team}, which the
structlog bridge drops on the floor — the trap notify.py documents. Switched to
a bound logger with team as a kwarg.
This commit is contained in:
@@ -21,6 +21,7 @@ from svcforge_core.adapters import helm
|
||||
from svcforge_core.adapters.helm import (
|
||||
MANAGED_BY_LABEL,
|
||||
MANAGED_BY_VALUE,
|
||||
HelmError,
|
||||
HelmProvisioner,
|
||||
)
|
||||
from svcforge_core.domain.models import CatalogEntry, SizeSpec
|
||||
@@ -131,8 +132,10 @@ def _fake_api(
|
||||
|
||||
token = tmp_path / "token"
|
||||
token.write_text("tok", encoding="utf-8")
|
||||
ca = tmp_path / "ca.crt"
|
||||
ca.write_text("ca", encoding="utf-8") # a complete SA has both; the readability check needs it
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", token)
|
||||
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
|
||||
monkeypatch.setattr(helm, "_SA_CA", ca)
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_PORT_HTTPS", "443")
|
||||
|
||||
@@ -267,3 +270,89 @@ async def test_api_read_handles_kubernetes_serialising_empty_as_null(
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _NullClient)
|
||||
|
||||
assert await HelmProvisioner().list_releases() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_passes_tls_verify_and_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The API client must verify against the SA CA and carry the read timeout.
|
||||
|
||||
A refactor that dropped `verify` to the default (or None) is a TLS regression the happy
|
||||
path would not reveal, so it is pinned here off the captured client kwargs.
|
||||
"""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
|
||||
await HelmProvisioner().list_releases()
|
||||
|
||||
assert cap["client_kwargs"]["verify"] == str(helm._SA_CA)
|
||||
assert cap["client_kwargs"]["timeout"] == helm._API_TIMEOUT_S
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_sends_no_limit_param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""No `limit`, so the apiserver returns the full set and the single read is complete.
|
||||
|
||||
Pins the pagination invariant: adding `limit` without consuming `metadata.continue`
|
||||
would silently truncate the release list.
|
||||
"""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
|
||||
await HelmProvisioner().list_releases()
|
||||
|
||||
assert "limit" not in cap["params"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_error_raises_helmerror_and_does_not_fall_back(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A reachable-but-erroring apiserver raises HelmError; it must not shell out to helm.
|
||||
|
||||
Falling back would swap a visible error for the 330s helm timeout this method exists to
|
||||
remove. The fallback is only for a ServiceAccount that is not present at all.
|
||||
"""
|
||||
(tmp_path / "ca.crt").write_text("ca", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
|
||||
(tmp_path / "token").write_text("tok", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
class _ErrClient:
|
||||
def __init__(self, **kw: object) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self) -> _ErrClient:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def get(self, url: str, **kw: object) -> object:
|
||||
raise httpx.ConnectError("connection reset by peer")
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _ErrClient)
|
||||
|
||||
with pytest.raises(HelmError):
|
||||
await HelmProvisioner().list_releases()
|
||||
assert seen == [], "an API error must not fall back to `helm list`"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_ca_falls_back_to_helm(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""A token without a readable CA is a half-mounted SA: fall back rather than crash.
|
||||
|
||||
httpx loads the CA when the client is built, raising OSError that the API except clause
|
||||
does not catch, so the readability check has to happen before the request. A missing CA
|
||||
means "not in-cluster", the same as a missing token.
|
||||
"""
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
|
||||
(tmp_path / "token").write_text("tok", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "absent-ca.crt") # never created
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
|
||||
|
||||
assert seen and seen[0][:2] == ["helm", "list"]
|
||||
|
||||
Reference in New Issue
Block a user