fix: three correctness bugs found in review + a dropped-log

fail() blanket-marked the instance `failed` for every dead-lettered task kind.
Only provision is correct. The others each left the instance in a state the
UPDATE then corrupted:
  - deprovision: `deleting` -> `failed` stranded the instance, because
    due_for_deprovision only re-selects ready/deleting, leaking the release
    reconcile.py promises to reclaim.
  - upgrade: helm --atomic rolled back, so the instance was still `ready` and
    serving the old version; `failed` mislabelled a healthy service and dropped
    it off the upgrade work-list.
  - verify: handle_verify already halted the rollout; the instance was `ready`.
Now gated on kind == provision, with a regression test per kind (control-tested
against the blanket UPDATE, which fails all three).

The API exception handler was registered on fastapi.HTTPException, a subclass
of starlette's. Starlette matches handlers by walking type(exc).__mro__, so
framework-raised 404/405 never hit it and returned {"detail": ...} instead of
ErrorBody. Registered on the starlette parent, and added a
RequestValidationError handler so body-validation 422s share the shape too.
Tests assert the ErrorBody shape for framework 404, 405, and a forbidden field.

helm._list_releases_via_api built the httpx client with verify=<ca path>, which
loads the CA eagerly and raises OSError on a half-mounted ServiceAccount — an
error the except clause did not catch, crashing the reconciler tick as a bare
bug. Now the CA is checked for readability alongside the token, and a missing
one means "not in-cluster" and falls back to helm. Also documents the no-limit
pagination invariant and pins it with a test.

redis.py logged through stdlib logging with extra={"team": team}, which the
structlog bridge drops on the floor — the trap notify.py documents. Switched to
a bound logger with team as a kwarg.

tests/integration/test_api.py

@@ -298,6 +298,46 @@ async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) ->
     assert resp.status_code == 422
 
 
+def _is_error_body(payload: object) -> bool:
+    """The uniform error shape: a dict with `code` and `message`, and no default `detail`."""
+    return isinstance(payload, dict) and "code" in payload and "message" in payload
+
+
+async def test_framework_404_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
+    """A 404 raised by the router, not a handler, must still be ErrorBody.
+
+    Starlette raises its own HTTPException for an unknown route. The exception handler is
+    registered on that parent class precisely so this body is ErrorBody and not FastAPI's
+    default `{"detail": "Not Found"}` — one shape for every error.
+    """
+    resp = await client.get("/v1/no-such-route")
+    assert resp.status_code == 404
+    assert _is_error_body(resp.json()), resp.text
+
+
+async def test_framework_405_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
+    """A wrong-method 405 comes from the router too, and must be ErrorBody."""
+    resp = await client.delete("/v1/instances")  # collection route has no DELETE
+    assert resp.status_code == 405
+    assert _is_error_body(resp.json()), resp.text
+
+
+async def test_body_validation_422_uses_the_error_body_shape(client: httpx.AsyncClient, token: str) -> None:
+    """A RequestValidationError 422 must match the handler-raised 422 shape.
+
+    A forbidden extra field trips pydantic's `extra="forbid"` and raises
+    RequestValidationError, which the dedicated handler renders as ErrorBody rather than the
+    default `{"detail": [...]}`.
+    """
+    resp = await client.post(
+        "/v1/instances",
+        headers=auth(token),
+        json={"service_type": "redis", "size": "small", "surprise": "field"},
+    )
+    assert resp.status_code == 422
+    assert _is_error_body(resp.json()), resp.text
+
+
 # --------------------------------------------------------------------------- authn
 
 

tests/integration/test_tasks.py

@@ -110,6 +110,86 @@ async def test_fail_does_not_resurrect_a_deleted_instance(pool: DictPool) -> Non
     assert inst.error is None
 
 
+async def test_fail_of_deprovision_leaves_the_instance_deleting_to_be_retried(
+    pool: DictPool,
+) -> None:
+    """A dead-lettered deprovision must not strand the instance in `failed`.
+
+    The instance is `deleting`, which is one of the states that CAN legally become `failed`,
+    so the naive blanket UPDATE would move it there. due_for_deprovision only re-selects
+    `ready`(expired) and `deleting`, so `failed` would take the instance out of the recovery
+    sweep and leak the helm release forever. reconcile.py documents that a deprovision which
+    exhausts its retries stays re-enqueueable; this pins that guarantee.
+    """
+    iid = await make_instance(pool, state=InstanceState.DELETING)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.DEPROVISION)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "cluster unreachable", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.DELETING, "a stranded deprovision leaks the release"
+    assert inst.error is None
+
+
+async def test_fail_of_upgrade_leaves_a_working_instance_ready(pool: DictPool) -> None:
+    """A dead-lettered upgrade must not mark a healthy instance `failed`.
+
+    helm --atomic rolls the release back, so after a failed upgrade the instance is still
+    `ready` and serving the previous version. Marking it `failed` mislabels a working
+    service and drops it off the upgrade work-list. check_version_drift retries on the next
+    window; the dead-letter metric is the operator signal.
+    """
+    iid = await make_instance(pool, state=InstanceState.READY)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.UPGRADE)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "upgrade to 1.4.0 kept timing out", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.READY, "a failed upgrade mislabelled a healthy instance"
+    assert inst.error is None
+
+
+async def test_fail_of_verify_leaves_the_instance_ready(pool: DictPool) -> None:
+    """A dead-lettered verify must not mark the instance `failed`.
+
+    handle_verify halts the rollout for the service type; the instance itself is `ready`,
+    and drift re-provisions it if its release vanished. `failed` would take it out of both
+    recovery paths.
+    """
+    iid = await make_instance(pool, state=InstanceState.READY)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.VERIFY)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "release vanished after upgrade", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.READY
+    assert inst.error is None
+
+
 async def test_fail_truncates_error_to_2kb(pool: DictPool) -> None:
     iid = await make_instance(pool)
     repo = TaskRepo(pool)

tests/unit/test_helm_argv.py

@@ -21,6 +21,7 @@ from svcforge_core.adapters import helm
 from svcforge_core.adapters.helm import (
     MANAGED_BY_LABEL,
     MANAGED_BY_VALUE,
+    HelmError,
     HelmProvisioner,
 )
 from svcforge_core.domain.models import CatalogEntry, SizeSpec
@@ -131,8 +132,10 @@ def _fake_api(
 
     token = tmp_path / "token"
     token.write_text("tok", encoding="utf-8")
+    ca = tmp_path / "ca.crt"
+    ca.write_text("ca", encoding="utf-8")  # a complete SA has both; the readability check needs it
     monkeypatch.setattr(helm, "_SA_TOKEN", token)
-    monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
+    monkeypatch.setattr(helm, "_SA_CA", ca)
     monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
     monkeypatch.setenv("KUBERNETES_SERVICE_PORT_HTTPS", "443")
 
@@ -267,3 +270,89 @@ async def test_api_read_handles_kubernetes_serialising_empty_as_null(
     monkeypatch.setattr(httpx, "AsyncClient", _NullClient)
 
     assert await HelmProvisioner().list_releases() == []
+
+
+@pytest.mark.asyncio
+async def test_api_read_passes_tls_verify_and_timeout(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """The API client must verify against the SA CA and carry the read timeout.
+
+    A refactor that dropped `verify` to the default (or None) is a TLS regression the happy
+    path would not reveal, so it is pinned here off the captured client kwargs.
+    """
+    cap = _fake_api(monkeypatch, tmp_path, [])
+
+    await HelmProvisioner().list_releases()
+
+    assert cap["client_kwargs"]["verify"] == str(helm._SA_CA)
+    assert cap["client_kwargs"]["timeout"] == helm._API_TIMEOUT_S
+
+
+@pytest.mark.asyncio
+async def test_api_read_sends_no_limit_param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    """No `limit`, so the apiserver returns the full set and the single read is complete.
+
+    Pins the pagination invariant: adding `limit` without consuming `metadata.continue`
+    would silently truncate the release list.
+    """
+    cap = _fake_api(monkeypatch, tmp_path, [])
+
+    await HelmProvisioner().list_releases()
+
+    assert "limit" not in cap["params"]
+
+
+@pytest.mark.asyncio
+async def test_api_error_raises_helmerror_and_does_not_fall_back(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """A reachable-but-erroring apiserver raises HelmError; it must not shell out to helm.
+
+    Falling back would swap a visible error for the 330s helm timeout this method exists to
+    remove. The fallback is only for a ServiceAccount that is not present at all.
+    """
+    (tmp_path / "ca.crt").write_text("ca", encoding="utf-8")
+    monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
+    (tmp_path / "token").write_text("tok", encoding="utf-8")
+    monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
+    monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
+    seen = _capture(monkeypatch)
+
+    class _ErrClient:
+        def __init__(self, **kw: object) -> None:
+            pass
+
+        async def __aenter__(self) -> _ErrClient:
+            return self
+
+        async def __aexit__(self, *exc: object) -> None:
+            return None
+
+        async def get(self, url: str, **kw: object) -> object:
+            raise httpx.ConnectError("connection reset by peer")
+
+    monkeypatch.setattr(httpx, "AsyncClient", _ErrClient)
+
+    with pytest.raises(HelmError):
+        await HelmProvisioner().list_releases()
+    assert seen == [], "an API error must not fall back to `helm list`"
+
+
+@pytest.mark.asyncio
+async def test_missing_ca_falls_back_to_helm(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    """A token without a readable CA is a half-mounted SA: fall back rather than crash.
+
+    httpx loads the CA when the client is built, raising OSError that the API except clause
+    does not catch, so the readability check has to happen before the request. A missing CA
+    means "not in-cluster", the same as a missing token.
+    """
+    monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
+    (tmp_path / "token").write_text("tok", encoding="utf-8")
+    monkeypatch.setattr(helm, "_SA_CA", tmp_path / "absent-ca.crt")  # never created
+    monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
+    seen = _capture(monkeypatch)
+
+    await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
+
+    assert seen and seen[0][:2] == ["helm", "list"]