fix: three correctness bugs found in review + a dropped-log

fail() blanket-marked the instance `failed` for every dead-lettered task kind.
Only provision is correct. The others each left the instance in a state the
UPDATE then corrupted:
  - deprovision: `deleting` -> `failed` stranded the instance, because
    due_for_deprovision only re-selects ready/deleting, leaking the release
    reconcile.py promises to reclaim.
  - upgrade: helm --atomic rolled back, so the instance was still `ready` and
    serving the old version; `failed` mislabelled a healthy service and dropped
    it off the upgrade work-list.
  - verify: handle_verify already halted the rollout; the instance was `ready`.
Now gated on kind == provision, with a regression test per kind (control-tested
against the blanket UPDATE, which fails all three).

The API exception handler was registered on fastapi.HTTPException, a subclass
of starlette's. Starlette matches handlers by walking type(exc).__mro__, so
framework-raised 404/405 never hit it and returned {"detail": ...} instead of
ErrorBody. Registered on the starlette parent, and added a
RequestValidationError handler so body-validation 422s share the shape too.
Tests assert the ErrorBody shape for framework 404, 405, and a forbidden field.

helm._list_releases_via_api built the httpx client with verify=<ca path>, which
loads the CA eagerly and raises OSError on a half-mounted ServiceAccount — an
error the except clause did not catch, crashing the reconciler tick as a bare
bug. Now the CA is checked for readability alongside the token, and a missing
one means "not in-cluster" and falls back to helm. Also documents the no-limit
pagination invariant and pins it with a test.

redis.py logged through stdlib logging with extra={"team": team}, which the
structlog bridge drops on the floor — the trap notify.py documents. Switched to
a bound logger with team as a kwarg.

tests/integration/test_api.py

@@ -298,6 +298,46 @@ async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) ->
     assert resp.status_code == 422
 
 
+def _is_error_body(payload: object) -> bool:
+    """The uniform error shape: a dict with `code` and `message`, and no default `detail`."""
+    return isinstance(payload, dict) and "code" in payload and "message" in payload
+
+
+async def test_framework_404_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
+    """A 404 raised by the router, not a handler, must still be ErrorBody.
+
+    Starlette raises its own HTTPException for an unknown route. The exception handler is
+    registered on that parent class precisely so this body is ErrorBody and not FastAPI's
+    default `{"detail": "Not Found"}` — one shape for every error.
+    """
+    resp = await client.get("/v1/no-such-route")
+    assert resp.status_code == 404
+    assert _is_error_body(resp.json()), resp.text
+
+
+async def test_framework_405_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
+    """A wrong-method 405 comes from the router too, and must be ErrorBody."""
+    resp = await client.delete("/v1/instances")  # collection route has no DELETE
+    assert resp.status_code == 405
+    assert _is_error_body(resp.json()), resp.text
+
+
+async def test_body_validation_422_uses_the_error_body_shape(client: httpx.AsyncClient, token: str) -> None:
+    """A RequestValidationError 422 must match the handler-raised 422 shape.
+
+    A forbidden extra field trips pydantic's `extra="forbid"` and raises
+    RequestValidationError, which the dedicated handler renders as ErrorBody rather than the
+    default `{"detail": [...]}`.
+    """
+    resp = await client.post(
+        "/v1/instances",
+        headers=auth(token),
+        json={"service_type": "redis", "size": "small", "surprise": "field"},
+    )
+    assert resp.status_code == 422
+    assert _is_error_body(resp.json()), resp.text
+
+
 # --------------------------------------------------------------------------- authn
 
 

tests/integration/test_tasks.py

@@ -110,6 +110,86 @@ async def test_fail_does_not_resurrect_a_deleted_instance(pool: DictPool) -> Non
     assert inst.error is None
 
 
+async def test_fail_of_deprovision_leaves_the_instance_deleting_to_be_retried(
+    pool: DictPool,
+) -> None:
+    """A dead-lettered deprovision must not strand the instance in `failed`.
+
+    The instance is `deleting`, which is one of the states that CAN legally become `failed`,
+    so the naive blanket UPDATE would move it there. due_for_deprovision only re-selects
+    `ready`(expired) and `deleting`, so `failed` would take the instance out of the recovery
+    sweep and leak the helm release forever. reconcile.py documents that a deprovision which
+    exhausts its retries stays re-enqueueable; this pins that guarantee.
+    """
+    iid = await make_instance(pool, state=InstanceState.DELETING)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.DEPROVISION)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "cluster unreachable", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.DELETING, "a stranded deprovision leaks the release"
+    assert inst.error is None
+
+
+async def test_fail_of_upgrade_leaves_a_working_instance_ready(pool: DictPool) -> None:
+    """A dead-lettered upgrade must not mark a healthy instance `failed`.
+
+    helm --atomic rolls the release back, so after a failed upgrade the instance is still
+    `ready` and serving the previous version. Marking it `failed` mislabels a working
+    service and drops it off the upgrade work-list. check_version_drift retries on the next
+    window; the dead-letter metric is the operator signal.
+    """
+    iid = await make_instance(pool, state=InstanceState.READY)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.UPGRADE)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "upgrade to 1.4.0 kept timing out", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.READY, "a failed upgrade mislabelled a healthy instance"
+    assert inst.error is None
+
+
+async def test_fail_of_verify_leaves_the_instance_ready(pool: DictPool) -> None:
+    """A dead-lettered verify must not mark the instance `failed`.
+
+    handle_verify halts the rollout for the service type; the instance itself is `ready`,
+    and drift re-provisions it if its release vanished. `failed` would take it out of both
+    recovery paths.
+    """
+    iid = await make_instance(pool, state=InstanceState.READY)
+    tasks, instances = TaskRepo(pool), InstanceRepo(pool)
+    tid = await tasks.enqueue_standalone(iid, TaskKind.VERIFY)
+
+    claimed = await tasks.claim("w1")
+    assert claimed is not None
+    async with pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
+
+    assert await tasks.fail(tid, "release vanished after upgrade", "w1", max_attempts=5) is True
+
+    assert (await _task_row(pool, tid))["state"] == "failed"
+    inst = await instances.get(iid, team="platform")
+    assert inst is not None
+    assert inst.state is InstanceState.READY
+    assert inst.error is None
+
+
 async def test_fail_truncates_error_to_2kb(pool: DictPool) -> None:
     iid = await make_instance(pool)
     repo = TaskRepo(pool)