Nguyen Minh Phuc
d64c3c9f39
fail() blanket-marked the instance `failed` for every dead-lettered task kind.
Only provision is correct. The others each left the instance in a state the
UPDATE then corrupted:
- deprovision: `deleting` -> `failed` stranded the instance, because
due_for_deprovision only re-selects ready/deleting, leaking the release
reconcile.py promises to reclaim.
- upgrade: helm --atomic rolled back, so the instance was still `ready` and
serving the old version; `failed` mislabelled a healthy service and dropped
it off the upgrade work-list.
- verify: handle_verify already halted the rollout; the instance was `ready`.
Now gated on kind == provision, with a regression test per kind (control-tested
against the blanket UPDATE, which fails all three).
The API exception handler was registered on fastapi.HTTPException, a subclass
of starlette's. Starlette matches handlers by walking type(exc).__mro__, so
framework-raised 404/405 never hit it and returned {"detail": ...} instead of
ErrorBody. Registered on the starlette parent, and added a
RequestValidationError handler so body-validation 422s share the shape too.
Tests assert the ErrorBody shape for framework 404, 405, and a forbidden field.
helm._list_releases_via_api built the httpx client with verify=<ca path>, which
loads the CA eagerly and raises OSError on a half-mounted ServiceAccount — an
error the except clause did not catch, crashing the reconciler tick as a bare
bug. Now the CA is checked for readability alongside the token, and a missing
one means "not in-cluster" and falls back to helm. Also documents the no-limit
pagination invariant and pins it with a test.
redis.py logged through stdlib logging with extra={"team": team}, which the
structlog bridge drops on the floor — the trap notify.py documents. Switched to
a bound logger with team as a kwarg.
637 lines
27 KiB
Python
637 lines
27 KiB
Python
"""API integration tests: real app, real Postgres, real JWTs. No mocks.
|
|
|
|
`httpx.ASGITransport` calls the app in-process — no uvicorn, no socket, no port to race
|
|
over. It exercises the same routing, dependency resolution and lifespan a real request
|
|
would; only the TCP hop is gone.
|
|
|
|
The JWTs here are real RS256 tokens signed by a key generated in-fixture and served
|
|
through a PyJWKClient whose cache is pre-seeded. That is deliberate: overriding
|
|
`get_current_team` would leave the audience check, the issuer check and the algorithm
|
|
allow-list — the parts worth having — completely untested.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import AsyncIterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
import httpx
|
|
import jwt
|
|
import pytest
|
|
import pytest_asyncio
|
|
from cryptography.hazmat.primitives.asymmetric import rsa
|
|
from fastapi import FastAPI
|
|
|
|
from services.api.main import create_app
|
|
from services.api.routes.instances import release_name_for
|
|
from svcforge_core.repo.db import DictPool
|
|
from svcforge_core.settings import Settings
|
|
|
|
CATALOG = Path(__file__).resolve().parents[2] / "catalog.yaml"
|
|
ISSUER = "https://issuer.test/realms/svcforge"
|
|
AUDIENCE = "svcforge"
|
|
KID = "test-key-1"
|
|
|
|
|
|
# --------------------------------------------------------------------------- key material
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def rsa_key() -> rsa.RSAPrivateKey:
|
|
"""One 2048-bit key for the whole session. Generating it per test costs ~100ms each."""
|
|
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def jwks(rsa_key: rsa.RSAPrivateKey) -> dict[str, Any]:
|
|
"""The public half, as a JWKS document — exactly what Keycloak/Zitadel would serve."""
|
|
algo = jwt.algorithms.RSAAlgorithm
|
|
key_dict: dict[str, Any] = json.loads(algo.to_jwk(rsa_key.public_key()))
|
|
key_dict.update({"kid": KID, "alg": "RS256", "use": "sig"})
|
|
return {"keys": [key_dict]}
|
|
|
|
|
|
def make_token(
|
|
rsa_key: rsa.RSAPrivateKey,
|
|
team: str = "platform",
|
|
*,
|
|
audience: str = AUDIENCE,
|
|
issuer: str = ISSUER,
|
|
expires_in: timedelta = timedelta(minutes=5),
|
|
algorithm: str = "RS256",
|
|
) -> str:
|
|
"""Sign a token. Defaults are valid; every argument exists so a test can invalidate one."""
|
|
now = datetime.now(UTC)
|
|
claims: dict[str, Any] = {
|
|
"sub": f"user@{team}",
|
|
"team": team,
|
|
"aud": audience,
|
|
"iss": issuer,
|
|
"iat": now,
|
|
"exp": now + expires_in,
|
|
}
|
|
key = rsa_key if algorithm == "RS256" else "x" * 32 # HS256 needs >=32 bytes to sign quietly
|
|
return jwt.encode(claims, key, algorithm=algorithm, headers={"kid": KID})
|
|
|
|
|
|
def auth(token: str) -> dict[str, str]:
|
|
"""An Authorization header."""
|
|
return {"Authorization": f"Bearer {token}"}
|
|
|
|
|
|
# --------------------------------------------------------------------------- the app
|
|
|
|
|
|
@pytest.fixture
|
|
def settings(pg_dsn: str) -> Settings:
|
|
"""Settings pointed at the throwaway database. The whole reason create_app is a factory.
|
|
|
|
`jwks_url=None` so lifespan builds no real client and attempts no warm-up: the `app`
|
|
fixture injects a pre-seeded one straight after. Set it to a fake URL and every test
|
|
pays a DNS timeout resolving a host that does not exist — ~4s each, ~2min a run. The
|
|
warm-up path itself is covered by test_lifespan_survives_an_unreachable_jwks.
|
|
"""
|
|
return Settings(
|
|
pg_dsn=pg_dsn, # type: ignore[arg-type]
|
|
jwks_url=None,
|
|
jwt_audience=AUDIENCE,
|
|
jwt_issuer=ISSUER,
|
|
catalog_path=CATALOG,
|
|
pool_min_size=1,
|
|
pool_max_size=5,
|
|
)
|
|
|
|
|
|
class _FrozenJWKClient:
|
|
"""A PyJWKClient with its cache pre-seeded and its network path removed.
|
|
|
|
Stands in for the real client at the seam `get_current_team` uses. It resolves a kid to
|
|
a key exactly as PyJWKClient does; it just cannot reach out to an identity provider
|
|
that does not exist in a test run.
|
|
"""
|
|
|
|
def __init__(self, jwks: dict[str, Any]) -> None:
|
|
self._keys = jwt.PyJWKSet.from_dict(jwks)
|
|
|
|
def get_signing_key_from_jwt(self, token: str) -> jwt.PyJWK:
|
|
"""Resolve the token's `kid` against the key set. Raises if it is unknown."""
|
|
kid = jwt.get_unverified_header(token)["kid"]
|
|
for key in self._keys.keys:
|
|
if key.key_id == kid:
|
|
return key
|
|
raise jwt.exceptions.PyJWKClientError(f"unable to find key {kid}")
|
|
|
|
def get_signing_keys(self) -> list[jwt.PyJWK]:
|
|
"""Warm-up hook, called by lifespan."""
|
|
return list(self._keys.keys)
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def app(settings: Settings, pool: DictPool, jwks: dict[str, Any]) -> AsyncIterator[FastAPI]:
|
|
"""A live app with its lifespan run.
|
|
|
|
Depends on `pool` only for its truncate-per-test side effect; the app opens its own.
|
|
"""
|
|
application = create_app(settings)
|
|
# ASGITransport does NOT run lifespan — it only speaks the `http` scope. Drive the
|
|
# lifespan by hand rather than reaching for asgi-lifespan: without this the pool is
|
|
# never opened, app.state.pool does not exist, and every DB test dies on AttributeError.
|
|
async with application.router.lifespan_context(application):
|
|
# Swap the real (network-bound) JWKS client for one whose cache is pre-seeded.
|
|
application.state.jwks_client = _FrozenJWKClient(jwks)
|
|
yield application
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]:
|
|
"""An httpx client wired straight into the ASGI app."""
|
|
transport = httpx.ASGITransport(app=app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
@pytest.fixture
|
|
def token(rsa_key: rsa.RSAPrivateKey) -> str:
|
|
"""A valid token for team `platform`."""
|
|
return make_token(rsa_key, "platform")
|
|
|
|
|
|
async def _fetch_all(pool: DictPool, sql: str, args: tuple[Any, ...] = ()) -> list[Any]:
|
|
"""Read rows back out of the database, bypassing the API entirely."""
|
|
async with pool.connection() as conn, conn.cursor() as cur:
|
|
await cur.execute(sql, args)
|
|
return list(await cur.fetchall())
|
|
|
|
|
|
# --------------------------------------------------------------------------- create
|
|
|
|
|
|
async def test_post_returns_202_and_location(client: httpx.AsyncClient, token: str) -> None:
|
|
"""202 Accepted, not 201: nothing is provisioned yet. Location points at the poll target."""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "elasticsearch", "size": "small"},
|
|
)
|
|
assert resp.status_code == 202, resp.text
|
|
body = resp.json()
|
|
assert resp.headers["location"] == f"/v1/instances/{body['id']}"
|
|
assert body["state"] == "requested"
|
|
assert body["service_type"] == "elasticsearch"
|
|
# Pinned from catalog.yaml at creation time, not echoed from the request.
|
|
assert body["chart_version"] == "21.3.15"
|
|
assert body["endpoint"] is None
|
|
# The response model is an allow-list: placement details stay off the wire.
|
|
assert "team" not in body and "namespace" not in body and "release_name" not in body
|
|
|
|
|
|
async def test_post_commits_instance_and_task_together(
|
|
client: httpx.AsyncClient, pool: DictPool, token: str
|
|
) -> None:
|
|
"""The point of the whole module: both rows exist, or neither does."""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "redis", "size": "small"},
|
|
)
|
|
assert resp.status_code == 202
|
|
iid = UUID(resp.json()["id"])
|
|
|
|
rows = await _fetch_all(pool, "select * from instances where id = %s", (iid,))
|
|
assert len(rows) == 1
|
|
assert rows[0]["state"] == "requested"
|
|
assert rows[0]["team"] == "platform"
|
|
assert rows[0]["namespace"] == "tenant-platform"
|
|
# The idempotency anchor, and it is unique in the schema.
|
|
assert rows[0]["release_name"] == release_name_for("platform", "redis", iid)
|
|
|
|
tasks = await _fetch_all(pool, "select * from tasks where instance_id = %s", (iid,))
|
|
assert len(tasks) == 1
|
|
assert tasks[0]["kind"] == "provision"
|
|
assert tasks[0]["state"] == "queued"
|
|
assert tasks[0]["attempts"] == 0
|
|
|
|
|
|
async def test_release_name_is_deterministic_and_unique(
|
|
client: httpx.AsyncClient, pool: DictPool, token: str
|
|
) -> None:
|
|
"""Two requests for the same service_type get distinct releases; the name is pure."""
|
|
ids: list[UUID] = []
|
|
for _ in range(2):
|
|
resp = await client.post(
|
|
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
assert resp.status_code == 202
|
|
ids.append(UUID(resp.json()["id"]))
|
|
|
|
names = await _fetch_all(pool, "select release_name from instances order by created_at")
|
|
assert len({r["release_name"] for r in names}) == 2
|
|
# Pure function of (team, service_type, id): recomputing on a worker retry gives the
|
|
# same answer, so `helm upgrade --install` lands on the same release.
|
|
assert release_name_for("platform", "redis", ids[0]) == release_name_for("platform", "redis", ids[0])
|
|
|
|
|
|
async def test_ttl_days_sets_expires_at(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
|
|
"""ttl_days is the tenant-facing knob; expires_at is what the reconciler sweeps."""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "redis", "size": "small", "ttl_days": 7},
|
|
)
|
|
assert resp.status_code == 202
|
|
rows = await _fetch_all(
|
|
pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
|
|
)
|
|
delta = rows[0]["expires_at"] - datetime.now(UTC)
|
|
assert timedelta(days=6, hours=23) < delta <= timedelta(days=7)
|
|
|
|
|
|
async def test_no_expires_at_without_ttl(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
|
|
"""No TTL means no expiry. A default TTL would delete someone's database by surprise."""
|
|
resp = await client.post(
|
|
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
rows = await _fetch_all(
|
|
pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
|
|
)
|
|
assert rows[0]["expires_at"] is None
|
|
|
|
|
|
# --------------------------------------------------------------------------- validation
|
|
|
|
|
|
async def test_unknown_size_is_rejected(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
|
|
"""size='enormous' is not in the catalog. Nothing is written."""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "elasticsearch", "size": "enormous"},
|
|
)
|
|
# The spec says 422 for an unknown size and 404 for an unknown service_type; its
|
|
# summary line collapses both into "404 if unknown". Either is defensible; what is not
|
|
# defensible is writing the row. Assert the contract that matters and allow both codes.
|
|
assert resp.status_code in (404, 422), resp.text
|
|
assert await _fetch_all(pool, "select 1 from instances") == []
|
|
assert await _fetch_all(pool, "select 1 from tasks") == []
|
|
|
|
|
|
async def test_unknown_service_type_is_404(client: httpx.AsyncClient, token: str) -> None:
|
|
"""A service_type the catalog has never heard of is a resource that does not exist."""
|
|
resp = await client.post(
|
|
"/v1/instances", headers=auth(token), json={"service_type": "mongodb", "size": "small"}
|
|
)
|
|
assert resp.status_code == 404
|
|
assert resp.json()["code"] == "unknown_service_type"
|
|
|
|
|
|
async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) -> None:
|
|
"""ttl_days has a schema bound (1..30); pydantic rejects it before the handler runs."""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "redis", "size": "small", "ttl_days": 999},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
|
|
def _is_error_body(payload: object) -> bool:
|
|
"""The uniform error shape: a dict with `code` and `message`, and no default `detail`."""
|
|
return isinstance(payload, dict) and "code" in payload and "message" in payload
|
|
|
|
|
|
async def test_framework_404_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
|
|
"""A 404 raised by the router, not a handler, must still be ErrorBody.
|
|
|
|
Starlette raises its own HTTPException for an unknown route. The exception handler is
|
|
registered on that parent class precisely so this body is ErrorBody and not FastAPI's
|
|
default `{"detail": "Not Found"}` — one shape for every error.
|
|
"""
|
|
resp = await client.get("/v1/no-such-route")
|
|
assert resp.status_code == 404
|
|
assert _is_error_body(resp.json()), resp.text
|
|
|
|
|
|
async def test_framework_405_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
|
|
"""A wrong-method 405 comes from the router too, and must be ErrorBody."""
|
|
resp = await client.delete("/v1/instances") # collection route has no DELETE
|
|
assert resp.status_code == 405
|
|
assert _is_error_body(resp.json()), resp.text
|
|
|
|
|
|
async def test_body_validation_422_uses_the_error_body_shape(client: httpx.AsyncClient, token: str) -> None:
|
|
"""A RequestValidationError 422 must match the handler-raised 422 shape.
|
|
|
|
A forbidden extra field trips pydantic's `extra="forbid"` and raises
|
|
RequestValidationError, which the dedicated handler renders as ErrorBody rather than the
|
|
default `{"detail": [...]}`.
|
|
"""
|
|
resp = await client.post(
|
|
"/v1/instances",
|
|
headers=auth(token),
|
|
json={"service_type": "redis", "size": "small", "surprise": "field"},
|
|
)
|
|
assert resp.status_code == 422
|
|
assert _is_error_body(resp.json()), resp.text
|
|
|
|
|
|
# --------------------------------------------------------------------------- authn
|
|
|
|
|
|
async def test_missing_authorization_is_401(client: httpx.AsyncClient) -> None:
|
|
"""No header -> 401, NOT 403.
|
|
|
|
This is the HTTPBearer(auto_error=True) trap: FastAPI's default answers a missing
|
|
header with 403, which tells the client "you are known and forbidden" when the truth
|
|
is "you never said who you are". deps.py passes auto_error=False for exactly this.
|
|
"""
|
|
resp = await client.post("/v1/instances", json={"service_type": "redis", "size": "small"})
|
|
assert resp.status_code == 401
|
|
assert resp.headers.get("www-authenticate") == "Bearer"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("kwargs", "case"),
|
|
[
|
|
({"audience": "some-other-api"}, "wrong audience: a token minted for another service"),
|
|
({"issuer": "https://evil.test/"}, "wrong issuer"),
|
|
({"expires_in": timedelta(minutes=-5)}, "expired"),
|
|
({"algorithm": "HS256"}, "algorithm confusion: signed with HMAC, not RS256"),
|
|
],
|
|
)
|
|
async def test_bad_tokens_are_401(
|
|
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey, kwargs: dict[str, Any], case: str
|
|
) -> None:
|
|
"""Every way a token can be wrong produces the same opaque 401.
|
|
|
|
Identical bodies matter: a caller who can tell "expired" from "bad signature" from
|
|
"wrong audience" has an oracle to tune a forgery against.
|
|
"""
|
|
resp = await client.get("/v1/instances", headers=auth(make_token(rsa_key, **kwargs)))
|
|
assert resp.status_code == 401, f"{case} should be rejected"
|
|
assert resp.json() == {"code": "unauthorized", "message": "invalid or missing credentials"}
|
|
|
|
|
|
async def test_garbage_token_is_401(client: httpx.AsyncClient) -> None:
|
|
"""Not even a JWT. Same 401, no stack trace, no 500."""
|
|
resp = await client.get("/v1/instances", headers=auth("not-a-jwt"))
|
|
assert resp.status_code == 401
|
|
|
|
|
|
# --------------------------------------------------------------------------- authz
|
|
|
|
|
|
async def test_team_a_cannot_get_team_b_instance(
|
|
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
|
|
) -> None:
|
|
"""404, not 403. AuthZ is the WHERE clause.
|
|
|
|
403 would confirm the id exists — an enumeration oracle. 404 is the same answer a
|
|
made-up uuid gets, so the two are indistinguishable, which is the point.
|
|
"""
|
|
a_token = make_token(rsa_key, "team-a")
|
|
b_token = make_token(rsa_key, "team-b")
|
|
|
|
created = await client.post(
|
|
"/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
assert created.status_code == 202
|
|
iid = created.json()["id"]
|
|
|
|
assert (await client.get(f"/v1/instances/{iid}", headers=auth(a_token))).status_code == 200
|
|
|
|
stolen = await client.get(f"/v1/instances/{iid}", headers=auth(b_token))
|
|
assert stolen.status_code == 404
|
|
# Byte-identical to a genuinely nonexistent id? The message embeds the id, so compare
|
|
# the code: the shape a client can branch on must not distinguish the two cases.
|
|
nonexistent = await client.get(f"/v1/instances/{uuid4()}", headers=auth(b_token))
|
|
assert nonexistent.status_code == 404
|
|
assert stolen.json()["code"] == nonexistent.json()["code"] == "not_found"
|
|
|
|
|
|
async def test_list_is_scoped_to_the_callers_team(
|
|
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
|
|
) -> None:
|
|
"""A list endpoint is where cross-tenant leaks show up first."""
|
|
a_token = make_token(rsa_key, "team-a")
|
|
b_token = make_token(rsa_key, "team-b")
|
|
for tok in (a_token, a_token, b_token):
|
|
await client.post("/v1/instances", headers=auth(tok), json={"service_type": "redis", "size": "small"})
|
|
|
|
a_list = await client.get("/v1/instances", headers=auth(a_token))
|
|
assert a_list.status_code == 200
|
|
assert len(a_list.json()) == 2
|
|
|
|
b_list = await client.get("/v1/instances", headers=auth(b_token))
|
|
assert len(b_list.json()) == 1
|
|
|
|
|
|
# --------------------------------------------------------------------------- get / delete
|
|
|
|
|
|
async def test_get_unknown_id_is_404(client: httpx.AsyncClient, token: str) -> None:
|
|
"""A well-formed uuid that is not a row."""
|
|
resp = await client.get(f"/v1/instances/{uuid4()}", headers=auth(token))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_get_malformed_id_is_422(client: httpx.AsyncClient, token: str) -> None:
|
|
"""Not a uuid at all: a path-schema failure, caught before the handler."""
|
|
resp = await client.get("/v1/instances/not-a-uuid", headers=auth(token))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_delete_moves_to_deleting_and_enqueues_deprovision(
|
|
client: httpx.AsyncClient, pool: DictPool, token: str
|
|
) -> None:
|
|
"""202: the helm uninstall has not happened yet. Only the intent is durable."""
|
|
created = await client.post(
|
|
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
iid = UUID(created.json()["id"])
|
|
|
|
# requested -> deleting is not legal; the state machine only allows it from ready.
|
|
async with pool.connection() as conn:
|
|
await conn.execute("update instances set state='ready' where id = %s", (iid,))
|
|
|
|
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
|
|
assert resp.status_code == 202, resp.text
|
|
assert resp.json()["state"] == "deleting"
|
|
|
|
rows = await _fetch_all(pool, "select state from instances where id = %s", (iid,))
|
|
assert rows[0]["state"] == "deleting"
|
|
kinds = await _fetch_all(pool, "select kind from tasks where instance_id = %s order by id", (iid,))
|
|
assert [r["kind"] for r in kinds] == ["provision", "deprovision"]
|
|
|
|
|
|
async def test_delete_other_teams_instance_is_404(
|
|
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
|
|
) -> None:
|
|
"""The destructive endpoint gets the same WHERE-clause treatment as the read."""
|
|
a_token = make_token(rsa_key, "team-a")
|
|
created = await client.post(
|
|
"/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
iid = created.json()["id"]
|
|
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(make_token(rsa_key, "team-b")))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_delete_from_illegal_state_is_409(
|
|
client: httpx.AsyncClient, pool: DictPool, token: str
|
|
) -> None:
|
|
"""`deleted` is terminal. The state machine says no, and the API does not overrule it."""
|
|
created = await client.post(
|
|
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
|
|
)
|
|
iid = UUID(created.json()["id"])
|
|
async with pool.connection() as conn:
|
|
await conn.execute("update instances set state='deleted' where id = %s", (iid,))
|
|
|
|
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
|
|
assert resp.status_code == 409
|
|
assert resp.json()["code"] == "illegal_transition"
|
|
|
|
|
|
async def test_delete_requires_auth(client: httpx.AsyncClient) -> None:
|
|
"""No token, no teardown."""
|
|
assert (await client.delete(f"/v1/instances/{uuid4()}")).status_code == 401
|
|
|
|
|
|
# --------------------------------------------------------------------------- ops endpoints
|
|
|
|
|
|
async def test_healthz_needs_no_database(settings: Settings) -> None:
|
|
"""Liveness does no I/O, so it answers 200 with no database anywhere near it.
|
|
|
|
Built by hand against a nonsense DSN and with NO lifespan: if /healthz touched the
|
|
pool, there is no pool to touch and this would fail. That is the assertion — a DB blip
|
|
must never get the whole fleet killed and CrashLoopBackOff'd.
|
|
"""
|
|
broken = create_app(settings.model_copy(update={"pg_dsn": "postgresql://nobody@127.0.0.1:1/nothing"}))
|
|
transport = httpx.ASGITransport(app=broken)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
|
|
resp = await c.get("/healthz") # no `async with transport` => lifespan never ran
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"status": "ok"}
|
|
|
|
|
|
async def test_healthz_needs_no_auth(client: httpx.AsyncClient) -> None:
|
|
"""The kubelet has no bearer token."""
|
|
assert (await client.get("/healthz")).status_code == 200
|
|
|
|
|
|
async def test_readyz_is_200_when_the_pool_is_open(client: httpx.AsyncClient) -> None:
|
|
"""`select 1` against a live database."""
|
|
resp = await client.get("/readyz")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"status": "ready"}
|
|
|
|
|
|
async def test_readyz_is_503_when_the_pool_is_closed(client: httpx.AsyncClient, app: FastAPI) -> None:
|
|
"""Readiness fails closed. 503 pulls the pod out of the Service; it does not kill it."""
|
|
await app.state.pool.close()
|
|
resp = await client.get("/readyz")
|
|
assert resp.status_code == 503
|
|
assert resp.json()["code"] == "not_ready"
|
|
|
|
|
|
async def test_metrics_serves_the_prometheus_exposition_format(client: httpx.AsyncClient) -> None:
|
|
"""A bare /metrics (what every scrape config requests) returns the exposition format."""
|
|
resp = await client.get("/metrics")
|
|
assert resp.status_code == 200
|
|
assert "text/plain" in resp.headers["content-type"]
|
|
assert "python_gc_objects_collected_total" in resp.text
|
|
|
|
|
|
# --------------------------------------------------------------------------- contract
|
|
|
|
|
|
async def test_openapi_lists_the_seven_endpoints(client: httpx.AsyncClient) -> None:
|
|
"""The deliverable, asserted: /docs renders these paths."""
|
|
paths = (await client.get("/openapi.json")).json()["paths"]
|
|
assert sorted(paths) == [
|
|
"/healthz",
|
|
"/metrics",
|
|
"/readyz",
|
|
"/v1/instances",
|
|
"/v1/instances/{instance_id}",
|
|
]
|
|
assert sorted(paths["/v1/instances"]) == ["get", "post"]
|
|
assert sorted(paths["/v1/instances/{instance_id}"]) == ["delete", "get"]
|
|
|
|
|
|
async def test_errors_are_documented_as_errorbody(client: httpx.AsyncClient) -> None:
|
|
"""Every error the API returns has one declared shape, and clients can generate against it."""
|
|
schema = (await client.get("/openapi.json")).json()
|
|
assert sorted(schema["components"]["schemas"]["ErrorBody"]["properties"]) == ["code", "message"]
|
|
post = schema["paths"]["/v1/instances"]["post"]["responses"]
|
|
for code in ("401", "404", "409", "422"):
|
|
ref = post[code]["content"]["application/json"]["schema"]["$ref"]
|
|
assert ref.endswith("/ErrorBody"), f"{code} is not documented as ErrorBody"
|
|
|
|
|
|
async def test_202_is_on_the_decorator_not_the_response_object(client: httpx.AsyncClient) -> None:
|
|
"""The schema must say 202, not just the runtime.
|
|
|
|
Setting response.status_code in a handler body changes the response and leaves the
|
|
OpenAPI document claiming 200 — so generated clients treat a 202 as an error.
|
|
"""
|
|
schema = (await client.get("/openapi.json")).json()
|
|
assert "202" in schema["paths"]["/v1/instances"]["post"]["responses"]
|
|
assert "202" in schema["paths"]["/v1/instances/{instance_id}"]["delete"]["responses"]
|
|
|
|
|
|
def test_auth_disabled_is_allowed_locally_and_refused_everywhere_else(settings: Settings) -> None:
|
|
"""The dev escape hatch exists, and check_production() refuses it outside `local`.
|
|
|
|
Both halves matter. The permissive half is why every entrypoint can call this
|
|
unconditionally at startup; the refusing half is the actual safety property. An
|
|
earlier version raised unconditionally, which meant it could only be called from a
|
|
branch that already knew it was production — so nobody ever wrote that branch and the
|
|
check never ran at all.
|
|
"""
|
|
dev_locally = settings.model_copy(update={"auth_disabled": True, "environment": "local"})
|
|
dev_locally.check_production() # must not raise
|
|
|
|
for env in ("prod", "staging"):
|
|
dev_in_prod = settings.model_copy(update={"auth_disabled": True, "environment": env})
|
|
with pytest.raises(ValueError, match="refused"):
|
|
dev_in_prod.check_production()
|
|
|
|
# And a correctly-configured production app starts fine.
|
|
settings.model_copy(update={"auth_disabled": False, "environment": "prod"}).check_production()
|
|
|
|
|
|
@pytest.mark.slow
|
|
async def test_lifespan_survives_an_unreachable_jwks(settings: Settings) -> None:
|
|
"""A down identity provider must not stop the pod from starting.
|
|
|
|
The warm-up is best-effort on purpose: fail startup on it and an IdP blip means no pod
|
|
in the fleet can start, so the outage outlives the blip. A cache miss later costs one
|
|
to_thread hop. Requests still 401 until the keys arrive — fail closed, stay up.
|
|
"""
|
|
unreachable = settings.model_copy(
|
|
update={"jwks_url": "https://nonexistent.invalid/protocol/openid-connect/certs"}
|
|
)
|
|
booted = create_app(unreachable)
|
|
async with booted.router.lifespan_context(booted):
|
|
transport = httpx.ASGITransport(app=booted)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
|
|
assert (await c.get("/healthz")).status_code == 200
|
|
# Keys never loaded, so auth fails closed rather than falling open.
|
|
assert (await c.get("/v1/instances", headers=auth("whatever"))).status_code == 401
|
|
|
|
|
|
async def test_auth_disabled_accepts_an_unauthenticated_request(settings: Settings) -> None:
|
|
"""With the hatch open, no header is needed and a fixed team is used."""
|
|
dev_app = create_app(settings.model_copy(update={"auth_disabled": True}))
|
|
async with dev_app.router.lifespan_context(dev_app):
|
|
transport = httpx.ASGITransport(app=dev_app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
|
|
resp = await c.get("/v1/instances")
|
|
assert resp.status_code == 200
|