svcforge/tests/integration/test_api.py

"""API integration tests: real app, real Postgres, real JWTs. No mocks.

`httpx.ASGITransport` calls the app in-process — no uvicorn, no socket, no port to race
over. It exercises the same routing, dependency resolution and lifespan a real request
would; only the TCP hop is gone.

The JWTs here are real RS256 tokens signed by a key generated in-fixture and served
through a PyJWKClient whose cache is pre-seeded. That is deliberate: overriding
`get_current_team` would leave the audience check, the issuer check and the algorithm
allow-list — the parts worth having — completely untested.
"""

from __future__ import annotations

import json
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import UUID, uuid4

import httpx
import jwt
import pytest
import pytest_asyncio
from cryptography.hazmat.primitives.asymmetric import rsa
from fastapi import FastAPI

from services.api.main import create_app
from services.api.routes.instances import release_name_for
from svcforge_core.repo.db import DictPool
from svcforge_core.settings import Settings

CATALOG = Path(__file__).resolve().parents[2] / "catalog.yaml"
ISSUER = "https://issuer.test/realms/svcforge"
AUDIENCE = "svcforge"
KID = "test-key-1"


# --------------------------------------------------------------------------- key material


@pytest.fixture(scope="session")
def rsa_key() -> rsa.RSAPrivateKey:
    """One 2048-bit key for the whole session. Generating it per test costs ~100ms each."""
    return rsa.generate_private_key(public_exponent=65537, key_size=2048)


@pytest.fixture(scope="session")
def jwks(rsa_key: rsa.RSAPrivateKey) -> dict[str, Any]:
    """The public half, as a JWKS document — exactly what Keycloak/Zitadel would serve."""
    algo = jwt.algorithms.RSAAlgorithm
    key_dict: dict[str, Any] = json.loads(algo.to_jwk(rsa_key.public_key()))
    key_dict.update({"kid": KID, "alg": "RS256", "use": "sig"})
    return {"keys": [key_dict]}


def make_token(
    rsa_key: rsa.RSAPrivateKey,
    team: str = "platform",
    *,
    audience: str = AUDIENCE,
    issuer: str = ISSUER,
    expires_in: timedelta = timedelta(minutes=5),
    algorithm: str = "RS256",
) -> str:
    """Sign a token. Defaults are valid; every argument exists so a test can invalidate one."""
    now = datetime.now(UTC)
    claims: dict[str, Any] = {
        "sub": f"user@{team}",
        "team": team,
        "aud": audience,
        "iss": issuer,
        "iat": now,
        "exp": now + expires_in,
    }
    key = rsa_key if algorithm == "RS256" else "x" * 32  # HS256 needs >=32 bytes to sign quietly
    return jwt.encode(claims, key, algorithm=algorithm, headers={"kid": KID})


def auth(token: str) -> dict[str, str]:
    """An Authorization header."""
    return {"Authorization": f"Bearer {token}"}


# --------------------------------------------------------------------------- the app


@pytest.fixture
def settings(pg_dsn: str) -> Settings:
    """Settings pointed at the throwaway database. The whole reason create_app is a factory.

    `jwks_url=None` so lifespan builds no real client and attempts no warm-up: the `app`
    fixture injects a pre-seeded one straight after. Set it to a fake URL and every test
    pays a DNS timeout resolving a host that does not exist — ~4s each, ~2min a run. The
    warm-up path itself is covered by test_lifespan_survives_an_unreachable_jwks.
    """
    return Settings(
        pg_dsn=pg_dsn,  # type: ignore[arg-type]
        jwks_url=None,
        jwt_audience=AUDIENCE,
        jwt_issuer=ISSUER,
        catalog_path=CATALOG,
        pool_min_size=1,
        pool_max_size=5,
    )


class _FrozenJWKClient:
    """A PyJWKClient with its cache pre-seeded and its network path removed.

    Stands in for the real client at the seam `get_current_team` uses. It resolves a kid to
    a key exactly as PyJWKClient does; it just cannot reach out to an identity provider
    that does not exist in a test run.
    """

    def __init__(self, jwks: dict[str, Any]) -> None:
        self._keys = jwt.PyJWKSet.from_dict(jwks)

    def get_signing_key_from_jwt(self, token: str) -> jwt.PyJWK:
        """Resolve the token's `kid` against the key set. Raises if it is unknown."""
        kid = jwt.get_unverified_header(token)["kid"]
        for key in self._keys.keys:
            if key.key_id == kid:
                return key
        raise jwt.exceptions.PyJWKClientError(f"unable to find key {kid}")

    def get_signing_keys(self) -> list[jwt.PyJWK]:
        """Warm-up hook, called by lifespan."""
        return list(self._keys.keys)


@pytest_asyncio.fixture
async def app(settings: Settings, pool: DictPool, jwks: dict[str, Any]) -> AsyncIterator[FastAPI]:
    """A live app with its lifespan run.

    Depends on `pool` only for its truncate-per-test side effect; the app opens its own.
    """
    application = create_app(settings)
    # ASGITransport does NOT run lifespan — it only speaks the `http` scope. Drive the
    # lifespan by hand rather than reaching for asgi-lifespan: without this the pool is
    # never opened, app.state.pool does not exist, and every DB test dies on AttributeError.
    async with application.router.lifespan_context(application):
        # Swap the real (network-bound) JWKS client for one whose cache is pre-seeded.
        application.state.jwks_client = _FrozenJWKClient(jwks)
        yield application


@pytest_asyncio.fixture
async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]:
    """An httpx client wired straight into the ASGI app."""
    transport = httpx.ASGITransport(app=app)
    async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
        yield c


@pytest.fixture
def token(rsa_key: rsa.RSAPrivateKey) -> str:
    """A valid token for team `platform`."""
    return make_token(rsa_key, "platform")


async def _fetch_all(pool: DictPool, sql: str, args: tuple[Any, ...] = ()) -> list[Any]:
    """Read rows back out of the database, bypassing the API entirely."""
    async with pool.connection() as conn, conn.cursor() as cur:
        await cur.execute(sql, args)
        return list(await cur.fetchall())


# --------------------------------------------------------------------------- create


async def test_post_returns_202_and_location(client: httpx.AsyncClient, token: str) -> None:
    """202 Accepted, not 201: nothing is provisioned yet. Location points at the poll target."""
    resp = await client.post(
        "/v1/instances",
        headers=auth(token),
        json={"service_type": "elasticsearch", "size": "small"},
    )
    assert resp.status_code == 202, resp.text
    body = resp.json()
    assert resp.headers["location"] == f"/v1/instances/{body['id']}"
    assert body["state"] == "requested"
    assert body["service_type"] == "elasticsearch"
    # Pinned from catalog.yaml at creation time, not echoed from the request.
    assert body["chart_version"] == "21.3.15"
    assert body["endpoint"] is None
    # The response model is an allow-list: placement details stay off the wire.
    assert "team" not in body and "namespace" not in body and "release_name" not in body


async def test_post_commits_instance_and_task_together(
    client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
    """The point of the whole module: both rows exist, or neither does."""
    resp = await client.post(
        "/v1/instances",
        headers=auth(token),
        json={"service_type": "redis", "size": "small"},
    )
    assert resp.status_code == 202
    iid = UUID(resp.json()["id"])

    rows = await _fetch_all(pool, "select * from instances where id = %s", (iid,))
    assert len(rows) == 1
    assert rows[0]["state"] == "requested"
    assert rows[0]["team"] == "platform"
    assert rows[0]["namespace"] == "tenant-platform"
    # The idempotency anchor, and it is unique in the schema.
    assert rows[0]["release_name"] == release_name_for("platform", "redis", iid)

    tasks = await _fetch_all(pool, "select * from tasks where instance_id = %s", (iid,))
    assert len(tasks) == 1
    assert tasks[0]["kind"] == "provision"
    assert tasks[0]["state"] == "queued"
    assert tasks[0]["attempts"] == 0


async def test_release_name_is_deterministic_and_unique(
    client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
    """Two requests for the same service_type get distinct releases; the name is pure."""
    ids: list[UUID] = []
    for _ in range(2):
        resp = await client.post(
            "/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
        )
        assert resp.status_code == 202
        ids.append(UUID(resp.json()["id"]))

    names = await _fetch_all(pool, "select release_name from instances order by created_at")
    assert len({r["release_name"] for r in names}) == 2
    # Pure function of (team, service_type, id): recomputing on a worker retry gives the
    # same answer, so `helm upgrade --install` lands on the same release.
    assert release_name_for("platform", "redis", ids[0]) == release_name_for("platform", "redis", ids[0])


async def test_ttl_days_sets_expires_at(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
    """ttl_days is the tenant-facing knob; expires_at is what the reconciler sweeps."""
    resp = await client.post(
        "/v1/instances",
        headers=auth(token),
        json={"service_type": "redis", "size": "small", "ttl_days": 7},
    )
    assert resp.status_code == 202
    rows = await _fetch_all(
        pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
    )
    delta = rows[0]["expires_at"] - datetime.now(UTC)
    assert timedelta(days=6, hours=23) < delta <= timedelta(days=7)


async def test_no_expires_at_without_ttl(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
    """No TTL means no expiry. A default TTL would delete someone's database by surprise."""
    resp = await client.post(
        "/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
    )
    rows = await _fetch_all(
        pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
    )
    assert rows[0]["expires_at"] is None


# --------------------------------------------------------------------------- validation


async def test_unknown_size_is_rejected(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
    """size='enormous' is not in the catalog. Nothing is written."""
    resp = await client.post(
        "/v1/instances",
        headers=auth(token),
        json={"service_type": "elasticsearch", "size": "enormous"},
    )
    # The spec says 422 for an unknown size and 404 for an unknown service_type; its
    # summary line collapses both into "404 if unknown". Either is defensible; what is not
    # defensible is writing the row. Assert the contract that matters and allow both codes.
    assert resp.status_code in (404, 422), resp.text
    assert await _fetch_all(pool, "select 1 from instances") == []
    assert await _fetch_all(pool, "select 1 from tasks") == []


async def test_unknown_service_type_is_404(client: httpx.AsyncClient, token: str) -> None:
    """A service_type the catalog has never heard of is a resource that does not exist."""
    resp = await client.post(
        "/v1/instances", headers=auth(token), json={"service_type": "mongodb", "size": "small"}
    )
    assert resp.status_code == 404
    assert resp.json()["code"] == "unknown_service_type"


async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) -> None:
    """ttl_days has a schema bound (1..30); pydantic rejects it before the handler runs."""
    resp = await client.post(
        "/v1/instances",
        headers=auth(token),
        json={"service_type": "redis", "size": "small", "ttl_days": 999},
    )
    assert resp.status_code == 422


# --------------------------------------------------------------------------- authn


async def test_missing_authorization_is_401(client: httpx.AsyncClient) -> None:
    """No header -> 401, NOT 403.

    This is the HTTPBearer(auto_error=True) trap: FastAPI's default answers a missing
    header with 403, which tells the client "you are known and forbidden" when the truth
    is "you never said who you are". deps.py passes auto_error=False for exactly this.
    """
    resp = await client.post("/v1/instances", json={"service_type": "redis", "size": "small"})
    assert resp.status_code == 401
    assert resp.headers.get("www-authenticate") == "Bearer"


@pytest.mark.parametrize(
    ("kwargs", "case"),
    [
        ({"audience": "some-other-api"}, "wrong audience: a token minted for another service"),
        ({"issuer": "https://evil.test/"}, "wrong issuer"),
        ({"expires_in": timedelta(minutes=-5)}, "expired"),
        ({"algorithm": "HS256"}, "algorithm confusion: signed with HMAC, not RS256"),
    ],
)
async def test_bad_tokens_are_401(
    client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey, kwargs: dict[str, Any], case: str
) -> None:
    """Every way a token can be wrong produces the same opaque 401.

    Identical bodies matter: a caller who can tell "expired" from "bad signature" from
    "wrong audience" has an oracle to tune a forgery against.
    """
    resp = await client.get("/v1/instances", headers=auth(make_token(rsa_key, **kwargs)))
    assert resp.status_code == 401, f"{case} should be rejected"
    assert resp.json() == {"code": "unauthorized", "message": "invalid or missing credentials"}


async def test_garbage_token_is_401(client: httpx.AsyncClient) -> None:
    """Not even a JWT. Same 401, no stack trace, no 500."""
    resp = await client.get("/v1/instances", headers=auth("not-a-jwt"))
    assert resp.status_code == 401


# --------------------------------------------------------------------------- authz


async def test_team_a_cannot_get_team_b_instance(
    client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
    """404, not 403. AuthZ is the WHERE clause.

    403 would confirm the id exists — an enumeration oracle. 404 is the same answer a
    made-up uuid gets, so the two are indistinguishable, which is the point.
    """
    a_token = make_token(rsa_key, "team-a")
    b_token = make_token(rsa_key, "team-b")

    created = await client.post(
        "/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
    )
    assert created.status_code == 202
    iid = created.json()["id"]

    assert (await client.get(f"/v1/instances/{iid}", headers=auth(a_token))).status_code == 200

    stolen = await client.get(f"/v1/instances/{iid}", headers=auth(b_token))
    assert stolen.status_code == 404
    # Byte-identical to a genuinely nonexistent id? The message embeds the id, so compare
    # the code: the shape a client can branch on must not distinguish the two cases.
    nonexistent = await client.get(f"/v1/instances/{uuid4()}", headers=auth(b_token))
    assert nonexistent.status_code == 404
    assert stolen.json()["code"] == nonexistent.json()["code"] == "not_found"


async def test_list_is_scoped_to_the_callers_team(
    client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
    """A list endpoint is where cross-tenant leaks show up first."""
    a_token = make_token(rsa_key, "team-a")
    b_token = make_token(rsa_key, "team-b")
    for tok in (a_token, a_token, b_token):
        await client.post("/v1/instances", headers=auth(tok), json={"service_type": "redis", "size": "small"})

    a_list = await client.get("/v1/instances", headers=auth(a_token))
    assert a_list.status_code == 200
    assert len(a_list.json()) == 2

    b_list = await client.get("/v1/instances", headers=auth(b_token))
    assert len(b_list.json()) == 1


# --------------------------------------------------------------------------- get / delete


async def test_get_unknown_id_is_404(client: httpx.AsyncClient, token: str) -> None:
    """A well-formed uuid that is not a row."""
    resp = await client.get(f"/v1/instances/{uuid4()}", headers=auth(token))
    assert resp.status_code == 404


async def test_get_malformed_id_is_422(client: httpx.AsyncClient, token: str) -> None:
    """Not a uuid at all: a path-schema failure, caught before the handler."""
    resp = await client.get("/v1/instances/not-a-uuid", headers=auth(token))
    assert resp.status_code == 422


async def test_delete_moves_to_deleting_and_enqueues_deprovision(
    client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
    """202: the helm uninstall has not happened yet. Only the intent is durable."""
    created = await client.post(
        "/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
    )
    iid = UUID(created.json()["id"])

    # requested -> deleting is not legal; the state machine only allows it from ready.
    async with pool.connection() as conn:
        await conn.execute("update instances set state='ready' where id = %s", (iid,))

    resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
    assert resp.status_code == 202, resp.text
    assert resp.json()["state"] == "deleting"

    rows = await _fetch_all(pool, "select state from instances where id = %s", (iid,))
    assert rows[0]["state"] == "deleting"
    kinds = await _fetch_all(pool, "select kind from tasks where instance_id = %s order by id", (iid,))
    assert [r["kind"] for r in kinds] == ["provision", "deprovision"]


async def test_delete_other_teams_instance_is_404(
    client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
    """The destructive endpoint gets the same WHERE-clause treatment as the read."""
    a_token = make_token(rsa_key, "team-a")
    created = await client.post(
        "/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
    )
    iid = created.json()["id"]
    resp = await client.delete(f"/v1/instances/{iid}", headers=auth(make_token(rsa_key, "team-b")))
    assert resp.status_code == 404


async def test_delete_from_illegal_state_is_409(
    client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
    """`deleted` is terminal. The state machine says no, and the API does not overrule it."""
    created = await client.post(
        "/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
    )
    iid = UUID(created.json()["id"])
    async with pool.connection() as conn:
        await conn.execute("update instances set state='deleted' where id = %s", (iid,))

    resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
    assert resp.status_code == 409
    assert resp.json()["code"] == "illegal_transition"


async def test_delete_requires_auth(client: httpx.AsyncClient) -> None:
    """No token, no teardown."""
    assert (await client.delete(f"/v1/instances/{uuid4()}")).status_code == 401


# --------------------------------------------------------------------------- ops endpoints


async def test_healthz_needs_no_database(settings: Settings) -> None:
    """Liveness does no I/O, so it answers 200 with no database anywhere near it.

    Built by hand against a nonsense DSN and with NO lifespan: if /healthz touched the
    pool, there is no pool to touch and this would fail. That is the assertion — a DB blip
    must never get the whole fleet killed and CrashLoopBackOff'd.
    """
    broken = create_app(settings.model_copy(update={"pg_dsn": "postgresql://nobody@127.0.0.1:1/nothing"}))
    transport = httpx.ASGITransport(app=broken)
    async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
        resp = await c.get("/healthz")  # no `async with transport` => lifespan never ran
    assert resp.status_code == 200
    assert resp.json() == {"status": "ok"}


async def test_healthz_needs_no_auth(client: httpx.AsyncClient) -> None:
    """The kubelet has no bearer token."""
    assert (await client.get("/healthz")).status_code == 200


async def test_readyz_is_200_when_the_pool_is_open(client: httpx.AsyncClient) -> None:
    """`select 1` against a live database."""
    resp = await client.get("/readyz")
    assert resp.status_code == 200
    assert resp.json() == {"status": "ready"}


async def test_readyz_is_503_when_the_pool_is_closed(client: httpx.AsyncClient, app: FastAPI) -> None:
    """Readiness fails closed. 503 pulls the pod out of the Service; it does not kill it."""
    await app.state.pool.close()
    resp = await client.get("/readyz")
    assert resp.status_code == 503
    assert resp.json()["code"] == "not_ready"


async def test_metrics_serves_the_prometheus_exposition_format(client: httpx.AsyncClient) -> None:
    """A bare /metrics (what every scrape config requests) returns the exposition format."""
    resp = await client.get("/metrics")
    assert resp.status_code == 200
    assert "text/plain" in resp.headers["content-type"]
    assert "python_gc_objects_collected_total" in resp.text


# --------------------------------------------------------------------------- contract


async def test_openapi_lists_the_seven_endpoints(client: httpx.AsyncClient) -> None:
    """The deliverable, asserted: /docs renders these paths."""
    paths = (await client.get("/openapi.json")).json()["paths"]
    assert sorted(paths) == [
        "/healthz",
        "/metrics",
        "/readyz",
        "/v1/instances",
        "/v1/instances/{instance_id}",
    ]
    assert sorted(paths["/v1/instances"]) == ["get", "post"]
    assert sorted(paths["/v1/instances/{instance_id}"]) == ["delete", "get"]


async def test_errors_are_documented_as_errorbody(client: httpx.AsyncClient) -> None:
    """Every error the API returns has one declared shape, and clients can generate against it."""
    schema = (await client.get("/openapi.json")).json()
    assert sorted(schema["components"]["schemas"]["ErrorBody"]["properties"]) == ["code", "message"]
    post = schema["paths"]["/v1/instances"]["post"]["responses"]
    for code in ("401", "404", "409", "422"):
        ref = post[code]["content"]["application/json"]["schema"]["$ref"]
        assert ref.endswith("/ErrorBody"), f"{code} is not documented as ErrorBody"


async def test_202_is_on_the_decorator_not_the_response_object(client: httpx.AsyncClient) -> None:
    """The schema must say 202, not just the runtime.

    Setting response.status_code in a handler body changes the response and leaves the
    OpenAPI document claiming 200 — so generated clients treat a 202 as an error.
    """
    schema = (await client.get("/openapi.json")).json()
    assert "202" in schema["paths"]["/v1/instances"]["post"]["responses"]
    assert "202" in schema["paths"]["/v1/instances/{instance_id}"]["delete"]["responses"]


def test_auth_disabled_is_allowed_locally_and_refused_everywhere_else(settings: Settings) -> None:
    """The dev escape hatch exists, and check_production() refuses it outside `local`.

    Both halves matter. The permissive half is why every entrypoint can call this
    unconditionally at startup; the refusing half is the actual safety property. An
    earlier version raised unconditionally, which meant it could only be called from a
    branch that already knew it was production — so nobody ever wrote that branch and the
    check never ran at all.
    """
    dev_locally = settings.model_copy(update={"auth_disabled": True, "environment": "local"})
    dev_locally.check_production()  # must not raise

    for env in ("prod", "staging"):
        dev_in_prod = settings.model_copy(update={"auth_disabled": True, "environment": env})
        with pytest.raises(ValueError, match="refused"):
            dev_in_prod.check_production()

    # And a correctly-configured production app starts fine.
    settings.model_copy(update={"auth_disabled": False, "environment": "prod"}).check_production()


@pytest.mark.slow
async def test_lifespan_survives_an_unreachable_jwks(settings: Settings) -> None:
    """A down identity provider must not stop the pod from starting.

    The warm-up is best-effort on purpose: fail startup on it and an IdP blip means no pod
    in the fleet can start, so the outage outlives the blip. A cache miss later costs one
    to_thread hop. Requests still 401 until the keys arrive — fail closed, stay up.
    """
    unreachable = settings.model_copy(
        update={"jwks_url": "https://nonexistent.invalid/protocol/openid-connect/certs"}
    )
    booted = create_app(unreachable)
    async with booted.router.lifespan_context(booted):
        transport = httpx.ASGITransport(app=booted)
        async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
            assert (await c.get("/healthz")).status_code == 200
            # Keys never loaded, so auth fails closed rather than falling open.
            assert (await c.get("/v1/instances", headers=auth("whatever"))).status_code == 401


async def test_auth_disabled_accepts_an_unauthenticated_request(settings: Settings) -> None:
    """With the hatch open, no header is needed and a fixed team is used."""
    dev_app = create_app(settings.model_copy(update={"auth_disabled": True}))
    async with dev_app.router.lifespan_context(dev_app):
        transport = httpx.ASGITransport(app=dev_app)
        async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
            resp = await c.get("/v1/instances")
    assert resp.status_code == 200