svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

services/api/routes/__init__.py

@@ -0,0 +1 @@
+"""HTTP routers."""

services/api/routes/health.py

@@ -0,0 +1,81 @@
+"""Liveness, readiness, metrics.
+
+The distinction between the first two is not pedantry, it is the difference between a
+30-second blip and a fleet-wide outage:
+
+* `/healthz` (liveness) answers "is this process wedged?" A failure here gets the
+  container KILLED. It must therefore touch NOTHING external. Wire it to the DB and a
+  20-second Postgres failover restarts every pod at once; they come back, find the DB
+  still down, and CrashLoopBackOff with exponential restart delays — so the fleet is now
+  down for minutes after the database recovered.
+* `/readyz` (readiness) answers "should this pod get traffic?" A failure here only removes
+  it from the Service endpoints. It is allowed to check dependencies, and it recovers by
+  itself the moment the check passes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Request, Response, status
+from prometheus_client import REGISTRY
+from prometheus_client.exposition import choose_encoder
+
+from services.api.deps import PoolDep
+from services.api.models import ErrorBody
+
+router = APIRouter(tags=["ops"])
+
+# No PROMETHEUS_MULTIPROC_DIR here, deliberately: it exists for prefork servers where each
+# worker process holds a slice of the counters. One uvicorn process per container means
+# the default in-process registry is already correct, and multiproc mode would add a
+# shared temp dir, a cleanup obligation, and a class of stale-file bugs for nothing.
+
+
+@router.get("/healthz", status_code=status.HTTP_200_OK)
+async def healthz() -> dict[str, str]:
+    """Liveness. No I/O. If the event loop can run this, the process is alive."""
+    return {"status": "ok"}
+
+
+@router.get(
+    "/readyz",
+    responses={503: {"model": ErrorBody, "description": "A dependency is unavailable"}},
+)
+async def readyz(pool: PoolDep) -> dict[str, str]:
+    """Readiness. Postgres only.
+
+    Postgres-only is the rule, and Redis is the temptation. Redis holds derived state —
+    rate-limit buckets, caches — and everything degrades gracefully without it. Put it in
+    this check and an Upstash hiccup marks every pod unready, Kubernetes empties the
+    Service, and a cache outage becomes a total API outage.
+    """
+    try:
+        async with pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute("select 1")
+            row: Any = await cur.fetchone()
+        if row is None:
+            raise RuntimeError("select 1 returned no row")
+    except Exception as exc:  # closed pool, timeout, dead DB — all mean the same 'not ready'
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={"code": "not_ready", "message": "database unavailable"},
+        ) from exc
+    return {"status": "ready"}
+
+
+@router.get("/metrics", response_class=Response)
+async def metrics(request: Request) -> Response:
+    """The Prometheus scrape endpoint.
+
+    A route rather than `app.mount("/metrics", make_asgi_app())`, for two reasons. A
+    Starlette `Mount` compiles to `^/metrics(?P<path>/.*)$`, which does not match a bare
+    `/metrics` — the exact URL every scrape config uses — and a `Mount` is invisible to
+    OpenAPI, while the deliverable asks for `/metrics` in `openapi.json`.
+
+    The encoding is still prometheus_client's: `choose_encoder` reads the Accept header and
+    picks the exposition format (Prometheus text vs OpenMetrics) with its matching content
+    type. Hand-rolling either is how you end up serving text/plain that a scraper rejects.
+    """
+    encoder, content_type = choose_encoder(request.headers.get("Accept", ""))
+    return Response(content=encoder(REGISTRY), media_type=content_type)

services/api/routes/instances.py

@@ -0,0 +1,218 @@
+"""The tenant-facing API.
+
+Two rules run through every handler here:
+
+* **AuthZ is the WHERE clause.** No handler ever compares `inst.team` to the caller's
+  team, because the repo never returns another team's row to compare. A wrong-team id is
+  a 404. 403 would confirm the id exists, which is the leak.
+* **The instance and its task commit together.** A committed instance with no task is an
+  instance that never provisions and that nothing will ever retry.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import UUID, uuid4
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
+
+from services.api.deps import (
+    CatalogDep,
+    InstanceRepoDep,
+    PoolDep,
+    TaskRepoDep,
+    TeamDep,
+    idempotency_key,
+    rate_limit,
+)
+from services.api.models import CreateInstanceRequest, ErrorBody, InstanceResponse
+from svcforge_core.domain.models import CatalogEntry, Instance, TaskKind
+from svcforge_core.domain.states import IllegalTransition, InstanceState, transition
+
+# Declared on the router so every error shape lands in openapi.json under ErrorBody.
+# The exception handler already renders this at runtime; without declaring it, generated
+# clients see the contract for 2xx only and invent their own guess for the rest.
+ERROR_RESPONSES: dict[int | str, dict[str, Any]] = {
+    401: {"model": ErrorBody, "description": "Missing or invalid credentials"},
+    404: {"model": ErrorBody, "description": "No such instance, or not this team's"},
+    409: {"model": ErrorBody, "description": "Instance is not in a state that allows this"},
+    422: {"model": ErrorBody, "description": "The body is well-formed but cannot be processed"},
+}
+
+router = APIRouter(prefix="/v1/instances", tags=["instances"], responses=ERROR_RESPONSES)
+
+
+def release_name_for(team: str, service_type: str, instance_id: UUID) -> str:
+    """The helm release name. Deterministic, and `unique` in the schema.
+
+    This is the idempotency anchor. A worker that dies after `helm install` but before it
+    marks the task done will retry, compute the same name, and `helm upgrade --install`
+    onto the same release instead of creating a second one. Derive it from anything that
+    is not already durable — a timestamp, a random suffix, the retry count — and a retry
+    provisions a duplicate.
+
+    Truncated to the uuid's first 8 chars to stay inside the 53-char limit helm imposes
+    on release names (Kubernetes label values, minus room for chart-generated suffixes).
+    """
+    return f"{team}-{service_type}-{str(instance_id)[:8]}"
+
+
+def namespace_for(team: str) -> str:
+    """One namespace per tenant. The blast radius of a bad chart is one team."""
+    return f"tenant-{team}"
+
+
+def _resolve(catalog: dict[str, CatalogEntry], service_type: str, size: str) -> CatalogEntry:
+    """Look up service_type + size, or raise the right 4xx.
+
+    The two failures are different HTTP problems and the spec asks for different codes:
+    an unknown service_type is a resource that does not exist (404); an unknown size for a
+    real service_type is a body the server understood and cannot process (422).
+    """
+    entry = catalog.get(service_type)
+    if entry is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={
+                "code": "unknown_service_type",
+                "message": f"no such service_type: {service_type}",
+            },
+        )
+    if size not in entry.sizes:
+        raise HTTPException(
+            status_code=422,  # starlette renamed the 422 constant; the number never moved
+            detail={
+                "code": "unknown_size",
+                "message": f"{service_type} has no size {size!r}; available: {sorted(entry.sizes)}",
+            },
+        )
+    return entry
+
+
+@router.post(
+    "",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=InstanceResponse,
+    dependencies=[Depends(rate_limit), Depends(idempotency_key)],
+)
+async def create_instance(
+    body: CreateInstanceRequest,
+    response: Response,
+    team: TeamDep,
+    pool: PoolDep,
+    instances: InstanceRepoDep,
+    tasks: TaskRepoDep,
+    catalog: CatalogDep,
+) -> Instance:
+    """Accept a provisioning request. 202, never 201.
+
+    Nothing is provisioned when this returns. The row exists and a task is queued; a
+    worker will do the work seconds or minutes from now. 201 Created would be a lie about
+    a resource that does not exist yet, and clients would stop polling.
+    """
+    entry = _resolve(catalog, body.service_type, body.size)
+
+    instance_id = uuid4()
+    now = datetime.now(UTC)
+    inst = Instance(
+        id=instance_id,
+        team=team,
+        service_type=body.service_type,
+        size=body.size,
+        state=InstanceState.REQUESTED,
+        namespace=namespace_for(team),
+        release_name=release_name_for(team, body.service_type, instance_id),
+        # Pinned from the catalog AT CREATION TIME, not read from the catalog later.
+        # This column records what is actually deployed; bumping catalog.yaml must show up
+        # as drift the reconciler can see, not silently rewrite history.
+        chart_version=entry.chart_version,
+        expires_at=now + timedelta(days=body.ttl_days) if body.ttl_days is not None else None,
+        created_at=now,
+        updated_at=now,
+    )
+
+    # The transaction. Both writes go through THIS conn, or the atomicity is theatre.
+    async with pool.connection() as conn, conn.transaction():
+        created = await instances.create(conn, inst)
+        await tasks.enqueue(conn, created.id, TaskKind.PROVISION)
+
+    response.headers["Location"] = f"/v1/instances/{created.id}"
+    return created
+
+
+@router.get("/{instance_id}", response_model=InstanceResponse)
+async def get_instance(
+    instance_id: UUID,
+    team: TeamDep,
+    instances: InstanceRepoDep,
+) -> Instance:
+    """404 if the repo returns None. A wrong-team id is a 404, not a 403."""
+    inst = await instances.get(instance_id, team)
+    if inst is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
+        )
+    return inst
+
+
+@router.get("", response_model=list[InstanceResponse])
+async def list_instances(
+    team: TeamDep,
+    instances: InstanceRepoDep,
+    limit: int = Query(default=50, ge=1, le=200),
+) -> list[Instance]:
+    """The caller's instances, newest first. Bounded: no endpoint returns 'all rows'."""
+    return await instances.list(team, limit=limit)
+
+
+@router.delete(
+    "/{instance_id}",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=InstanceResponse,
+    dependencies=[Depends(rate_limit)],
+)
+async def delete_instance(
+    instance_id: UUID,
+    team: TeamDep,
+    instances: InstanceRepoDep,
+    tasks: TaskRepoDep,
+) -> Instance:
+    """state -> deleting, enqueue deprovision. 202: the helm uninstall has not happened yet.
+
+    Ordering note. `InstanceRepo.update_state` owns its own connection, so the CAS and the
+    enqueue cannot share one transaction without reaching around the repo. Given two
+    statements, the order is chosen for its failure mode: CAS first, enqueue second. A
+    crash in between leaves an instance in `deleting` with no task, which the reconciler's
+    sweep re-enqueues. The other order leaves a deprovision task pointing at a `ready`
+    instance, and a worker would tear down a live service nobody asked to delete.
+    """
+    inst = await instances.get(instance_id, team)
+    if inst is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
+        )
+
+    try:
+        target = transition(inst.state, InstanceState.DELETING)
+    except IllegalTransition as exc:
+        # Already deleting or already deleted. Not an error the tenant can fix.
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={
+                "code": "illegal_transition",
+                "message": f"instance is {inst.state}; delete is not a legal transition",
+            },
+        ) from exc
+
+    if not await instances.update_state(inst.id, expect=inst.state, to=target):
+        # Someone moved the row between the read and the CAS.
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={"code": "conflict", "message": "instance changed concurrently; retry"},
+        )
+    await tasks.enqueue_standalone(inst.id, TaskKind.DEPROVISION)
+
+    return inst.model_copy(update={"state": target})