svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

services/__init__.py

@@ -0,0 +1 @@
+"""svcforge services: api, worker, reconciler."""

services/api/Dockerfile

@@ -0,0 +1,55 @@
+# syntax=docker/dockerfile:1.10
+#
+# svcforge api. Build from the REPO ROOT:
+#     docker buildx build -f services/api/Dockerfile -t svcforge/api:dev .
+# `COPY ../..` is illegal, so the context must be the root. There is no other option.
+#
+# Two syncs, not one: deps change rarely and our own code changes every commit, so the
+# expensive layer (third-party wheels) must land before the cheap one (our source).
+
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+# --- layer 1: third-party dependencies only -------------------------------------------
+# --no-install-project skips the root; --no-install-package skips our path dependency.
+# Without the latter, uv would try to build svcforge-core here, where its source is not
+# yet in the context, and the build would fail.
+COPY pyproject.toml uv.lock ./
+COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable \
+        --no-install-project --no-install-package svcforge-core
+
+# --- layer 2: our code ----------------------------------------------------------------
+COPY libs/ libs/
+COPY services/api/ services/api/
+COPY catalog.yaml ./
+# --no-editable is what turns svcforge-core into a real wheel in site-packages.
+# pyproject.toml declares it `editable = true` for local dev; an editable install in an
+# image points at /app/libs, which is a source tree that need not survive the final stage.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable && \
+    /app/.venv/bin/python -c 'import svcforge_core, sys; \
+p = svcforge_core.__file__; \
+sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
+
+# --- runtime --------------------------------------------------------------------------
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
+
+ARG BUILD_SHA=unknown
+LABEL org.opencontainers.image.title="svcforge-api" \
+      org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
+      org.opencontainers.image.revision="${BUILD_SHA}"
+
+RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
+WORKDIR /app
+COPY --from=builder --chown=10001:10001 /app /app
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+USER 10001
+ENTRYPOINT ["python", "-m", "services.api"]

services/api/__init__.py

@@ -0,0 +1 @@
+"""The api service: HTTP transport over the domain."""

services/api/__main__.py

@@ -0,0 +1,29 @@
+"""`python -m services.api` — what the container's ENTRYPOINT runs.
+
+One uvicorn worker per container, on purpose. Replicas are Kubernetes' job: `--workers N`
+forks processes the orchestrator cannot see, size, or drain, and it breaks the in-process
+Prometheus registry that /metrics depends on.
+"""
+
+from __future__ import annotations
+
+import uvicorn
+
+from svcforge_core.settings import load_settings
+
+
+def main() -> None:
+    """Load settings (fail fast if the env is wrong), then serve."""
+    settings = load_settings()
+    uvicorn.run(
+        "services.api.main:app",
+        factory=True,
+        host="0.0.0.0",  # noqa: S104 - a container binds all interfaces; the pod is the boundary
+        port=8000,
+        log_level=settings.log_level,
+        access_log=False,  # structlog owns request logging (Module 7); two sources would double it
+    )
+
+
+if __name__ == "__main__":
+    main()

services/api/deps.py

@@ -0,0 +1,171 @@
+"""Dependency injection: how a handler gets a pool, a repo, a catalog, and a team.
+
+Everything expensive — the pool, the JWKS client, the parsed catalog — is built once in
+`lifespan` and parked on `app.state`. These functions only hand it out. A `Depends` that
+does I/O per request is a `Depends` that does that I/O on every request forever.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Annotated, Any
+
+import jwt
+from fastapi import Depends, HTTPException, Request, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from jwt import PyJWKClient
+
+from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.repo.db import DictPool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings
+
+# The algorithm allow-list is the whole point of naming algorithms explicitly.
+# `jwt.decode(..., algorithms=...)` without it accepts whatever the *token* claims in its
+# own header — including `none`, and including HS256 verified with the RSA public key as
+# an HMAC secret. Both are forgery. The list is not configuration.
+ALLOWED_ALGORITHMS = ["RS256"]
+
+# What `auth_disabled` returns. Settings.check_production() refuses that flag in prod.
+DEV_TEAM = "platform"
+
+TEAM_CLAIM = "team"
+
+# auto_error=False is load-bearing. HTTPBearer(auto_error=True) answers a *missing*
+# Authorization header with 403, not 401 — an old FastAPI wart. The spec (and every
+# client that knows what to do about it) wants 401, so the error is raised here.
+_bearer = HTTPBearer(auto_error=False)
+
+
+def _unauthorized() -> HTTPException:
+    """One shape for every auth failure.
+
+    Expired, wrong issuer, wrong audience, bad signature, malformed, no header: all the
+    same 401 with the same body. Telling a caller *which* one turns the endpoint into an
+    oracle they can tune a forgery against.
+    """
+    return HTTPException(
+        status_code=status.HTTP_401_UNAUTHORIZED,
+        detail={"code": "unauthorized", "message": "invalid or missing credentials"},
+        headers={"WWW-Authenticate": "Bearer"},
+    )
+
+
+def get_settings(request: Request) -> Settings:
+    """The Settings that create_app() was handed."""
+    settings: Settings = request.app.state.settings
+    return settings
+
+
+async def get_pool(request: Request) -> DictPool:
+    """Return the pool that lifespan put on app.state."""
+    pool: DictPool = request.app.state.pool
+    return pool
+
+
+def get_catalog(request: Request) -> dict[str, CatalogEntry]:
+    """The catalog, parsed once at startup.
+
+    Read from disk per request and a mid-flight edit to catalog.yaml changes the answer
+    between two requests of the same deploy. Load it at startup; a change is a restart.
+    """
+    catalog: dict[str, CatalogEntry] = request.app.state.catalog
+    return catalog
+
+
+def get_instance_repo(pool: Annotated[DictPool, Depends(get_pool)]) -> InstanceRepo:
+    """An InstanceRepo bound to the app's pool. Cheap: it is a handle, not a connection."""
+    return InstanceRepo(pool)
+
+
+def get_task_repo(pool: Annotated[DictPool, Depends(get_pool)]) -> TaskRepo:
+    """A TaskRepo bound to the app's pool."""
+    return TaskRepo(pool)
+
+
+async def get_current_team(
+    request: Request,
+    creds: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> str:
+    """Verify the JWT against the cached JWKS. Check aud/iss/exp and the alg allow-list.
+
+    Returns the team claim. Raises HTTPException(401) on any failure — never leaks why.
+    """
+    if settings.auth_disabled:
+        return DEV_TEAM
+
+    if creds is None or not creds.credentials:
+        raise _unauthorized()
+
+    jwks_client: PyJWKClient | None = getattr(request.app.state, "jwks_client", None)
+    if jwks_client is None:
+        # Auth is on but there is no key source. Fail closed. Answering 500 here would be
+        # honest about the cause and would also let a misconfigured deploy be told apart
+        # from a bad token; 401 is the same answer a forger gets.
+        raise _unauthorized()
+
+    try:
+        # PyJWKClient keeps its own TTL cache, so this is a dict lookup on the hot path.
+        # It is only blocking on a cache MISS (key rotation) — hence to_thread, which
+        # costs a thread hop we take a handful of times a day rather than an event loop
+        # stalled on someone else's HTTP call once per rotation.
+        signing_key = await _signing_key(jwks_client, creds.credentials)
+        claims: dict[str, Any] = jwt.decode(
+            creds.credentials,
+            signing_key.key,
+            algorithms=ALLOWED_ALGORITHMS,
+            audience=settings.jwt_audience,
+            issuer=settings.jwt_issuer,
+            options={
+                "require": ["exp", "aud", "iss"],
+                "verify_exp": True,
+                "verify_aud": True,
+                "verify_iss": settings.jwt_issuer is not None,
+                "verify_signature": True,
+            },
+        )
+    except Exception as exc:  # deliberate catch-all: every failure becomes one opaque 401
+        raise _unauthorized() from exc
+
+    team = claims.get(TEAM_CLAIM)
+    if not isinstance(team, str) or not team:
+        raise _unauthorized()
+    return team
+
+
+async def _signing_key(client: PyJWKClient, token: str) -> jwt.PyJWK:
+    """Fetch the signing key off the event loop.
+
+    PyJWKClient.get_signing_key_from_jwt() does a synchronous urlopen on a cache miss.
+    Called directly from `async def`, that blocks the loop — every other in-flight request
+    on this worker stops until the identity provider answers, and if it hangs, so does the
+    pod, and /readyz keeps saying it is fine.
+    """
+    return await asyncio.to_thread(client.get_signing_key_from_jwt, token)
+
+
+async def rate_limit(team: Annotated[str, Depends(get_current_team)]) -> None:
+    """Per-team rate limiting. Seam only — Module 10 fills this in (Redis, Lua, token bucket).
+
+    It exists now, wired into the routes, so that turning it on is an edit to one function
+    body rather than a change to every handler signature.
+    """
+    return None
+
+
+async def idempotency_key(request: Request) -> str | None:
+    """`Idempotency-Key` handling. Seam only — Module 10 fills this in (Redis store).
+
+    Until then the real idempotency anchor is `instances.release_name`, which is unique in
+    the schema and deterministic from (team, service_type, id).
+    """
+    return request.headers.get("Idempotency-Key")
+
+
+PoolDep = Annotated[DictPool, Depends(get_pool)]
+TeamDep = Annotated[str, Depends(get_current_team)]
+InstanceRepoDep = Annotated[InstanceRepo, Depends(get_instance_repo)]
+TaskRepoDep = Annotated[TaskRepo, Depends(get_task_repo)]
+CatalogDep = Annotated[dict[str, CatalogEntry], Depends(get_catalog)]

services/api/main.py

@@ -0,0 +1,121 @@
+"""The app factory and its lifespan.
+
+`create_app(settings)` is a factory, not a module-level `app = FastAPI()`, for one reason:
+a test needs an app pointed at a throwaway Postgres, and an import-time app reads the real
+environment at import time — before any fixture can say otherwise.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse
+from jwt import PyJWKClient
+
+from services.api.models import ErrorBody
+from services.api.routes import health, instances
+from svcforge_core.domain.catalog import load_catalog
+from svcforge_core.repo.db import make_pool
+from svcforge_core.settings import Settings, load_settings
+
+log = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Open the pool, yield, close the pool.
+
+    A lifespan context, not the deprecated startup/shutdown event decorators: those cannot
+    express "this resource lives for exactly as long as the app", and give you no place to
+    put the teardown next to the setup. Closing the pool matters — an unclosed pool means
+    connections linger server-side after SIGTERM, and on a pooled Postgres with a small
+    connection budget a few rolling deploys exhaust it.
+
+    (The old decorator's name is spelled nowhere in this package on purpose: CI greps for
+    the literal string, and a comment quoting it fails the gate just as loudly as a call.)
+    """
+    settings: Settings = app.state.settings
+
+    app.state.catalog = load_catalog(settings.catalog_path)
+
+    pool = make_pool(str(settings.pg_dsn), settings.pool_min_size, settings.pool_max_size)
+    # wait=True fails NOW, loudly, if the DSN is wrong — instead of at the first request,
+    # as a PoolTimeout, in front of a user.
+    await pool.open(wait=True)
+    app.state.pool = pool
+
+    # The pool is open from here on, so everything below is inside the try: an exception
+    # in JWKS setup must still close it, or a crash-looping pod leaks a connection per
+    # restart until the database refuses new ones.
+    try:
+        if settings.jwks_url and not settings.auth_disabled:
+            client = PyJWKClient(settings.jwks_url, cache_keys=True, lifespan=300)
+            app.state.jwks_client = client
+            # Warm the cache off the loop so the first authenticated request does not pay
+            # a blocking urlopen. Best-effort: a slow identity provider must not stop the
+            # pod from starting — a cache miss later just costs one to_thread hop.
+            try:
+                await asyncio.to_thread(client.get_signing_keys)
+            except Exception:  # deliberate catch-all: startup must not hinge on the IdP being up
+                log.warning("JWKS warm-up failed; keys will be fetched on first use", exc_info=True)
+        else:
+            app.state.jwks_client = None
+
+        yield
+    finally:
+        await pool.close()
+
+
+async def _http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Render HTTPException bodies as ErrorBody, so every error has one shape.
+
+    Handlers raise `detail={"code": ..., "message": ...}`; FastAPI's default would nest
+    that under `{"detail": {...}}`. Plain-string details (raised by FastAPI itself, e.g.
+    a 405) are wrapped so clients never have to branch on the body's type.
+    """
+    assert isinstance(exc, HTTPException)  # noqa: S101 - registered only for HTTPException
+    # Widened to object deliberately. Starlette types `detail` as str, but FastAPI passes
+    # through whatever a handler raised — our handlers raise dicts. Narrowing off the
+    # declared type would make mypy call the dict branch unreachable and delete it.
+    detail: object = exc.detail
+    if isinstance(detail, dict) and "code" in detail and "message" in detail:
+        body = ErrorBody(code=str(detail["code"]), message=str(detail["message"]))
+    else:
+        body = ErrorBody(code=f"http_{exc.status_code}", message=str(detail))
+    return JSONResponse(status_code=exc.status_code, content=body.model_dump(), headers=exc.headers)
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    """App factory: lifespan, routers, exception handler, /metrics."""
+    settings = settings or load_settings()
+
+    app = FastAPI(
+        title="svcforge",
+        version="0.1.0",
+        summary="X-as-a-Service control plane",
+        lifespan=lifespan,
+    )
+    app.state.settings = settings
+
+    # /metrics is a normal route on health.router, not an app.mount — see health.metrics
+    # for why the mount does not actually serve a bare /metrics.
+    app.include_router(health.router)
+    app.include_router(instances.router)
+
+    app.add_exception_handler(HTTPException, _http_exception_handler)
+    return app
+
+
+def app() -> FastAPI:
+    """Entry point for `uvicorn services.api.main:app --factory`."""
+    return create_app()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import uvicorn
+
+    uvicorn.run("services.api.main:app", factory=True, host="0.0.0.0", port=8000)  # noqa: S104

services/api/models.py

@@ -0,0 +1,52 @@
+"""Wire types.
+
+These are deliberately NOT the domain models. `Instance` carries `team`, `namespace` and
+`release_name` — placement details a tenant has no business seeing and no business
+setting. The response model is the allow-list that keeps them off the wire, which is why
+it is written out by hand instead of derived from `Instance`.
+"""
+
+from __future__ import annotations
+
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from svcforge_core.domain.states import InstanceState
+
+
+class CreateInstanceRequest(BaseModel):
+    """What a tenant may ask for.
+
+    `service_type` and `size` are plain strings, not enums: the catalog is data loaded at
+    runtime, so baking its keys into a type would mean a redeploy to add a service type,
+    and a 422 (schema) where the spec wants a 404 (unknown resource). They are validated
+    against the catalog in the handler.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    service_type: str = Field(min_length=1)
+    size: str
+    ttl_days: int | None = Field(default=None, ge=1, le=30)
+
+
+class InstanceResponse(BaseModel):
+    """What a tenant gets back. A subset of Instance, on purpose."""
+
+    model_config = ConfigDict(from_attributes=True)
+
+    id: UUID
+    state: InstanceState
+    service_type: str
+    size: str
+    endpoint: str | None
+    chart_version: str
+    error: str | None
+
+
+class ErrorBody(BaseModel):
+    """Every non-2xx body. `code` is for machines, `message` is for humans."""
+
+    code: str
+    message: str

services/api/routes/__init__.py

@@ -0,0 +1 @@
+"""HTTP routers."""

services/api/routes/health.py

@@ -0,0 +1,81 @@
+"""Liveness, readiness, metrics.
+
+The distinction between the first two is not pedantry, it is the difference between a
+30-second blip and a fleet-wide outage:
+
+* `/healthz` (liveness) answers "is this process wedged?" A failure here gets the
+  container KILLED. It must therefore touch NOTHING external. Wire it to the DB and a
+  20-second Postgres failover restarts every pod at once; they come back, find the DB
+  still down, and CrashLoopBackOff with exponential restart delays — so the fleet is now
+  down for minutes after the database recovered.
+* `/readyz` (readiness) answers "should this pod get traffic?" A failure here only removes
+  it from the Service endpoints. It is allowed to check dependencies, and it recovers by
+  itself the moment the check passes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Request, Response, status
+from prometheus_client import REGISTRY
+from prometheus_client.exposition import choose_encoder
+
+from services.api.deps import PoolDep
+from services.api.models import ErrorBody
+
+router = APIRouter(tags=["ops"])
+
+# No PROMETHEUS_MULTIPROC_DIR here, deliberately: it exists for prefork servers where each
+# worker process holds a slice of the counters. One uvicorn process per container means
+# the default in-process registry is already correct, and multiproc mode would add a
+# shared temp dir, a cleanup obligation, and a class of stale-file bugs for nothing.
+
+
+@router.get("/healthz", status_code=status.HTTP_200_OK)
+async def healthz() -> dict[str, str]:
+    """Liveness. No I/O. If the event loop can run this, the process is alive."""
+    return {"status": "ok"}
+
+
+@router.get(
+    "/readyz",
+    responses={503: {"model": ErrorBody, "description": "A dependency is unavailable"}},
+)
+async def readyz(pool: PoolDep) -> dict[str, str]:
+    """Readiness. Postgres only.
+
+    Postgres-only is the rule, and Redis is the temptation. Redis holds derived state —
+    rate-limit buckets, caches — and everything degrades gracefully without it. Put it in
+    this check and an Upstash hiccup marks every pod unready, Kubernetes empties the
+    Service, and a cache outage becomes a total API outage.
+    """
+    try:
+        async with pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute("select 1")
+            row: Any = await cur.fetchone()
+        if row is None:
+            raise RuntimeError("select 1 returned no row")
+    except Exception as exc:  # closed pool, timeout, dead DB — all mean the same 'not ready'
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={"code": "not_ready", "message": "database unavailable"},
+        ) from exc
+    return {"status": "ready"}
+
+
+@router.get("/metrics", response_class=Response)
+async def metrics(request: Request) -> Response:
+    """The Prometheus scrape endpoint.
+
+    A route rather than `app.mount("/metrics", make_asgi_app())`, for two reasons. A
+    Starlette `Mount` compiles to `^/metrics(?P<path>/.*)$`, which does not match a bare
+    `/metrics` — the exact URL every scrape config uses — and a `Mount` is invisible to
+    OpenAPI, while the deliverable asks for `/metrics` in `openapi.json`.
+
+    The encoding is still prometheus_client's: `choose_encoder` reads the Accept header and
+    picks the exposition format (Prometheus text vs OpenMetrics) with its matching content
+    type. Hand-rolling either is how you end up serving text/plain that a scraper rejects.
+    """
+    encoder, content_type = choose_encoder(request.headers.get("Accept", ""))
+    return Response(content=encoder(REGISTRY), media_type=content_type)

services/api/routes/instances.py

@@ -0,0 +1,218 @@
+"""The tenant-facing API.
+
+Two rules run through every handler here:
+
+* **AuthZ is the WHERE clause.** No handler ever compares `inst.team` to the caller's
+  team, because the repo never returns another team's row to compare. A wrong-team id is
+  a 404. 403 would confirm the id exists, which is the leak.
+* **The instance and its task commit together.** A committed instance with no task is an
+  instance that never provisions and that nothing will ever retry.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import UUID, uuid4
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
+
+from services.api.deps import (
+    CatalogDep,
+    InstanceRepoDep,
+    PoolDep,
+    TaskRepoDep,
+    TeamDep,
+    idempotency_key,
+    rate_limit,
+)
+from services.api.models import CreateInstanceRequest, ErrorBody, InstanceResponse
+from svcforge_core.domain.models import CatalogEntry, Instance, TaskKind
+from svcforge_core.domain.states import IllegalTransition, InstanceState, transition
+
+# Declared on the router so every error shape lands in openapi.json under ErrorBody.
+# The exception handler already renders this at runtime; without declaring it, generated
+# clients see the contract for 2xx only and invent their own guess for the rest.
+ERROR_RESPONSES: dict[int | str, dict[str, Any]] = {
+    401: {"model": ErrorBody, "description": "Missing or invalid credentials"},
+    404: {"model": ErrorBody, "description": "No such instance, or not this team's"},
+    409: {"model": ErrorBody, "description": "Instance is not in a state that allows this"},
+    422: {"model": ErrorBody, "description": "The body is well-formed but cannot be processed"},
+}
+
+router = APIRouter(prefix="/v1/instances", tags=["instances"], responses=ERROR_RESPONSES)
+
+
+def release_name_for(team: str, service_type: str, instance_id: UUID) -> str:
+    """The helm release name. Deterministic, and `unique` in the schema.
+
+    This is the idempotency anchor. A worker that dies after `helm install` but before it
+    marks the task done will retry, compute the same name, and `helm upgrade --install`
+    onto the same release instead of creating a second one. Derive it from anything that
+    is not already durable — a timestamp, a random suffix, the retry count — and a retry
+    provisions a duplicate.
+
+    Truncated to the uuid's first 8 chars to stay inside the 53-char limit helm imposes
+    on release names (Kubernetes label values, minus room for chart-generated suffixes).
+    """
+    return f"{team}-{service_type}-{str(instance_id)[:8]}"
+
+
+def namespace_for(team: str) -> str:
+    """One namespace per tenant. The blast radius of a bad chart is one team."""
+    return f"tenant-{team}"
+
+
+def _resolve(catalog: dict[str, CatalogEntry], service_type: str, size: str) -> CatalogEntry:
+    """Look up service_type + size, or raise the right 4xx.
+
+    The two failures are different HTTP problems and the spec asks for different codes:
+    an unknown service_type is a resource that does not exist (404); an unknown size for a
+    real service_type is a body the server understood and cannot process (422).
+    """
+    entry = catalog.get(service_type)
+    if entry is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={
+                "code": "unknown_service_type",
+                "message": f"no such service_type: {service_type}",
+            },
+        )
+    if size not in entry.sizes:
+        raise HTTPException(
+            status_code=422,  # starlette renamed the 422 constant; the number never moved
+            detail={
+                "code": "unknown_size",
+                "message": f"{service_type} has no size {size!r}; available: {sorted(entry.sizes)}",
+            },
+        )
+    return entry
+
+
+@router.post(
+    "",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=InstanceResponse,
+    dependencies=[Depends(rate_limit), Depends(idempotency_key)],
+)
+async def create_instance(
+    body: CreateInstanceRequest,
+    response: Response,
+    team: TeamDep,
+    pool: PoolDep,
+    instances: InstanceRepoDep,
+    tasks: TaskRepoDep,
+    catalog: CatalogDep,
+) -> Instance:
+    """Accept a provisioning request. 202, never 201.
+
+    Nothing is provisioned when this returns. The row exists and a task is queued; a
+    worker will do the work seconds or minutes from now. 201 Created would be a lie about
+    a resource that does not exist yet, and clients would stop polling.
+    """
+    entry = _resolve(catalog, body.service_type, body.size)
+
+    instance_id = uuid4()
+    now = datetime.now(UTC)
+    inst = Instance(
+        id=instance_id,
+        team=team,
+        service_type=body.service_type,
+        size=body.size,
+        state=InstanceState.REQUESTED,
+        namespace=namespace_for(team),
+        release_name=release_name_for(team, body.service_type, instance_id),
+        # Pinned from the catalog AT CREATION TIME, not read from the catalog later.
+        # This column records what is actually deployed; bumping catalog.yaml must show up
+        # as drift the reconciler can see, not silently rewrite history.
+        chart_version=entry.chart_version,
+        expires_at=now + timedelta(days=body.ttl_days) if body.ttl_days is not None else None,
+        created_at=now,
+        updated_at=now,
+    )
+
+    # The transaction. Both writes go through THIS conn, or the atomicity is theatre.
+    async with pool.connection() as conn, conn.transaction():
+        created = await instances.create(conn, inst)
+        await tasks.enqueue(conn, created.id, TaskKind.PROVISION)
+
+    response.headers["Location"] = f"/v1/instances/{created.id}"
+    return created
+
+
+@router.get("/{instance_id}", response_model=InstanceResponse)
+async def get_instance(
+    instance_id: UUID,
+    team: TeamDep,
+    instances: InstanceRepoDep,
+) -> Instance:
+    """404 if the repo returns None. A wrong-team id is a 404, not a 403."""
+    inst = await instances.get(instance_id, team)
+    if inst is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
+        )
+    return inst
+
+
+@router.get("", response_model=list[InstanceResponse])
+async def list_instances(
+    team: TeamDep,
+    instances: InstanceRepoDep,
+    limit: int = Query(default=50, ge=1, le=200),
+) -> list[Instance]:
+    """The caller's instances, newest first. Bounded: no endpoint returns 'all rows'."""
+    return await instances.list(team, limit=limit)
+
+
+@router.delete(
+    "/{instance_id}",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=InstanceResponse,
+    dependencies=[Depends(rate_limit)],
+)
+async def delete_instance(
+    instance_id: UUID,
+    team: TeamDep,
+    instances: InstanceRepoDep,
+    tasks: TaskRepoDep,
+) -> Instance:
+    """state -> deleting, enqueue deprovision. 202: the helm uninstall has not happened yet.
+
+    Ordering note. `InstanceRepo.update_state` owns its own connection, so the CAS and the
+    enqueue cannot share one transaction without reaching around the repo. Given two
+    statements, the order is chosen for its failure mode: CAS first, enqueue second. A
+    crash in between leaves an instance in `deleting` with no task, which the reconciler's
+    sweep re-enqueues. The other order leaves a deprovision task pointing at a `ready`
+    instance, and a worker would tear down a live service nobody asked to delete.
+    """
+    inst = await instances.get(instance_id, team)
+    if inst is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
+        )
+
+    try:
+        target = transition(inst.state, InstanceState.DELETING)
+    except IllegalTransition as exc:
+        # Already deleting or already deleted. Not an error the tenant can fix.
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={
+                "code": "illegal_transition",
+                "message": f"instance is {inst.state}; delete is not a legal transition",
+            },
+        ) from exc
+
+    if not await instances.update_state(inst.id, expect=inst.state, to=target):
+        # Someone moved the row between the read and the CAS.
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={"code": "conflict", "message": "instance changed concurrently; retry"},
+        )
+    await tasks.enqueue_standalone(inst.id, TaskKind.DEPROVISION)
+
+    return inst.model_copy(update={"state": target})

services/cli/main.py

@@ -0,0 +1,170 @@
+"""svcforge — the control plane client.
+
+This talks to the API over HTTP and never touches the database. That restraint is the
+whole design: if the CLI could write to Postgres, every invariant the API enforces
+(the state machine, the one-transaction create, AuthZ in the WHERE clause) would have a
+back door, and the first 3am incident would go through it.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+import uuid
+from enum import StrEnum
+from typing import Annotated, Any
+
+import httpx
+import typer
+
+from svcforge_core.domain.states import InstanceState
+from svcforge_core.settings import load_settings
+
+app = typer.Typer(help="svcforge control plane client", no_args_is_help=True)
+
+_TERMINAL = {InstanceState.READY, InstanceState.FAILED, InstanceState.DELETED}
+
+
+class ServiceType(StrEnum):
+    """What the catalog offers. Kept as an enum so typer can complete and validate it."""
+
+    ELASTICSEARCH = "elasticsearch"
+    REDIS = "redis"
+    POSTGRES = "postgres"
+
+
+class Size(StrEnum):
+    SMALL = "small"
+    MEDIUM = "medium"
+
+
+def _client() -> httpx.Client:
+    settings = load_settings()
+    headers = {"authorization": f"Bearer {settings.api_token}"} if settings.api_token else {}
+    return httpx.Client(base_url=settings.api_url, headers=headers, timeout=10.0)
+
+
+def _die(msg: str) -> None:
+    typer.secho(msg, fg=typer.colors.RED, err=True)
+    raise typer.Exit(code=1)
+
+
+def _check(resp: httpx.Response) -> Any:  # noqa: ANN401 - a decoded JSON body genuinely is Any
+    if resp.status_code == 401:
+        _die("401 unauthorized: check SVCFORGE_API_TOKEN")
+    if resp.status_code == 404:
+        _die("404 not found")
+    if resp.status_code == 429:
+        _die("429 rate limited: slow down")
+    if resp.status_code >= 400:
+        _die(f"{resp.status_code}: {resp.text[:400]}")
+    return resp.json()
+
+
+def _parse_ttl(ttl: str | None) -> int | None:
+    """'7d' -> 7. Only days, because the API only takes days."""
+    if ttl is None:
+        return None
+    if not ttl.endswith("d") or not ttl[:-1].isdigit():
+        _die(f"bad --ttl {ttl!r}: expected something like '7d'")
+    return int(ttl[:-1])
+
+
+def _print_table(rows: list[dict[str, Any]]) -> None:
+    if not rows:
+        typer.echo("(none)")
+        return
+    cols = ["id", "service_type", "size", "state", "chart_version", "endpoint"]
+    widths = {c: max(len(c), *(len(str(r.get(c) or "-")) for r in rows)) for c in cols}
+    typer.echo("  ".join(c.ljust(widths[c]) for c in cols))
+    typer.echo("  ".join("-" * widths[c] for c in cols))
+    for r in rows:
+        typer.echo("  ".join(str(r.get(c) or "-").ljust(widths[c]) for c in cols))
+
+
+@app.command()
+def create(
+    service_type: Annotated[ServiceType, typer.Argument(help="what to provision")],
+    size: Annotated[Size, typer.Option()] = Size.SMALL,
+    ttl: Annotated[str | None, typer.Option(help="e.g. 7d")] = None,
+    wait: Annotated[bool, typer.Option(help="poll until ready or failed")] = False,
+) -> None:
+    """Request an instance. Returns as soon as the API accepts it (202)."""
+    body: dict[str, Any] = {"service_type": service_type.value, "size": size.value}
+    ttl_days = _parse_ttl(ttl)
+    if ttl_days is not None:
+        body["ttl_days"] = ttl_days
+
+    with _client() as c:
+        data = _check(c.post("/v1/instances", json=body))
+        typer.echo(f"{data['id']}  {data['state']}")
+        if not wait:
+            return
+
+        # 202 means "accepted", not "done". Polling is the client's job precisely because
+        # the API refused to block on a helm install that takes four minutes.
+        instance_id = data["id"]
+        deadline = time.monotonic() + 600
+        state = data["state"]
+        while time.monotonic() < deadline:
+            time.sleep(2)
+            data = _check(c.get(f"/v1/instances/{instance_id}"))
+            if data["state"] != state:
+                state = data["state"]
+                typer.echo(f"  -> {state}")
+            if state in _TERMINAL:
+                break
+        else:
+            _die("timed out waiting; the task may still be running — check `svcforge status`")
+
+    if state == InstanceState.FAILED:
+        _die(f"failed: {data.get('error') or 'no error recorded'}")
+    typer.echo(f"endpoint: {data.get('endpoint') or '-'}")
+
+
+@app.command("list")
+def list_instances(
+    state: Annotated[InstanceState | None, typer.Option(help="filter by state")] = None,
+) -> None:
+    """List your team's instances."""
+    with _client() as c:
+        rows = _check(c.get("/v1/instances"))
+    if state is not None:
+        rows = [r for r in rows if r["state"] == state.value]
+    _print_table(rows)
+
+
+@app.command()
+def status(instance_id: Annotated[uuid.UUID, typer.Argument()]) -> None:
+    """Show one instance. Exits non-zero if it is failed, so scripts can branch on it."""
+    with _client() as c:
+        data = _check(c.get(f"/v1/instances/{instance_id}"))
+    _print_table([data])
+    if data["state"] == InstanceState.FAILED:
+        typer.secho(f"error: {data.get('error')}", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def delete(
+    instance_id: Annotated[uuid.UUID, typer.Argument()],
+    yes: Annotated[bool, typer.Option("--yes", "-y", help="skip the confirmation")] = False,
+) -> None:
+    """Deprovision an instance."""
+    if not yes and not typer.confirm(f"delete {instance_id}?"):
+        raise typer.Abort
+    with _client() as c:
+        data = _check(c.delete(f"/v1/instances/{instance_id}"))
+    typer.echo(f"{data['id']}  {data['state']}")
+
+
+def main() -> None:  # pragma: no cover - console-script entrypoint
+    try:
+        app()
+    except httpx.ConnectError:
+        typer.secho("cannot reach the API: check SVCFORGE_API_URL", fg=typer.colors.RED, err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

services/reconciler/Dockerfile

@@ -0,0 +1,53 @@
+# syntax=docker/dockerfile:1.10
+#
+# svcforge reconciler. Build from the REPO ROOT:
+#     docker buildx build -f services/reconciler/Dockerfile -t svcforge/reconciler:dev .
+#
+# Reads helm state to detect drift, so it carries helm — but never kubectl, and it never
+# writes: the four checks enqueue tasks, they do not provision. Orphans are logged, never
+# deleted.
+
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable \
+        --no-install-project --no-install-package svcforge-core
+
+COPY libs/ libs/
+COPY services/reconciler/ services/reconciler/
+COPY catalog.yaml ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable && \
+    /app/.venv/bin/python -c 'import svcforge_core, sys; \
+p = svcforge_core.__file__; \
+sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
+
+# --- runtime --------------------------------------------------------------------------
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
+
+ARG BUILD_SHA=unknown
+LABEL org.opencontainers.image.title="svcforge-reconciler" \
+      org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
+      org.opencontainers.image.revision="${BUILD_SHA}"
+
+RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
+WORKDIR /app
+COPY --from=builder --chown=10001:10001 /app /app
+
+COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
+
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    HELM_CACHE_HOME=/tmp/helm/cache \
+    HELM_CONFIG_HOME=/tmp/helm/config \
+    HELM_DATA_HOME=/tmp/helm/data
+USER 10001
+ENTRYPOINT ["python", "-m", "services.reconciler.main"]

services/reconciler/__init__.py

@@ -0,0 +1 @@
+"""The reconciler service: one singleton loop that makes the world match the database."""

services/reconciler/main.py

@@ -0,0 +1,410 @@
+"""The control loop.
+
+Every other service in svcforge is edge-triggered: a tenant POSTs, a row appears, a worker
+claims it. Edge-triggered systems are correct exactly as long as nothing is ever missed —
+and things are missed. A worker is SIGKILLed holding a lease. An operator runs
+`helm uninstall` by hand. A pod dies between the CAS and the enqueue. Nobody sends an event
+for any of that, because the thing that would have sent it is the thing that died.
+
+So: level-triggered. Every 60 seconds, compare the world to the database and enqueue what
+is missing. The four checks below do not know or care what went wrong, or whether anything
+did; they are the same code on the happy path and after an outage. That property is the
+entire reason this service exists, and it is why each check is written as a *query for
+work*, never as a reaction to an event.
+
+Three rules hold the design together:
+
+* **Singleton.** `replicas: 1`, `strategy: Recreate` in the chart. Two reconcilers
+  double-enqueue drift and race on TTL. There is no leader election here on purpose: the
+  correct lease for that lives in Postgres next to the data, not in a Redis lock, and
+  until there is a second replica to elect between, an election is a subsystem that can
+  only fail. One pod, and the `SvcforgeReconcilerStale` alert is what notices it is gone.
+* **Each check is independent.** One failing check must not skip the other three. A helm
+  binary that cannot reach the API server must not stop TTLs from expiring.
+* **Enqueue, never act.** The reconciler diagnoses; workers treat. It writes task rows and
+  instance states, and never calls `helm install`. The one exception is reading — the drift
+  check runs `helm list`, because seeing reality is the job.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+
+import typer
+
+from svcforge_core.adapters.clock import Clock, SystemClock
+from svcforge_core.adapters.helm import HelmProvisioner, Provisioner
+from svcforge_core.adapters.notify import LogNotifier, Notifier
+from svcforge_core.domain.catalog import load_catalog
+from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.domain.windows import BadWindow, parse_window, schedule_upgrade_at
+from svcforge_core.obs import (
+    INSTANCES,
+    QUEUE_DEPTH,
+    RECONCILER_LAST_TICK,
+    get_logger,
+    setup,
+    start_metrics_server,
+    tracer,
+)
+from svcforge_core.repo.db import DictPool, make_pool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.reconcile import ReconcileRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings, load_settings
+
+log = get_logger("svcforge.reconciler")
+
+# The chart's PodMonitor scrapes the port named `metrics` on 9000. Keep them in step.
+DEFAULT_METRICS_PORT = 9000
+
+
+@dataclass(frozen=True)
+class ReconcilerDeps:
+    """Everything a check is allowed to touch. Built once in main(), passed down.
+
+    Same shape as `WorkerDeps` for the same reason: the checks take `deps` instead of
+    reaching for globals, so the integration tests below run every check against a real
+    Postgres and a `FakeProvisioner` without a cluster anywhere in sight.
+    """
+
+    pool: DictPool
+    instances: InstanceRepo
+    tasks: TaskRepo
+    reconcile: ReconcileRepo
+    provisioner: Provisioner
+    notifier: Notifier
+    clock: Clock
+    catalog: dict[str, CatalogEntry]
+    settings: Settings
+    # Whose instances go first in the day-2 work list. Eating your own dog food is an
+    # `order by`, not a policy document — see InstanceRepo.list_upgradable.
+    own_team: str
+    # A config value, not a scheduler. Leave it at 1 until 1 is too slow.
+    max_in_flight: int
+
+
+# --- The four checks ---------------------------------------------------------------------
+
+
+async def check_drift(deps: ReconcilerDeps) -> None:
+    """`helm list -A -o json` versus what the database believes.
+
+    This is the only check that looks outside Postgres, and the only one that can catch the
+    failure nothing else can: someone ran `helm uninstall` by hand, or a node was drained
+    and the release never came back. The DB still says `ready` and still hands the tenant an
+    endpoint that resolves to nothing.
+
+    Two directions, two very different answers:
+
+    * **Release gone, DB says `ready`** -> re-enqueue provision. Safe, because provisioning
+      is `helm upgrade --install` against a deterministic release name: converging on
+      desired state, not a blind re-install.
+    * **Release exists, DB knows nothing** -> log at error with release and namespace, and
+      stop. **Never delete in v1.** The reconciler's view of "the DB knows nothing" is one
+      query against one database; the release might belong to another team, another tool,
+      or a migration half-finished. Deleting on that evidence is how an automated system
+      takes down production faster than any human could. A human reads the log and decides.
+    """
+    with tracer().start_as_current_span("helm.list"):
+        releases = await deps.provisioner.list_releases()
+    live = {(r.name, r.namespace) for r in releases}
+
+    for inst in await deps.reconcile.ready_instances():
+        if (inst.release_name, inst.namespace) in live:
+            continue
+        reason = f"drift: helm release {inst.release_name} missing from namespace {inst.namespace}"
+        task_id = await deps.reconcile.enqueue_reprovision(inst.id, reason)
+        if task_id is None:
+            continue  # already being dealt with, or the row moved under us
+        log.warning(
+            "drift.release_missing",
+            instance_id=str(inst.id),
+            team=inst.team,
+            release=inst.release_name,
+            namespace=inst.namespace,
+            task_id=task_id,
+        )
+        try:
+            await deps.notifier.send(
+                "drift.release_missing",
+                f"re-provisioning {inst.id}: release {inst.release_name} vanished",
+                {"instance_id": str(inst.id), "team": inst.team},
+            )
+        except Exception:
+            # The task is already committed; the notification is a courtesy. A webhook
+            # timing out must not abandon the rest of the sweep — the instances after this
+            # one in the loop have the same problem and nobody else is coming to find them.
+            log.exception("notify.failed", instance_id=str(inst.id))
+
+    known = await deps.reconcile.known_releases()
+    for name, namespace in sorted(live - known):
+        # error, not warning: this is a resource nobody is billing for and nobody owns.
+        # It will sit here every 60s until a human deletes it or adopts it. That is the
+        # intended pressure.
+        log.error("drift.orphan_release", release=name, namespace=namespace, action="none (v1 never deletes)")
+
+
+async def check_lease_expiry(deps: ReconcilerDeps) -> None:
+    """Tasks whose worker died -> back to `queued`.
+
+    A lease, not a lock. No lock survives a power cut: a worker SIGKILLed mid-provision
+    leaves `state='running'` with `locked_by` set and nobody running it, and no amount of
+    cleanup code in the worker helps, because the worker is the part that died. `locked_at`
+    plus a timeout is the only thing that recovers the row, which is why `locked_at` exists.
+
+    The 5-minute default is not arbitrary: it must exceed the longest a healthy task can
+    hold a lease, or the reconciler hands a still-running provision to a second worker.
+    Handlers are idempotent, so that is survivable rather than fatal — but survivable is
+    not free, and `lease_seconds` sits above helm's `--timeout` for that reason.
+    """
+    freed = await deps.tasks.reset_expired_leases(deps.settings.lease_seconds)
+    if freed:
+        log.warning("lease.expired", tasks_freed=freed, lease_seconds=deps.settings.lease_seconds)
+
+
+async def check_ttl(deps: ReconcilerDeps) -> None:
+    """Expired instances -> `deleting`, plus a deprovision task.
+
+    The line item that stops a demo cluster from becoming a permanent cloud bill. Also the
+    sweep the API's DELETE route depends on: it CASes to `deleting` and enqueues in two
+    statements, and a crash in between lands here on the next tick.
+
+    Idempotent by construction — the work list excludes anything that already has a queued
+    or running deprovision, and the CAS and the insert share one transaction. Without that
+    guard, a deprovision that takes longer than 60 seconds gets a second task on the next
+    tick, and a third on the tick after.
+    """
+    for inst in await deps.reconcile.due_for_deprovision():
+        task_id = await deps.reconcile.enqueue_deprovision(inst.id)
+        if task_id is None:
+            continue
+        log.info(
+            "ttl.expired",
+            instance_id=str(inst.id),
+            team=inst.team,
+            expires_at=inst.expires_at.isoformat() if inst.expires_at else None,
+            previous_state=inst.state.value,
+            task_id=task_id,
+        )
+
+
+async def check_version_drift(deps: ReconcilerDeps) -> None:
+    """The day-2 rollout: the work-list query, one service type at a time.
+
+    Everything that makes this safe is somewhere else, which is the point:
+
+    * `list_upgradable` limits to `max_in_flight` and returns nothing while
+      `rollout_state='halted'`, so a bad chart stops after one tenant.
+    * `schedule_upgrade_at` turns the tenant's maintenance window into a `run_after`; the
+      queue does the waiting, in `where run_after <= now()`. There is no scheduler here and
+      there must not be one — a task parked in Postgres until 03:00 Sunday survives a
+      reconciler restart, and an in-memory timer does not.
+    * `security: true` in the catalog bypasses the window. A CVE with a public exploit does
+      not wait until Sunday.
+
+    A bad window spec is this instance's problem, not the fleet's: log it and move to the
+    next one. Failing the whole check would let one tenant's typo freeze everyone's
+    security rollout.
+    """
+    now = deps.clock.now()
+
+    for service_type, entry in deps.catalog.items():
+        candidates = await deps.instances.list_upgradable(
+            service_type=service_type,
+            catalog_version=entry.chart_version,
+            own_team=deps.own_team,
+            max_in_flight=deps.max_in_flight,
+        )
+        for candidate in candidates:
+            inst = candidate.instance
+            try:
+                window = parse_window(candidate.maintenance_window)
+            except BadWindow:
+                log.exception(
+                    "upgrade.bad_window",
+                    instance_id=str(inst.id),
+                    team=inst.team,
+                    maintenance_window=candidate.maintenance_window,
+                )
+                continue
+
+            run_after = schedule_upgrade_at(window, security=entry.security, now=now)
+            task_id = await deps.reconcile.enqueue_upgrade(inst.id, run_after)
+            if task_id is None:
+                continue  # already queued or running; this is the max_in_flight guard
+            log.info(
+                "upgrade.scheduled",
+                instance_id=str(inst.id),
+                team=inst.team,
+                service_type=service_type,
+                from_version=inst.chart_version,
+                to_version=entry.chart_version,
+                run_after=run_after.isoformat(),
+                security=entry.security,
+                task_id=task_id,
+            )
+
+
+CHECKS: dict[str, Callable[[ReconcilerDeps], Awaitable[None]]] = {
+    "drift": check_drift,
+    "lease_expiry": check_lease_expiry,
+    "ttl": check_ttl,
+    "version_drift": check_version_drift,
+}
+
+
+# --- The tick ----------------------------------------------------------------------------
+
+
+async def tick(deps: ReconcilerDeps) -> None:
+    """One pass: all four checks, then the gauges, then the heartbeat.
+
+    Checks first, gauges second: `svcforge_queue_depth` is read straight after the checks
+    that add to the queue, so the value scraped is the value the tick left behind rather
+    than one from before its own work.
+
+    The heartbeat is set unconditionally, and that is deliberate. It answers "is the loop
+    running", not "is everything fine" — the checks have their own alerts. Gating it on
+    success would make `SvcforgeReconcilerStale` fire for a helm blip and mean two things
+    at once, and an alert that means two things gets muted.
+
+    The whole tick runs inside one span, which is a considered exception to "manual spans go
+    around helm calls only". That rule exists so the API does not hand-roll spans that
+    `opentelemetry-instrument` already creates for it. Nothing auto-instruments the
+    reconciler: without a span here it emits no traces at all, and — because
+    `inject_traceparent` serialises the *active* context — every task it enqueues would be
+    written with a null `traceparent` and be unjoinable to the tick that decided to create
+    it. One span per tick is what makes "why was this instance re-provisioned at 03:00?" a
+    question the traces can answer.
+    """
+    with tracer().start_as_current_span("reconciler.tick"):
+        await _run_checks(deps)
+
+    RECONCILER_LAST_TICK.set(deps.clock.now().timestamp())
+    log.info("tick.done")
+
+
+async def _run_checks(deps: ReconcilerDeps) -> None:
+    """The four checks and the gauges. Split out so `tick` reads as span + heartbeat."""
+    for name, check in CHECKS.items():
+        try:
+            await check(deps)
+        except Exception:  # the tick is the error boundary
+            # The swallow is the design. These four checks share nothing but a database
+            # handle, and the value of a level-triggered loop is that it keeps running: an
+            # unreachable cluster must not stop TTLs from expiring, and one tenant's broken
+            # window spec must not stop drift detection. This means "this check achieved
+            # nothing for 60 seconds", which the log says out loud. It never means "the
+            # reconciler stops".
+            log.exception("check.failed", check=name)
+
+    try:
+        QUEUE_DEPTH.set(await deps.reconcile.queue_depth())
+        counts = await deps.reconcile.instance_counts()
+        for state, n in counts.items():
+            INSTANCES.labels(state=state).set(n)
+    except Exception:  # gauges are diagnostics; a failed read is not a failed tick
+        log.exception("gauges.failed")
+
+
+async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
+    """Sleep, but wake immediately on SIGTERM. A 60s nap must not cost 60s of shutdown."""
+    with contextlib.suppress(TimeoutError):
+        await asyncio.wait_for(stop.wait(), timeout=seconds)
+
+
+async def run_reconciler(deps: ReconcilerDeps, stop: asyncio.Event) -> None:
+    """Tick, sleep, repeat, until told to stop.
+
+    Tick first, then sleep: a pod that has just been restarted should reconcile now, not in
+    sixty seconds. Fixed interval rather than a fixed period — a tick that overruns simply
+    delays the next one, instead of stacking a second tick on top of the first, which for a
+    singleton would be exactly the concurrent reconciler `replicas: 1` exists to prevent.
+    """
+    while not stop.is_set():
+        await tick(deps)
+        await _sleep_or_stop(stop, deps.settings.reconcile_interval_s)
+
+
+def build_deps(
+    pool: DictPool,
+    settings: Settings,
+    own_team: str,
+    max_in_flight: int,
+) -> ReconcilerDeps:
+    """Wire the real collaborators. The only place that names concrete classes."""
+    return ReconcilerDeps(
+        pool=pool,
+        instances=InstanceRepo(pool),
+        tasks=TaskRepo(pool),
+        reconcile=ReconcileRepo(pool),
+        provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
+        notifier=LogNotifier(),
+        clock=SystemClock(),
+        catalog=load_catalog(settings.catalog_path),
+        settings=settings,
+        own_team=own_team,
+        max_in_flight=max_in_flight,
+    )
+
+
+async def _amain(once: bool, metrics_port: int, own_team: str, max_in_flight: int) -> None:
+    settings = load_settings()
+    setup("svcforge-reconciler", settings)
+
+    pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
+    await pool.open(wait=True)
+    deps = build_deps(pool, settings, own_team, max_in_flight)
+
+    try:
+        if once:
+            # One pass and exit: the acceptance path, and how you drive a reconcile by hand
+            # from a shell. No metrics server — nothing would ever scrape it.
+            await tick(deps)
+            return
+
+        start_metrics_server(metrics_port)
+
+        stop = asyncio.Event()
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            # add_signal_handler, NOT signal.signal. signal.signal fires the handler at an
+            # arbitrary bytecode boundary on the main thread and the loop does not notice
+            # until its next timer — which here is up to a full 60s tick away. This one is
+            # scheduled as an ordinary loop callback, so the `stop.wait()` above returns
+            # immediately.
+            loop.add_signal_handler(sig, stop.set)
+
+        await run_reconciler(deps, stop)
+    finally:
+        await pool.close()
+
+
+app = typer.Typer(add_completion=False, help="svcforge reconciler: the control loop.")
+
+
+@app.command()
+def main(
+    once: bool = typer.Option(False, "--once", help="Run one tick and exit."),
+    metrics_port: int = typer.Option(
+        DEFAULT_METRICS_PORT, envvar="SVCFORGE_METRICS_PORT", help="Port for /metrics."
+    ),
+    own_team: str = typer.Option(
+        "platform", envvar="SVCFORGE_OWN_TEAM", help="Team whose instances upgrade first."
+    ),
+    max_in_flight: int = typer.Option(
+        1, envvar="SVCFORGE_MAX_IN_FLIGHT", min=1, help="Concurrent upgrades across the fleet."
+    ),
+) -> None:
+    """Run the reconciler."""
+    # One asyncio.run, at the top, never nested. Everything below it is already async.
+    asyncio.run(_amain(once, metrics_port, own_team, max_in_flight))
+
+
+if __name__ == "__main__":
+    app()

services/worker/Dockerfile

@@ -0,0 +1,54 @@
+# syntax=docker/dockerfile:1.10
+#
+# svcforge worker. Build from the REPO ROOT:
+#     docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
+#
+# The only service that shells out to helm/kubectl, so the only one carrying those two
+# binaries. They are copied from pinned images rather than curl'd, so the version is a
+# reviewable line in a Dockerfile instead of a network call at build time.
+
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable \
+        --no-install-project --no-install-package svcforge-core
+
+COPY libs/ libs/
+COPY services/worker/ services/worker/
+COPY catalog.yaml ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable && \
+    /app/.venv/bin/python -c 'import svcforge_core, sys; \
+p = svcforge_core.__file__; \
+sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
+
+# --- runtime --------------------------------------------------------------------------
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
+
+ARG BUILD_SHA=unknown
+LABEL org.opencontainers.image.title="svcforge-worker" \
+      org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
+      org.opencontainers.image.revision="${BUILD_SHA}"
+
+RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
+WORKDIR /app
+COPY --from=builder --chown=10001:10001 /app /app
+
+COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
+COPY --from=bitnamilegacy/kubectl:1.31.2@sha256:0eab9ec8f5e0f75271277467ebb7513b36dea0122bc68615d18c47fece4fd82c /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
+
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    HELM_CACHE_HOME=/tmp/helm/cache \
+    HELM_CONFIG_HOME=/tmp/helm/config \
+    HELM_DATA_HOME=/tmp/helm/data
+USER 10001
+ENTRYPOINT ["python", "-m", "services.worker.main"]

services/worker/deps.py

@@ -0,0 +1,33 @@
+"""What a worker needs to do its job.
+
+One frozen bag of collaborators, constructed once in main() and passed down. Handlers
+take `deps` rather than reaching for globals, which is the entire reason the worker tests
+run in milliseconds against a FakeProvisioner instead of needing a cluster.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from svcforge_core.adapters.clock import Clock
+from svcforge_core.adapters.helm import Provisioner
+from svcforge_core.adapters.notify import Notifier
+from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.repo.db import DictPool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings
+
+
+@dataclass(frozen=True)
+class WorkerDeps:
+    """Everything a handler is allowed to touch."""
+
+    pool: DictPool
+    instances: InstanceRepo
+    tasks: TaskRepo
+    provisioner: Provisioner
+    notifier: Notifier
+    clock: Clock
+    catalog: dict[str, CatalogEntry]
+    settings: Settings

services/worker/handlers.py

@@ -0,0 +1,161 @@
+"""Task handlers.
+
+Every handler here obeys one rule: running it twice must equal running it once.
+
+That is not a nicety. A worker can be SIGKILLed after helm has installed the release but
+before the DB row says so; the lease expires; another worker claims the same task and runs
+this function again. If the handler is not idempotent, the tenant gets two Elasticsearches
+and you get a bill. Idempotency is what makes the crash safe, and it is bought in two
+places: a deterministic `release_name`, and adapters that state desired state
+(`helm upgrade --install`) instead of issuing imperative commands.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from services.worker.deps import WorkerDeps
+from svcforge_core.domain.models import CatalogEntry, Instance, Task, TaskKind
+from svcforge_core.domain.states import InstanceState
+
+
+class HandlerError(RuntimeError):
+    """A task failed in a way worth retrying. The message lands in tasks.last_error."""
+
+
+async def _load_instance(task: Task, deps: WorkerDeps) -> Instance:
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            """select id, team, service_type, size, state, namespace, release_name,
+                      chart_version, endpoint, error, expires_at, created_at, updated_at
+                 from instances where id = %s""",
+            (task.instance_id,),
+        )
+        row = await cur.fetchone()
+    if row is None:
+        raise HandlerError(f"instance {task.instance_id} vanished")
+    return Instance.model_validate(row)
+
+
+def _values_for(inst: Instance, entry: CatalogEntry) -> dict[str, Any]:
+    """Turn a catalog size into helm values."""
+    size = entry.sizes.get(inst.size)
+    if size is None:
+        raise HandlerError(f"size {inst.size!r} not in catalog for {inst.service_type!r}")
+    return {"replicaCount": size.replicas, "resources": size.resources}
+
+
+async def handle_provision(task: Task, deps: WorkerDeps) -> None:
+    """Install the release and mark the instance ready. Idempotent."""
+    inst = await _load_instance(task, deps)
+
+    if inst.state is InstanceState.READY:
+        # A previous attempt already finished; the crash was after the work, before the
+        # bookkeeping. Nothing to do — and re-installing would be the bug.
+        return
+
+    entry = deps.catalog.get(inst.service_type)
+    if entry is None:
+        raise HandlerError(f"unknown service_type {inst.service_type!r}")
+
+    # Best-effort CAS. It returning False means someone else moved the row; the helm call
+    # below is idempotent either way, so this is bookkeeping, not a lock.
+    await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
+
+    await deps.provisioner.install(
+        release=inst.release_name,
+        ns=inst.namespace,
+        entry=entry,
+        values=_values_for(inst, entry),
+    )
+
+    endpoint = f"http://{inst.release_name}.{inst.namespace}.svc.cluster.local"
+    ok = await deps.instances.update_state(
+        inst.id, InstanceState.PROVISIONING, InstanceState.READY, endpoint=endpoint
+    )
+    if ok:
+        await deps.notifier.send(
+            "instance.ready",
+            f"instance {inst.id} is ready at {endpoint}",
+            {"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
+        )
+
+
+async def handle_deprovision(task: Task, deps: WorkerDeps) -> None:
+    """Remove the release and mark the instance deleted. Idempotent."""
+    inst = await _load_instance(task, deps)
+
+    if inst.state is InstanceState.DELETED:
+        return
+
+    # `helm uninstall` of an already-gone release is not an error to us: the adapter
+    # swallows not-found, because the desired state — no release — is already true.
+    await deps.provisioner.uninstall(release=inst.release_name, ns=inst.namespace)
+    await deps.instances.update_state(inst.id, InstanceState.DELETING, InstanceState.DELETED)
+
+
+async def handle_upgrade(task: Task, deps: WorkerDeps) -> None:
+    """Upgrade the release to the catalog's pinned version, then record it.
+
+    `instances.chart_version` is written only AFTER helm reports success. That column is
+    what the day-2 work-list query compares against, so writing it optimistically would
+    make the fleet look upgraded while it isn't.
+    """
+    inst = await _load_instance(task, deps)
+    entry = deps.catalog.get(inst.service_type)
+    if entry is None:
+        raise HandlerError(f"unknown service_type {inst.service_type!r}")
+
+    if inst.chart_version == entry.chart_version:
+        return  # already there
+
+    await deps.provisioner.install(
+        release=inst.release_name,
+        ns=inst.namespace,
+        entry=entry,
+        values=_values_for(inst, entry),
+    )
+
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            "update instances set chart_version = %s, updated_at = now() where id = %s",
+            (entry.chart_version, inst.id),
+        )
+
+
+async def handle_verify(task: Task, deps: WorkerDeps) -> None:
+    """Post-upgrade health probe. On failure, halt the whole rollout for this service type.
+
+    One column decides whether the fleet keeps rolling. The work-list query returns nothing
+    while `rollout_state='halted'`, so a bad chart stops after the first tenant instead of
+    after all of them. You clear it with SQL, deliberately: an automatic un-halt would just
+    resume breaking things.
+    """
+    inst = await _load_instance(task, deps)
+    releases = {r.name for r in await deps.provisioner.list_releases()}
+
+    if inst.release_name in releases:
+        return
+
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            """insert into catalog_versions (service_type, rollout_state)
+                    values (%s, 'halted')
+               on conflict (service_type) do update set rollout_state = 'halted'""",
+            (inst.service_type,),
+        )
+    await deps.notifier.send(
+        "rollout.halted",
+        f"rollout halted for {inst.service_type}: {inst.release_name} failed verify",
+        {"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
+    )
+    raise HandlerError(f"verify failed for {inst.release_name}; rollout halted")
+
+
+HANDLERS: dict[TaskKind, Callable[[Task, WorkerDeps], Awaitable[None]]] = {
+    TaskKind.PROVISION: handle_provision,
+    TaskKind.DEPROVISION: handle_deprovision,
+    TaskKind.UPGRADE: handle_upgrade,
+    TaskKind.VERIFY: handle_verify,
+}

services/worker/main.py

@@ -0,0 +1,175 @@
+"""The claim loop.
+
+Poll every 5 seconds. Claim while a semaphore slot is free. Run the handler. Report.
+That is the whole design, and the restraint is the point: LISTEN/NOTIFY would shave the
+latency, is fire-and-forget so it can never replace the poll anyway, is strictly extra
+code, and does not exist on pgbouncer's transaction pooler. The poll is not a placeholder
+for something better.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+import time
+
+from opentelemetry import trace
+
+from services.worker.deps import WorkerDeps
+from services.worker.handlers import HANDLERS
+from svcforge_core import obs
+from svcforge_core.adapters.clock import SystemClock
+from svcforge_core.adapters.helm import HelmProvisioner
+from svcforge_core.adapters.notify import LogNotifier
+from svcforge_core.domain.catalog import load_catalog
+from svcforge_core.domain.models import Task
+from svcforge_core.repo.db import make_pool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings, load_settings
+
+log = obs.get_logger("svcforge.worker")
+
+
+async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
+    """Sleep, but wake immediately on shutdown.
+
+    `await asyncio.sleep(5)` would make every SIGTERM cost up to five seconds of
+    Kubernetes waiting on terminationGracePeriod for no reason.
+    """
+    with contextlib.suppress(TimeoutError):
+        await asyncio.wait_for(stop.wait(), timeout=seconds)
+
+
+async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None:
+    """Run one task to a terminal report. Never lets an exception escape the TaskGroup."""
+    try:
+        # Every log line from here carries instance_id/task_id/team. Bound once, at claim,
+        # rather than passed down: the alternative is threading three arguments through
+        # every function that might log, and the first one anyone forgets is the one you
+        # need at 3am.
+        obs.bind_task_context(task.instance_id, task.id, team=task.team or "unknown")
+        log.info("task claimed", kind=task.kind.value, attempt=task.attempts)
+        obs.TASKS_CLAIMED.labels(kind=task.kind.value).inc()
+
+        handler = HANDLERS.get(task.kind)
+        if handler is None:
+            await deps.tasks.fail(task.id, f"no handler for {task.kind}", max_attempts=1)
+            return
+
+        # Re-parent to the span that enqueued this task. Without the stored traceparent
+        # the worker's span starts a brand-new trace, and the POST that caused the work
+        # is in a different trace to the helm call that did it.
+        ctx = obs.context_from_traceparent(task.traceparent)
+        started = time.monotonic()
+        with obs.tracer().start_as_current_span(
+            f"task.{task.kind.value}",
+            context=ctx,
+            kind=trace.SpanKind.CONSUMER,
+        ) as span:
+            span.set_attribute("task.id", task.id)
+            span.set_attribute("task.kind", task.kind.value)
+            span.set_attribute("instance.id", str(task.instance_id))
+            try:
+                await handler(task, deps)
+            except asyncio.CancelledError:
+                # CancelledError inherits from BaseException, so `except Exception` below
+                # would never see it. Catch it only to release the claim, then ALWAYS
+                # re-raise: swallowing it breaks cancellation for everyone above us.
+                await deps.tasks.fail(task.id, "cancelled", max_attempts=deps.settings.max_attempts)
+                raise
+            except Exception as exc:
+                log.exception("task failed", kind=task.kind.value, error=str(exc))
+                span.record_exception(exc)
+                span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc)))
+                obs.TASKS_FAILED.labels(kind=task.kind.value).inc()
+                await deps.tasks.fail(task.id, str(exc), max_attempts=deps.settings.max_attempts)
+            else:
+                obs.PROVISION_TIME.observe(time.monotonic() - started)
+                await deps.tasks.complete(task.id)
+    finally:
+        sem.release()
+
+
+async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
+    """Claim and run until told to stop, then drain what is in flight.
+
+    Draining is what makes a rolling deploy invisible. Exiting the `async with` block
+    awaits every in-flight handler, so a pod that is being replaced finishes the provision
+    it already started instead of abandoning it half-done for the lease to clean up
+    five minutes later.
+    """
+    sem = asyncio.Semaphore(deps.settings.worker_concurrency)
+    worker_id = deps.settings.worker_id
+
+    async with asyncio.TaskGroup() as tg:
+        while not stop.is_set():
+            await sem.acquire()
+            if stop.is_set():
+                sem.release()
+                break
+
+            try:
+                task = await deps.tasks.claim(worker_id)
+            except Exception:
+                # A DB blip must not kill the worker; back off and try again.
+                log.exception("claim failed")
+                sem.release()
+                await _sleep_or_stop(stop, deps.settings.poll_interval_s)
+                continue
+
+            if task is None:
+                sem.release()
+                await _sleep_or_stop(stop, deps.settings.poll_interval_s)
+                continue
+
+            tg.create_task(_run_one(deps, task, sem))
+    # TaskGroup.__aexit__ awaited the in-flight handlers. Now it is safe to exit 0.
+
+
+async def _amain() -> None:
+    settings: Settings = load_settings()
+
+    # Before anything else: nothing logged above this line is structured, and the metrics
+    # the SvcforgeTaskFailed / SvcforgeProvisionSlow alerts query do not exist until the
+    # registry is up.
+    obs.setup("svcforge-worker", settings)
+    obs.start_metrics_server(settings.metrics_port)
+
+    pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
+    await pool.open(wait=True)
+
+    deps = WorkerDeps(
+        pool=pool,
+        instances=InstanceRepo(pool),
+        tasks=TaskRepo(pool),
+        provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
+        notifier=LogNotifier(),
+        clock=SystemClock(),
+        catalog=load_catalog(settings.catalog_path),
+        settings=settings,
+    )
+
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        # add_signal_handler, NOT signal.signal. signal.signal runs the handler at an
+        # arbitrary bytecode boundary on whatever thread the C-level handler lands on,
+        # and the event loop will not notice until its next timer fires. This one is
+        # loop-safe: the callback runs as a normal loop callback.
+        loop.add_signal_handler(sig, stop.set)
+
+    try:
+        await run_worker(deps, stop)
+    finally:
+        await pool.close()
+
+
+def main() -> None:
+    """One asyncio.run, at the top, never nested."""
+    asyncio.run(_amain())
+
+
+if __name__ == "__main__":
+    main()