svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

libs/svcforge_core/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "svcforge-core"
+version = "0.1.0"
+description = "svcforge shared core: domain, repo, adapters"
+requires-python = ">=3.12"
+dependencies = [
+    "pydantic>=2.9",
+    "pydantic-settings>=2.6",
+    "pyyaml>=6.0",
+    "psycopg[binary,pool]>=3.2",
+    "redis>=5.2",
+    "structlog>=24.4",
+    "prometheus-client>=0.21",
+    "opentelemetry-api>=1.28",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["svcforge_core"]

libs/svcforge_core/svcforge_core/adapters/__init__.py

@@ -0,0 +1,14 @@
+"""The outside world: helm/kubectl subprocesses, notifications, the clock.
+
+Everything in here is I/O the domain must never learn about. Each adapter exposes a
+Protocol with exactly two implementations — the real one here, the fake in `tests/fakes.py`.
+A Protocol with one implementation is an interface nobody asked for; delete it.
+
+The subprocess discipline lives in :func:`svcforge_core.adapters.helm._run` and is shared
+by every adapter that shells out (`k8s.py` included):
+
+* `create_subprocess_exec`, never a shell — `team` is tenant input that reaches a release name.
+* `start_new_session=True` at spawn time, so a timeout can kill the whole process group.
+* `communicate()`, never `wait()` with pipes attached.
+* stderr truncated to a fixed tail at this boundary, because it lands in `instances.error`.
+"""

libs/svcforge_core/svcforge_core/adapters/clock.py

@@ -0,0 +1,41 @@
+"""Time, as a dependency.
+
+The centrepiece of the Day-2 module, and it is nine lines. `datetime.now()` called from
+inside domain logic is an untestable global read: a maintenance-window test that wants
+"03:00 next Sunday" would have to either sleep until Sunday or monkeypatch a stdlib symbol
+and hope nothing else in the process noticed. Passing a Clock makes the same test a
+`FakeClock(start=...)` and an `advance()`.
+
+Aware UTC, always. A naive datetime is a bug that survives every test on a UTC CI box and
+detonates the first time it meets a tenant in Asia/Ho_Chi_Minh: `datetime.utcnow()` returns
+a naive value, and comparing it to a `timestamptz` from Postgres raises TypeError, or worse,
+silently compares wrong after somebody "fixes" it with a `.replace(tzinfo=...)`.
+
+The fake lives in `tests/fakes.py`, not here: shipping test doubles in the production
+package is how they end up imported by production code.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Protocol
+
+
+class Clock(Protocol):
+    """Implemented by SystemClock (here) and FakeClock (tests/fakes.py)."""
+
+    def now(self) -> datetime:
+        """Current time, aware, UTC."""
+        ...
+
+
+class SystemClock:
+    """The wall clock. The only thing in the codebase allowed to read it."""
+
+    def now(self) -> datetime:
+        """Current time, aware, UTC.
+
+        `datetime.now(UTC)`, never `datetime.utcnow()` — the latter is naive and deprecated
+        in 3.12 for exactly this reason.
+        """
+        return datetime.now(UTC)

libs/svcforge_core/svcforge_core/adapters/helm.py

@@ -0,0 +1,245 @@
+"""Driving helm from asyncio, with timeouts that actually kill helm.
+
+The whole module exists for `_run`. Everything above it is argv construction.
+
+Four things go wrong when you spawn a process from an event loop, and all four are
+handled here rather than in the caller:
+
+1. `subprocess.run` blocks the loop. Use `create_subprocess_exec`.
+2. `stdout=PIPE` with `proc.wait()` and nobody draining deadlocks at ~64 KB of output —
+   `helm --debug` clears that in one install. Use `communicate()`.
+3. `asyncio.wait_for` cancels the *coroutine*. The process does not know it was waited on:
+   helm keeps running and keeps mutating the cluster. The timeout has to kill it.
+4. `proc.kill()` signals the direct child. `helm` forks; its children reparent to init and
+   survive. Only `killpg` gets the whole tree, and only if the group exists — which needs
+   `start_new_session=True` **at spawn time**, because setsid can only run in the window
+   between fork and exec.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shutil
+import signal
+import tempfile
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any, Protocol
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from svcforge_core.domain.models import CatalogEntry
+
+# How long the process group gets to honour SIGTERM before SIGKILL. Helm traps SIGTERM
+# and tries to leave the release in a coherent state; give it a moment to do so.
+_TERM_GRACE_S = 5.0
+
+# `_run` is the backstop, not the primary timeout: helm gets its own `--timeout` so that
+# `--atomic` can roll back cleanly. `_run` only fires when helm itself is wedged, so its
+# deadline sits this far past helm's.
+_RUN_TIMEOUT_MARGIN_S = 30
+
+_STDERR_TAIL_BYTES = 2048
+
+
+class HelmError(RuntimeError):
+    """Non-zero exit. str(self) is the stderr tail that lands in instances.error."""
+
+
+class ReleaseInfo(BaseModel):
+    """One row of `helm list -o json`."""
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    namespace: str
+    chart: str
+    status: str
+    revision: int = Field(default=0, ge=0)
+    app_version: str = ""
+
+
+class Provisioner(Protocol):
+    """What the worker needs from a cluster. Implemented by HelmProvisioner and FakeProvisioner."""
+
+    async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None: ...
+
+    async def uninstall(self, release: str, ns: str) -> None: ...
+
+    async def list_releases(self) -> list[ReleaseInfo]: ...
+
+
+def _tail(raw: bytes, tail_bytes: int) -> str:
+    """The last `tail_bytes` of a stream, as text.
+
+    Truncation happens here, at the adapter boundary, and nowhere else. A helm failure can
+    emit megabytes; `instances.error` is a text column read by humans. Slice the bytes, not
+    the decoded string, then decode with `replace` — the cut can land mid-codepoint.
+    """
+    return raw[-tail_bytes:].decode("utf-8", errors="replace").strip()
+
+
+def _signal_group(proc: asyncio.subprocess.Process, sig: int) -> None:
+    """Signal the process's whole group. No-op if it has already exited.
+
+    `os.getpgid` rather than `proc.pid`: with `start_new_session=True` they are equal, but
+    that equality is an implementation detail, and asking the kernel costs nothing.
+    """
+    if proc.returncode is not None:
+        return
+    try:
+        os.killpg(os.getpgid(proc.pid), sig)
+    except (ProcessLookupError, PermissionError):
+        # Exited and reaped between the check and the call, or reparented out of reach.
+        return
+
+
+async def _terminate_group(proc: asyncio.subprocess.Process) -> None:
+    """SIGTERM the group, grace, SIGKILL the group. Then reap, so we leave no zombie."""
+    _signal_group(proc, signal.SIGTERM)
+    try:
+        await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_S)
+    except TimeoutError:
+        _signal_group(proc, signal.SIGKILL)
+        await proc.wait()
+
+
+async def _run(argv: Sequence[str], timeout_s: int, tail_bytes: int = _STDERR_TAIL_BYTES) -> str:
+    """create_subprocess_exec(*argv, stdout=PIPE, stderr=PIPE, start_new_session=True).
+
+    wait_for the drain. On TimeoutError: killpg(TERM), grace, killpg(KILL), raise.
+    Non-zero rc: raise HelmError(stderr tail). Return stdout.
+    """
+    proc = await asyncio.create_subprocess_exec(
+        *argv,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        start_new_session=True,
+    )
+
+    try:
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
+    except TimeoutError:
+        # wait_for cancelled the drain, not the process. Kill the group and re-raise.
+        await _terminate_group(proc)
+        raise
+    except asyncio.CancelledError:
+        # Our caller is being torn down (SIGTERM to the worker). Do not await anything here:
+        # a second cancellation would land on that await and leave helm running. Signal and go.
+        _signal_group(proc, signal.SIGKILL)
+        raise
+
+    if proc.returncode != 0:
+        detail = _tail(stderr, tail_bytes) or _tail(stdout, tail_bytes)
+        raise HelmError(
+            f"{argv[0]} exited {proc.returncode}: {detail}"
+            if detail
+            else f"{argv[0]} exited {proc.returncode}"
+        )
+    return stdout.decode("utf-8", errors="replace")
+
+
+class HelmProvisioner:
+    """The real thing. One helm binary, one kubeconfig, one deadline."""
+
+    def __init__(
+        self, *, helm_bin: str = "helm", kubeconfig: Path | None = None, timeout_s: int = 600
+    ) -> None:
+        """kubeconfig=None means the ambient config: $KUBECONFIG, ~/.kube/config, or the
+        in-cluster service account when svcforge runs as a pod. Keyword-only so that the
+        three arguments can never be swapped by position at a call site.
+        """
+        self._helm_bin = helm_bin
+        self._kubeconfig = kubeconfig
+        self._timeout_s = timeout_s
+
+    @property
+    def _run_timeout_s(self) -> int:
+        return self._timeout_s + _RUN_TIMEOUT_MARGIN_S
+
+    def _base_argv(self, *args: str) -> list[str]:
+        argv = [self._helm_bin, *args]
+        if self._kubeconfig is not None:
+            argv += ["--kubeconfig", str(self._kubeconfig)]
+        return argv
+
+    async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None:
+        """helm upgrade --install --wait --timeout. Idempotent by construction."""
+        # `upgrade --install` is why this is idempotent: a retried task after a crash mid-provision
+        # converges on the same release instead of erroring with "release already exists".
+        # `--wait` is why `ready` in the DB means ready — it returns when the pods are up.
+        # `--atomic` rolls back a failed upgrade; it doubles the worst case, which is what
+        # `_RUN_TIMEOUT_MARGIN_S` and helm's own `--timeout` are sized around.
+        with _values_file(values) as path:
+            argv = self._base_argv(
+                "upgrade",
+                "--install",
+                release,
+                entry.chart,
+                "--namespace",
+                ns,
+                "--version",
+                entry.chart_version,
+                "--values",
+                str(path),
+                "--wait",
+                "--atomic",
+                "--timeout",
+                f"{self._timeout_s}s",
+            )
+            await _run(argv, timeout_s=self._run_timeout_s)
+
+    async def uninstall(self, release: str, ns: str) -> None:
+        """helm uninstall --wait. `--ignore-not-found` makes the retry of a half-done delete a no-op."""
+        argv = self._base_argv(
+            "uninstall",
+            release,
+            "--namespace",
+            ns,
+            "--ignore-not-found",
+            "--wait",
+            "--timeout",
+            f"{self._timeout_s}s",
+        )
+        await _run(argv, timeout_s=self._run_timeout_s)
+
+    async def list_releases(self) -> list[ReleaseInfo]:
+        """Every release helm knows about, in every namespace. The reconciler's view of reality."""
+        argv = self._base_argv("list", "--all-namespaces", "--output", "json")
+        raw = await _run(argv, timeout_s=self._run_timeout_s)
+        try:
+            parsed: Any = json.loads(raw or "[]")
+        except json.JSONDecodeError as exc:
+            raise HelmError(f"helm list returned non-JSON: {_tail(raw.encode(), 256)}") from exc
+        if not isinstance(parsed, list):
+            raise HelmError(f"helm list returned {type(parsed).__name__}, expected a list")
+        return [ReleaseInfo.model_validate(row) for row in parsed]
+
+
+class _ValuesFile:
+    """Context manager yielding a path to a values.yaml written from a dict.
+
+    A file, not `--set`: `--set` has its own escaping grammar (commas, dots, backslashes) and
+    values carry tenant-shaped strings. Serialising YAML sidesteps the grammar entirely.
+    """
+
+    def __init__(self, values: dict[str, Any]) -> None:
+        self._values = values
+        self._dir: str | None = None
+
+    def __enter__(self) -> Path:
+        self._dir = tempfile.mkdtemp(prefix="svcforge-values-")
+        path = Path(self._dir) / "values.yaml"
+        path.write_text(yaml.safe_dump(self._values, default_flow_style=False), encoding="utf-8")
+        return path
+
+    def __exit__(self, *exc: object) -> None:
+        if self._dir is not None:
+            shutil.rmtree(self._dir, ignore_errors=True)
+
+
+def _values_file(values: dict[str, Any]) -> _ValuesFile:
+    return _ValuesFile(values)

libs/svcforge_core/svcforge_core/adapters/k8s.py

@@ -0,0 +1,136 @@
+"""kubectl, for the two things helm will not do: make a namespace, read a secret back.
+
+Same subprocess discipline as helm — it is literally the same `_run`, imported rather than
+copied, because the process-group handling is the part that is easy to get subtly wrong twice.
+
+Both operations are declarative and therefore idempotent: `apply` of a namespace that exists
+is a no-op, and a `get` has no effect at all. A worker that dies mid-provision and retries
+must not find a "namespace already exists" error waiting for it.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from svcforge_core.adapters.helm import HelmError, _run
+
+_KUBECTL_TIMEOUT_S = 60
+
+
+class K8sError(RuntimeError):
+    """kubectl failed. str(self) is the stderr tail, already truncated by `_run`."""
+
+
+class SecretNotFound(K8sError):
+    """The secret is not there (yet). Distinct because a caller may want to retry rather than fail."""
+
+
+class KubectlClient:
+    """A kubectl binary and a kubeconfig. No client-go, no in-cluster config, no CRDs."""
+
+    def __init__(
+        self,
+        *,
+        kubectl_bin: str = "kubectl",
+        kubeconfig: Path | None = None,
+        timeout_s: int = _KUBECTL_TIMEOUT_S,
+    ) -> None:
+        """Mirrors HelmProvisioner: kubeconfig=None means the ambient/in-cluster config."""
+        self._kubectl_bin = kubectl_bin
+        self._kubeconfig = kubeconfig
+        self._timeout_s = timeout_s
+
+    def _base_argv(self, *args: str) -> list[str]:
+        argv = [self._kubectl_bin, *args]
+        if self._kubeconfig is not None:
+            argv += ["--kubeconfig", str(self._kubeconfig)]
+        return argv
+
+    async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
+        """Create the namespace if absent, leave it alone if present.
+
+        `apply -f` of a manifest rather than `create namespace` (which errors on the second
+        call) or `create --dry-run=client -o yaml | apply -f -` (which needs a shell, and a
+        shell is exactly what tenant input must never reach).
+        """
+        manifest: dict[str, Any] = {
+            "apiVersion": "v1",
+            "kind": "Namespace",
+            "metadata": {
+                "name": ns,
+                "labels": {"app.kubernetes.io/managed-by": "svcforge", **(labels or {})},
+            },
+        }
+        with _manifest_file(manifest) as path:
+            await self._kubectl("apply", "--filename", str(path))
+
+    async def read_secret(self, ns: str, name: str) -> dict[str, str]:
+        """The secret's `data`, base64-decoded. Chart-generated passwords come back through here.
+
+        Returns str, not bytes: every value svcforge reads (passwords, hosts, ports) is text.
+        A genuinely binary value raises rather than silently mangling into replacement chars.
+        """
+        try:
+            raw = await self._kubectl("get", "secret", name, "--namespace", ns, "--output", "json")
+        except K8sError as exc:
+            if "notfound" in str(exc).lower().replace(" ", ""):
+                raise SecretNotFound(f"secret {ns}/{name} not found") from exc
+            raise
+
+        try:
+            parsed: Any = json.loads(raw)
+        except json.JSONDecodeError as exc:
+            raise K8sError(f"kubectl get secret {ns}/{name} returned non-JSON") from exc
+
+        data: Any = parsed.get("data") or {}
+        if not isinstance(data, dict):
+            raise K8sError(f"secret {ns}/{name}: 'data' is {type(data).__name__}, expected a mapping")
+
+        out: dict[str, str] = {}
+        for key, encoded in data.items():
+            try:
+                out[str(key)] = base64.b64decode(str(encoded), validate=True).decode("utf-8")
+            except (binascii.Error, ValueError) as exc:
+                raise K8sError(f"secret {ns}/{name}: key {key!r} is not base64-encoded utf-8") from exc
+        return out
+
+    async def _kubectl(self, *args: str) -> str:
+        """Run kubectl through helm's `_run`, translating its error type at this boundary.
+
+        `_run` is spec'd to live in helm.py and to raise HelmError; nothing outside adapters
+        should have to know that kubectl failures arrive wearing a helm-shaped exception.
+        """
+        try:
+            return await _run(self._base_argv(*args), timeout_s=self._timeout_s)
+        except HelmError as exc:
+            raise K8sError(str(exc)) from exc
+
+
+class _ManifestFile:
+    """A temp file holding one YAML manifest, removed on exit."""
+
+    def __init__(self, manifest: dict[str, Any]) -> None:
+        self._manifest = manifest
+        self._dir: str | None = None
+
+    def __enter__(self) -> Path:
+        self._dir = tempfile.mkdtemp(prefix="svcforge-manifest-")
+        path = Path(self._dir) / "manifest.yaml"
+        path.write_text(yaml.safe_dump(self._manifest, default_flow_style=False), encoding="utf-8")
+        return path
+
+    def __exit__(self, *exc: object) -> None:
+        if self._dir is not None:
+            shutil.rmtree(self._dir, ignore_errors=True)
+
+
+def _manifest_file(manifest: dict[str, Any]) -> _ManifestFile:
+    return _ManifestFile(manifest)

libs/svcforge_core/svcforge_core/adapters/notify.py

@@ -0,0 +1,77 @@
+"""Telling someone a provision finished, or didn't.
+
+Best-effort by construction: `send` never raises. A notifier that can fail a task is a
+notifier that lets a Slack outage roll back a successful provision. The instance is ready;
+the DB says so; failing the task would re-run helm for nothing. Delivery failures are
+logged and dropped on the floor, which is the correct amount of ceremony for a webhook.
+
+Two implementations, so the Protocol earns its place: LogNotifier (the default, and what
+tests and local dev get) and WebhookNotifier (the one that leaves the process).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Protocol
+
+import httpx
+
+_log = logging.getLogger(__name__)
+
+_DEFAULT_TIMEOUT_S = 5.0
+
+
+class Notifier(Protocol):
+    """Implemented by LogNotifier, WebhookNotifier, and FakeNotifier (tests/fakes.py)."""
+
+    async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
+        """Announce `event`. Must not raise: delivery is never worth failing a task over."""
+        ...
+
+
+class LogNotifier:
+    """Writes the event to the log. The default: structured logs are already shipped somewhere."""
+
+    async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
+        # NOT extra={"message": ...}. `message` is a reserved LogRecord attribute, and
+        # logging raises KeyError("Attempt to overwrite 'message' in LogRecord") at call
+        # time — so the notifier would crash the caller it was meant to inform. Same trap
+        # for `msg`, `args`, `name`, `levelname`, `exc_info`.
+        _log.info("notify", extra={"event": event, "detail": message, "fields": fields or {}})
+
+
+class WebhookNotifier:
+    """POSTs a JSON body at a URL. Slack-shaped, but anything that accepts JSON will do."""
+
+    def __init__(
+        self, url: str, *, client: httpx.AsyncClient | None = None, timeout_s: float = _DEFAULT_TIMEOUT_S
+    ) -> None:
+        """Pass `client` to share a connection pool with the rest of the process.
+
+        An owned client is closed by `aclose`; an injected one is the caller's to close.
+        """
+        self._url = url
+        self._timeout_s = timeout_s
+        self._client = client
+        self._owns_client = client is None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self._timeout_s)
+        return self._client
+
+    async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
+        payload = {"event": event, "message": message, "fields": fields or {}}
+        try:
+            client = await self._get_client()
+            resp = await client.post(self._url, json=payload, timeout=self._timeout_s)
+            resp.raise_for_status()
+        except httpx.HTTPError as exc:
+            # Deliberately swallowed. See the module docstring: the work already succeeded.
+            _log.warning("notify webhook failed", extra={"event": event, "error": str(exc)})
+
+    async def aclose(self) -> None:
+        """Close the client, if we made it."""
+        if self._client is not None and self._owns_client:
+            await self._client.aclose()
+            self._client = None

libs/svcforge_core/svcforge_core/adapters/redis.py

@@ -0,0 +1,427 @@
+"""Redis: derived state only. Never the truth, never the queue.
+
+Everything in here is a shortcut past Postgres, and every one of them is optional. Postgres
+holds the instances, the tasks, the leases and the `release_name` UNIQUE constraint. Redis
+holds a counter, a claim marker and a copy — all of it rebuildable by doing nothing and
+waiting for a TTL.
+
+That framing is not philosophy, it decides the error handling, and the error handling is
+the module. Each class below catches `RedisError` and returns a *safe* answer rather than
+raising:
+
+| Path        | Redis is down            | Why                                              |
+|-------------|--------------------------|--------------------------------------------------|
+| Cache       | miss -> read Postgres    | It was an optimisation. Nobody notices.          |
+| Rate limit  | **allow**                | An internal platform that refuses every request  |
+|             |                          | because the limiter is sick is worse than one    |
+|             |                          | that is briefly unmetered.                       |
+| Idempotency | fall through to the DB   | `instances.release_name` is UNIQUE. That is the  |
+|             |                          | real guarantee; this is the fast path.           |
+
+Consequently nothing here raises out to a caller, and `/readyz` stays Postgres-only. A
+Redis outage must not make a single pod unready — that would convert "the cache is down"
+into "the platform is down", which is the exact inversion this module exists to prevent.
+
+**The budget is a design constraint, not a footnote.** Upstash free tier:
+
+    500,000 commands / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
+
+One worker polling Redis every five seconds spends 518,400/month: the entire budget,
+producing nothing. So the rule is structural — **Redis lives on the request path only**,
+where volume is bounded by the number of humans with an API token, and never inside a poll
+or control loop. That is also why the limiter is a Lua script: `GET`/`INCR`/`EXPIRE` is
+three billed commands and a race; one `EVALSHA` is one billed command and atomic. A
+pipeline would not help — it batches round trips but still bills N.
+
+Every key gets a TTL. 256 MB with no expiry is a slow leak that ends by evicting the keys
+you cared about.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Protocol
+from uuid import UUID
+
+from prometheus_client import Counter
+from pydantic import ValidationError
+from redis.asyncio import Redis
+from redis.exceptions import RedisError
+
+from svcforge_core.adapters.clock import Clock, SystemClock
+from svcforge_core.domain.models import Instance
+
+if TYPE_CHECKING:
+    from redis.commands.core import AsyncScript
+
+    from svcforge_core.settings import Settings
+
+_log = logging.getLogger(__name__)
+
+# --- The budget metric ------------------------------------------------------------------
+#
+# The counter that `scripts/redis_budget.py` projects month-end burn from. It counts
+# commands we *send*, incremented next to each call, because the number that matters is
+# the one Upstash bills — not the number of times a method was called. A cache miss calls
+# `get()` once and spends one command; a `put()` after it spends another.
+#
+# The dangerous failure this makes visible: when Redis is down, every path degrades
+# silently and correctly, so nothing pages. Nothing fails until the month rolls over and
+# every Redis call starts erroring at once. A counter you can extrapolate from is the only
+# warning you get.
+
+REDIS_COMMANDS = Counter(
+    "svcforge_redis_commands_total",
+    "Redis commands issued, as Upstash bills them. One EVALSHA is one.",
+    ["op"],
+)
+
+REDIS_ERRORS = Counter(
+    "svcforge_redis_errors_total",
+    "Redis calls that failed and were degraded past. Never surfaced to the caller.",
+    ["op"],
+)
+
+# A hung Redis must not hang the request path. Without these, a TCP connection that is
+# open but unanswered blocks the handler until the client gives up — which turns "Redis is
+# slow" into "the API is down", the same inversion the fail-open policy prevents. Upstash
+# steady-state RTT is ~2.4 ms; two seconds is already pathological.
+_SOCKET_TIMEOUT_S = 2.0
+_CONNECT_TIMEOUT_S = 2.0
+
+# Errors that mean "Redis did not answer". Every public method below turns these into a
+# safe default. `OSError` because a DNS failure at connect time need not arrive wrapped,
+# `TimeoutError` because the socket timeouts above raise it.
+_REDIS_DOWN = (RedisError, OSError, asyncio.TimeoutError)
+
+
+def _as_text(value: bytes | str) -> str:
+    """`decode_responses=True` already did this; redis-py's annotations do not know it.
+
+    The client is configured for text, so the `bytes` branch is unreachable in this
+    process. It stays because the type says it is reachable, and a `cast` here would hide
+    the day someone builds a client without `decode_responses` and gets a `UUID(b'...')`
+    TypeError from three frames away instead of a value that just works.
+    """
+    return value.decode() if isinstance(value, bytes) else value
+
+
+def make_redis(settings: Settings) -> Redis | None:
+    """One client per process, opened in lifespan next to the psycopg pool, closed on exit.
+
+    `None` when no DSN is configured, and that is a supported way to run: every consumer
+    below is optional by construction, so "no Redis" and "Redis is down" take the same
+    code path. The signature is `Redis | None` rather than `Redis` precisely so that
+    "unconfigured" cannot be faked with a client pointed at nothing.
+
+    Two settings are not negotiable:
+
+    `decode_responses=True` — the first bug everyone hits. Without it every read is
+    `bytes` and the traceback is `AttributeError: 'bytes' object has no attribute
+    'encode'`, several frames away from the cause.
+
+    `rediss://` (TLS) — Upstash rejects plaintext. The handshake is ~56 ms against a
+    ~2.4 ms steady-state RTT, which is the whole argument for one pooled client per
+    process: a client per request pays the handshake every time and turns a cache into a
+    latency regression.
+    """
+    if settings.redis_dsn is None:
+        return None
+    return Redis.from_url(
+        str(settings.redis_dsn),
+        decode_responses=True,
+        socket_timeout=_SOCKET_TIMEOUT_S,
+        socket_connect_timeout=_CONNECT_TIMEOUT_S,
+    )
+
+
+# --- Rate limiting ----------------------------------------------------------------------
+
+# One INCR; EXPIRE only when the counter is new. The `== 1` test is the entire trick: set
+# the TTL unconditionally and every request slides the window forward, so a caller at
+# steady load is never reset and the "window" is a sliding refusal that never lets up.
+#
+# KEYS and ARGV arrive as 1-based tables — Lua indexes from 1, and `ARGV[0]` is silently
+# nil rather than an error, which reads as "the limit is nil" and compares false forever.
+#
+# Everything derivable in Python is derived in Python: `reset_at` comes from the window
+# number the caller already computed, so there is no TTL round trip. One command, total.
+_RATE_LIMIT_LUA = """
+local n = redis.call('INCR', KEYS[1])
+if n == 1 then
+  redis.call('EXPIRE', KEYS[1], ARGV[2])
+end
+local limit = tonumber(ARGV[1])
+local remaining = limit - n
+if remaining < 0 then
+  remaining = 0
+end
+local allowed = 0
+if n <= limit then
+  allowed = 1
+end
+return {allowed, remaining}
+"""
+
+
+@dataclass(frozen=True)
+class RateLimitResult:
+    """The limiter's answer. `degraded` is how the caller knows it did not really check."""
+
+    allowed: bool
+    limit: int
+    remaining: int
+    reset_at: datetime
+    degraded: bool = False
+
+    @property
+    def retry_after_s(self) -> int:
+        """Seconds until the window rolls over, for the `Retry-After` header on a 429.
+
+        Rounded up and floored at one: `Retry-After: 0` invites an immediate retry into
+        the same closed window, which is a busy loop with extra steps.
+        """
+        delta = (self.reset_at - datetime.now(UTC)).total_seconds()
+        return max(1, math.ceil(delta))
+
+
+class RateLimiterProto(Protocol):
+    """Implemented by RateLimiter (here) and FakeRateLimiter (tests/fakes.py)."""
+
+    async def check(self, team: str) -> RateLimitResult:
+        """Count this request against `team`'s window. Must not raise."""
+        ...
+
+
+class RateLimiter:
+    """Fixed-window limiter. One EVALSHA per check. Fails OPEN.
+
+    Fixed window, not a token bucket or a sliding log, because the window boundary is the
+    only thing a fixed window gets wrong and the cost of getting it wrong is that a caller
+    can spend 2x the limit across a boundary. A sliding log is a sorted set, an
+    `ZREMRANGEBYSCORE`, an `ZADD` and a `ZCARD` — four billed commands and unbounded key
+    size — to fix a burst nobody is paying for. The limit is a courtesy, not a security
+    control; the security control is the JWT.
+    """
+
+    def __init__(self, r: Redis, limit: int, window_s: int, *, clock: Clock | None = None) -> None:
+        """`clock` is injectable so the window boundary is testable without sleeping."""
+        if limit < 1:
+            raise ValueError("limit must be >= 1")
+        if window_s < 1:
+            raise ValueError("window_s must be >= 1")
+        self._limit = limit
+        self._window_s = window_s
+        self._clock = clock or SystemClock()
+        # register_script() is local: it hashes the source and returns a callable. No round
+        # trip here, and none wasted at import. The first call sends EVALSHA; redis-py
+        # catches NOSCRIPT and replays it as EVAL, which is why a restarted Redis costs one
+        # extra command once rather than an outage.
+        self._script: AsyncScript = r.register_script(_RATE_LIMIT_LUA)
+
+    def _window(self) -> tuple[int, datetime]:
+        """The current window number and when it ends. Pure arithmetic, no Redis."""
+        now = self._clock.now()
+        epoch = int(now.timestamp())
+        window = epoch // self._window_s
+        reset_at = datetime.fromtimestamp((window + 1) * self._window_s, tz=UTC)
+        return window, reset_at
+
+    async def check(self, team: str) -> RateLimitResult:
+        """Count one request against `team`. Never raises.
+
+        On any Redis error: allow, log loudly, count it. The metric is the point — a
+        limiter that fails open silently is indistinguishable from no limiter at all, and
+        you find out which one you shipped during the incident.
+        """
+        window, reset_at = self._window()
+        key = f"rl:{team}:{window}"
+        try:
+            REDIS_COMMANDS.labels(op="ratelimit").inc()
+            allowed, remaining = await self._script(keys=[key], args=[self._limit, self._window_s])
+        except _REDIS_DOWN:
+            REDIS_ERRORS.labels(op="ratelimit").inc()
+            _log.warning(
+                "rate limiter degraded: redis unavailable, failing OPEN",
+                exc_info=True,
+                extra={"team": team},
+            )
+            return RateLimitResult(
+                allowed=True,
+                limit=self._limit,
+                remaining=self._limit,
+                reset_at=reset_at,
+                degraded=True,
+            )
+        return RateLimitResult(
+            allowed=bool(allowed),
+            limit=self._limit,
+            remaining=int(remaining),
+            reset_at=reset_at,
+        )
+
+
+# --- Idempotency ------------------------------------------------------------------------
+
+
+class IdempotencyStoreProto(Protocol):
+    """Implemented by IdempotencyStore (here) and FakeIdempotencyStore (tests/fakes.py)."""
+
+    async def claim(self, key: str, instance_id: UUID) -> UUID | None:
+        """None = we won and the caller creates. A UUID = it already exists. Must not raise."""
+        ...
+
+
+class IdempotencyStore:
+    """`SET NX EX`. Maps an `Idempotency-Key` to the instance UUID it created.
+
+    Claimed BEFORE the DB transaction, so the marker exists before the row it names. The
+    inversion matters: claim after the commit and a crash in between leaves a created
+    instance with no marker, and the client's retry creates a second one.
+
+    Claiming first has its own hole — a crash after the claim and before the commit leaves
+    a marker pointing at an instance that never existed, and the retry is told "already
+    done" about nothing. It is the better hole: the client polls the id, gets a 404, and
+    retries with a fresh key. The alternative loses money to a duplicate Elasticsearch.
+
+    And neither hole is load-bearing, because `instances.release_name` is UNIQUE and
+    deterministic from (team, service_type, id). **That constraint is the guarantee.** This
+    class only saves the round trip to find out.
+    """
+
+    def __init__(self, r: Redis, ttl_s: int = 86400) -> None:
+        """A day is the window a client might reasonably retry in; then the key is garbage."""
+        if ttl_s < 1:
+            raise ValueError("ttl_s must be >= 1")
+        self._r = r
+        self._ttl_s = ttl_s
+
+    async def claim(self, key: str, instance_id: UUID) -> UUID | None:
+        """Try to bind `key` to `instance_id`. Never raises.
+
+        `None` from the happy path means "you won, go create it". `None` from a Redis
+        failure means the same thing — the caller creates, and the UNIQUE constraint
+        catches an actual duplicate. Degrading to "create it" is safe *only* because that
+        constraint exists; without it this would have to fail closed.
+
+        One command when we win, which is the common case and the one the budget is sized
+        for. Two when we lose: the loser pays a GET, and losers are rare by definition.
+        """
+        redis_key = f"idem:{key}"
+        try:
+            REDIS_COMMANDS.labels(op="idempotency").inc()
+            won = await self._r.set(redis_key, str(instance_id), nx=True, ex=self._ttl_s)
+            if won:
+                return None
+            REDIS_COMMANDS.labels(op="idempotency").inc()
+            existing = await self._r.get(redis_key)
+        except _REDIS_DOWN:
+            REDIS_ERRORS.labels(op="idempotency").inc()
+            _log.warning(
+                "idempotency degraded: redis unavailable, falling through to the DB constraint",
+                exc_info=True,
+            )
+            return None
+
+        if existing is None:
+            # The key expired between the SET and the GET. Vanishingly rare, and the honest
+            # answer is "no winner recorded" — let the caller create and let Postgres decide.
+            return None
+        try:
+            return UUID(_as_text(existing))
+        except ValueError:
+            _log.warning("idempotency key holds a non-UUID value; ignoring it")
+            return None
+
+
+# --- Read cache -------------------------------------------------------------------------
+
+
+class InstanceCacheProto(Protocol):
+    """Implemented by InstanceCache (here) and FakeInstanceCache (tests/fakes.py)."""
+
+    async def get(self, instance_id: UUID) -> Instance | None:
+        """The cached instance, or None for a miss. Must not raise."""
+        ...
+
+    async def put(self, inst: Instance) -> None:
+        """Cache `inst` for the TTL. Must not raise."""
+        ...
+
+    async def invalidate(self, instance_id: UUID) -> None:
+        """Drop the entry. Must not raise."""
+        ...
+
+
+class InstanceCache:
+    """Cache-aside for `GET /v1/instances/{id}`. TTL 30s.
+
+    Hit costs one command, miss costs two (the GET, then the SET after Postgres answers).
+    That is ~1 per read at any useful hit rate, which is what keeps a read-heavy poller
+    inside the budget.
+
+    The TTL is short on purpose and is the actual correctness argument. `invalidate()` on
+    every state transition is the fast path, not the guarantee: the worker can crash
+    between the UPDATE and the DEL, and then the cache is wrong. Thirty seconds bounds how
+    wrong. Trusting the invalidation instead — and raising the TTL to an hour — is how a
+    deleted instance stays `ready` in the API for an hour.
+    """
+
+    def __init__(self, r: Redis, ttl_s: int = 30) -> None:
+        if ttl_s < 1:
+            raise ValueError("ttl_s must be >= 1")
+        self._r = r
+        self._ttl_s = ttl_s
+
+    @staticmethod
+    def _key(instance_id: UUID) -> str:
+        return f"inst:{instance_id}"
+
+    async def get(self, instance_id: UUID) -> Instance | None:
+        """One GET. A miss, a Redis outage and a corrupt entry are all the same answer.
+
+        Which is the point: the caller writes `cache.get() or repo.get()` and has no branch
+        for "Redis is broken", because there is nothing different to do about it.
+        """
+        try:
+            REDIS_COMMANDS.labels(op="cache_get").inc()
+            raw = await self._r.get(self._key(instance_id))
+        except _REDIS_DOWN:
+            REDIS_ERRORS.labels(op="cache_get").inc()
+            _log.warning("cache read degraded: redis unavailable, falling through to Postgres")
+            return None
+        if raw is None:
+            return None
+        try:
+            return Instance.model_validate_json(raw)
+        except ValidationError:
+            # A model change deployed over a warm cache. Treat it as a miss and let the TTL
+            # take the old shape out. Not an error: the truth is in Postgres either way.
+            _log.info("cache entry failed validation; treating as a miss")
+            return None
+
+    async def put(self, inst: Instance) -> None:
+        """One SET with `EX`. Never write a key here without a TTL."""
+        try:
+            REDIS_COMMANDS.labels(op="cache_put").inc()
+            await self._r.set(self._key(inst.id), inst.model_dump_json(), ex=self._ttl_s)
+        except _REDIS_DOWN:
+            REDIS_ERRORS.labels(op="cache_put").inc()
+            _log.warning("cache write degraded: redis unavailable")
+
+    async def invalidate(self, instance_id: UUID) -> None:
+        """One DEL. Called by the worker inside the code path that writes the state.
+
+        Inside that path, not after it and not from a subscriber: an invalidation that can
+        be skipped by an early return is an invalidation that will be.
+        """
+        try:
+            REDIS_COMMANDS.labels(op="cache_del").inc()
+            await self._r.delete(self._key(instance_id))
+        except _REDIS_DOWN:
+            REDIS_ERRORS.labels(op="cache_del").inc()
+            _log.warning("cache invalidate degraded: redis unavailable; entry expires within the TTL")

libs/svcforge_core/svcforge_core/domain/__init__.py

@@ -0,0 +1,5 @@
+"""Pure domain logic: state machine, models, backoff math, catalog rules.
+
+No I/O lives here except :func:`svcforge_core.domain.catalog.load_catalog`, which reads
+the one path it is handed. Nothing in this package is async.
+"""

libs/svcforge_core/svcforge_core/domain/backoff.py

@@ -0,0 +1,24 @@
+"""Retry backoff math. Pure: `now` and `rand` are injected, never called for you."""
+
+import random
+from collections.abc import Callable
+from datetime import datetime, timedelta
+
+
+def next_attempt_at(
+    attempt: int,
+    *,
+    now: datetime,
+    base_s: float = 2.0,
+    cap_s: float = 300.0,
+    rand: Callable[[], float] = random.random,
+) -> datetime:
+    """Exponential backoff with full jitter, capped. Pure: takes `now` and `rand`, never calls them itself.
+
+    Delay is uniform in [0, min(cap_s, base_s * 2**attempt)]. attempt is 0-based; raise ValueError if < 0.
+    """
+    if attempt < 0:
+        raise ValueError(f"attempt must be >= 0, got {attempt}")
+    ceiling = min(cap_s, base_s * 2.0**attempt)
+    delay_s = ceiling * rand()
+    return now + timedelta(seconds=delay_s)

libs/svcforge_core/svcforge_core/domain/catalog.py

@@ -0,0 +1,58 @@
+"""Load and validate catalog.yaml into CatalogEntry objects.
+
+The only I/O in `domain/`: reading the one path handed to :func:`load_catalog`.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from svcforge_core.domain.models import CatalogEntry
+
+
+class CatalogError(Exception):
+    """A catalog file could not be parsed or validated.
+
+    `key` names the offending service type, or None when the failure is file-level.
+    """
+
+    def __init__(self, message: str, *, key: str | None = None) -> None:
+        self.key = key
+        super().__init__(message if key is None else f"{key}: {message}")
+
+
+def load_catalog(path: Path) -> dict[str, CatalogEntry]:
+    """Parse catalog.yaml -> {service_type: CatalogEntry}. yaml.safe_load, never yaml.load.
+
+    Raise CatalogError (typed, with the offending key) on a bad file.
+    """
+    try:
+        raw: Any = yaml.safe_load(path.read_text())
+    except OSError as exc:
+        raise CatalogError(f"cannot read catalog at {path}: {exc}") from exc
+    except yaml.YAMLError as exc:
+        raise CatalogError(f"catalog at {path} is not valid YAML: {exc}") from exc
+
+    if not isinstance(raw, dict):
+        raise CatalogError(
+            f"catalog at {path} must be a mapping of service_type -> entry, got {type(raw).__name__}"
+        )
+
+    services: Any = raw.get("services", raw)
+    if not isinstance(services, dict):
+        raise CatalogError(f"catalog at {path}: 'services' must be a mapping, got {type(services).__name__}")
+
+    catalog: dict[str, CatalogEntry] = {}
+    for key, body in services.items():
+        service_type = str(key)
+        if not isinstance(body, dict):
+            raise CatalogError(f"entry must be a mapping, got {type(body).__name__}", key=service_type)
+        try:
+            catalog[service_type] = CatalogEntry(service_type=service_type, **body)
+        except ValidationError as exc:
+            raise CatalogError(f"invalid entry: {exc}", key=service_type) from exc
+        except TypeError as exc:
+            raise CatalogError(f"invalid entry: {exc}", key=service_type) from exc
+    return catalog

libs/svcforge_core/svcforge_core/domain/models.py

@@ -0,0 +1,101 @@
+"""Domain models. Pydantic v2, all frozen.
+
+`frozen=True` is not transitive: a `dict` field (e.g. `SizeSpec.resources`) stays mutable
+in place. Treat those dicts as read-only by convention.
+"""
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from svcforge_core.domain.states import InstanceState
+
+TEAM_PATTERN = r"^[a-z0-9-]+$"
+
+
+class TaskKind(StrEnum):
+    """What a worker is being asked to do."""
+
+    PROVISION = "provision"
+    DEPROVISION = "deprovision"
+    UPGRADE = "upgrade"
+    VERIFY = "verify"
+
+
+class TaskState(StrEnum):
+    """Where a task is in the claim loop."""
+
+    QUEUED = "queued"
+    RUNNING = "running"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class SizeSpec(BaseModel):
+    """One t-shirt size of a catalog entry."""
+
+    model_config = ConfigDict(frozen=True)
+
+    replicas: int = Field(ge=1)
+    resources: dict[str, Any]
+
+
+class CatalogEntry(BaseModel):
+    """One offerable service type and the sizes it comes in."""
+
+    model_config = ConfigDict(frozen=True)
+
+    service_type: str = Field(min_length=1)
+    chart: str = Field(min_length=1)
+    chart_version: str = Field(min_length=1)
+    sizes: dict[str, SizeSpec]
+    # Bypass tenant maintenance windows for this entry's upgrades. Defaults False: a
+    # normal version bump waits for 03:00 Sunday; a CVE with a public exploit does not.
+    security: bool = False
+
+
+class Instance(BaseModel):
+    """A provisioned (or in-flight) service instance owned by a team."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: UUID
+    team: str = Field(min_length=1, pattern=TEAM_PATTERN)
+    service_type: str
+    size: str
+    state: InstanceState
+    namespace: str
+    release_name: str
+    chart_version: str
+    endpoint: str | None = None
+    error: str | None = None
+    expires_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class Task(BaseModel):
+    """A unit of work against an instance, claimed by exactly one worker at a time."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id: int
+    instance_id: UUID
+    kind: TaskKind
+    state: TaskState
+    attempts: int = Field(ge=0)
+    run_after: datetime
+    locked_by: str | None = None
+    last_error: str | None = None
+
+    # The W3C trace context of whoever enqueued this. Null is normal, not an error: a task
+    # the reconciler raised on its own tick has no inbound request to belong to.
+    traceparent: str | None = None
+
+    # Denormalised from `instances` by the claim query. It is here so a worker can bind
+    # `team` onto its log context at claim time, before it has loaded anything — the point
+    # of structured logs is that the FIRST line already tells you whose tenant broke.
+    team: str | None = None

libs/svcforge_core/svcforge_core/domain/states.py

@@ -0,0 +1,37 @@
+"""The instance lifecycle state machine, encoded as data."""
+
+from enum import StrEnum
+from typing import Final
+
+
+class InstanceState(StrEnum):
+    """Lifecycle of a provisioned service instance."""
+
+    REQUESTED = "requested"
+    PROVISIONING = "provisioning"
+    READY = "ready"
+    DELETING = "deleting"
+    DELETED = "deleted"
+    FAILED = "failed"
+
+
+class IllegalTransition(Exception):
+    """Raised by transition() when cur -> nxt is not in LEGAL."""
+
+
+LEGAL: Final[dict[InstanceState, frozenset[InstanceState]]] = {
+    InstanceState.REQUESTED: frozenset({InstanceState.PROVISIONING, InstanceState.FAILED}),
+    InstanceState.PROVISIONING: frozenset({InstanceState.READY, InstanceState.FAILED}),
+    InstanceState.READY: frozenset({InstanceState.DELETING, InstanceState.FAILED}),
+    InstanceState.DELETING: frozenset({InstanceState.DELETED, InstanceState.FAILED}),
+    # `deleted` is terminal: an empty frozenset, not a missing key.
+    InstanceState.DELETED: frozenset(),
+    InstanceState.FAILED: frozenset({InstanceState.PROVISIONING, InstanceState.DELETING}),
+}
+
+
+def transition(cur: InstanceState, nxt: InstanceState) -> InstanceState:
+    """Return nxt if LEGAL[cur] contains it, else raise IllegalTransition. Pure. No DB."""
+    if nxt not in LEGAL[cur]:
+        raise IllegalTransition(f"{cur} -> {nxt} is not a legal transition")
+    return nxt

libs/svcforge_core/svcforge_core/domain/windows.py

@@ -0,0 +1,111 @@
+"""Maintenance windows: when a tenant will tolerate an upgrade.
+
+Pure domain. No I/O, no `Clock`, no `datetime.now()` anywhere in this file — `now` arrives
+as a parameter and every datetime crossing this module's boundary is aware and UTC. That
+is the whole discipline: `datetime.now()` is naive and lies, comparing aware to naive
+raises TypeError at 03:00 on a Sunday, and mypy will not catch it for you.
+
+Local time is where the arithmetic has to happen, though. A window means "03:00 as the
+tenant reads a clock", which is not a fixed UTC offset in any zone that observes DST. So
+the cron is evaluated in the window's own zone and the result is converted back to UTC at
+the door.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from croniter import croniter
+
+CRON_FIELDS = 5
+SEPARATOR = "|"
+
+
+class BadWindow(ValueError):
+    """A maintenance window spec is not a 5-field cron plus a known IANA zone."""
+
+
+@dataclass(frozen=True)
+class MaintenanceWindow:
+    """When a tenant's instance may be upgraded."""
+
+    cron: str  # standard 5-field cron
+    tz: str  # IANA name, e.g. 'Asia/Ho_Chi_Minh'
+
+
+def parse_window(spec: str | None) -> MaintenanceWindow | None:
+    """'0 3 * * 0|Asia/Ho_Chi_Minh' -> MaintenanceWindow. None -> None (upgrade any time).
+
+    Raise BadWindow on an invalid cron expression or unknown IANA zone. Validation happens
+    here, once, on the way in — not at 03:00 in a worker, where the failure is a task that
+    dies inside someone else's maintenance window.
+    """
+    if spec is None or not spec.strip():
+        return None
+
+    cron, sep, tz = spec.partition(SEPARATOR)
+    if not sep:
+        raise BadWindow(f"window {spec!r} is not 'CRON{SEPARATOR}IANA_ZONE'")
+
+    cron, tz = cron.strip(), tz.strip()
+
+    # croniter's is_valid() accepts a 6-field form (with seconds). The column is documented
+    # as 5-field, so a 6th field is a typo, not a feature.
+    if len(cron.split()) != CRON_FIELDS:
+        raise BadWindow(f"cron {cron!r} must have exactly {CRON_FIELDS} fields")
+    if not croniter.is_valid(cron):
+        raise BadWindow(f"cron {cron!r} is not a valid cron expression")
+
+    try:
+        ZoneInfo(tz)
+    except (ZoneInfoNotFoundError, ValueError) as exc:
+        # ZoneInfoNotFoundError in a slim image means tzdata is missing, not that the zone
+        # is fictional. Same exception, so the message names both possibilities.
+        raise BadWindow(f"timezone {tz!r} is not a known IANA zone (is tzdata installed?)") from exc
+
+    return MaintenanceWindow(cron=cron, tz=tz)
+
+
+def next_window_open(window: MaintenanceWindow | None, now: datetime) -> datetime:
+    """Next time the window opens, as an aware UTC datetime.
+
+    `now` must be aware; raise ValueError if it is naive. window=None -> return now.
+
+    Inclusive of `now`: if the window opens at this exact instant, that instant is the
+    answer. Otherwise an upgrade that became due precisely at 03:00 would be pushed a
+    whole week.
+    """
+    if now.tzinfo is None or now.utcoffset() is None:
+        raise ValueError(f"now must be an aware datetime, got naive {now!r}")
+
+    if window is None:
+        return now.astimezone(UTC)
+
+    local_now = now.astimezone(ZoneInfo(window.tz))
+
+    # croniter.get_next() is strictly greater than its start. Backing the start off by one
+    # second makes a `now` that lands exactly on a cron minute return itself; a `now` at
+    # 03:00:30 still rolls to the next occurrence, because cron times are minute-aligned
+    # and 03:00:00 is already behind 03:00:29.
+    start = local_now - timedelta(seconds=1)
+    local_next: datetime = croniter(window.cron, start).get_next(datetime)
+
+    return local_next.astimezone(UTC)
+
+
+def schedule_upgrade_at(window: MaintenanceWindow | None, security: bool, now: datetime) -> datetime:
+    """run_after for an upgrade task. security=True bypasses the window -> now.
+
+    A security fix with a public exploit does not wait until Sunday. That is the entire
+    reason `security:` exists in the catalog.
+
+    `now` must be aware here too, bypass or not: the naive-datetime rule does not get a
+    hole punched in it by the branch that skips the window.
+    """
+    if now.tzinfo is None or now.utcoffset() is None:
+        raise ValueError(f"now must be an aware datetime, got naive {now!r}")
+    if security:
+        return now.astimezone(UTC)
+    return next_window_open(window, now)

libs/svcforge_core/svcforge_core/migrate.py

@@ -0,0 +1,81 @@
+"""Migration runner. Twenty lines of psycopg, not Alembic.
+
+Three rules this file exists to enforce:
+
+1. Never migrate on app startup. N replicas would race. This runs as a Helm
+   pre-upgrade/pre-install hook Job, once, before any new pod serves traffic.
+2. Forward-only. There are no down scripts. A mistake is fixed by a new migration.
+3. Expand/contract. Add nullable, backfill, switch reads, drop the old column in a
+   LATER release. A rename is three deploys, never one.
+
+Run:  python -m svcforge_core.migrate
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import psycopg
+
+from svcforge_core.settings import load_settings
+
+MIGRATIONS_DIR = Path(__file__).resolve().parents[3] / "migrations"
+
+# Advisory lock: two Jobs racing (a retried hook, a hand-run) must not both apply DDL.
+# Session-scoped, so this needs the session pooler (5432), not pgbouncer (6543).
+_LOCK_KEY = 0x5643_464F  # "SVCFO"
+
+_BOOTSTRAP = """
+create table if not exists schema_migrations (
+    filename    text        primary key,
+    applied_at  timestamptz not null default now()
+);
+"""
+
+
+def _pending(conn: psycopg.Connection[tuple[str, ...]], files: list[Path]) -> list[Path]:
+    with conn.cursor() as cur:
+        cur.execute("select filename from schema_migrations")
+        done = {row[0] for row in cur.fetchall()}
+    return [f for f in files if f.name not in done]
+
+
+def main() -> int:
+    settings = load_settings()
+    files = sorted(MIGRATIONS_DIR.glob("*.sql"))
+    if not files:
+        print(f"no migrations found in {MIGRATIONS_DIR}", file=sys.stderr)
+        return 1
+
+    with psycopg.connect(settings.migration_dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            cur.execute("select pg_advisory_lock(%s)", (_LOCK_KEY,))
+        try:
+            with conn.cursor() as cur:
+                cur.execute(_BOOTSTRAP)
+
+            pending = _pending(conn, files)
+            if not pending:
+                print("up to date, nothing to apply")
+                return 0
+
+            for path in pending:
+                sql = path.read_text(encoding="utf-8")
+                # One transaction per file: a file applies completely or not at all.
+                with conn.transaction():
+                    with conn.cursor() as cur:
+                        cur.execute(sql)
+                        cur.execute(
+                            "insert into schema_migrations (filename) values (%s)",
+                            (path.name,),
+                        )
+                print(f"applied: {path.name}")
+        finally:
+            with conn.cursor() as cur:
+                cur.execute("select pg_advisory_unlock(%s)", (_LOCK_KEY,))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

libs/svcforge_core/svcforge_core/obs.py

@@ -0,0 +1,308 @@
+"""Logs, traces, metrics. One setup() call, made once, before anything else.
+
+Three libraries, one module, because the three are one decision. A log line without the
+trace id it belongs to is a log line you cannot join to anything; a span without the
+`instance_id` the request is about is a span you cannot search for. They are wired here
+together so that no service can configure two of the three and ship.
+
+The three things that make this module worth reading:
+
+1. **Context does not cross a queue.** A trace is a chain of parent/child span contexts
+   passed in-process or over a wire header. `POST /v1/instances` inserts a row and
+   returns; the worker picks that row up ninety seconds later in a different pod. Nothing
+   carries the context across — unless we carry it ourselves. So: `inject_traceparent()`
+   at enqueue, a `traceparent` column, `context_from_traceparent()` at claim. Two
+   disconnected traces in Tempo is the symptom of skipping this.
+
+2. **Histogram buckets are a domain decision.** prometheus_client's defaults top out at
+   10 seconds because they were chosen for HTTP handlers. A provision is `helm --wait` on
+   a StatefulSet: minutes. With the defaults every observation lands in `+Inf`,
+   `histogram_quantile` interpolates inside a bucket that spans 10s→infinity, and the p95
+   it prints is a number with no relationship to reality. The buckets below are sized for
+   what is being measured.
+
+3. **One process per pod.** prometheus_client keeps its registry in process memory. Run
+   `uvicorn --workers 4` and Prometheus scrapes whichever of the four children the socket
+   happens to hand it, so counters appear to jump backwards. There are two fixes:
+   `PROMETHEUS_MULTIPROC_DIR` + `MultiProcessCollector` (a shared mmap directory, a
+   gauge-mode decision at every call site, and dead files to garbage-collect after every
+   crash), or one process per pod and scale with replicas. This repo takes the second.
+   `PROMETHEUS_MULTIPROC_DIR` is deliberately not set, and nothing here reads it.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+import structlog
+from opentelemetry import trace
+from opentelemetry.context import Context
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from prometheus_client import Counter, Gauge, Histogram, start_http_server
+
+if TYPE_CHECKING:
+    from svcforge_core.settings import Settings
+
+# --- Metrics ----------------------------------------------------------------------------
+#
+# Module level, created exactly once at import. A second registration of the same name
+# against the default registry raises ValueError, which is a feature: it turns "two modules
+# each defined their own copy of this counter" into an ImportError at startup instead of a
+# metric that silently reports half the truth.
+
+TASKS_CLAIMED = Counter(
+    "svcforge_tasks_claimed_total",
+    "Tasks claimed off the queue by a worker.",
+    ["kind"],
+)
+
+TASKS_FAILED = Counter(
+    "svcforge_tasks_failed_total",
+    "Tasks that exhausted their attempts and went to 'failed'.",
+    ["kind"],
+)
+
+PROVISION_TIME = Histogram(
+    "svcforge_provision_duration_seconds",
+    "Wall time of a provision task, claim to terminal report.",
+    # NOT the defaults. See the module docstring: the defaults end at 10s and a provision
+    # takes minutes. The top finite bucket is 1800 because helm's own --timeout is 600 and
+    # a provision past thirty minutes is not slow, it is broken and belongs in +Inf.
+    buckets=(10, 30, 60, 120, 300, 600, 1800, float("inf")),
+)
+
+QUEUE_DEPTH = Gauge(
+    "svcforge_queue_depth",
+    "Runnable tasks waiting in the queue. Set by the reconciler each tick.",
+)
+
+INSTANCES = Gauge(
+    "svcforge_instances",
+    "Instances by lifecycle state. Set by the reconciler each tick.",
+    ["state"],
+)
+
+RECONCILER_LAST_TICK = Gauge(
+    "svcforge_reconciler_last_tick_timestamp_seconds",
+    "Unix time of the reconciler's last completed tick. The liveness signal that matters.",
+)
+
+# --- Wiring -----------------------------------------------------------------------------
+
+_TRACER_NAME = "svcforge"
+
+# setup() is idempotent because it is called from three entrypoints and from tests, and
+# because configuring structlog twice silently discards the first configuration while
+# adding a second stdout handler to the root logger — every line then prints twice.
+_configured = False
+
+_propagator = TraceContextTextMapPropagator()
+
+
+def _add_trace_ids(
+    _logger: Any,  # noqa: ANN401 - structlog's Processor signature; the logger is untyped
+    _method: str,
+    event_dict: structlog.typing.EventDict,
+) -> structlog.typing.EventDict:
+    """Stamp the active trace/span id onto the line, if there is one.
+
+    This is the join key. Without it, "find the logs for this trace" is a full-text search
+    over a time window and a guess; with it, it is one query. Hex-formatted to the widths
+    the W3C spec uses, so the value pasted from Tempo matches the value in Loki.
+    """
+    span = trace.get_current_span()
+    ctx = span.get_span_context()
+    if ctx.is_valid:
+        event_dict["trace_id"] = format(ctx.trace_id, "032x")
+        event_dict["span_id"] = format(ctx.span_id, "016x")
+    return event_dict
+
+
+def setup(service_name: str, settings: Settings) -> None:
+    """Configure structlog, the tracer provider, and the metric registry. Idempotent.
+
+    Called once from each service's entrypoint, before anything else — "before anything
+    else" because any logger bound before this runs keeps the default configuration
+    (`cache_logger_on_first_use`), and a module-level `log = structlog.get_logger()` in an
+    import that lands first will print unstructured text forever.
+    """
+    global _configured  # process-wide config is process-wide state
+    if _configured:
+        return
+
+    _setup_logging(service_name, settings)
+    _setup_tracing(service_name, settings)
+    _configured = True
+
+
+def _setup_logging(service_name: str, settings: Settings) -> None:
+    """structlog + stdlib logging, both rendering JSON to stdout through one handler.
+
+    The stdlib half is not optional. `psycopg`, `httpx`, `uvicorn` and the OTEL SDK all log
+    through `logging`; without the ProcessorFormatter bridge below, their lines arrive as
+    bare text on the same stdout and every one of them is a parse failure in the collector.
+    """
+    level = getattr(logging, settings.log_level.upper(), logging.INFO)
+
+    shared: list[structlog.typing.Processor] = [
+        # FIRST. This is what puts instance_id/task_id/team on every line, including the
+        # lines written by code that has never heard of them.
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        structlog.processors.TimeStamper(fmt="iso", utc=True),
+        _add_trace_ids,
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+    ]
+
+    structlog.configure(
+        processors=[
+            *shared,
+            # Hands off to the stdlib formatter below, which appends the renderer. This is
+            # what lets one handler render both structlog and foreign logs identically.
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    # The renderer goes LAST and nothing follows it: it turns the event dict into a string,
+    # so any processor after it receives a str where it expects a dict and raises.
+    renderer: structlog.typing.Processor = (
+        structlog.processors.JSONRenderer() if settings.log_json else structlog.dev.ConsoleRenderer()
+    )
+    formatter = structlog.stdlib.ProcessorFormatter(
+        foreign_pre_chain=shared,  # applied to records from logging.getLogger(...) callers
+        processors=[structlog.stdlib.ProcessorFormatter.remove_processors_meta, renderer],
+    )
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    root = logging.getLogger()
+    # Replace rather than append: basicConfig may already have run, and two handlers means
+    # two copies of every line. stdout only — a container writes logs to stdout and the
+    # collector tails them from there. A log file inside a pod is deleted with the pod.
+    root.handlers = [handler]
+    root.setLevel(level)
+
+    structlog.contextvars.bind_contextvars(service=service_name)
+
+
+def _setup_tracing(service_name: str, settings: Settings) -> None:
+    """Set the global tracer provider, exporting over OTLP when an endpoint is configured.
+
+    Skipped entirely when something already set a provider: the API runs under
+    `opentelemetry-instrument`, whose auto-instrumentation installs one before our
+    `main()` is reached. Overwriting it drops the FastAPI and psycopg instrumentation's
+    spans on the floor, and the SDK only logs a warning about it.
+    """
+    if isinstance(trace.get_tracer_provider(), TracerProvider):
+        return
+
+    provider = TracerProvider(resource=Resource.create({"service.name": service_name}))
+
+    if settings.otel_endpoint:
+        exporter = _otlp_exporter(settings.otel_endpoint)
+        if exporter is not None:
+            # Batch, not Simple: SimpleSpanProcessor exports inline on span end, so every
+            # helm span would block on a network round trip to the collector.
+            provider.add_span_processor(BatchSpanProcessor(exporter))
+
+    trace.set_tracer_provider(provider)
+
+
+def _otlp_exporter(endpoint: str) -> Any | None:  # noqa: ANN401 - one of two exporter classes
+    """The OTLP exporter, if the optional exporter package is installed.
+
+    Optional on purpose. In the cluster the API runs under `opentelemetry-instrument`,
+    which brings its own exporter and configures it from `OTEL_EXPORTER_OTLP_*`. Making it
+    a hard dependency of the shared library would mean every unit test imports gRPC.
+    """
+    try:
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+    except ImportError:
+        logging.getLogger(__name__).warning(
+            "otel_endpoint is set but no OTLP exporter is installed; traces stay in-process",
+            extra={"endpoint": endpoint},
+        )
+        return None
+    return OTLPSpanExporter(endpoint=endpoint)
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    """A bound logger. Call it inside a function, not at import time — see setup()."""
+    logger: structlog.stdlib.BoundLogger = structlog.get_logger(name)
+    return logger
+
+
+def tracer() -> trace.Tracer:
+    """The svcforge tracer. Manual spans wrap helm calls, and nothing else.
+
+    Everything else is auto-instrumented (FastAPI, psycopg). A hand-rolled span around a
+    function that the SDK already wraps is a duplicated span and a maintenance cost.
+    """
+    return trace.get_tracer(_TRACER_NAME)
+
+
+def start_metrics_server(port: int) -> None:
+    """Expose /metrics on `port`, for the services with no HTTP server of their own.
+
+    The worker and reconciler have no Service; the chart's PodMonitor scrapes them by pod
+    on the port named `metrics`. The API does not use this — it mounts the same registry on
+    its own ASGI app.
+    """
+    start_http_server(port)
+
+
+# --- Context that has to cross a process boundary ---------------------------------------
+
+
+def bind_task_context(instance_id: UUID, task_id: int, team: str) -> None:
+    """Bind the three keys every log line in a task must carry. Called at claim time.
+
+    `clear_contextvars()` first, and this is the whole reason the function exists rather
+    than three `bind_contextvars` calls at the call site. A worker coroutine reuses its
+    context across loop iterations; without the clear, task 41's `instance_id` is still
+    bound when task 42 starts logging, and the log for the incident you are debugging
+    names the wrong tenant. Contextvars are per-task in asyncio, which makes this safe
+    under the concurrency semaphore: two handlers running at once do not see each other's.
+    """
+    structlog.contextvars.clear_contextvars()
+    structlog.contextvars.bind_contextvars(
+        instance_id=str(instance_id),
+        task_id=task_id,
+        team=team,
+    )
+
+
+def inject_traceparent() -> str | None:
+    """Serialise the active span context to a W3C traceparent, for the tasks row.
+
+    None when there is no recording span — a task enqueued by the reconciler's own tick has
+    no inbound request to be part of. Nullable column, nullable return: an untraced task is
+    normal, not an error.
+    """
+    carrier: dict[str, str] = {}
+    _propagator.inject(carrier)
+    return carrier.get("traceparent")
+
+
+def context_from_traceparent(traceparent: str | None) -> Context:
+    """Inverse of inject_traceparent. Used at claim to parent the worker span to the API's.
+
+    An empty Context for None or for a malformed value — `extract` does not raise on a
+    traceparent that fails to parse, it returns the carrier's context unchanged, and the
+    resulting span starts a new trace. A bad header must never fail a provision.
+    """
+    if not traceparent:
+        return Context()
+    return _propagator.extract({"traceparent": traceparent})

libs/svcforge_core/svcforge_core/repo/__init__.py

@@ -0,0 +1 @@
+"""SQL. Rows in, domain objects out. Knows psycopg; knows nothing about HTTP."""

libs/svcforge_core/svcforge_core/repo/db.py

@@ -0,0 +1,57 @@
+"""The connection pool.
+
+Two settings here are load-bearing on Supabase's transaction pooler (port 6543) and
+cost hours if you get them wrong. Both are documented at the call site below.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import AsyncConnection
+from psycopg.rows import dict_row
+from psycopg_pool import AsyncConnectionPool
+
+# The pool hands out dict-row connections because of `row_factory=dict_row` below. Say so
+# in the type system too, or every `row["attempts"]` in this codebase is a mypy error
+# against a bare `AsyncConnectionPool`, which resolves to tuple rows. The runtime was
+# always right; without these aliases the annotations quietly disagree with it, and the
+# fix people reach for is `# type: ignore`, which throws away the checking entirely.
+type DictRow = dict[str, Any]
+type DictConnection = AsyncConnection[DictRow]
+type DictPool = AsyncConnectionPool[DictConnection]
+
+
+def make_pool(dsn: str, min_size: int = 1, max_size: int = 5) -> DictPool:
+    """Construct the pool. Does NOT open it — the caller owns open/close.
+
+    `open=False` is deliberate: the constructor does zero I/O, so building a pool at
+    import time and never opening it fails later as a PoolTimeout at first use, far
+    from the cause. The caller (a FastAPI lifespan, a worker main) opens and closes it.
+
+    kwargs are per-connection:
+
+    * `prepare_threshold=None` — REQUIRED through pgbouncer in transaction mode.
+      psycopg3 auto-prepares a statement after it sees it 5 times. pgbouncer may hand
+      the next execution to a different backend, which has never heard of that prepared
+      statement. Symptom: everything works for exactly five calls, then
+      `prepared statement "_pg3_0" does not exist` — intermittent, only under
+      concurrency, never in a unit test.
+
+    * `row_factory=dict_row` — rows arrive as dicts, so `Instance.model_validate(row)`
+      works directly instead of unpacking tuples by position.
+
+    Also gone on 6543: LISTEN/NOTIFY, session-level SET, cross-statement advisory locks.
+    `SELECT ... FOR UPDATE SKIP LOCKED` inside one transaction is unaffected — which is
+    exactly why the queue is built on it. Use the session pooler (5432) for migrations.
+
+    max_size is a database-capacity decision, not a throughput knob: the free tier has a
+    small connection budget, and replicas multiply this number.
+    """
+    return AsyncConnectionPool(
+        conninfo=dsn,
+        min_size=min_size,
+        max_size=max_size,
+        open=False,
+        kwargs={"prepare_threshold": None, "row_factory": dict_row, "autocommit": False},
+    )

libs/svcforge_core/svcforge_core/repo/instances.py

@@ -0,0 +1,192 @@
+"""Instance persistence.
+
+AuthZ lives in the WHERE clause. `get(id, team)` filters by team in SQL rather than
+fetching the row and comparing in Python: a wrong-team id must be indistinguishable
+from a nonexistent one, and a check you can forget to write is a check you will forget
+to write.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+from uuid import UUID
+
+from psycopg import AsyncConnection
+from psycopg.rows import dict_row
+
+from svcforge_core.domain.models import Instance
+from svcforge_core.domain.states import InstanceState
+from svcforge_core.repo.db import DictPool
+
+_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
+              endpoint, error, expires_at, created_at, updated_at"""
+
+
+@dataclass(frozen=True)
+class UpgradeCandidate:
+    """One row of the day-2 work list: an instance, plus the raw window spec to schedule in.
+
+    `maintenance_window` rides alongside rather than inside `Instance` because `create()`
+    never writes it — folding it into the model would mean every freshly created Instance
+    reports `maintenance_window=None` whether or not the row has one. A column only the
+    work list reads is a column only the work list carries.
+    """
+
+    instance: Instance
+    maintenance_window: str | None
+
+
+class InstanceRepo:
+    """Reads and writes `instances`."""
+
+    def __init__(self, pool: DictPool) -> None:
+        self._pool = pool
+
+    async def create(self, conn: AsyncConnection[dict[str, Any]], inst: Instance) -> Instance:
+        """Insert one instance.
+
+        Takes a `conn` rather than using the pool so the caller can share a transaction
+        with TaskRepo.enqueue — the instance row and its provision task must commit
+        together or not at all. That single fact is why the queue is in Postgres.
+        """
+        async with conn.cursor() as cur:
+            await cur.execute(
+                f"""insert into instances (id, team, service_type, size, state, namespace,
+                                           release_name, chart_version, endpoint, error, expires_at)
+                    values (%(id)s, %(team)s, %(service_type)s, %(size)s, %(state)s, %(namespace)s,
+                            %(release_name)s, %(chart_version)s, %(endpoint)s, %(error)s, %(expires_at)s)
+                    returning {_COLUMNS}""",  # noqa: S608 - _COLUMNS is a module constant, not input
+                {
+                    "id": inst.id,
+                    "team": inst.team,
+                    "service_type": inst.service_type,
+                    "size": inst.size,
+                    "state": inst.state.value,
+                    "namespace": inst.namespace,
+                    "release_name": inst.release_name,
+                    "chart_version": inst.chart_version,
+                    "endpoint": inst.endpoint,
+                    "error": inst.error,
+                    "expires_at": inst.expires_at,
+                },
+            )
+            row = await cur.fetchone()
+        assert row is not None  # noqa: S101 - `returning` always yields a row or raises
+        return Instance.model_validate(row)
+
+    async def get(self, id: UUID, team: str) -> Instance | None:
+        """Fetch one instance owned by `team`. None if it does not exist OR is not theirs."""
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                f"select {_COLUMNS} from instances where id = %s and team = %s",  # noqa: S608
+                (id, team),
+            )
+            row = await cur.fetchone()
+        return Instance.model_validate(row) if row else None
+
+    async def list(self, team: str, limit: int = 50) -> list[Instance]:
+        """The team's instances, newest first."""
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                f"""select {_COLUMNS} from instances
+                    where team = %s order by created_at desc limit %s""",  # noqa: S608
+                (team, limit),
+            )
+            rows = await cur.fetchall()
+        return [Instance.model_validate(r) for r in rows]
+
+    async def update_state(
+        self,
+        id: UUID,
+        expect: InstanceState,
+        to: InstanceState,
+        error: str | None = None,
+        endpoint: str | None = None,
+    ) -> bool:
+        """Compare-and-set. False if the row moved under you.
+
+        `where id=%s and state=%s` is the whole trick: two workers racing to move the
+        same instance means exactly one UPDATE matches a row. The loser gets False and
+        must not treat it as an error — it means someone else already did the work.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                """update instances
+                      set state = %(to)s,
+                          error = %(error)s,
+                          endpoint = coalesce(%(endpoint)s, endpoint),
+                          updated_at = now()
+                    where id = %(id)s and state = %(expect)s""",
+                {
+                    "id": id,
+                    "expect": expect.value,
+                    "to": to.value,
+                    "error": error,
+                    "endpoint": endpoint,
+                },
+            )
+            return cur.rowcount == 1
+
+    async def set_error(self, id: UUID, error: str) -> None:
+        """Record a terminal failure message. Used when a task exhausts its attempts."""
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                "update instances set error = %s, state = %s, updated_at = now() where id = %s",
+                (error[-2000:], InstanceState.FAILED.value, id),
+            )
+
+    async def list_upgradable(
+        self,
+        service_type: str,
+        catalog_version: str,
+        own_team: str,
+        max_in_flight: int = 1,
+    ) -> Sequence[UpgradeCandidate]:
+        """The day-2 work list: `ready` instances not yet on the catalog's pinned version.
+
+        The whole rollout is this query. Three clauses carry it:
+
+        `not exists (... rollout_state = 'halted')` — a failed `verify` writes one column
+        and this query goes empty for that service type. That is the stop button: no
+        rollouts table, no state machine, no pause/resume CLI. You clear it with SQL.
+
+        `order by team = %(own_team)s desc` — your own instances upgrade first, so you are
+        the tenant who finds out the chart is broken. Eating your own dog food is a
+        `order by`, not a policy document.
+
+        `limit %(max_in_flight)s` — a config value, not a scheduler. It stays at 1 until 1
+        is too slow, and 1 is what makes the halt meaningful: the fleet stops after the
+        first casualty rather than after all of them.
+        """
+        # `row_factory=dict_row` is already the pool's per-connection default; naming it
+        # here is for mypy, which cannot see a row factory passed through a kwargs dict
+        # and would otherwise type these rows as tuples.
+        async with self._pool.connection() as conn, conn.cursor(row_factory=dict_row) as cur:
+            await cur.execute(
+                f"""select {_COLUMNS}, maintenance_window from instances
+                     where state = 'ready'
+                       and service_type = %(service_type)s
+                       and chart_version <> %(catalog_version)s
+                       and not exists (
+                             select 1 from catalog_versions cv
+                              where cv.service_type = %(service_type)s
+                                and cv.rollout_state = 'halted')
+                     order by team = %(own_team)s desc, created_at
+                     limit %(max_in_flight)s""",  # noqa: S608 - _COLUMNS is a module constant, not input
+                {
+                    "service_type": service_type,
+                    "catalog_version": catalog_version,
+                    "own_team": own_team,
+                    "max_in_flight": max_in_flight,
+                },
+            )
+            rows = await cur.fetchall()
+        return [
+            UpgradeCandidate(
+                instance=Instance.model_validate(r),
+                maintenance_window=r["maintenance_window"],
+            )
+            for r in rows
+        ]

libs/svcforge_core/svcforge_core/repo/reconcile.py

@@ -0,0 +1,276 @@
+"""The reconciler's SQL.
+
+Why this file exists rather than the queries living in `services/reconciler/main.py`: the
+layer rule says transport knows nothing about SQL, and the reconciler is transport — a CLI
+entrypoint. It gets its own repo module rather than growing `InstanceRepo` and `TaskRepo`
+because everything here is a *sweep*: it reads rows nobody asked about and it writes an
+instance state and a task row in the same transaction. `InstanceRepo.update_state` owns its
+own connection by design, so the reconciler cannot get atomicity from it without reaching
+around the repo — which is the thing the layer rule exists to prevent.
+
+The recurring shape below is: lock the row, re-check the condition under the lock, act.
+The re-check is not paranoia about concurrency — the reconciler is a singleton. It is what
+makes the sweep idempotent against *itself*: a tick that crashes after the insert and
+before the commit must leave nothing behind, and the next tick must not double-enqueue.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from psycopg import AsyncCursor
+
+from svcforge_core.domain.models import Instance, TaskKind, TaskState
+from svcforge_core.domain.states import InstanceState, transition
+from svcforge_core.obs import inject_traceparent
+from svcforge_core.repo.db import DictPool
+
+_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
+              endpoint, error, expires_at, created_at, updated_at"""
+
+# A task nobody will ever run again. The idempotency guard on every enqueue below asks
+# "is one already outstanding?", and 'done'/'failed' are not outstanding: a failed
+# deprovision that exhausted its attempts must be re-enqueueable by the next sweep, or a
+# transient cluster outage would permanently strand the instance.
+_UNFINISHED = (TaskState.QUEUED.value, TaskState.RUNNING.value)
+
+
+class ReconcileRepo:
+    """Fleet-wide sweeps. Read-mostly, and every write is one transaction."""
+
+    def __init__(self, pool: DictPool) -> None:
+        self._pool = pool
+
+    # --- Gauges -------------------------------------------------------------------------
+
+    async def queue_depth(self) -> int:
+        """Tasks waiting to be claimed.
+
+        Counts every `queued` row, not just the runnable ones (`run_after <= now()`). The
+        alert on this gauge is `deriv(...) > 0` — "the backlog is growing" — and a backlog
+        of tasks parked on backoff is exactly the backlog you want to see growing.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute("select count(*) as n from tasks where state = %s", (TaskState.QUEUED.value,))
+            row = await cur.fetchone()
+        return int(row["n"]) if row else 0
+
+    async def instance_counts(self) -> dict[str, int]:
+        """Instances per lifecycle state. States with no rows are absent, not zero."""
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute("select state, count(*) as n from instances group by state")
+            rows = await cur.fetchall()
+        return {str(r["state"]): int(r["n"]) for r in rows}
+
+    # --- Drift --------------------------------------------------------------------------
+
+    async def ready_instances(self) -> list[Instance]:
+        """Every instance the DB believes is running. The drift check's expectation."""
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                f"select {_COLUMNS} from instances where state = %s",  # noqa: S608 - module constant
+                (InstanceState.READY.value,),
+            )
+            rows = await cur.fetchall()
+        return [Instance.model_validate(r) for r in rows]
+
+    async def known_releases(self) -> set[tuple[str, str]]:
+        """(release_name, namespace) for every instance row, in any state.
+
+        Any state, deliberately. An instance that is still `requested` has no release yet,
+        but a worker may be installing it *right now* — treating it as unknown would
+        report a healthy in-flight provision as an orphan on every tick.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute("select release_name, namespace from instances")
+            rows = await cur.fetchall()
+        return {(str(r["release_name"]), str(r["namespace"])) for r in rows}
+
+    async def enqueue_reprovision(self, instance_id: UUID, reason: str) -> int | None:
+        """`ready` instance whose release vanished -> back to `provisioning`, with a task.
+
+        None if the row moved, or if a provision is already outstanding.
+
+        The two-hop state change is the interesting part, and it is not busywork:
+
+        * `LEGAL` has no `ready -> provisioning` edge. The tenant-visible lifecycle only
+          leaves `ready` through `deleting` or `failed`, and drift is a failure — the
+          service the tenant is paying for is gone. So: `ready -> failed -> provisioning`,
+          both edges legal, asserted below by the domain function rather than assumed.
+        * The row must land in `provisioning`, not `failed`, before the worker sees the
+          task. `handle_provision` CASes `requested -> provisioning` best-effort and then
+          CASes `provisioning -> ready` for real; hand it a `failed` row and helm runs, the
+          final CAS matches nothing, and the instance sits in `failed` forever with a
+          healthy release behind it.
+
+        Both hops and the insert are one transaction, so the row is never observable in the
+        intermediate `failed` state and a crash mid-sweep leaves nothing half-done.
+        """
+        async with self._pool.connection() as conn:
+            async with conn.transaction(), conn.cursor() as cur:
+                await cur.execute(
+                    "select state from instances where id = %s and state = %s for update",
+                    (instance_id, InstanceState.READY.value),
+                )
+                if await cur.fetchone() is None:
+                    return None
+                if await _has_unfinished(cur, instance_id, TaskKind.PROVISION):
+                    return None
+
+                # Assert the path through the state machine instead of trusting the SQL.
+                # If someone edits LEGAL, this raises here rather than corrupting rows.
+                failed = transition(InstanceState.READY, InstanceState.FAILED)
+                provisioning = transition(failed, InstanceState.PROVISIONING)
+
+                await cur.execute(
+                    "update instances set state = %s, error = %s, updated_at = now() where id = %s",
+                    (provisioning.value, reason[-2000:], instance_id),
+                )
+                return await _insert_task(cur, instance_id, TaskKind.PROVISION)
+
+    # --- TTL ----------------------------------------------------------------------------
+
+    async def due_for_deprovision(self) -> list[Instance]:
+        """Instances that should be torn down and have no deprovision task outstanding.
+
+        Two populations, one query:
+
+        * `ready` and past `expires_at` — the TTL sweep proper. The whole reason a
+          throwaway Elasticsearch does not become a permanent line on the cloud bill.
+        * `deleting` with nothing to do the deleting — the API CASes to `deleting` and then
+          enqueues in a second statement, and a crash between the two leaves exactly this.
+          That ordering is chosen *because* this sweep exists; the other order would leave
+          a deprovision task pointing at a `ready` instance, and a worker would tear down a
+          live service nobody asked to delete.
+
+        Note the parentheses around the OR. Without them, `and not exists (...)` binds to
+        the second branch alone and the query re-enqueues a deprovision for every deleting
+        instance on every tick, forever.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                f"""select {_COLUMNS} from instances i
+                     where ((i.state = %(ready)s and i.expires_at < now()) or i.state = %(deleting)s)
+                       and not exists (
+                             select 1 from tasks t
+                              where t.instance_id = i.id
+                                and t.kind = %(kind)s
+                                and t.state = any(%(unfinished)s))
+                     order by i.created_at""",  # noqa: S608 - module constant
+                {
+                    "ready": InstanceState.READY.value,
+                    "deleting": InstanceState.DELETING.value,
+                    "kind": TaskKind.DEPROVISION.value,
+                    "unfinished": list(_UNFINISHED),
+                },
+            )
+            rows = await cur.fetchall()
+        return [Instance.model_validate(r) for r in rows]
+
+    async def enqueue_deprovision(self, instance_id: UUID) -> int | None:
+        """CAS to `deleting` if needed, and enqueue the task. One transaction. None if moot.
+
+        The instance must be in `deleting` before the worker claims the task, for the same
+        reason as `enqueue_reprovision`: `handle_deprovision` finishes with a
+        `deleting -> deleted` CAS, and a `ready` row would make helm uninstall the release
+        and the DB keep advertising an endpoint that no longer resolves.
+        """
+        async with self._pool.connection() as conn:
+            async with conn.transaction(), conn.cursor() as cur:
+                await cur.execute(
+                    """select state from instances
+                        where id = %(id)s
+                          and ((state = %(ready)s and expires_at < now()) or state = %(deleting)s)
+                          for update""",
+                    {
+                        "id": instance_id,
+                        "ready": InstanceState.READY.value,
+                        "deleting": InstanceState.DELETING.value,
+                    },
+                )
+                row = await cur.fetchone()
+                if row is None:
+                    return None
+                if await _has_unfinished(cur, instance_id, TaskKind.DEPROVISION):
+                    return None
+
+                if row["state"] == InstanceState.READY.value:
+                    deleting = transition(InstanceState.READY, InstanceState.DELETING)
+                    await cur.execute(
+                        "update instances set state = %s, updated_at = now() where id = %s",
+                        (deleting.value, instance_id),
+                    )
+                return await _insert_task(cur, instance_id, TaskKind.DEPROVISION)
+
+    # --- Version drift ------------------------------------------------------------------
+
+    async def enqueue_upgrade(self, instance_id: UUID, run_after: datetime) -> int | None:
+        """Enqueue an upgrade unless one is already outstanding. None if it is.
+
+        The guard is what keeps the fleet at `max_in_flight`. The work list is a query over
+        `chart_version`, and that column is only written *after* helm reports success — so
+        an instance stays on the work list for the entire duration of its own upgrade, and
+        for the hours it spends parked waiting for its 03:00 window. Without this check the
+        sweep enqueues one more upgrade for the same instance every 60 seconds, and
+        `max_in_flight=1` becomes sixty tasks an hour against one release.
+
+        `verify` counts as outstanding too: an upgrade whose verify has not reported is an
+        upgrade still in progress, and re-enqueueing it would race the probe that decides
+        whether the whole rollout halts.
+        """
+        async with self._pool.connection() as conn:
+            async with conn.transaction(), conn.cursor() as cur:
+                if await _has_unfinished(cur, instance_id, TaskKind.UPGRADE, TaskKind.VERIFY):
+                    return None
+                return await _insert_task(cur, instance_id, TaskKind.UPGRADE, run_after)
+
+
+async def _has_unfinished(
+    cur: AsyncCursor[dict[str, Any]],
+    instance_id: UUID,
+    *kinds: TaskKind,
+) -> bool:
+    """Is a task of any of these kinds queued or running for this instance?
+
+    Takes the caller's cursor on purpose: the answer is only true for as long as the
+    transaction that asked, and checking on a separate connection would be a check against
+    a different snapshot than the insert that follows it.
+    """
+    await cur.execute(
+        """select 1 from tasks
+            where instance_id = %(id)s and kind = any(%(kinds)s) and state = any(%(unfinished)s)
+            limit 1""",
+        {
+            "id": instance_id,
+            "kinds": [k.value for k in kinds],
+            "unfinished": list(_UNFINISHED),
+        },
+    )
+    return await cur.fetchone() is not None
+
+
+async def _insert_task(
+    cur: AsyncCursor[dict[str, Any]],
+    instance_id: UUID,
+    kind: TaskKind,
+    run_after: datetime | None = None,
+) -> int:
+    """Insert one task in the caller's transaction, carrying the current trace context.
+
+    `traceparent` is written here rather than left to `TaskRepo.enqueue` because these rows
+    are inserted inside a transaction the reconciler owns. Nothing propagates a trace
+    through a table on its own — see `obs.inject_traceparent`. It is null when the sweep is
+    not itself inside a span, which is fine and expected: a nullable column for an untraced
+    task.
+    """
+    await cur.execute(
+        """insert into tasks (instance_id, kind, run_after, traceparent)
+           values (%s, %s, coalesce(%s, now()), %s)
+           returning id""",
+        (instance_id, kind.value, run_after, inject_traceparent()),
+    )
+    row = await cur.fetchone()
+    assert row is not None  # noqa: S101 - `returning` always yields a row or raises
+    return int(row["id"])

libs/svcforge_core/svcforge_core/repo/tasks.py

@@ -0,0 +1,219 @@
+"""The queue.
+
+The queue is a Postgres table, not Redis. The reason is one sentence: a task and the
+instance state it describes must commit atomically. Split them across two stores and you
+own a distributed commit problem that has no winning move — the process can die between
+the two writes, and whichever you write first is the one that lies.
+
+Everything else here follows from that.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any, Final
+from uuid import UUID
+
+from psycopg import AsyncConnection
+
+from svcforge_core.domain.backoff import next_attempt_at
+from svcforge_core.domain.models import Task, TaskKind
+from svcforge_core.domain.states import LEGAL, InstanceState
+from svcforge_core.obs import inject_traceparent
+from svcforge_core.repo.db import DictPool
+
+# Which states may legally become `failed`, derived from the domain's own table rather
+# than restated here. Without this guard the UPDATE below would happily move a `deleted`
+# instance to `failed` — a transition domain.transition() explicitly forbids, performed
+# by raw SQL that never asks it. The state machine has to be the same one everywhere, or
+# it is decoration.
+_CAN_FAIL: Final[tuple[str, ...]] = tuple(
+    state.value for state, allowed in LEGAL.items() if InstanceState.FAILED in allowed
+)
+
+# The claim query. Do not "simplify" this into two statements.
+#
+# Postgres has no `UPDATE ... LIMIT`, so the row is chosen by a subquery. That subquery
+# takes a row lock (`for update`) and steps over rows other workers already hold
+# (`skip locked`) instead of blocking behind them — which is what makes N workers scale
+# instead of queueing single-file behind the oldest task.
+#
+# The whole thing is ONE statement on purpose. Select-then-update as two statements
+# leaves a gap in which a second worker reads the same id, and both provision. The gap
+# is small, which means you will not hit it in testing and will hit it in production.
+#
+# The `with claimed as (...)` wrapper is the ONLY addition to the canonical form, and it
+# changes nothing about the locking: the UPDATE and its `for update skip locked` subquery
+# are still one statement, executed once. The outer SELECT only joins `instances.team`
+# onto the row that was already claimed, so the worker can bind `team` to its log context
+# before it has loaded anything. A data-modifying CTE runs exactly once and cannot claim
+# twice. The alternative — a second SELECT for the team — would be a second round trip per
+# task to fetch a column the database already had in hand.
+_CLAIM_SQL = """
+with claimed as (
+  update tasks set state='running', attempts=attempts+1, locked_by=%(worker)s, locked_at=now()
+  where id = (
+    select id from tasks
+    where state='queued' and run_after <= now()
+    order by run_after
+    for update skip locked
+    limit 1
+  )
+  returning *
+)
+select claimed.*, instances.team
+  from claimed join instances on instances.id = claimed.instance_id;
+"""
+
+
+class TaskRepo:
+    """Reads and writes `tasks`. Claiming is the only interesting part."""
+
+    def __init__(self, pool: DictPool) -> None:
+        self._pool = pool
+
+    async def enqueue(
+        self,
+        conn: AsyncConnection[dict[str, Any]],
+        instance_id: UUID,
+        kind: TaskKind,
+        run_after: datetime | None = None,
+    ) -> Task:
+        """Insert a task inside the CALLER's transaction.
+
+        Takes `conn` so the API can insert the instance and enqueue its provision task in
+        one transaction. Rolling back must lose both, or you get an orphan task pointing
+        at an instance that was never committed.
+
+        The `traceparent` is captured here, at enqueue time, because this is the last
+        moment the caller's span context still exists. Trace context does NOT survive a
+        queue on its own: the worker picks the row up in a different process, minutes
+        later, with no ambient context. Writing the W3C traceparent onto the row is the
+        thread that lets the worker re-parent its span to the POST that caused it — the
+        difference between one trace spanning API → queue → helm and two unrelated ones.
+        """
+        async with conn.cursor() as cur:
+            await cur.execute(
+                """insert into tasks (instance_id, kind, run_after, traceparent)
+                   values (%s, %s, coalesce(%s, now()), %s)
+                   returning id, instance_id, kind, state, attempts, run_after, locked_by, last_error""",
+                (instance_id, kind.value, run_after, inject_traceparent()),
+            )
+            row = await cur.fetchone()
+        assert row is not None  # noqa: S101 - `returning` always yields a row or raises
+        return Task.model_validate(row)
+
+    async def enqueue_standalone(
+        self,
+        instance_id: UUID,
+        kind: TaskKind,
+        run_after: datetime | None = None,
+    ) -> int:
+        """Enqueue in its own transaction, returning the new task id.
+
+        For callers with nothing to commit alongside it — the reconciler, tests. The
+        module specs disagree about enqueue's shape (Module 2 passes a conn, Module 4
+        does not); rather than making `conn` optional and quietly hiding the transaction
+        question, both callers get an honest method name.
+        """
+        async with self._pool.connection() as conn:
+            task = await self.enqueue(conn, instance_id, kind, run_after)
+        return task.id
+
+    async def claim(self, worker_id: str) -> Task | None:
+        """Claim one runnable task, or None if nothing is runnable.
+
+        `attempts` increments HERE, at claim time, not on failure. A worker that dies
+        mid-task without reporting anything has still burned an attempt, so a task that
+        reliably kills its worker cannot retry forever.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(_CLAIM_SQL, {"worker": worker_id})
+            row = await cur.fetchone()
+        return Task.model_validate(row) if row else None
+
+    async def complete(
+        self,
+        task_id: int,
+        conn: AsyncConnection[dict[str, Any]] | None = None,
+    ) -> None:
+        """Mark done. Pass `conn` to commit alongside the caller's instance update.
+
+        Completing the task and recording what it accomplished belong in one transaction:
+        commit them separately and a crash in between leaves a task marked done whose
+        work never landed, or work that landed and will be redone.
+        """
+        sql = "update tasks set state='done', locked_by=null where id = %s"
+        if conn is not None:
+            async with conn.cursor() as cur:
+                await cur.execute(sql, (task_id,))
+            return
+        async with self._pool.connection() as own, own.cursor() as cur:
+            await cur.execute(sql, (task_id,))
+
+    async def fail(self, task_id: int, err: str, max_attempts: int = 5) -> None:
+        """Retry with backoff, or give up.
+
+        Under max_attempts: back to 'queued' with run_after pushed out by exponential
+        backoff with full jitter. Jitter matters — a cluster-wide outage fails every task
+        at once, and without it every worker retries in the same instant, forever.
+
+        At max_attempts: 'failed', and the error is copied onto the instance so the tenant
+        can see it. A dead-letter state, not an infinite retry: a task that cannot succeed
+        must stop and become someone's problem.
+        """
+        now = datetime.now(UTC)
+        async with self._pool.connection() as conn:
+            async with conn.transaction(), conn.cursor() as cur:
+                await cur.execute(
+                    "select attempts, instance_id from tasks where id = %s for update",
+                    (task_id,),
+                )
+                row = await cur.fetchone()
+                if row is None:
+                    return
+                attempts = int(row["attempts"])
+                instance_id = row["instance_id"]
+
+                if attempts < max_attempts:
+                    await cur.execute(
+                        """update tasks
+                              set state='queued', locked_by=null, locked_at=null,
+                                  last_error=%s, run_after=%s
+                            where id = %s""",
+                        (err[-2000:], next_attempt_at(attempts - 1, now=now), task_id),
+                    )
+                    return
+
+                await cur.execute(
+                    """update tasks
+                          set state='failed', locked_by=null, locked_at=null, last_error=%s
+                        where id = %s""",
+                    (err[-2000:], task_id),
+                )
+                # `state = any(%s)` keeps this honest: a deprovision that exhausts its
+                # retries against an already-deleted instance records nothing rather than
+                # resurrecting it into `failed`.
+                await cur.execute(
+                    """update instances set error=%s, state=%s, updated_at=now()
+                        where id=%s and state = any(%s)""",
+                    (err[-2000:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
+                )
+
+    async def reset_expired_leases(self, lease_seconds: int) -> int:
+        """Return tasks whose worker died back to the queue. Called by the reconciler.
+
+        No distributed lock survives a power cut. A worker that is SIGKILLed leaves
+        `state='running'` and `locked_by` set with nobody running it, and that row would
+        sit there forever. The lease is the only thing that recovers it: not a lock, a
+        timeout. This is why `locked_at` exists.
+        """
+        async with self._pool.connection() as conn, conn.cursor() as cur:
+            await cur.execute(
+                """update tasks
+                      set state='queued', locked_by=null, locked_at=null
+                    where state='running'
+                      and locked_at < now() - make_interval(secs => %s)""",
+                (lease_seconds,),
+            )
+            return cur.rowcount

libs/svcforge_core/svcforge_core/settings.py

@@ -0,0 +1,91 @@
+"""Typed configuration. Environment in, validated object out, fails fast at startup.
+
+The whole point: a missing or malformed DSN kills the process on line one with a readable
+error, instead of surfacing as a PoolTimeout twenty minutes into a provision.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import Field, PostgresDsn, RedisDsn
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Every knob svcforge has. Read once, at startup, and passed down explicitly."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="SVCFORGE_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+        frozen=True,
+    )
+
+    # --- Postgres -----------------------------------------------------------------
+    # Transaction pooler (6543 on Supabase). Everything the services do at runtime.
+    pg_dsn: PostgresDsn
+    # Session pooler (5432). Migrations and psql only: DDL and advisory locks need a
+    # session that outlives a single transaction.
+    pg_dsn_session: PostgresDsn | None = None
+
+    pool_min_size: int = Field(default=1, ge=0)
+    pool_max_size: int = Field(default=5, ge=1)
+
+    # --- Redis (derived state only; never the source of truth) ---------------------
+    redis_dsn: RedisDsn | None = None
+
+    # --- API ----------------------------------------------------------------------
+    jwks_url: str | None = None
+    jwt_audience: str = "svcforge"
+    jwt_issuer: str | None = None
+    # Dev escape hatch: skip JWT verification. Refused in prod by check_production().
+    auth_disabled: bool = False
+
+    # --- Worker -------------------------------------------------------------------
+    worker_id: str = Field(default="worker-local", min_length=1)
+    worker_concurrency: int = Field(default=4, ge=1)
+    poll_interval_s: float = Field(default=5.0, gt=0)
+    max_attempts: int = Field(default=5, ge=1)
+    lease_seconds: int = Field(default=300, ge=1)
+
+    # --- Reconciler ---------------------------------------------------------------
+    reconcile_interval_s: float = Field(default=60.0, gt=0)
+
+    # --- Catalog / helm -----------------------------------------------------------
+    catalog_path: Path = Path("catalog.yaml")
+    helm_bin: str = "helm"
+    kubectl_bin: str = "kubectl"
+    helm_timeout_s: float = Field(default=300.0, gt=0)
+
+    # --- CLI ----------------------------------------------------------------------
+    # The CLI is an API client and nothing more. It gets a URL and a token; it does not
+    # get a DSN, because the moment a human can reach the database directly, someone will
+    # "just fix one row" and the state machine stops being true.
+    api_url: str = "http://localhost:8000"
+    api_token: str | None = None
+
+    # --- Observability ------------------------------------------------------------
+    log_level: str = "info"
+    log_json: bool = True
+    otel_endpoint: str | None = None
+    service_name: str = "svcforge"
+    # The worker and reconciler have no HTTP server of their own, so they start a tiny one
+    # just for /metrics. 9000 matches the chart's PodMonitor; change both or neither.
+    metrics_port: int = Field(default=9000, ge=1, le=65535)
+
+    @property
+    def migration_dsn(self) -> str:
+        """Migrations need a session-mode connection; fall back to the runtime DSN locally."""
+        return str(self.pg_dsn_session or self.pg_dsn)
+
+    def check_production(self) -> None:
+        """Refuse the dev escape hatches when they would matter."""
+        if self.auth_disabled:
+            raise ValueError("SVCFORGE_AUTH_DISABLED=true is refused outside local development")
+
+
+def load_settings() -> Settings:
+    """Read the environment. Raises ValidationError — loudly — if anything is missing."""
+    return Settings()  # type: ignore[call-arg]  # pydantic-settings fills these from env