svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

services/reconciler/Dockerfile

@@ -0,0 +1,53 @@
+# syntax=docker/dockerfile:1.10
+#
+# svcforge reconciler. Build from the REPO ROOT:
+#     docker buildx build -f services/reconciler/Dockerfile -t svcforge/reconciler:dev .
+#
+# Reads helm state to detect drift, so it carries helm — but never kubectl, and it never
+# writes: the four checks enqueue tasks, they do not provision. Orphans are logged, never
+# deleted.
+
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable \
+        --no-install-project --no-install-package svcforge-core
+
+COPY libs/ libs/
+COPY services/reconciler/ services/reconciler/
+COPY catalog.yaml ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable && \
+    /app/.venv/bin/python -c 'import svcforge_core, sys; \
+p = svcforge_core.__file__; \
+sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
+
+# --- runtime --------------------------------------------------------------------------
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
+
+ARG BUILD_SHA=unknown
+LABEL org.opencontainers.image.title="svcforge-reconciler" \
+      org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
+      org.opencontainers.image.revision="${BUILD_SHA}"
+
+RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
+WORKDIR /app
+COPY --from=builder --chown=10001:10001 /app /app
+
+COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
+
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    HELM_CACHE_HOME=/tmp/helm/cache \
+    HELM_CONFIG_HOME=/tmp/helm/config \
+    HELM_DATA_HOME=/tmp/helm/data
+USER 10001
+ENTRYPOINT ["python", "-m", "services.reconciler.main"]

services/reconciler/__init__.py

@@ -0,0 +1 @@
+"""The reconciler service: one singleton loop that makes the world match the database."""

services/reconciler/main.py

@@ -0,0 +1,410 @@
+"""The control loop.
+
+Every other service in svcforge is edge-triggered: a tenant POSTs, a row appears, a worker
+claims it. Edge-triggered systems are correct exactly as long as nothing is ever missed —
+and things are missed. A worker is SIGKILLed holding a lease. An operator runs
+`helm uninstall` by hand. A pod dies between the CAS and the enqueue. Nobody sends an event
+for any of that, because the thing that would have sent it is the thing that died.
+
+So: level-triggered. Every 60 seconds, compare the world to the database and enqueue what
+is missing. The four checks below do not know or care what went wrong, or whether anything
+did; they are the same code on the happy path and after an outage. That property is the
+entire reason this service exists, and it is why each check is written as a *query for
+work*, never as a reaction to an event.
+
+Three rules hold the design together:
+
+* **Singleton.** `replicas: 1`, `strategy: Recreate` in the chart. Two reconcilers
+  double-enqueue drift and race on TTL. There is no leader election here on purpose: the
+  correct lease for that lives in Postgres next to the data, not in a Redis lock, and
+  until there is a second replica to elect between, an election is a subsystem that can
+  only fail. One pod, and the `SvcforgeReconcilerStale` alert is what notices it is gone.
+* **Each check is independent.** One failing check must not skip the other three. A helm
+  binary that cannot reach the API server must not stop TTLs from expiring.
+* **Enqueue, never act.** The reconciler diagnoses; workers treat. It writes task rows and
+  instance states, and never calls `helm install`. The one exception is reading — the drift
+  check runs `helm list`, because seeing reality is the job.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+
+import typer
+
+from svcforge_core.adapters.clock import Clock, SystemClock
+from svcforge_core.adapters.helm import HelmProvisioner, Provisioner
+from svcforge_core.adapters.notify import LogNotifier, Notifier
+from svcforge_core.domain.catalog import load_catalog
+from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.domain.windows import BadWindow, parse_window, schedule_upgrade_at
+from svcforge_core.obs import (
+    INSTANCES,
+    QUEUE_DEPTH,
+    RECONCILER_LAST_TICK,
+    get_logger,
+    setup,
+    start_metrics_server,
+    tracer,
+)
+from svcforge_core.repo.db import DictPool, make_pool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.reconcile import ReconcileRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings, load_settings
+
+log = get_logger("svcforge.reconciler")
+
+# The chart's PodMonitor scrapes the port named `metrics` on 9000. Keep them in step.
+DEFAULT_METRICS_PORT = 9000
+
+
+@dataclass(frozen=True)
+class ReconcilerDeps:
+    """Everything a check is allowed to touch. Built once in main(), passed down.
+
+    Same shape as `WorkerDeps` for the same reason: the checks take `deps` instead of
+    reaching for globals, so the integration tests below run every check against a real
+    Postgres and a `FakeProvisioner` without a cluster anywhere in sight.
+    """
+
+    pool: DictPool
+    instances: InstanceRepo
+    tasks: TaskRepo
+    reconcile: ReconcileRepo
+    provisioner: Provisioner
+    notifier: Notifier
+    clock: Clock
+    catalog: dict[str, CatalogEntry]
+    settings: Settings
+    # Whose instances go first in the day-2 work list. Eating your own dog food is an
+    # `order by`, not a policy document — see InstanceRepo.list_upgradable.
+    own_team: str
+    # A config value, not a scheduler. Leave it at 1 until 1 is too slow.
+    max_in_flight: int
+
+
+# --- The four checks ---------------------------------------------------------------------
+
+
+async def check_drift(deps: ReconcilerDeps) -> None:
+    """`helm list -A -o json` versus what the database believes.
+
+    This is the only check that looks outside Postgres, and the only one that can catch the
+    failure nothing else can: someone ran `helm uninstall` by hand, or a node was drained
+    and the release never came back. The DB still says `ready` and still hands the tenant an
+    endpoint that resolves to nothing.
+
+    Two directions, two very different answers:
+
+    * **Release gone, DB says `ready`** -> re-enqueue provision. Safe, because provisioning
+      is `helm upgrade --install` against a deterministic release name: converging on
+      desired state, not a blind re-install.
+    * **Release exists, DB knows nothing** -> log at error with release and namespace, and
+      stop. **Never delete in v1.** The reconciler's view of "the DB knows nothing" is one
+      query against one database; the release might belong to another team, another tool,
+      or a migration half-finished. Deleting on that evidence is how an automated system
+      takes down production faster than any human could. A human reads the log and decides.
+    """
+    with tracer().start_as_current_span("helm.list"):
+        releases = await deps.provisioner.list_releases()
+    live = {(r.name, r.namespace) for r in releases}
+
+    for inst in await deps.reconcile.ready_instances():
+        if (inst.release_name, inst.namespace) in live:
+            continue
+        reason = f"drift: helm release {inst.release_name} missing from namespace {inst.namespace}"
+        task_id = await deps.reconcile.enqueue_reprovision(inst.id, reason)
+        if task_id is None:
+            continue  # already being dealt with, or the row moved under us
+        log.warning(
+            "drift.release_missing",
+            instance_id=str(inst.id),
+            team=inst.team,
+            release=inst.release_name,
+            namespace=inst.namespace,
+            task_id=task_id,
+        )
+        try:
+            await deps.notifier.send(
+                "drift.release_missing",
+                f"re-provisioning {inst.id}: release {inst.release_name} vanished",
+                {"instance_id": str(inst.id), "team": inst.team},
+            )
+        except Exception:
+            # The task is already committed; the notification is a courtesy. A webhook
+            # timing out must not abandon the rest of the sweep — the instances after this
+            # one in the loop have the same problem and nobody else is coming to find them.
+            log.exception("notify.failed", instance_id=str(inst.id))
+
+    known = await deps.reconcile.known_releases()
+    for name, namespace in sorted(live - known):
+        # error, not warning: this is a resource nobody is billing for and nobody owns.
+        # It will sit here every 60s until a human deletes it or adopts it. That is the
+        # intended pressure.
+        log.error("drift.orphan_release", release=name, namespace=namespace, action="none (v1 never deletes)")
+
+
+async def check_lease_expiry(deps: ReconcilerDeps) -> None:
+    """Tasks whose worker died -> back to `queued`.
+
+    A lease, not a lock. No lock survives a power cut: a worker SIGKILLed mid-provision
+    leaves `state='running'` with `locked_by` set and nobody running it, and no amount of
+    cleanup code in the worker helps, because the worker is the part that died. `locked_at`
+    plus a timeout is the only thing that recovers the row, which is why `locked_at` exists.
+
+    The 5-minute default is not arbitrary: it must exceed the longest a healthy task can
+    hold a lease, or the reconciler hands a still-running provision to a second worker.
+    Handlers are idempotent, so that is survivable rather than fatal — but survivable is
+    not free, and `lease_seconds` sits above helm's `--timeout` for that reason.
+    """
+    freed = await deps.tasks.reset_expired_leases(deps.settings.lease_seconds)
+    if freed:
+        log.warning("lease.expired", tasks_freed=freed, lease_seconds=deps.settings.lease_seconds)
+
+
+async def check_ttl(deps: ReconcilerDeps) -> None:
+    """Expired instances -> `deleting`, plus a deprovision task.
+
+    The line item that stops a demo cluster from becoming a permanent cloud bill. Also the
+    sweep the API's DELETE route depends on: it CASes to `deleting` and enqueues in two
+    statements, and a crash in between lands here on the next tick.
+
+    Idempotent by construction — the work list excludes anything that already has a queued
+    or running deprovision, and the CAS and the insert share one transaction. Without that
+    guard, a deprovision that takes longer than 60 seconds gets a second task on the next
+    tick, and a third on the tick after.
+    """
+    for inst in await deps.reconcile.due_for_deprovision():
+        task_id = await deps.reconcile.enqueue_deprovision(inst.id)
+        if task_id is None:
+            continue
+        log.info(
+            "ttl.expired",
+            instance_id=str(inst.id),
+            team=inst.team,
+            expires_at=inst.expires_at.isoformat() if inst.expires_at else None,
+            previous_state=inst.state.value,
+            task_id=task_id,
+        )
+
+
+async def check_version_drift(deps: ReconcilerDeps) -> None:
+    """The day-2 rollout: the work-list query, one service type at a time.
+
+    Everything that makes this safe is somewhere else, which is the point:
+
+    * `list_upgradable` limits to `max_in_flight` and returns nothing while
+      `rollout_state='halted'`, so a bad chart stops after one tenant.
+    * `schedule_upgrade_at` turns the tenant's maintenance window into a `run_after`; the
+      queue does the waiting, in `where run_after <= now()`. There is no scheduler here and
+      there must not be one — a task parked in Postgres until 03:00 Sunday survives a
+      reconciler restart, and an in-memory timer does not.
+    * `security: true` in the catalog bypasses the window. A CVE with a public exploit does
+      not wait until Sunday.
+
+    A bad window spec is this instance's problem, not the fleet's: log it and move to the
+    next one. Failing the whole check would let one tenant's typo freeze everyone's
+    security rollout.
+    """
+    now = deps.clock.now()
+
+    for service_type, entry in deps.catalog.items():
+        candidates = await deps.instances.list_upgradable(
+            service_type=service_type,
+            catalog_version=entry.chart_version,
+            own_team=deps.own_team,
+            max_in_flight=deps.max_in_flight,
+        )
+        for candidate in candidates:
+            inst = candidate.instance
+            try:
+                window = parse_window(candidate.maintenance_window)
+            except BadWindow:
+                log.exception(
+                    "upgrade.bad_window",
+                    instance_id=str(inst.id),
+                    team=inst.team,
+                    maintenance_window=candidate.maintenance_window,
+                )
+                continue
+
+            run_after = schedule_upgrade_at(window, security=entry.security, now=now)
+            task_id = await deps.reconcile.enqueue_upgrade(inst.id, run_after)
+            if task_id is None:
+                continue  # already queued or running; this is the max_in_flight guard
+            log.info(
+                "upgrade.scheduled",
+                instance_id=str(inst.id),
+                team=inst.team,
+                service_type=service_type,
+                from_version=inst.chart_version,
+                to_version=entry.chart_version,
+                run_after=run_after.isoformat(),
+                security=entry.security,
+                task_id=task_id,
+            )
+
+
+CHECKS: dict[str, Callable[[ReconcilerDeps], Awaitable[None]]] = {
+    "drift": check_drift,
+    "lease_expiry": check_lease_expiry,
+    "ttl": check_ttl,
+    "version_drift": check_version_drift,
+}
+
+
+# --- The tick ----------------------------------------------------------------------------
+
+
+async def tick(deps: ReconcilerDeps) -> None:
+    """One pass: all four checks, then the gauges, then the heartbeat.
+
+    Checks first, gauges second: `svcforge_queue_depth` is read straight after the checks
+    that add to the queue, so the value scraped is the value the tick left behind rather
+    than one from before its own work.
+
+    The heartbeat is set unconditionally, and that is deliberate. It answers "is the loop
+    running", not "is everything fine" — the checks have their own alerts. Gating it on
+    success would make `SvcforgeReconcilerStale` fire for a helm blip and mean two things
+    at once, and an alert that means two things gets muted.
+
+    The whole tick runs inside one span, which is a considered exception to "manual spans go
+    around helm calls only". That rule exists so the API does not hand-roll spans that
+    `opentelemetry-instrument` already creates for it. Nothing auto-instruments the
+    reconciler: without a span here it emits no traces at all, and — because
+    `inject_traceparent` serialises the *active* context — every task it enqueues would be
+    written with a null `traceparent` and be unjoinable to the tick that decided to create
+    it. One span per tick is what makes "why was this instance re-provisioned at 03:00?" a
+    question the traces can answer.
+    """
+    with tracer().start_as_current_span("reconciler.tick"):
+        await _run_checks(deps)
+
+    RECONCILER_LAST_TICK.set(deps.clock.now().timestamp())
+    log.info("tick.done")
+
+
+async def _run_checks(deps: ReconcilerDeps) -> None:
+    """The four checks and the gauges. Split out so `tick` reads as span + heartbeat."""
+    for name, check in CHECKS.items():
+        try:
+            await check(deps)
+        except Exception:  # the tick is the error boundary
+            # The swallow is the design. These four checks share nothing but a database
+            # handle, and the value of a level-triggered loop is that it keeps running: an
+            # unreachable cluster must not stop TTLs from expiring, and one tenant's broken
+            # window spec must not stop drift detection. This means "this check achieved
+            # nothing for 60 seconds", which the log says out loud. It never means "the
+            # reconciler stops".
+            log.exception("check.failed", check=name)
+
+    try:
+        QUEUE_DEPTH.set(await deps.reconcile.queue_depth())
+        counts = await deps.reconcile.instance_counts()
+        for state, n in counts.items():
+            INSTANCES.labels(state=state).set(n)
+    except Exception:  # gauges are diagnostics; a failed read is not a failed tick
+        log.exception("gauges.failed")
+
+
+async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
+    """Sleep, but wake immediately on SIGTERM. A 60s nap must not cost 60s of shutdown."""
+    with contextlib.suppress(TimeoutError):
+        await asyncio.wait_for(stop.wait(), timeout=seconds)
+
+
+async def run_reconciler(deps: ReconcilerDeps, stop: asyncio.Event) -> None:
+    """Tick, sleep, repeat, until told to stop.
+
+    Tick first, then sleep: a pod that has just been restarted should reconcile now, not in
+    sixty seconds. Fixed interval rather than a fixed period — a tick that overruns simply
+    delays the next one, instead of stacking a second tick on top of the first, which for a
+    singleton would be exactly the concurrent reconciler `replicas: 1` exists to prevent.
+    """
+    while not stop.is_set():
+        await tick(deps)
+        await _sleep_or_stop(stop, deps.settings.reconcile_interval_s)
+
+
+def build_deps(
+    pool: DictPool,
+    settings: Settings,
+    own_team: str,
+    max_in_flight: int,
+) -> ReconcilerDeps:
+    """Wire the real collaborators. The only place that names concrete classes."""
+    return ReconcilerDeps(
+        pool=pool,
+        instances=InstanceRepo(pool),
+        tasks=TaskRepo(pool),
+        reconcile=ReconcileRepo(pool),
+        provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
+        notifier=LogNotifier(),
+        clock=SystemClock(),
+        catalog=load_catalog(settings.catalog_path),
+        settings=settings,
+        own_team=own_team,
+        max_in_flight=max_in_flight,
+    )
+
+
+async def _amain(once: bool, metrics_port: int, own_team: str, max_in_flight: int) -> None:
+    settings = load_settings()
+    setup("svcforge-reconciler", settings)
+
+    pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
+    await pool.open(wait=True)
+    deps = build_deps(pool, settings, own_team, max_in_flight)
+
+    try:
+        if once:
+            # One pass and exit: the acceptance path, and how you drive a reconcile by hand
+            # from a shell. No metrics server — nothing would ever scrape it.
+            await tick(deps)
+            return
+
+        start_metrics_server(metrics_port)
+
+        stop = asyncio.Event()
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            # add_signal_handler, NOT signal.signal. signal.signal fires the handler at an
+            # arbitrary bytecode boundary on the main thread and the loop does not notice
+            # until its next timer — which here is up to a full 60s tick away. This one is
+            # scheduled as an ordinary loop callback, so the `stop.wait()` above returns
+            # immediately.
+            loop.add_signal_handler(sig, stop.set)
+
+        await run_reconciler(deps, stop)
+    finally:
+        await pool.close()
+
+
+app = typer.Typer(add_completion=False, help="svcforge reconciler: the control loop.")
+
+
+@app.command()
+def main(
+    once: bool = typer.Option(False, "--once", help="Run one tick and exit."),
+    metrics_port: int = typer.Option(
+        DEFAULT_METRICS_PORT, envvar="SVCFORGE_METRICS_PORT", help="Port for /metrics."
+    ),
+    own_team: str = typer.Option(
+        "platform", envvar="SVCFORGE_OWN_TEAM", help="Team whose instances upgrade first."
+    ),
+    max_in_flight: int = typer.Option(
+        1, envvar="SVCFORGE_MAX_IN_FLIGHT", min=1, help="Concurrent upgrades across the fleet."
+    ),
+) -> None:
+    """Run the reconciler."""
+    # One asyncio.run, at the top, never nested. Everything below it is already async.
+    asyncio.run(_amain(once, metrics_port, own_team, max_in_flight))
+
+
+if __name__ == "__main__":
+    app()