"""The control loop. Every other service in svcforge is edge-triggered: a tenant POSTs, a row appears, a worker claims it. Edge-triggered systems are correct exactly as long as nothing is ever missed — and things are missed. A worker is SIGKILLed holding a lease. An operator runs `helm uninstall` by hand. A pod dies between the CAS and the enqueue. Nobody sends an event for any of that, because the thing that would have sent it is the thing that died. So: level-triggered. Every 60 seconds, compare the world to the database and enqueue what is missing. The four checks below do not know or care what went wrong, or whether anything did; they are the same code on the happy path and after an outage. That property is the entire reason this service exists, and it is why each check is written as a *query for work*, never as a reaction to an event. Three rules hold the design together: * **Singleton.** `replicas: 1`, `strategy: Recreate` in the chart. Two reconcilers double-enqueue drift and race on TTL. There is no leader election here on purpose: the correct lease for that lives in Postgres next to the data, not in a Redis lock, and until there is a second replica to elect between, an election is a subsystem that can only fail. One pod, and the `SvcforgeReconcilerStale` alert is what notices it is gone. * **Each check is independent.** One failing check must not skip the other three. A helm binary that cannot reach the API server must not stop TTLs from expiring. * **Enqueue, never act.** The reconciler diagnoses; workers treat. It writes task rows and instance states, and never calls `helm install`. The one exception is reading — the drift check runs `helm list`, because seeing reality is the job. """ from __future__ import annotations import asyncio import contextlib import signal from collections.abc import Awaitable, Callable from dataclasses import dataclass import typer from svcforge_core.adapters.clock import Clock, SystemClock from svcforge_core.adapters.helm import HelmProvisioner, Provisioner from svcforge_core.adapters.notify import LogNotifier, Notifier from svcforge_core.domain.catalog import load_catalog from svcforge_core.domain.models import CatalogEntry from svcforge_core.domain.windows import BadWindow, parse_window, schedule_upgrade_at from svcforge_core.obs import ( INSTANCES, QUEUE_DEPTH, RECONCILER_LAST_TICK, get_logger, setup, start_metrics_server, tracer, ) from svcforge_core.repo.db import DictPool, make_pool from svcforge_core.repo.instances import InstanceRepo from svcforge_core.repo.reconcile import ReconcileRepo from svcforge_core.repo.tasks import TaskRepo from svcforge_core.settings import Settings, load_settings log = get_logger("svcforge.reconciler") # The chart's PodMonitor scrapes the port named `metrics` on 9000. Keep them in step. DEFAULT_METRICS_PORT = 9000 @dataclass(frozen=True) class ReconcilerDeps: """Everything a check is allowed to touch. Built once in main(), passed down. Same shape as `WorkerDeps` for the same reason: the checks take `deps` instead of reaching for globals, so the integration tests below run every check against a real Postgres and a `FakeProvisioner` without a cluster anywhere in sight. """ pool: DictPool instances: InstanceRepo tasks: TaskRepo reconcile: ReconcileRepo provisioner: Provisioner notifier: Notifier clock: Clock catalog: dict[str, CatalogEntry] settings: Settings # Whose instances go first in the day-2 work list. Eating your own dog food is an # `order by`, not a policy document — see InstanceRepo.list_upgradable. own_team: str # A config value, not a scheduler. Leave it at 1 until 1 is too slow. max_in_flight: int # --- The four checks --------------------------------------------------------------------- async def check_drift(deps: ReconcilerDeps) -> None: """`helm list -A -o json` versus what the database believes. This is the only check that looks outside Postgres, and the only one that can catch the failure nothing else can: someone ran `helm uninstall` by hand, or a node was drained and the release never came back. The DB still says `ready` and still hands the tenant an endpoint that resolves to nothing. Two directions, two very different answers: * **Release gone, DB says `ready`** -> re-enqueue provision. Safe, because provisioning is `helm upgrade --install` against a deterministic release name: converging on desired state, not a blind re-install. * **Release exists, DB knows nothing** -> log at error with release and namespace, and stop. **Never delete in v1.** The reconciler's view of "the DB knows nothing" is one query against one database; the release might belong to another team, another tool, or a migration half-finished. Deleting on that evidence is how an automated system takes down production faster than any human could. A human reads the log and decides. """ with tracer().start_as_current_span("helm.list"): releases = await deps.provisioner.list_releases() live = {(r.name, r.namespace) for r in releases} for inst in await deps.reconcile.ready_instances(): if (inst.release_name, inst.namespace) in live: continue reason = f"drift: helm release {inst.release_name} missing from namespace {inst.namespace}" task_id = await deps.reconcile.enqueue_reprovision(inst.id, reason) if task_id is None: continue # already being dealt with, or the row moved under us log.warning( "drift.release_missing", instance_id=str(inst.id), team=inst.team, release=inst.release_name, namespace=inst.namespace, task_id=task_id, ) try: await deps.notifier.send( "drift.release_missing", f"re-provisioning {inst.id}: release {inst.release_name} vanished", {"instance_id": str(inst.id), "team": inst.team}, ) except Exception: # The task is already committed; the notification is a courtesy. A webhook # timing out must not abandon the rest of the sweep — the instances after this # one in the loop have the same problem and nobody else is coming to find them. log.exception("notify.failed", instance_id=str(inst.id)) known = await deps.reconcile.known_releases() for name, namespace in sorted(live - known): # error, not warning: this is a resource nobody is billing for and nobody owns. # It will sit here every 60s until a human deletes it or adopts it. That is the # intended pressure. log.error("drift.orphan_release", release=name, namespace=namespace, action="none (v1 never deletes)") async def check_lease_expiry(deps: ReconcilerDeps) -> None: """Tasks whose worker died -> back to `queued`. A lease, not a lock. No lock survives a power cut: a worker SIGKILLed mid-provision leaves `state='running'` with `locked_by` set and nobody running it, and no amount of cleanup code in the worker helps, because the worker is the part that died. `locked_at` plus a timeout is the only thing that recovers the row, which is why `locked_at` exists. The 5-minute default is not arbitrary: it must exceed the longest a healthy task can hold a lease, or the reconciler hands a still-running provision to a second worker. Handlers are idempotent, so that is survivable rather than fatal — but survivable is not free, and `lease_seconds` sits above helm's `--timeout` for that reason. """ freed = await deps.tasks.reset_expired_leases(deps.settings.lease_seconds) if freed: log.warning("lease.expired", tasks_freed=freed, lease_seconds=deps.settings.lease_seconds) async def check_ttl(deps: ReconcilerDeps) -> None: """Expired instances -> `deleting`, plus a deprovision task. The line item that stops a demo cluster from becoming a permanent cloud bill. Also the sweep the API's DELETE route depends on: it CASes to `deleting` and enqueues in two statements, and a crash in between lands here on the next tick. Idempotent by construction — the work list excludes anything that already has a queued or running deprovision, and the CAS and the insert share one transaction. Without that guard, a deprovision that takes longer than 60 seconds gets a second task on the next tick, and a third on the tick after. """ for inst in await deps.reconcile.due_for_deprovision(): task_id = await deps.reconcile.enqueue_deprovision(inst.id) if task_id is None: continue log.info( "ttl.expired", instance_id=str(inst.id), team=inst.team, expires_at=inst.expires_at.isoformat() if inst.expires_at else None, previous_state=inst.state.value, task_id=task_id, ) async def check_version_drift(deps: ReconcilerDeps) -> None: """The day-2 rollout: the work-list query, one service type at a time. Everything that makes this safe is somewhere else, which is the point: * `list_upgradable` limits to `max_in_flight` and returns nothing while `rollout_state='halted'`, so a bad chart stops after one tenant. * `schedule_upgrade_at` turns the tenant's maintenance window into a `run_after`; the queue does the waiting, in `where run_after <= now()`. There is no scheduler here and there must not be one — a task parked in Postgres until 03:00 Sunday survives a reconciler restart, and an in-memory timer does not. * `security: true` in the catalog bypasses the window. A CVE with a public exploit does not wait until Sunday. A bad window spec is this instance's problem, not the fleet's: log it and move to the next one. Failing the whole check would let one tenant's typo freeze everyone's security rollout. """ now = deps.clock.now() for service_type, entry in deps.catalog.items(): candidates = await deps.instances.list_upgradable( service_type=service_type, catalog_version=entry.chart_version, own_team=deps.own_team, max_in_flight=deps.max_in_flight, ) for candidate in candidates: inst = candidate.instance try: window = parse_window(candidate.maintenance_window) except BadWindow: log.exception( "upgrade.bad_window", instance_id=str(inst.id), team=inst.team, maintenance_window=candidate.maintenance_window, ) continue run_after = schedule_upgrade_at(window, security=entry.security, now=now) task_id = await deps.reconcile.enqueue_upgrade(inst.id, run_after) if task_id is None: continue # already queued or running; this is the max_in_flight guard log.info( "upgrade.scheduled", instance_id=str(inst.id), team=inst.team, service_type=service_type, from_version=inst.chart_version, to_version=entry.chart_version, run_after=run_after.isoformat(), security=entry.security, task_id=task_id, ) CHECKS: dict[str, Callable[[ReconcilerDeps], Awaitable[None]]] = { "drift": check_drift, "lease_expiry": check_lease_expiry, "ttl": check_ttl, "version_drift": check_version_drift, } # --- The tick ---------------------------------------------------------------------------- async def tick(deps: ReconcilerDeps) -> None: """One pass: all four checks, then the gauges, then the heartbeat. Checks first, gauges second: `svcforge_queue_depth` is read straight after the checks that add to the queue, so the value scraped is the value the tick left behind rather than one from before its own work. The heartbeat is set unconditionally, and that is deliberate. It answers "is the loop running", not "is everything fine" — the checks have their own alerts. Gating it on success would make `SvcforgeReconcilerStale` fire for a helm blip and mean two things at once, and an alert that means two things gets muted. The whole tick runs inside one span, which is a considered exception to "manual spans go around helm calls only". That rule exists so the API does not hand-roll spans that `opentelemetry-instrument` already creates for it. Nothing auto-instruments the reconciler: without a span here it emits no traces at all, and — because `inject_traceparent` serialises the *active* context — every task it enqueues would be written with a null `traceparent` and be unjoinable to the tick that decided to create it. One span per tick is what makes "why was this instance re-provisioned at 03:00?" a question the traces can answer. """ with tracer().start_as_current_span("reconciler.tick"): await _run_checks(deps) RECONCILER_LAST_TICK.set(deps.clock.now().timestamp()) log.info("tick.done") async def _run_checks(deps: ReconcilerDeps) -> None: """The four checks and the gauges. Split out so `tick` reads as span + heartbeat.""" for name, check in CHECKS.items(): try: await check(deps) except Exception: # the tick is the error boundary # The swallow is the design. These four checks share nothing but a database # handle, and the value of a level-triggered loop is that it keeps running: an # unreachable cluster must not stop TTLs from expiring, and one tenant's broken # window spec must not stop drift detection. This means "this check achieved # nothing for 60 seconds", which the log says out loud. It never means "the # reconciler stops". log.exception("check.failed", check=name) try: QUEUE_DEPTH.set(await deps.reconcile.queue_depth()) counts = await deps.reconcile.instance_counts() for state, n in counts.items(): INSTANCES.labels(state=state).set(n) except Exception: # gauges are diagnostics; a failed read is not a failed tick log.exception("gauges.failed") async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None: """Sleep, but wake immediately on SIGTERM. A 60s nap must not cost 60s of shutdown.""" with contextlib.suppress(TimeoutError): await asyncio.wait_for(stop.wait(), timeout=seconds) async def run_reconciler(deps: ReconcilerDeps, stop: asyncio.Event) -> None: """Tick, sleep, repeat, until told to stop. Tick first, then sleep: a pod that has just been restarted should reconcile now, not in sixty seconds. Fixed interval rather than a fixed period — a tick that overruns simply delays the next one, instead of stacking a second tick on top of the first, which for a singleton would be exactly the concurrent reconciler `replicas: 1` exists to prevent. """ while not stop.is_set(): await tick(deps) await _sleep_or_stop(stop, deps.settings.reconcile_interval_s) def build_deps( pool: DictPool, settings: Settings, own_team: str, max_in_flight: int, ) -> ReconcilerDeps: """Wire the real collaborators. The only place that names concrete classes.""" return ReconcilerDeps( pool=pool, instances=InstanceRepo(pool), tasks=TaskRepo(pool), reconcile=ReconcileRepo(pool), provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)), notifier=LogNotifier(), clock=SystemClock(), catalog=load_catalog(settings.catalog_path), settings=settings, own_team=own_team, max_in_flight=max_in_flight, ) async def _amain(once: bool, metrics_port: int, own_team: str, max_in_flight: int) -> None: settings = load_settings() setup("svcforge-reconciler", settings) pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size) await pool.open(wait=True) deps = build_deps(pool, settings, own_team, max_in_flight) try: if once: # One pass and exit: the acceptance path, and how you drive a reconcile by hand # from a shell. No metrics server — nothing would ever scrape it. await tick(deps) return start_metrics_server(metrics_port) stop = asyncio.Event() loop = asyncio.get_running_loop() for sig in (signal.SIGTERM, signal.SIGINT): # add_signal_handler, NOT signal.signal. signal.signal fires the handler at an # arbitrary bytecode boundary on the main thread and the loop does not notice # until its next timer — which here is up to a full 60s tick away. This one is # scheduled as an ordinary loop callback, so the `stop.wait()` above returns # immediately. loop.add_signal_handler(sig, stop.set) await run_reconciler(deps, stop) finally: await pool.close() app = typer.Typer(add_completion=False, help="svcforge reconciler: the control loop.") @app.command() def main( once: bool = typer.Option(False, "--once", help="Run one tick and exit."), metrics_port: int = typer.Option( DEFAULT_METRICS_PORT, envvar="SVCFORGE_METRICS_PORT", help="Port for /metrics." ), own_team: str = typer.Option( "platform", envvar="SVCFORGE_OWN_TEAM", help="Team whose instances upgrade first." ), max_in_flight: int = typer.Option( 1, envvar="SVCFORGE_MAX_IN_FLIGHT", min=1, help="Concurrent upgrades across the fleet." ), ) -> None: """Run the reconciler.""" # One asyncio.run, at the top, never nested. Everything below it is already async. asyncio.run(_amain(once, metrics_port, own_team, max_in_flight)) if __name__ == "__main__": app()