svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

services/worker/Dockerfile

@@ -0,0 +1,54 @@
+# syntax=docker/dockerfile:1.10
+#
+# svcforge worker. Build from the REPO ROOT:
+#     docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
+#
+# The only service that shells out to helm/kubectl, so the only one carrying those two
+# binaries. They are copied from pinned images rather than curl'd, so the version is a
+# reviewable line in a Dockerfile instead of a network call at build time.
+
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable \
+        --no-install-project --no-install-package svcforge-core
+
+COPY libs/ libs/
+COPY services/worker/ services/worker/
+COPY catalog.yaml ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-editable && \
+    /app/.venv/bin/python -c 'import svcforge_core, sys; \
+p = svcforge_core.__file__; \
+sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
+
+# --- runtime --------------------------------------------------------------------------
+FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
+
+ARG BUILD_SHA=unknown
+LABEL org.opencontainers.image.title="svcforge-worker" \
+      org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
+      org.opencontainers.image.revision="${BUILD_SHA}"
+
+RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
+WORKDIR /app
+COPY --from=builder --chown=10001:10001 /app /app
+
+COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
+COPY --from=bitnamilegacy/kubectl:1.31.2@sha256:0eab9ec8f5e0f75271277467ebb7513b36dea0122bc68615d18c47fece4fd82c /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
+
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    HELM_CACHE_HOME=/tmp/helm/cache \
+    HELM_CONFIG_HOME=/tmp/helm/config \
+    HELM_DATA_HOME=/tmp/helm/data
+USER 10001
+ENTRYPOINT ["python", "-m", "services.worker.main"]

services/worker/deps.py

@@ -0,0 +1,33 @@
+"""What a worker needs to do its job.
+
+One frozen bag of collaborators, constructed once in main() and passed down. Handlers
+take `deps` rather than reaching for globals, which is the entire reason the worker tests
+run in milliseconds against a FakeProvisioner instead of needing a cluster.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from svcforge_core.adapters.clock import Clock
+from svcforge_core.adapters.helm import Provisioner
+from svcforge_core.adapters.notify import Notifier
+from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.repo.db import DictPool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings
+
+
+@dataclass(frozen=True)
+class WorkerDeps:
+    """Everything a handler is allowed to touch."""
+
+    pool: DictPool
+    instances: InstanceRepo
+    tasks: TaskRepo
+    provisioner: Provisioner
+    notifier: Notifier
+    clock: Clock
+    catalog: dict[str, CatalogEntry]
+    settings: Settings

services/worker/handlers.py

@@ -0,0 +1,161 @@
+"""Task handlers.
+
+Every handler here obeys one rule: running it twice must equal running it once.
+
+That is not a nicety. A worker can be SIGKILLed after helm has installed the release but
+before the DB row says so; the lease expires; another worker claims the same task and runs
+this function again. If the handler is not idempotent, the tenant gets two Elasticsearches
+and you get a bill. Idempotency is what makes the crash safe, and it is bought in two
+places: a deterministic `release_name`, and adapters that state desired state
+(`helm upgrade --install`) instead of issuing imperative commands.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from services.worker.deps import WorkerDeps
+from svcforge_core.domain.models import CatalogEntry, Instance, Task, TaskKind
+from svcforge_core.domain.states import InstanceState
+
+
+class HandlerError(RuntimeError):
+    """A task failed in a way worth retrying. The message lands in tasks.last_error."""
+
+
+async def _load_instance(task: Task, deps: WorkerDeps) -> Instance:
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            """select id, team, service_type, size, state, namespace, release_name,
+                      chart_version, endpoint, error, expires_at, created_at, updated_at
+                 from instances where id = %s""",
+            (task.instance_id,),
+        )
+        row = await cur.fetchone()
+    if row is None:
+        raise HandlerError(f"instance {task.instance_id} vanished")
+    return Instance.model_validate(row)
+
+
+def _values_for(inst: Instance, entry: CatalogEntry) -> dict[str, Any]:
+    """Turn a catalog size into helm values."""
+    size = entry.sizes.get(inst.size)
+    if size is None:
+        raise HandlerError(f"size {inst.size!r} not in catalog for {inst.service_type!r}")
+    return {"replicaCount": size.replicas, "resources": size.resources}
+
+
+async def handle_provision(task: Task, deps: WorkerDeps) -> None:
+    """Install the release and mark the instance ready. Idempotent."""
+    inst = await _load_instance(task, deps)
+
+    if inst.state is InstanceState.READY:
+        # A previous attempt already finished; the crash was after the work, before the
+        # bookkeeping. Nothing to do — and re-installing would be the bug.
+        return
+
+    entry = deps.catalog.get(inst.service_type)
+    if entry is None:
+        raise HandlerError(f"unknown service_type {inst.service_type!r}")
+
+    # Best-effort CAS. It returning False means someone else moved the row; the helm call
+    # below is idempotent either way, so this is bookkeeping, not a lock.
+    await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
+
+    await deps.provisioner.install(
+        release=inst.release_name,
+        ns=inst.namespace,
+        entry=entry,
+        values=_values_for(inst, entry),
+    )
+
+    endpoint = f"http://{inst.release_name}.{inst.namespace}.svc.cluster.local"
+    ok = await deps.instances.update_state(
+        inst.id, InstanceState.PROVISIONING, InstanceState.READY, endpoint=endpoint
+    )
+    if ok:
+        await deps.notifier.send(
+            "instance.ready",
+            f"instance {inst.id} is ready at {endpoint}",
+            {"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
+        )
+
+
+async def handle_deprovision(task: Task, deps: WorkerDeps) -> None:
+    """Remove the release and mark the instance deleted. Idempotent."""
+    inst = await _load_instance(task, deps)
+
+    if inst.state is InstanceState.DELETED:
+        return
+
+    # `helm uninstall` of an already-gone release is not an error to us: the adapter
+    # swallows not-found, because the desired state — no release — is already true.
+    await deps.provisioner.uninstall(release=inst.release_name, ns=inst.namespace)
+    await deps.instances.update_state(inst.id, InstanceState.DELETING, InstanceState.DELETED)
+
+
+async def handle_upgrade(task: Task, deps: WorkerDeps) -> None:
+    """Upgrade the release to the catalog's pinned version, then record it.
+
+    `instances.chart_version` is written only AFTER helm reports success. That column is
+    what the day-2 work-list query compares against, so writing it optimistically would
+    make the fleet look upgraded while it isn't.
+    """
+    inst = await _load_instance(task, deps)
+    entry = deps.catalog.get(inst.service_type)
+    if entry is None:
+        raise HandlerError(f"unknown service_type {inst.service_type!r}")
+
+    if inst.chart_version == entry.chart_version:
+        return  # already there
+
+    await deps.provisioner.install(
+        release=inst.release_name,
+        ns=inst.namespace,
+        entry=entry,
+        values=_values_for(inst, entry),
+    )
+
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            "update instances set chart_version = %s, updated_at = now() where id = %s",
+            (entry.chart_version, inst.id),
+        )
+
+
+async def handle_verify(task: Task, deps: WorkerDeps) -> None:
+    """Post-upgrade health probe. On failure, halt the whole rollout for this service type.
+
+    One column decides whether the fleet keeps rolling. The work-list query returns nothing
+    while `rollout_state='halted'`, so a bad chart stops after the first tenant instead of
+    after all of them. You clear it with SQL, deliberately: an automatic un-halt would just
+    resume breaking things.
+    """
+    inst = await _load_instance(task, deps)
+    releases = {r.name for r in await deps.provisioner.list_releases()}
+
+    if inst.release_name in releases:
+        return
+
+    async with deps.pool.connection() as conn, conn.cursor() as cur:
+        await cur.execute(
+            """insert into catalog_versions (service_type, rollout_state)
+                    values (%s, 'halted')
+               on conflict (service_type) do update set rollout_state = 'halted'""",
+            (inst.service_type,),
+        )
+    await deps.notifier.send(
+        "rollout.halted",
+        f"rollout halted for {inst.service_type}: {inst.release_name} failed verify",
+        {"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
+    )
+    raise HandlerError(f"verify failed for {inst.release_name}; rollout halted")
+
+
+HANDLERS: dict[TaskKind, Callable[[Task, WorkerDeps], Awaitable[None]]] = {
+    TaskKind.PROVISION: handle_provision,
+    TaskKind.DEPROVISION: handle_deprovision,
+    TaskKind.UPGRADE: handle_upgrade,
+    TaskKind.VERIFY: handle_verify,
+}

services/worker/main.py

@@ -0,0 +1,175 @@
+"""The claim loop.
+
+Poll every 5 seconds. Claim while a semaphore slot is free. Run the handler. Report.
+That is the whole design, and the restraint is the point: LISTEN/NOTIFY would shave the
+latency, is fire-and-forget so it can never replace the poll anyway, is strictly extra
+code, and does not exist on pgbouncer's transaction pooler. The poll is not a placeholder
+for something better.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import signal
+import time
+
+from opentelemetry import trace
+
+from services.worker.deps import WorkerDeps
+from services.worker.handlers import HANDLERS
+from svcforge_core import obs
+from svcforge_core.adapters.clock import SystemClock
+from svcforge_core.adapters.helm import HelmProvisioner
+from svcforge_core.adapters.notify import LogNotifier
+from svcforge_core.domain.catalog import load_catalog
+from svcforge_core.domain.models import Task
+from svcforge_core.repo.db import make_pool
+from svcforge_core.repo.instances import InstanceRepo
+from svcforge_core.repo.tasks import TaskRepo
+from svcforge_core.settings import Settings, load_settings
+
+log = obs.get_logger("svcforge.worker")
+
+
+async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
+    """Sleep, but wake immediately on shutdown.
+
+    `await asyncio.sleep(5)` would make every SIGTERM cost up to five seconds of
+    Kubernetes waiting on terminationGracePeriod for no reason.
+    """
+    with contextlib.suppress(TimeoutError):
+        await asyncio.wait_for(stop.wait(), timeout=seconds)
+
+
+async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None:
+    """Run one task to a terminal report. Never lets an exception escape the TaskGroup."""
+    try:
+        # Every log line from here carries instance_id/task_id/team. Bound once, at claim,
+        # rather than passed down: the alternative is threading three arguments through
+        # every function that might log, and the first one anyone forgets is the one you
+        # need at 3am.
+        obs.bind_task_context(task.instance_id, task.id, team=task.team or "unknown")
+        log.info("task claimed", kind=task.kind.value, attempt=task.attempts)
+        obs.TASKS_CLAIMED.labels(kind=task.kind.value).inc()
+
+        handler = HANDLERS.get(task.kind)
+        if handler is None:
+            await deps.tasks.fail(task.id, f"no handler for {task.kind}", max_attempts=1)
+            return
+
+        # Re-parent to the span that enqueued this task. Without the stored traceparent
+        # the worker's span starts a brand-new trace, and the POST that caused the work
+        # is in a different trace to the helm call that did it.
+        ctx = obs.context_from_traceparent(task.traceparent)
+        started = time.monotonic()
+        with obs.tracer().start_as_current_span(
+            f"task.{task.kind.value}",
+            context=ctx,
+            kind=trace.SpanKind.CONSUMER,
+        ) as span:
+            span.set_attribute("task.id", task.id)
+            span.set_attribute("task.kind", task.kind.value)
+            span.set_attribute("instance.id", str(task.instance_id))
+            try:
+                await handler(task, deps)
+            except asyncio.CancelledError:
+                # CancelledError inherits from BaseException, so `except Exception` below
+                # would never see it. Catch it only to release the claim, then ALWAYS
+                # re-raise: swallowing it breaks cancellation for everyone above us.
+                await deps.tasks.fail(task.id, "cancelled", max_attempts=deps.settings.max_attempts)
+                raise
+            except Exception as exc:
+                log.exception("task failed", kind=task.kind.value, error=str(exc))
+                span.record_exception(exc)
+                span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc)))
+                obs.TASKS_FAILED.labels(kind=task.kind.value).inc()
+                await deps.tasks.fail(task.id, str(exc), max_attempts=deps.settings.max_attempts)
+            else:
+                obs.PROVISION_TIME.observe(time.monotonic() - started)
+                await deps.tasks.complete(task.id)
+    finally:
+        sem.release()
+
+
+async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
+    """Claim and run until told to stop, then drain what is in flight.
+
+    Draining is what makes a rolling deploy invisible. Exiting the `async with` block
+    awaits every in-flight handler, so a pod that is being replaced finishes the provision
+    it already started instead of abandoning it half-done for the lease to clean up
+    five minutes later.
+    """
+    sem = asyncio.Semaphore(deps.settings.worker_concurrency)
+    worker_id = deps.settings.worker_id
+
+    async with asyncio.TaskGroup() as tg:
+        while not stop.is_set():
+            await sem.acquire()
+            if stop.is_set():
+                sem.release()
+                break
+
+            try:
+                task = await deps.tasks.claim(worker_id)
+            except Exception:
+                # A DB blip must not kill the worker; back off and try again.
+                log.exception("claim failed")
+                sem.release()
+                await _sleep_or_stop(stop, deps.settings.poll_interval_s)
+                continue
+
+            if task is None:
+                sem.release()
+                await _sleep_or_stop(stop, deps.settings.poll_interval_s)
+                continue
+
+            tg.create_task(_run_one(deps, task, sem))
+    # TaskGroup.__aexit__ awaited the in-flight handlers. Now it is safe to exit 0.
+
+
+async def _amain() -> None:
+    settings: Settings = load_settings()
+
+    # Before anything else: nothing logged above this line is structured, and the metrics
+    # the SvcforgeTaskFailed / SvcforgeProvisionSlow alerts query do not exist until the
+    # registry is up.
+    obs.setup("svcforge-worker", settings)
+    obs.start_metrics_server(settings.metrics_port)
+
+    pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
+    await pool.open(wait=True)
+
+    deps = WorkerDeps(
+        pool=pool,
+        instances=InstanceRepo(pool),
+        tasks=TaskRepo(pool),
+        provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
+        notifier=LogNotifier(),
+        clock=SystemClock(),
+        catalog=load_catalog(settings.catalog_path),
+        settings=settings,
+    )
+
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        # add_signal_handler, NOT signal.signal. signal.signal runs the handler at an
+        # arbitrary bytecode boundary on whatever thread the C-level handler lands on,
+        # and the event loop will not notice until its next timer fires. This one is
+        # loop-safe: the callback runs as a normal loop callback.
+        loop.add_signal_handler(sig, stop.set)
+
+    try:
+        await run_worker(deps, stop)
+    finally:
+        await pool.close()
+
+
+def main() -> None:
+    """One asyncio.run, at the top, never nested."""
+    asyncio.run(_amain())
+
+
+if __name__ == "__main__":
+    main()