refactor: converge the patterns multiple authors left divergent

The codebase was written by several agents and had the same concept done more
than one way. This makes it read as one voice, with no behaviour change.

Dedup, each to a single canonical form:
  - INSTANCE_COLUMNS: the 13-column instances SELECT list existed as _COLUMNS in
    instances.py and reconcile.py (byte-identical) and inlined a third time in
    the worker. One exported constant now.
  - Settings.runtime_dsn: the three entrypoints each chose between str(pg_dsn)
    and pg_dsn.unicode_string(). One property.
  - yaml_tempfile: helm._ValuesFile and k8s._ManifestFile were the same
    write-yaml-to-a-temp-dir context manager. One helper in adapters/tempyaml.py.
  - services/_runtime.py: sleep_or_stop and install_stop_signals were copied
    between the worker and reconciler loops. One module, so shutdown behaviour
    cannot drift between them.
  - k8s.ensure_namespace used MANAGED_BY_LABEL/VALUE from helm.py instead of a
    hardcoded literal, so the managed-by label has one definition.
  - SvcforgeError is now the root of every svcforge exception (CatalogError,
    IllegalTransition, BadWindow, HandlerError), keeping each stdlib base in the
    MRO, so `except SvcforgeError` means what errors.py says it does.
  - ERROR_MAX_CHARS replaces the repeated `[-2000:]` truncation feeding the same
    error columns.
  - the reconciler reads settings.metrics_port like the worker, dropping its
    duplicate DEFAULT_METRICS_PORT and redundant --metrics-port option; the
    SVCFORGE_METRICS_PORT env override still applies through pydantic.

Two smaller correctness/consistency fixes:
  - RateLimitResult.retry_after_s computed its delta against datetime.now(UTC)
    while the limiter runs on an injectable clock, so it was meaningless under a
    FakeClock and drifted by request latency in production. It now carries a
    checked_at from the same clock as reset_at.
  - handle_provision's notifier.send is wrapped like the reconciler's: a flaky
    webhook after the READY CAS would fail the task, and the retry would hit the
    READY early-return and drop the notification, turning a good provision into a
    failed one.

libs/svcforge_core/svcforge_core/adapters/helm.py

@@ -21,17 +21,15 @@ from __future__ import annotations
 import asyncio
 import json
 import os
-import shutil
 import signal
-import tempfile
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Protocol
 
 import httpx
-import yaml
 from pydantic import BaseModel, ConfigDict, Field
 
+from svcforge_core.adapters.tempyaml import yaml_tempfile
 from svcforge_core.domain.models import CatalogEntry
 from svcforge_core.errors import SvcforgeError
 
@@ -220,7 +218,7 @@ class HelmProvisioner:
         # `--wait` is why `ready` in the DB means ready — it returns when the pods are up.
         # `--atomic` rolls back a failed upgrade; it doubles the worst case, which is what
         # `_RUN_TIMEOUT_MARGIN_S` and helm's own `--timeout` are sized around.
-        with _values_file(values) as path:
+        with yaml_tempfile(values, prefix="svcforge-values-", name="values.yaml") as path:
             argv = self._base_argv(
                 "upgrade",
                 "--install",
@@ -433,29 +431,3 @@ class HelmProvisioner:
                 ),
             )
         return [info for _, info in newest.values()]
-
-
-class _ValuesFile:
-    """Context manager yielding a path to a values.yaml written from a dict.
-
-    A file, not `--set`: `--set` has its own escaping grammar (commas, dots, backslashes) and
-    values carry tenant-shaped strings. Serialising YAML sidesteps the grammar entirely.
-    """
-
-    def __init__(self, values: dict[str, Any]) -> None:
-        self._values = values
-        self._dir: str | None = None
-
-    def __enter__(self) -> Path:
-        self._dir = tempfile.mkdtemp(prefix="svcforge-values-")
-        path = Path(self._dir) / "values.yaml"
-        path.write_text(yaml.safe_dump(self._values, default_flow_style=False), encoding="utf-8")
-        return path
-
-    def __exit__(self, *exc: object) -> None:
-        if self._dir is not None:
-            shutil.rmtree(self._dir, ignore_errors=True)
-
-
-def _values_file(values: dict[str, Any]) -> _ValuesFile:
-    return _ValuesFile(values)

libs/svcforge_core/svcforge_core/adapters/k8s.py

@@ -13,14 +13,11 @@ from __future__ import annotations
 import base64
 import binascii
 import json
-import shutil
-import tempfile
 from pathlib import Path
 from typing import Any
 
-import yaml
-
-from svcforge_core.adapters.helm import HelmError, _run
+from svcforge_core.adapters.helm import MANAGED_BY_LABEL, MANAGED_BY_VALUE, HelmError, _run
+from svcforge_core.adapters.tempyaml import yaml_tempfile
 from svcforge_core.errors import SvcforgeError
 
 _KUBECTL_TIMEOUT_S = 60
@@ -71,10 +68,10 @@ class KubectlClient:
             "kind": "Namespace",
             "metadata": {
                 "name": ns,
-                "labels": {"app.kubernetes.io/managed-by": "svcforge", **(labels or {})},
+                "labels": {MANAGED_BY_LABEL: MANAGED_BY_VALUE, **(labels or {})},
             },
         }
-        with _manifest_file(manifest) as path:
+        with yaml_tempfile(manifest, prefix="svcforge-manifest-", name="manifest.yaml") as path:
             await self._kubectl("apply", "--filename", str(path))
 
     async def read_secret(self, ns: str, name: str) -> dict[str, str]:
@@ -122,25 +119,3 @@ class KubectlClient:
             # timeout path does not come through HelmError. Without this clause a wedged
             # kubectl surfaces as TimeoutError past a caller written to `except K8sError`.
             raise K8sError(f"kubectl {args[0] if args else ''} timed out after {self._timeout_s}s") from exc
-
-
-class _ManifestFile:
-    """A temp file holding one YAML manifest, removed on exit."""
-
-    def __init__(self, manifest: dict[str, Any]) -> None:
-        self._manifest = manifest
-        self._dir: str | None = None
-
-    def __enter__(self) -> Path:
-        self._dir = tempfile.mkdtemp(prefix="svcforge-manifest-")
-        path = Path(self._dir) / "manifest.yaml"
-        path.write_text(yaml.safe_dump(self._manifest, default_flow_style=False), encoding="utf-8")
-        return path
-
-    def __exit__(self, *exc: object) -> None:
-        if self._dir is not None:
-            shutil.rmtree(self._dir, ignore_errors=True)
-
-
-def _manifest_file(manifest: dict[str, Any]) -> _ManifestFile:
-    return _ManifestFile(manifest)

libs/svcforge_core/svcforge_core/adapters/redis.py

@@ -178,6 +178,10 @@ class RateLimitResult:
     limit: int
     remaining: int
     reset_at: datetime
+    # When the limiter made this decision, from the same injected clock as reset_at. The two
+    # have to share a clock or retry_after_s (their difference) is meaningless under a
+    # FakeClock, and drifts by the request latency even in production.
+    checked_at: datetime
     degraded: bool = False
 
     @property
@@ -187,7 +191,7 @@ class RateLimitResult:
         Rounded up and floored at one: `Retry-After: 0` invites an immediate retry into
         the same closed window, which is a busy loop with extra steps.
         """
-        delta = (self.reset_at - datetime.now(UTC)).total_seconds()
+        delta = (self.reset_at - self.checked_at).total_seconds()
         return max(1, math.ceil(delta))
 
 
@@ -258,6 +262,7 @@ class RateLimiter:
                 limit=self._limit,
                 remaining=self._limit,
                 reset_at=reset_at,
+                checked_at=self._clock.now(),
                 degraded=True,
             )
         return RateLimitResult(
@@ -265,6 +270,7 @@ class RateLimiter:
             limit=self._limit,
             remaining=int(remaining),
             reset_at=reset_at,
+            checked_at=self._clock.now(),
         )
 
 

libs/svcforge_core/svcforge_core/adapters/tempyaml.py

@@ -0,0 +1,34 @@
+"""One temp YAML file, written from a dict and removed on exit.
+
+helm and kubectl both take their input as a file rather than on the command line: `--set`
+and inline manifests each have their own escaping grammar, and tenant-shaped values would
+have to be escaped into it. Serialising YAML to a file sidesteps the grammar entirely. Both
+adapters needed the same throwaway-file dance, so it lives here once.
+"""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@contextmanager
+def yaml_tempfile(payload: dict[str, Any], *, prefix: str, name: str) -> Iterator[Path]:
+    """Yield a path to `name` inside a fresh temp dir, holding `payload` as YAML.
+
+    The whole dir is removed on exit, `ignore_errors` so a cleanup race never masks the real
+    error from the block.
+    """
+    tmpdir = tempfile.mkdtemp(prefix=prefix)
+    try:
+        path = Path(tmpdir) / name
+        path.write_text(yaml.safe_dump(payload, default_flow_style=False), encoding="utf-8")
+        yield path
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)

libs/svcforge_core/svcforge_core/domain/catalog.py

@@ -10,9 +10,10 @@ import yaml
 from pydantic import ValidationError
 
 from svcforge_core.domain.models import CatalogEntry
+from svcforge_core.errors import SvcforgeError
 
 
-class CatalogError(Exception):
+class CatalogError(SvcforgeError):
     """A catalog file could not be parsed or validated.
 
     `key` names the offending service type, or None when the failure is file-level.

libs/svcforge_core/svcforge_core/domain/states.py

@@ -3,6 +3,8 @@
 from enum import StrEnum
 from typing import Final
 
+from svcforge_core.errors import SvcforgeError
+
 
 class InstanceState(StrEnum):
     """Lifecycle of a provisioned service instance."""
@@ -15,7 +17,7 @@ class InstanceState(StrEnum):
     FAILED = "failed"
 
 
-class IllegalTransition(Exception):
+class IllegalTransition(SvcforgeError):
     """Raised by transition() when cur -> nxt is not in LEGAL."""
 
 

libs/svcforge_core/svcforge_core/domain/windows.py

@@ -19,11 +19,13 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from croniter import croniter
 
+from svcforge_core.errors import SvcforgeError
+
 CRON_FIELDS = 5
 SEPARATOR = "|"
 
 
-class BadWindow(ValueError):
+class BadWindow(SvcforgeError, ValueError):
     """A maintenance window spec is not a 5-field cron plus a known IANA zone."""
 
 

libs/svcforge_core/svcforge_core/repo/db.py

@@ -17,6 +17,11 @@ from psycopg_pool import AsyncConnectionPool
 # against a bare `AsyncConnectionPool`, which resolves to tuple rows. The runtime was
 # always right; without these aliases the annotations quietly disagree with it, and the
 # fix people reach for is `# type: ignore`, which throws away the checking entirely.
+# The cap on error text written to the `instances.error` and `tasks.last_error` columns. A
+# helm failure can emit megabytes; these columns are read by humans. Defined once here rather
+# than as a bare 2000 at each write, so the two call paths that feed the same columns agree.
+ERROR_MAX_CHARS = 2000
+
 type DictRow = dict[str, Any]
 type DictConnection = AsyncConnection[DictRow]
 type DictPool = AsyncConnectionPool[DictConnection]

libs/svcforge_core/svcforge_core/repo/instances.py

@@ -20,7 +20,7 @@ from svcforge_core.domain.models import Instance
 from svcforge_core.domain.states import InstanceState
 from svcforge_core.repo.db import DictPool
 
-_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
+INSTANCE_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
               endpoint, error, expires_at, created_at, updated_at"""
 
 
@@ -57,7 +57,7 @@ class InstanceRepo:
                                            release_name, chart_version, endpoint, error, expires_at)
                     values (%(id)s, %(team)s, %(service_type)s, %(size)s, %(state)s, %(namespace)s,
                             %(release_name)s, %(chart_version)s, %(endpoint)s, %(error)s, %(expires_at)s)
-                    returning {_COLUMNS}""",  # noqa: S608 - _COLUMNS is a module constant, not input
+                    returning {INSTANCE_COLUMNS}""",  # noqa: S608 - INSTANCE_COLUMNS is a module constant, not input
                 {
                     "id": inst.id,
                     "team": inst.team,
@@ -80,7 +80,7 @@ class InstanceRepo:
         """Fetch one instance owned by `team`. None if it does not exist OR is not theirs."""
         async with self._pool.connection() as conn, conn.cursor() as cur:
             await cur.execute(
-                f"select {_COLUMNS} from instances where id = %s and team = %s",  # noqa: S608
+                f"select {INSTANCE_COLUMNS} from instances where id = %s and team = %s",  # noqa: S608
                 (id, team),
             )
             row = await cur.fetchone()
@@ -90,7 +90,7 @@ class InstanceRepo:
         """The team's instances, newest first."""
         async with self._pool.connection() as conn, conn.cursor() as cur:
             await cur.execute(
-                f"""select {_COLUMNS} from instances
+                f"""select {INSTANCE_COLUMNS} from instances
                     where team = %s order by created_at desc limit %s""",  # noqa: S608
                 (team, limit),
             )
@@ -157,7 +157,7 @@ class InstanceRepo:
         # and would otherwise type these rows as tuples.
         async with self._pool.connection() as conn, conn.cursor(row_factory=dict_row) as cur:
             await cur.execute(
-                f"""select {_COLUMNS}, maintenance_window from instances
+                f"""select {INSTANCE_COLUMNS}, maintenance_window from instances
                      where state = 'ready'
                        and service_type = %(service_type)s
                        and chart_version <> %(catalog_version)s
@@ -166,7 +166,7 @@ class InstanceRepo:
                               where cv.service_type = %(service_type)s
                                 and cv.rollout_state = 'halted')
                      order by team = %(own_team)s desc, created_at
-                     limit %(max_in_flight)s""",  # noqa: S608 - _COLUMNS is a module constant, not input
+                     limit %(max_in_flight)s""",  # noqa: S608 - INSTANCE_COLUMNS is a module constant, not input
                 {
                     "service_type": service_type,
                     "catalog_version": catalog_version,

libs/svcforge_core/svcforge_core/repo/reconcile.py

@@ -25,10 +25,8 @@ from psycopg import AsyncCursor
 from svcforge_core.domain.models import Instance, TaskKind, TaskState
 from svcforge_core.domain.states import InstanceState, transition
 from svcforge_core.obs import inject_traceparent
-from svcforge_core.repo.db import DictPool
-
-_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
-              endpoint, error, expires_at, created_at, updated_at"""
+from svcforge_core.repo.db import ERROR_MAX_CHARS, DictPool
+from svcforge_core.repo.instances import INSTANCE_COLUMNS
 
 # A task nobody will ever run again. The idempotency guard on every enqueue below asks
 # "is one already outstanding?", and 'done'/'failed' are not outstanding: a failed
@@ -70,7 +68,7 @@ class ReconcileRepo:
         """Every instance the DB believes is running. The drift check's expectation."""
         async with self._pool.connection() as conn, conn.cursor() as cur:
             await cur.execute(
-                f"select {_COLUMNS} from instances where state = %s",  # noqa: S608 - module constant
+                f"select {INSTANCE_COLUMNS} from instances where state = %s",  # noqa: S608 - module constant
                 (InstanceState.READY.value,),
             )
             rows = await cur.fetchall()
@@ -126,7 +124,7 @@ class ReconcileRepo:
 
                 await cur.execute(
                     "update instances set state = %s, error = %s, updated_at = now() where id = %s",
-                    (provisioning.value, reason[-2000:], instance_id),
+                    (provisioning.value, reason[-ERROR_MAX_CHARS:], instance_id),
                 )
                 return await _insert_task(cur, instance_id, TaskKind.PROVISION)
 
@@ -151,7 +149,7 @@ class ReconcileRepo:
         """
         async with self._pool.connection() as conn, conn.cursor() as cur:
             await cur.execute(
-                f"""select {_COLUMNS} from instances i
+                f"""select {INSTANCE_COLUMNS} from instances i
                      where ((i.state = %(ready)s and i.expires_at < now()) or i.state = %(deleting)s)
                        and not exists (
                              select 1 from tasks t

libs/svcforge_core/svcforge_core/repo/tasks.py

@@ -20,7 +20,7 @@ from svcforge_core.domain.backoff import next_attempt_at
 from svcforge_core.domain.models import Task, TaskKind
 from svcforge_core.domain.states import LEGAL, InstanceState
 from svcforge_core.obs import TASKS_DEAD_LETTERED, inject_traceparent
-from svcforge_core.repo.db import DictPool
+from svcforge_core.repo.db import ERROR_MAX_CHARS, DictPool
 
 # Which states may legally become `failed`, derived from the domain's own table rather
 # than restated here. Without this guard the UPDATE below would happily move a `deleted`
@@ -205,7 +205,7 @@ class TaskRepo:
                               set state='queued', locked_by=null, locked_at=null,
                                   last_error=%s, run_after=%s
                             where id = %s""",
-                        (err[-2000:], next_attempt_at(attempts - 1, now=now), task_id),
+                        (err[-ERROR_MAX_CHARS:], next_attempt_at(attempts - 1, now=now), task_id),
                     )
                     return True
 
@@ -213,7 +213,7 @@ class TaskRepo:
                     """update tasks
                           set state='failed', locked_by=null, locked_at=null, last_error=%s
                         where id = %s""",
-                    (err[-2000:], task_id),
+                    (err[-ERROR_MAX_CHARS:], task_id),
                 )
                 # Dead-lettering the task is correct for every kind. Moving the INSTANCE to
                 # `failed` is correct only for provision: a provisioning instance that never
@@ -235,7 +235,7 @@ class TaskRepo:
                     await cur.execute(
                         """update instances set error=%s, state=%s, updated_at=now()
                             where id=%s and state = any(%s)""",
-                        (err[-2000:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
+                        (err[-ERROR_MAX_CHARS:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
                     )
                 # Counted here, not in the worker: this is the only place that knows the
                 # difference between "attempt 2 of 5 failed" and "this task is done trying".

libs/svcforge_core/svcforge_core/settings.py

@@ -83,6 +83,15 @@ class Settings(BaseSettings):
     # just for /metrics. 9000 matches the chart's PodMonitor; change both or neither.
     metrics_port: int = Field(default=9000, ge=1, le=65535)
 
+    @property
+    def runtime_dsn(self) -> str:
+        """The transaction-pooler DSN the services open their pool against, as a string.
+
+        A property so the three entrypoints do not each choose between `str(pg_dsn)` and
+        `pg_dsn.unicode_string()` — the two spellings that were drifting across the services.
+        """
+        return str(self.pg_dsn)
+
     @property
     def migration_dsn(self) -> str:
         """Migrations need a session-mode connection; fall back to the runtime DSN locally."""