svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

scripts/bump-digests.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+#
+# CI's last act.
+#
+# Resolves the digest each service's commit-SHA tag points at, writes those digests into
+# deploy/chart/values.yaml, and commits. That commit is the deploy: ArgoCD is watching
+# master and picks it up. This script does not, and must not, talk to the cluster.
+#
+# Called by .gitea/workflows/ci.yaml on master only. Runnable by hand for a re-bump:
+#   REGISTRY=gitea.oci-oci.duckdns.org IMAGE_NS=gitea_admin IMAGE_TAG=<sha> ./scripts/bump-digests.sh
+#
+# -e   a failed inspect must not lead to committing a stale digest
+# -u   an unset REGISTRY would silently resolve the wrong image
+# -o pipefail   the digest comes out of a pipe; without this, a failing inspect that pipes
+#               into a successful grep exits 0 and writes garbage
+set -euo pipefail
+
+: "${REGISTRY:?REGISTRY must be set}"
+: "${IMAGE_NS:?IMAGE_NS must be set}"
+: "${IMAGE_TAG:?IMAGE_TAG must be set (the commit sha the images were built from)}"
+
+SERVICES=(api worker reconciler)
+CHART_VALUES="deploy/chart/values.yaml"
+
+# yq, pinned by digest. Not python+pyyaml: a yaml round-trip strips every comment in
+# values.yaml, and those comments are the only thing explaining why the digests are there.
+# yq edits in place and leaves the rest of the file alone.
+YQ_IMAGE="mikefarah/yq:4.44.6@sha256:b1d117c609ba990436ad1649299e2f6c378f62cb562caf30b6f2fb6144713422"
+
+WORKDIR="$(mktemp -d)"
+cleanup() {
+    rm -rf "${WORKDIR}"
+}
+trap cleanup EXIT
+
+yq() {
+    docker run --rm -v "${PWD}:/work" -w /work -u "$(id -u):$(id -g)" "${YQ_IMAGE}" "$@"
+}
+
+echo "==> resolving digests for tag ${IMAGE_TAG}"
+for svc in "${SERVICES[@]}"; do
+    image="${REGISTRY}/${IMAGE_NS}/svcforge-${svc}"
+    digest="$(docker buildx imagetools inspect "${image}:${IMAGE_TAG}" --format '{{.Manifest.Digest}}')"
+
+    # Defence against a silently empty inspect. Without this, `yq` would happily write an
+    # empty digest and the chart's own guard would fail the release later, further from
+    # the cause.
+    if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+        echo "!! ${svc}: refusing to write a non-digest: '${digest}'" >&2
+        exit 1
+    fi
+
+    echo "    ${svc} -> ${digest}"
+    echo "${digest}" > "${WORKDIR}/${svc}.digest"
+done
+
+echo "==> writing ${CHART_VALUES}"
+for svc in "${SERVICES[@]}"; do
+    digest="$(cat "${WORKDIR}/${svc}.digest")"
+    # env(...) rather than string interpolation: a digest is attacker-controlled only in
+    # theory, but yq expression injection is not a thing worth leaving open.
+    DIGEST="${digest}" yq -i ".image.${svc}.digest = strenv(DIGEST)" "${CHART_VALUES}"
+done
+
+if git diff --quiet -- "${CHART_VALUES}"; then
+    echo "==> no digest changed; nothing to commit"
+    exit 0
+fi
+
+echo "==> committing"
+git config user.name "svcforge-ci"
+git config user.email "ci@svcforge.invalid"
+git add "${CHART_VALUES}"
+git commit -m "ci: bump image digests to ${IMAGE_TAG}
+
+Built and scanned by ${IMAGE_TAG}. ArgoCD syncs from this commit.
+
+[skip ci]"
+git push origin HEAD:master
+
+echo "==> done. ArgoCD owns it from here."

scripts/load.py

@@ -0,0 +1,130 @@
+"""Throwaway load generator. Enqueue N instances, watch the queue drain, print three numbers.
+
+The point is not a benchmark. It is to find the ceiling on purpose, in a place where finding
+it is free, so that the number in RUNBOOK.md comes from an observation instead of a guess.
+
+The ceiling you are looking for is arithmetic, not mysterious:
+
+    total connections = (api_replicas + worker_replicas) x pool_max_size
+
+Supabase's free-tier pooler has a small connection budget. Cross it and the failure does not
+look like "too many connections" — it looks like slow claims, then PoolTimeout, then a queue
+that grows while every worker looks idle. Once you have watched it once, you recognise it in
+two seconds instead of an hour.
+
+Usage:
+    python -m scripts.load --count 200 --watch
+    python -m scripts.load --count 200 --direct        # skip the API, enqueue straight to the DB
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import time
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import psycopg
+from psycopg.rows import dict_row
+
+from svcforge_core.settings import load_settings
+
+
+async def _seed_direct(dsn: str, count: int) -> float:
+    """Insert `count` instances + provision tasks. Returns seconds taken.
+
+    --direct exists to separate two questions that a single POST run conflates: "how fast can
+    the API accept work" and "how fast can workers drain it". Measure them apart or you will
+    tune the wrong one.
+    """
+    started = time.monotonic()
+    async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
+        async with conn.transaction(), conn.cursor() as cur:
+            for _ in range(count):
+                iid = uuid4()
+                await cur.execute(
+                    """insert into instances (id, team, service_type, size, state, namespace,
+                                              release_name, chart_version)
+                       values (%s, 'loadtest', 'elasticsearch', 'small', 'requested',
+                               'tenant-loadtest', %s, '21.3.19')""",
+                    (iid, f"loadtest-elasticsearch-{str(iid)[:8]}"),
+                )
+                await cur.execute(
+                    "insert into tasks (instance_id, kind) values (%s, 'provision')",
+                    (iid,),
+                )
+    return time.monotonic() - started
+
+
+async def _depth(dsn: str) -> dict[str, int]:
+    async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
+        cur = await conn.execute("select state, count(*) as n from tasks group by 1")
+        return {str(r["state"]): int(r["n"]) for r in await cur.fetchall()}
+
+
+async def _watch(dsn: str, timeout_s: float) -> None:
+    """Print queue depth once a second until it drains. The slope is the number you want."""
+    started = time.monotonic()
+    peak = 0
+    print(f"{'t(s)':>6}  {'queued':>7} {'running':>8} {'done':>6} {'failed':>7}  slope/s")
+    prev_done, prev_t = 0, started
+
+    while time.monotonic() - started < timeout_s:
+        d = await _depth(dsn)
+        queued, running = d.get("queued", 0), d.get("running", 0)
+        done, failed = d.get("done", 0), d.get("failed", 0)
+        peak = max(peak, queued + running)
+
+        now = time.monotonic()
+        slope = (done - prev_done) / max(now - prev_t, 1e-9)
+        prev_done, prev_t = done, now
+
+        print(f"{now - started:6.1f}  {queued:7d} {running:8d} {done:6d} {failed:7d}  {slope:7.1f}")
+
+        if queued == 0 and running == 0:
+            elapsed = now - started
+            print(f"\ndrained in {elapsed:.1f}s   peak depth {peak}   throughput {done / elapsed:.1f} task/s")
+            if failed:
+                print(f"WARNING: {failed} tasks failed — the number above is not a clean drain")
+            return
+        await asyncio.sleep(1.0)
+
+    print(f"\nstill draining after {timeout_s}s — that IS the result. Record it.")
+
+
+async def _amain() -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--count", type=int, default=200)
+    ap.add_argument("--direct", action="store_true", help="enqueue via SQL instead of the API")
+    ap.add_argument("--watch", action="store_true", help="poll queue depth until drained")
+    ap.add_argument("--timeout", type=float, default=600.0)
+    ap.add_argument("--cleanup", action="store_true", help="delete loadtest rows and exit")
+    args = ap.parse_args()
+
+    settings = load_settings()
+    dsn = settings.pg_dsn.unicode_string()
+
+    if args.cleanup:
+        async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as conn:
+            await conn.execute("delete from instances where team = 'loadtest'")  # tasks cascade
+        print("loadtest rows deleted")
+        return
+
+    if not args.direct:
+        raise SystemExit(
+            "POST mode needs a token; use --direct for the drain measurement, or drive the API "
+            "with k6 (one dependency, not two — do not add locust for this)."
+        )
+
+    print(f"seeding {args.count} instances at {datetime.now(UTC).isoformat()} ...")
+    took = await _seed_direct(dsn, args.count)
+    print(f"enqueued {args.count} in {took:.2f}s ({args.count / took:.0f}/s)\n")
+
+    if args.watch:
+        await _watch(dsn, args.timeout)
+        print("\nremember: `python -m scripts.load --cleanup` when you are done.")
+
+
+if __name__ == "__main__":
+    asyncio.run(_amain())

scripts/redis_budget.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Project month-end Redis command burn from the live counter. Exit 1 if it blows the budget.
+
+    $ python3 scripts/redis_budget.py
+    $ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000
+
+Upstash's free tier is 500,000 commands/month, which sounds enormous and is not:
+
+    500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
+
+0.19 commands per second is the entire engineering constraint. One worker polling Redis
+every five seconds spends 518,400/month — the whole budget, to learn nothing. That is why
+Redis is only ever on the request path here, and why this script exists: the rule is easy
+to state and invisible to violate. A `cache.get()` added inside the reconciler's
+per-instance loop is one line in review and 2,160,000 commands/month in production.
+
+**Why a projection and not an alarm on the counter.** Exhausting the budget is a slow,
+silent failure with a cliff at the end: nothing degrades, nothing pages, every call
+succeeds, and then the month rolls over and every Redis call starts erroring at once. By
+then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible
+on day two, which is the only time it is cheap to fix.
+
+Stdlib only, on purpose — this is meant to run from CI, from a laptop, or from inside a
+pod that has nothing but python3, without an environment to activate first.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+import urllib.request
+from urllib.parse import urlparse
+
+# A 30-day month. Upstash bills on a calendar month; 30 days is the honest rounding and
+# errs slightly pessimistic on the long ones, which is the correct direction for a budget.
+_MONTH_S = 30 * 24 * 60 * 60
+_FREE_TIER_BUDGET = 500_000
+
+_COMMANDS_METRIC = "svcforge_redis_commands_total"
+_START_TIME_METRIC = "process_start_time_seconds"
+
+
+class BudgetError(RuntimeError):
+    """The metrics endpoint did not give us enough to project from."""
+
+
+def scrape(url: str, timeout_s: float = 5.0) -> str:
+    """GET the Prometheus text exposition. http/https only."""
+    if urlparse(url).scheme not in ("http", "https"):
+        raise BudgetError(f"refusing to fetch a non-http(s) url: {url}")
+    # S310 is satisfied by the scheme check above: this cannot open file:// or ftp://.
+    with urllib.request.urlopen(url, timeout=timeout_s) as resp:  # noqa: S310
+        body: str = resp.read().decode("utf-8")
+    return body
+
+
+def _parse_sample(line: str) -> tuple[str, dict[str, str], float] | None:
+    """One exposition line -> (name, labels, value). None for comments and blanks.
+
+    A deliberately small parser rather than prometheus_client's: importing the library
+    would mean this script only runs where the app's venv is already active, which is
+    exactly where you least need to check the budget.
+    """
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    head, _, raw_value = line.rpartition(" ")
+    if not head:
+        return None
+    try:
+        value = float(raw_value)
+    except ValueError:
+        return None
+
+    name, brace, rest = head.partition("{")
+    labels: dict[str, str] = {}
+    if brace:
+        for pair in rest.rstrip("}").split(","):
+            key, eq, val = pair.partition("=")
+            if eq:
+                labels[key.strip()] = val.strip().strip('"')
+    return name.strip(), labels, value
+
+
+def collect(text: str) -> tuple[dict[str, float], float]:
+    """Extract per-op command totals and the process start time from a scrape."""
+    per_op: dict[str, float] = {}
+    started_at: float | None = None
+
+    for line in text.splitlines():
+        parsed = _parse_sample(line)
+        if parsed is None:
+            continue
+        name, labels, value = parsed
+        # prometheus_client exposes counters with a `_total` suffix already; tolerate both
+        # spellings so this keeps working if the client library changes its mind.
+        if name in (_COMMANDS_METRIC, _COMMANDS_METRIC.removesuffix("_total")):
+            per_op[labels.get("op", "unknown")] = value
+        elif name == _START_TIME_METRIC:
+            started_at = value
+
+    if not per_op:
+        raise BudgetError(
+            f"{_COMMANDS_METRIC} is not exposed. Either the API never imported "
+            f"svcforge_core.adapters.redis, or you are scraping the wrong process."
+        )
+    if started_at is None:
+        raise BudgetError(
+            f"{_START_TIME_METRIC} is missing, so there is no window to project over. "
+            f"It comes from prometheus_client's default collector."
+        )
+    return per_op, started_at
+
+
+def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int:
+    """Print the projection. Returns the process exit code."""
+    elapsed_s = max(1.0, now - started_at)
+    total = sum(per_op.values())
+    rate = total / elapsed_s
+    projected = rate * _MONTH_S
+
+    print(f"window          {elapsed_s / 3600:.2f} h since process start")
+    print(f"commands        {total:,.0f}")
+    for op, value in sorted(per_op.items(), key=lambda kv: -kv[1]):
+        share = (value / total * 100) if total else 0.0
+        print(f"  {op:<14}{value:>12,.0f}  ({share:.1f}%)")
+    print(f"rate            {rate:.4f} /s   (budget allows {budget / _MONTH_S:.4f} /s sustained)")
+    print(f"projected       {projected:,.0f} / month")
+    print(f"budget          {budget:,} / month")
+
+    if total < 100:
+        # Extrapolating a month from a handful of commands is astrology. Say so rather than
+        # printing a confident number derived from six samples.
+        print("verdict         INCONCLUSIVE — fewer than 100 commands; let it run longer")
+        return 0
+    if projected >= budget:
+        headroom = projected / budget
+        print(f"verdict         OVER BUDGET — {headroom:.1f}x. Find the Redis call in a loop.")
+        return 1
+    share = projected / budget * 100
+    print(f"verdict         OK — {share:.1f}% of budget, {budget / projected:.1f}x headroom")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint")
+    parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month")
+    args = parser.parse_args(argv)
+
+    try:
+        per_op, started_at = collect(scrape(args.url))
+    except BudgetError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    except OSError as exc:
+        print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr)
+        return 2
+
+    return report(per_op, started_at, args.budget, time.time())
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())