svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

scripts/redis_budget.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Project month-end Redis command burn from the live counter. Exit 1 if it blows the budget.
+
+    $ python3 scripts/redis_budget.py
+    $ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000
+
+Upstash's free tier is 500,000 commands/month, which sounds enormous and is not:
+
+    500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
+
+0.19 commands per second is the entire engineering constraint. One worker polling Redis
+every five seconds spends 518,400/month — the whole budget, to learn nothing. That is why
+Redis is only ever on the request path here, and why this script exists: the rule is easy
+to state and invisible to violate. A `cache.get()` added inside the reconciler's
+per-instance loop is one line in review and 2,160,000 commands/month in production.
+
+**Why a projection and not an alarm on the counter.** Exhausting the budget is a slow,
+silent failure with a cliff at the end: nothing degrades, nothing pages, every call
+succeeds, and then the month rolls over and every Redis call starts erroring at once. By
+then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible
+on day two, which is the only time it is cheap to fix.
+
+Stdlib only, on purpose — this is meant to run from CI, from a laptop, or from inside a
+pod that has nothing but python3, without an environment to activate first.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+import urllib.request
+from urllib.parse import urlparse
+
+# A 30-day month. Upstash bills on a calendar month; 30 days is the honest rounding and
+# errs slightly pessimistic on the long ones, which is the correct direction for a budget.
+_MONTH_S = 30 * 24 * 60 * 60
+_FREE_TIER_BUDGET = 500_000
+
+_COMMANDS_METRIC = "svcforge_redis_commands_total"
+_START_TIME_METRIC = "process_start_time_seconds"
+
+
+class BudgetError(RuntimeError):
+    """The metrics endpoint did not give us enough to project from."""
+
+
+def scrape(url: str, timeout_s: float = 5.0) -> str:
+    """GET the Prometheus text exposition. http/https only."""
+    if urlparse(url).scheme not in ("http", "https"):
+        raise BudgetError(f"refusing to fetch a non-http(s) url: {url}")
+    # S310 is satisfied by the scheme check above: this cannot open file:// or ftp://.
+    with urllib.request.urlopen(url, timeout=timeout_s) as resp:  # noqa: S310
+        body: str = resp.read().decode("utf-8")
+    return body
+
+
+def _parse_sample(line: str) -> tuple[str, dict[str, str], float] | None:
+    """One exposition line -> (name, labels, value). None for comments and blanks.
+
+    A deliberately small parser rather than prometheus_client's: importing the library
+    would mean this script only runs where the app's venv is already active, which is
+    exactly where you least need to check the budget.
+    """
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    head, _, raw_value = line.rpartition(" ")
+    if not head:
+        return None
+    try:
+        value = float(raw_value)
+    except ValueError:
+        return None
+
+    name, brace, rest = head.partition("{")
+    labels: dict[str, str] = {}
+    if brace:
+        for pair in rest.rstrip("}").split(","):
+            key, eq, val = pair.partition("=")
+            if eq:
+                labels[key.strip()] = val.strip().strip('"')
+    return name.strip(), labels, value
+
+
+def collect(text: str) -> tuple[dict[str, float], float]:
+    """Extract per-op command totals and the process start time from a scrape."""
+    per_op: dict[str, float] = {}
+    started_at: float | None = None
+
+    for line in text.splitlines():
+        parsed = _parse_sample(line)
+        if parsed is None:
+            continue
+        name, labels, value = parsed
+        # prometheus_client exposes counters with a `_total` suffix already; tolerate both
+        # spellings so this keeps working if the client library changes its mind.
+        if name in (_COMMANDS_METRIC, _COMMANDS_METRIC.removesuffix("_total")):
+            per_op[labels.get("op", "unknown")] = value
+        elif name == _START_TIME_METRIC:
+            started_at = value
+
+    if not per_op:
+        raise BudgetError(
+            f"{_COMMANDS_METRIC} is not exposed. Either the API never imported "
+            f"svcforge_core.adapters.redis, or you are scraping the wrong process."
+        )
+    if started_at is None:
+        raise BudgetError(
+            f"{_START_TIME_METRIC} is missing, so there is no window to project over. "
+            f"It comes from prometheus_client's default collector."
+        )
+    return per_op, started_at
+
+
+def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int:
+    """Print the projection. Returns the process exit code."""
+    elapsed_s = max(1.0, now - started_at)
+    total = sum(per_op.values())
+    rate = total / elapsed_s
+    projected = rate * _MONTH_S
+
+    print(f"window          {elapsed_s / 3600:.2f} h since process start")
+    print(f"commands        {total:,.0f}")
+    for op, value in sorted(per_op.items(), key=lambda kv: -kv[1]):
+        share = (value / total * 100) if total else 0.0
+        print(f"  {op:<14}{value:>12,.0f}  ({share:.1f}%)")
+    print(f"rate            {rate:.4f} /s   (budget allows {budget / _MONTH_S:.4f} /s sustained)")
+    print(f"projected       {projected:,.0f} / month")
+    print(f"budget          {budget:,} / month")
+
+    if total < 100:
+        # Extrapolating a month from a handful of commands is astrology. Say so rather than
+        # printing a confident number derived from six samples.
+        print("verdict         INCONCLUSIVE — fewer than 100 commands; let it run longer")
+        return 0
+    if projected >= budget:
+        headroom = projected / budget
+        print(f"verdict         OVER BUDGET — {headroom:.1f}x. Find the Redis call in a loop.")
+        return 1
+    share = projected / budget * 100
+    print(f"verdict         OK — {share:.1f}% of budget, {budget / projected:.1f}x headroom")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint")
+    parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month")
+    args = parser.parse_args(argv)
+
+    try:
+        per_op, started_at = collect(scrape(args.url))
+    except BudgetError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    except OSError as exc:
+        print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr)
+        return 2
+
+    return report(per_op, started_at, args.budget, time.time())
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())