#!/usr/bin/env python3 """Project month-end Redis command burn from the live counter. Exit 1 if it blows the budget. $ python3 scripts/redis_budget.py $ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000 Upstash's free tier is 500,000 commands/month, which sounds enormous and is not: 500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained 0.19 commands per second is the entire engineering constraint. One worker polling Redis every five seconds spends 518,400/month — the whole budget, to learn nothing. That is why Redis is only ever on the request path here, and why this script exists: the rule is easy to state and invisible to violate. A `cache.get()` added inside the reconciler's per-instance loop is one line in review and 2,160,000 commands/month in production. **Why a projection and not an alarm on the counter.** Exhausting the budget is a slow, silent failure with a cliff at the end: nothing degrades, nothing pages, every call succeeds, and then the month rolls over and every Redis call starts erroring at once. By then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible on day two, which is the only time it is cheap to fix. Stdlib only, on purpose — this is meant to run from CI, from a laptop, or from inside a pod that has nothing but python3, without an environment to activate first. """ from __future__ import annotations import argparse import sys import time import urllib.request from urllib.parse import urlparse # A 30-day month. Upstash bills on a calendar month; 30 days is the honest rounding and # errs slightly pessimistic on the long ones, which is the correct direction for a budget. _MONTH_S = 30 * 24 * 60 * 60 _FREE_TIER_BUDGET = 500_000 _COMMANDS_METRIC = "svcforge_redis_commands_total" _START_TIME_METRIC = "process_start_time_seconds" class BudgetError(RuntimeError): """The metrics endpoint did not give us enough to project from.""" def scrape(url: str, timeout_s: float = 5.0) -> str: """GET the Prometheus text exposition. http/https only.""" if urlparse(url).scheme not in ("http", "https"): raise BudgetError(f"refusing to fetch a non-http(s) url: {url}") # S310 is satisfied by the scheme check above: this cannot open file:// or ftp://. with urllib.request.urlopen(url, timeout=timeout_s) as resp: # noqa: S310 body: str = resp.read().decode("utf-8") return body def _parse_sample(line: str) -> tuple[str, dict[str, str], float] | None: """One exposition line -> (name, labels, value). None for comments and blanks. A deliberately small parser rather than prometheus_client's: importing the library would mean this script only runs where the app's venv is already active, which is exactly where you least need to check the budget. """ line = line.strip() if not line or line.startswith("#"): return None head, _, raw_value = line.rpartition(" ") if not head: return None try: value = float(raw_value) except ValueError: return None name, brace, rest = head.partition("{") labels: dict[str, str] = {} if brace: for pair in rest.rstrip("}").split(","): key, eq, val = pair.partition("=") if eq: labels[key.strip()] = val.strip().strip('"') return name.strip(), labels, value def collect(text: str) -> tuple[dict[str, float], float]: """Extract per-op command totals and the process start time from a scrape.""" per_op: dict[str, float] = {} started_at: float | None = None for line in text.splitlines(): parsed = _parse_sample(line) if parsed is None: continue name, labels, value = parsed # prometheus_client exposes counters with a `_total` suffix already; tolerate both # spellings so this keeps working if the client library changes its mind. if name in (_COMMANDS_METRIC, _COMMANDS_METRIC.removesuffix("_total")): per_op[labels.get("op", "unknown")] = value elif name == _START_TIME_METRIC: started_at = value if not per_op: raise BudgetError( f"{_COMMANDS_METRIC} is not exposed. Either the API never imported " f"svcforge_core.adapters.redis, or you are scraping the wrong process." ) if started_at is None: raise BudgetError( f"{_START_TIME_METRIC} is missing, so there is no window to project over. " f"It comes from prometheus_client's default collector." ) return per_op, started_at def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int: """Print the projection. Returns the process exit code.""" elapsed_s = max(1.0, now - started_at) total = sum(per_op.values()) rate = total / elapsed_s projected = rate * _MONTH_S print(f"window {elapsed_s / 3600:.2f} h since process start") print(f"commands {total:,.0f}") for op, value in sorted(per_op.items(), key=lambda kv: -kv[1]): share = (value / total * 100) if total else 0.0 print(f" {op:<14}{value:>12,.0f} ({share:.1f}%)") print(f"rate {rate:.4f} /s (budget allows {budget / _MONTH_S:.4f} /s sustained)") print(f"projected {projected:,.0f} / month") print(f"budget {budget:,} / month") if total < 100: # Extrapolating a month from a handful of commands is astrology. Say so rather than # printing a confident number derived from six samples. print("verdict INCONCLUSIVE — fewer than 100 commands; let it run longer") return 0 if projected >= budget: headroom = projected / budget print(f"verdict OVER BUDGET — {headroom:.1f}x. Find the Redis call in a loop.") return 1 share = projected / budget * 100 print(f"verdict OK — {share:.1f}% of budget, {budget / projected:.1f}x headroom") return 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint") parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month") args = parser.parse_args(argv) try: per_op, started_at = collect(scrape(args.url)) except BudgetError as exc: print(f"error: {exc}", file=sys.stderr) return 2 except OSError as exc: print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr) return 2 return report(per_op, started_at, args.budget, time.time()) if __name__ == "__main__": raise SystemExit(main())