svcforge/scripts/redis_budget.py

#!/usr/bin/env python3
"""Project month-end Redis command burn from the live counter. Exit 1 if it blows the budget.

    $ python3 scripts/redis_budget.py
    $ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000

Upstash's free tier is 500,000 commands/month, which sounds enormous and is not:

    500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained

0.19 commands per second is the entire engineering constraint. One worker polling Redis
every five seconds spends 518,400/month — the whole budget, to learn nothing. That is why
Redis is only ever on the request path here, and why this script exists: the rule is easy
to state and invisible to violate. A `cache.get()` added inside the reconciler's
per-instance loop is one line in review and 2,160,000 commands/month in production.

**Why a projection and not an alarm on the counter.** Exhausting the budget is a slow,
silent failure with a cliff at the end: nothing degrades, nothing pages, every call
succeeds, and then the month rolls over and every Redis call starts erroring at once. By
then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible
on day two, which is the only time it is cheap to fix.

Stdlib only, on purpose — this is meant to run from CI, from a laptop, or from inside a
pod that has nothing but python3, without an environment to activate first.
"""

from __future__ import annotations

import argparse
import sys
import time
import urllib.request
from urllib.parse import urlparse

# A 30-day month. Upstash bills on a calendar month; 30 days is the honest rounding and
# errs slightly pessimistic on the long ones, which is the correct direction for a budget.
_MONTH_S = 30 * 24 * 60 * 60
_FREE_TIER_BUDGET = 500_000

_COMMANDS_METRIC = "svcforge_redis_commands_total"
_START_TIME_METRIC = "process_start_time_seconds"


class BudgetError(RuntimeError):
    """The metrics endpoint did not give us enough to project from."""


def scrape(url: str, timeout_s: float = 5.0) -> str:
    """GET the Prometheus text exposition. http/https only."""
    if urlparse(url).scheme not in ("http", "https"):
        raise BudgetError(f"refusing to fetch a non-http(s) url: {url}")
    # S310 is satisfied by the scheme check above: this cannot open file:// or ftp://.
    with urllib.request.urlopen(url, timeout=timeout_s) as resp:  # noqa: S310
        body: str = resp.read().decode("utf-8")
    return body


def _parse_sample(line: str) -> tuple[str, dict[str, str], float] | None:
    """One exposition line -> (name, labels, value). None for comments and blanks.

    A deliberately small parser rather than prometheus_client's: importing the library
    would mean this script only runs where the app's venv is already active, which is
    exactly where you least need to check the budget.
    """
    line = line.strip()
    if not line or line.startswith("#"):
        return None
    head, _, raw_value = line.rpartition(" ")
    if not head:
        return None
    try:
        value = float(raw_value)
    except ValueError:
        return None

    name, brace, rest = head.partition("{")
    labels: dict[str, str] = {}
    if brace:
        for pair in rest.rstrip("}").split(","):
            key, eq, val = pair.partition("=")
            if eq:
                labels[key.strip()] = val.strip().strip('"')
    return name.strip(), labels, value


def collect(text: str) -> tuple[dict[str, float], float]:
    """Extract per-op command totals and the process start time from a scrape."""
    per_op: dict[str, float] = {}
    started_at: float | None = None

    for line in text.splitlines():
        parsed = _parse_sample(line)
        if parsed is None:
            continue
        name, labels, value = parsed
        # prometheus_client exposes counters with a `_total` suffix already; tolerate both
        # spellings so this keeps working if the client library changes its mind.
        if name in (_COMMANDS_METRIC, _COMMANDS_METRIC.removesuffix("_total")):
            per_op[labels.get("op", "unknown")] = value
        elif name == _START_TIME_METRIC:
            started_at = value

    if not per_op:
        raise BudgetError(
            f"{_COMMANDS_METRIC} is not exposed. Either the API never imported "
            f"svcforge_core.adapters.redis, or you are scraping the wrong process."
        )
    if started_at is None:
        raise BudgetError(
            f"{_START_TIME_METRIC} is missing, so there is no window to project over. "
            f"It comes from prometheus_client's default collector."
        )
    return per_op, started_at


def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int:
    """Print the projection. Returns the process exit code."""
    elapsed_s = max(1.0, now - started_at)
    total = sum(per_op.values())
    rate = total / elapsed_s
    projected = rate * _MONTH_S

    print(f"window          {elapsed_s / 3600:.2f} h since process start")
    print(f"commands        {total:,.0f}")
    for op, value in sorted(per_op.items(), key=lambda kv: -kv[1]):
        share = (value / total * 100) if total else 0.0
        print(f"  {op:<14}{value:>12,.0f}  ({share:.1f}%)")
    print(f"rate            {rate:.4f} /s   (budget allows {budget / _MONTH_S:.4f} /s sustained)")
    print(f"projected       {projected:,.0f} / month")
    print(f"budget          {budget:,} / month")

    if total < 100:
        # Extrapolating a month from a handful of commands is astrology. Say so rather than
        # printing a confident number derived from six samples.
        print("verdict         INCONCLUSIVE — fewer than 100 commands; let it run longer")
        return 0
    if projected >= budget:
        headroom = projected / budget
        print(f"verdict         OVER BUDGET — {headroom:.1f}x. Find the Redis call in a loop.")
        return 1
    share = projected / budget * 100
    print(f"verdict         OK — {share:.1f}% of budget, {budget / projected:.1f}x headroom")
    return 0


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint")
    parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month")
    args = parser.parse_args(argv)

    try:
        per_op, started_at = collect(scrape(args.url))
    except BudgetError as exc:
        print(f"error: {exc}", file=sys.stderr)
        return 2
    except OSError as exc:
        print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr)
        return 2

    return report(per_op, started_at, args.budget, time.time())


if __name__ == "__main__":
    raise SystemExit(main())