gitea_admin/svcforge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

svcforge: reference implementation
ci / lint (push) Successful in 1m19s
Details
ci / unit (push) Failing after 1m2s
Details
ci / integration (push) Has been skipped
Details
ci / types (push) Successful in 1m37s
Details
ci / security (push) Failing after 38s
Details
ci / dockerfile (push) Successful in 14s
Details
ci / image (api) (push) Has been skipped
Details
ci / image (reconciler) (push) Has been skipped
Details
ci / image (worker) (push) Has been skipped
Details
ci / bump (push) Has been skipped
Details

Browse Source 
Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.
This commit is contained in:
Nguyen Minh Phuc
2026-07-17 10:44:54 +00:00
commit 50c2fe2a1e
102 changed files with 12018 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+31
View File
@@ -0,0 +1,31 @@
# Build context is the repo root (docker build -f services/<svc>/Dockerfile .).
# Everything here is either a host artifact, a secret, or dead weight in a layer.
.venv/
.git/
.github/
.gitea/
tests/
**/__pycache__
*.pyc
.mypy_cache/
.ruff_cache/
.pytest_cache/
.hypothesis/
.coverage
htmlcov/
dist/
build/
*.egg-info/
# secrets never enter a build context, let alone a layer
.env
*.env
!.env.example
# not needed at runtime
deploy/
docs/
scripts/
Makefile
RUNBOOK.md
README.md
.env.example
+18
View File
@@ -0,0 +1,18 @@
# Copy to .env and fill in. .env is gitignored and must never be committed.
# Real values live in ~/.config/svcforge/secrets.env (chmod 600).
# Postgres — transaction pooler (6543). Runtime traffic.
SVCFORGE_PG_DSN=postgresql://postgres.<project>:<password>@aws-1-ap-southeast-1.pooler.supabase.com:6543/postgres
# Postgres — session pooler (5432). Migrations and psql only.
SVCFORGE_PG_DSN_SESSION=postgresql://postgres.<project>:<password>@aws-1-ap-southeast-1.pooler.supabase.com:5432/postgres
# Redis — Upstash. Derived state only.
SVCFORGE_REDIS_DSN=rediss://default:<token>@<endpoint>.upstash.io:6379
# API
SVCFORGE_JWKS_URL=https://<issuer>/protocol/openid-connect/certs
SVCFORGE_JWT_ISSUER=https://<issuer>
SVCFORGE_JWT_AUDIENCE=svcforge
# Worker
SVCFORGE_WORKER_ID=worker-local
SVCFORGE_WORKER_CONCURRENCY=4
.gitea/workflows/ci.yaml
+279
View File
@@ -0,0 +1,279 @@
# svcforge CI.
#
# The contract, in one line: merge to master -> three images built and scanned -> the
# chart's image digests bumped -> ArgoCD syncs. CI never touches the cluster. There is no
# kubeconfig here and there must never be one; the pipeline's last act is a git commit.
#
# Rules this file exists to enforce:
# - Build once, promote the artifact. The digest that trivy scanned is the digest that
# lands in values.yaml is the digest that runs.
# - Deploy by digest, never a mutable tag.
# - Everything pinned: actions by SHA, tool images by digest, deps by uv.lock + --frozen.
# - Every gate required. None advisory. Fail the PR, not prod.
#
# Stage order is deliberate and matches the module: cheapest and most likely to fail first,
# so a formatting mistake costs 20 seconds instead of three minutes of image builds.
name: ci
on: [pull_request, push]
concurrency:
# A second push to the same branch makes the first run's answer irrelevant. Cancel it —
# except on master, where the run ends in a commit and must not be interrupted midway.
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
env:
REGISTRY: gitea.oci-oci.duckdns.org
IMAGE_NS: gitea_admin
UV_VERSION: "0.5.11"
jobs:
# --- stage 1: lint -- fast, fails first ---------------------------------------------
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
# The uv store is keyed on the lockfile: same lock, same wheels, cache hit.
cache-dependency-glob: uv.lock
- run: uv sync --frozen
- name: ruff
run: uv run ruff check . && uv run ruff format --check .
# --- stage 2: types -- your compiler ------------------------------------------------
types:
runs-on: ubuntu-latest
needs: [lint]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
cache-dependency-glob: uv.lock
- run: uv sync --frozen
- name: mypy --strict
run: uv run mypy --strict .
# --- stage 3: unit -- domain only, milliseconds, coverage gate -----------------------
unit:
runs-on: ubuntu-latest
needs: [lint]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
cache-dependency-glob: uv.lock
- run: uv sync --frozen
- name: pytest unit
# The gate is on domain/ alone, and only domain/. It is pure, has no I/O, and needs
# no mocks — there is no excuse for a gap there. Pointing this at the whole repo
# would let untested SQL be paid for by well-tested pure functions.
run: uv run pytest tests/unit --cov=libs/svcforge_core/domain --cov-fail-under=90
# --- stages 4+5: migrate, then integration against that schema -----------------------
integration:
runs-on: ubuntu-latest
needs: [unit]
services:
postgres:
image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: svcforge
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
# A scratch Postgres, so a literal password is correct here: it is not a secret, it
# is a fixture. Real DSNs live in Vault and reach the pods via external-secrets.
SVCFORGE_PG_DSN: postgresql://postgres:postgres@postgres:5432/svcforge
SVCFORGE_PG_DSN_SESSION: postgresql://postgres:postgres@postgres:5432/svcforge
# What tests/integration/conftest.py actually reads. Without it the fixture falls
# back to testcontainers and starts a SECOND Postgres inside the runner's docker,
# while the service container above sits unused — slower, and a different database
# to the one `migrate` just ran against.
SVCFORGE_TEST_DSN: postgresql://postgres:postgres@postgres:5432/svcforge
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
cache-dependency-glob: uv.lock
- run: uv sync --frozen
- name: migrate
# The same entrypoint the chart's pre-upgrade hook runs. If migrations only ever
# ran under testcontainers, CI would be testing a code path production never takes.
run: uv run python -m svcforge_core.migrate
- name: pytest integration
run: uv run pytest tests/integration
# --- stages 6+7+8: SAST, secrets, dependency CVEs ------------------------------------
# One job, three independent gates. They share a checkout and nothing else; each `run`
# step fails the job on its own.
security:
runs-on: ubuntu-latest
needs: [lint]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
# gitleaks scans history, not just the tip. A secret committed and then reverted
# is still a leaked secret, and a shallow clone cannot see it.
fetch-depth: 0
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
with:
version: ${{ env.UV_VERSION }}
enable-cache: true
cache-dependency-glob: uv.lock
- run: uv sync --frozen
- name: bandit (SAST)
# `--with`, not a dev dependency: bandit is a CI tool, not something the project
# imports, and ruff's S ruleset already runs its checks in the lint stage. This is
# the belt to that suspenders — -ll reports medium severity and above only.
run: uv run --with bandit bandit -r libs services -ll
- name: gitleaks (secret scan)
# Pinned by digest and run directly, so the command is the documented one rather
# than a marketplace action's opinion of it.
run: |
docker run --rm -v "$PWD:/repo" -w /repo \
ghcr.io/gitleaks/gitleaks:v8.21.2@sha256:0e99e8821643ea5b235718642b93bb32486af9c8162c8b8731f7cbdc951a7f46 \
detect --no-banner --source /repo
- name: pip-audit (dependency CVEs)
# --strict fails on an audit error rather than shrugging and reporting clean.
run: uv run --with pip-audit pip-audit --strict
# --- stage 9: hadolint ---------------------------------------------------------------
dockerfile:
runs-on: ubuntu-latest
needs: [lint]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: hadolint
uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0
with:
recursive: true
dockerfile: "services/*/Dockerfile"
failure-threshold: warning
# --- stage 10: build -> trivy -> push by digest --------------------------------------
image:
runs-on: ubuntu-latest
# Every gate above is required. An image is not built until all of them are green,
# which is what makes "the digest CI pushed is a digest that passed everything" true.
needs: [types, unit, integration, security, dockerfile]
permissions:
contents: read
strategy:
fail-fast: false
matrix:
svc: [api, worker, reconciler]
# Deliberately no `outputs:` here. Matrix legs share one outputs map and clobber each
# other — the merge is not per-key and not ordered, so two of the three digests would
# arrive empty or stale, intermittently. The bump job resolves the digests from the
# registry instead, which is a read, not a rebuild.
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
- name: registry login
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: build
# Context is the repo root and the Dockerfile is addressed with -f. It cannot be
# otherwise: the image needs pyproject.toml, uv.lock and libs/, all of which live
# above services/<svc>/, and COPY ../.. is illegal.
#
# Loaded locally, not pushed. Trivy scans this exact image next; only then does it
# get pushed. The alternative — push, scan, and hope nobody pulled meanwhile — is
# how a CRITICAL ends up in the registry with a green checkmark next to it.
run: |
docker buildx build \
-f services/${{ matrix.svc }}/Dockerfile \
--build-arg BUILD_SHA=${{ github.sha }} \
--cache-from type=gha,scope=${{ matrix.svc }} \
--cache-to type=gha,mode=max,scope=${{ matrix.svc }} \
--load \
-t svcforge/${{ matrix.svc }}:ci \
.
- name: trivy
uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
with:
image-ref: svcforge/${{ matrix.svc }}:ci
severity: HIGH,CRITICAL
# A CVE with no fix available is not something this PR can act on; failing on it
# only teaches people to add ignore entries. Rebuilding on a new base image picks
# the fix up the day it exists.
ignore-unfixed: true
exit-code: "1"
format: table
- name: push by digest
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
# Re-running buildx here is a cache hit on every layer, not a second build: the
# image is byte-identical to the one trivy just cleared. buildx cannot --load and
# --push in one invocation, which is the only reason this step exists.
#
# The commit-SHA tag is a handle for the bump job to resolve, not something anything
# deploys. What deploys is the digest that tag resolves to.
run: |
set -euo pipefail
IMAGE="${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}"
docker buildx build \
-f services/${{ matrix.svc }}/Dockerfile \
--build-arg BUILD_SHA=${{ github.sha }} \
--cache-from type=gha,scope=${{ matrix.svc }} \
--push \
-t "${IMAGE}:${GITHUB_SHA}" \
.
docker buildx imagetools inspect "${IMAGE}:${GITHUB_SHA}" \
--format '{{.Manifest.Digest}}'
# --- stage 11: bump the chart's digests. CI's last act. ------------------------------
bump:
runs-on: ubuntu-latest
needs: [image]
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
permissions:
contents: write
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
# A bot token with contents:write on this repo and nothing else. Note what is
# absent: no kubeconfig, no cluster credential, no ArgoCD API token. CI's maximum
# blast radius is a bad commit, which is revertable.
token: ${{ secrets.CI_BOT_TOKEN }}
ref: master
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: bump image digests in the chart
env:
REGISTRY: ${{ env.REGISTRY }}
IMAGE_NS: ${{ env.IMAGE_NS }}
IMAGE_TAG: ${{ github.sha }}
# And then it stops. No kubectl, no helm upgrade, no argocd app sync. ArgoCD is
# watching master and will have this within a minute.
run: ./scripts/bump-digests.sh
.gitignore
+18
View File
@@ -0,0 +1,18 @@
.venv/
__pycache__/
*.pyc
.pytest_cache/
.mypy_cache/
.ruff_cache/
.coverage
htmlcov/
dist/
build/
*.egg-info/
# secrets never land in the repo
.env
# local scratch
.coverage
.hypothesis/
.pre-commit-config.yaml
+12
View File
@@ -0,0 +1,12 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.8.4
hooks:
- id: ruff
args: [--fix]
- id: ruff-format
- repo: https://github.com/gitleaks/gitleaks
rev: v8.21.2
hooks:
- id: gitleaks
.python-version
+1
View File
@@ -0,0 +1 @@
3.12
Makefile
+47
View File
@@ -0,0 +1,47 @@
.PHONY: dev lint test migrate run-api run-worker run-reconciler integration ci chart
dev:
uv sync
lint:
uv run ruff check . && uv run ruff format --check . && uv run mypy .
test:
uv run pytest tests/unit
integration:
uv run pytest tests/integration
migrate:
uv run python -m svcforge_core.migrate
run-api:
uv run uvicorn services.api.main:create_app --factory --reload --port 8000
run-worker:
uv run python -m services.worker.main
run-reconciler:
uv run python -m services.reconciler.main
# Every gate CI runs, except the image build — that needs a docker daemon and three
# minutes, and the point of this target is to find out you failed before pushing.
# Same commands as .gitea/workflows/ci.yaml, deliberately: a gate that only exists in CI
# is a gate you debug through a web UI.
ci:
uv run ruff check .
uv run ruff format --check .
uv run mypy --strict .
uv run pytest tests/unit --cov=libs/svcforge_core/domain --cov-fail-under=90
uv run pytest tests/integration
uv run --with bandit bandit -r libs services -ll
uv run --with pip-audit pip-audit --strict
hadolint services/*/Dockerfile
helm lint deploy/chart
helm template svcforge deploy/chart >/dev/null
# The chart's own gates. `helm template` is not a formality here: the image helper calls
# `fail` on any digest that is not a sha256, so this catches an unbumped values.yaml.
chart:
helm lint deploy/chart
helm template svcforge deploy/chart
README.md
+126
View File
@@ -0,0 +1,126 @@
# svcforge — reference implementation
A complete, working, verified build of the system `../learn-python/` teaches you to build.
## Read this part first
**This folder can ruin the course, and it will if you let it.**
`learn-python/AGENTS.md` opens with a rule: *"Do not write his implementation code for him.
`domain/`, `repo/`, the claim loop, the handlers — those are the course."* This repo is
exactly that code. You asked for it deliberately, and the rule allows you to overrule it —
but the reason for the rule did not go away when you did.
Module 4 is five evenings of getting the claim query wrong. That is not a tax on the way to
the answer; it **is** the answer. The evening you spend watching two workers grab the same
task is the evening `FOR UPDATE SKIP LOCKED` stops being a phrase and starts being a thing
you understand. Reading `repo/tasks.py` here takes ninety seconds and teaches you close to
nothing, while feeling exactly like learning. That feeling is the trap.
So:
| Use it like this | Not like this |
|---|---|
| Attempt the module. Get stuck. Stay stuck 30 minutes. **Then** diff your version against this one. | Open this first "just to see the shape". |
| Steal the scaffolding — `pyproject.toml`, Dockerfiles, `ci.yaml`, the chart. Nothing is learned by fighting hatchling. | Copy `domain/`, `repo/`, `handlers.py`, or the claim loop. |
| Read the **comments**. They explain *why*, which is the part that transfers. | Read the code. It is the part that doesn't. |
| Use it to check an answer you already produced. | Use it to produce an answer. |
The scaffolding is where struggling teaches you nothing. The domain and the queue are the
whole point. Know which one you're reading.
## What is actually verified
Everything below was executed, not asserted:
| Claim | How it was proven |
|---|---|
| The claim query never double-claims | 50 concurrent workers, 50 tasks, real Postgres. Each claimed exactly once. |
| The transaction story is real | Instance + task roll back together on an abort. |
| SIGTERM drains in flight | Worker finishes a 2s provision after stop is set, exits 0. |
| Handlers are idempotent | Re-running `handle_provision` installs once, not twice. |
| helm timeouts kill the process **group** | A negative control with `proc.kill()` leaks two `sleep 300`s; the real `_run` leaves zero. |
| The state machine is enforced in SQL too | Guard test fails when the guard is removed (control-tested). |
| Redis degrades safely | Rate limit fails open, cache falls through, platform stays up with Redis dead. |
| The images ship a wheel, not an editable | Negative control proved `uv sync` alone ships `/app/libs`; `--no-editable` fixed it. |
| The chart is valid | `helm lint`, `helm template`, `hadolint` — all clean. |
Numbers from a real drain (400 tasks, local Postgres, FakeProvisioner) are in
[RUNBOOK.md](RUNBOOK.md#measured-numbers).
## Running it
**No Docker on this host, deliberately.** This box's `containerd` belongs to a Kubernetes
kubelet; installing `docker.io` would evict the runtime and take every pod with it. So the
integration tests take a DSN from the environment and only fall back to testcontainers
(the CI path) when it is absent:
```bash
sudo -u postgres psql -c "create role svcforge login password 'svcforge' superuser;"
sudo -u postgres psql -c "create database svcforge_test owner svcforge;"
export PATH="$HOME/.local/bin:$PATH"
export SVCFORGE_TEST_DSN="postgresql://svcforge:svcforge@127.0.0.1:5432/svcforge_test"
make dev # uv sync
make lint # ruff + ruff format + mypy --strict
uv run pytest -q -m "not e2e and not slow"
```
Each pytest process clones its own database, so concurrent runs don't truncate each other.
Redis tests marked `slow` hit real Upstash. **Mind the budget**: the free tier is 500K
commands/month = 0.19/sec sustained. The whole suite spends about 35.
```bash
set -a; . ~/.config/svcforge/secrets.env; set +a # never commit these
uv run pytest -q -m slow
uv run python -m scripts.redis_budget # projects month-end burn, exits 1 if over
```
## Where things live
| Module | Teaches | Read here |
|---|---|---|
| 0 | toolchain | `pyproject.toml`, `Makefile` |
| 1 | domain, pydantic, Protocol | `libs/svcforge_core/svcforge_core/domain/` |
| 2 | psycopg3, repos, migrations | `repo/{db,instances}.py`, `migrations/001_init.sql` |
| 3 | FastAPI, JWT, one transaction | `services/api/` |
| **4** | **the queue — the core** | **`repo/tasks.py`, `services/worker/`** |
| 5 | subprocess, timeouts, Protocols | `adapters/helm.py` (read `_run` twice) |
| 6 | day-2 fleet upgrades | `domain/windows.py`, `InstanceRepo.list_upgradable` |
| 7 | reconciler, obs | `services/reconciler/`, `obs.py` |
| 8 | packaging, CI, ArgoCD | `services/*/Dockerfile`, `.gitea/workflows/ci.yaml`, `deploy/` |
| 9 | chaos, load, CLI, runbook | `scripts/load.py`, `services/cli/`, `RUNBOOK.md` |
| 10 | Redis as a shortcut | `adapters/redis.py`, `scripts/redis_budget.py` |
## Where this deviates from the spec, and why
Honest list. Each was a real conflict, not a shortcut.
1. **`TaskRepo.enqueue` exists twice.** Module 2 specs `enqueue(conn, ...)`; Module 4 specs
`enqueue(instance_id, kind) -> int`. Both callers are real, so both exist:
`enqueue(conn, ...)` and `enqueue_standalone(...)`. Making `conn` optional would have
hidden the transaction question, which is the one thing that module is about.
2. **The claim query has a CTE wrapper.** The spec calls it byte-identical. The `update ...
for update skip locked` shape is untouched; an outer `select` joins `instances.team` so
the worker can bind `team` to its logs at claim time. The alternative was a second round
trip per task for a column the DB already had.
3. **The day-2 work-list query gained `service_type` and a halted check.** As printed it
compares every service type against one version, and never consults `catalog_versions`
despite the same module requiring halted → 0 rows.
4. **`Role` → `ClusterRole`.** The specced rules grant `create` on `namespaces`, which is
cluster-scoped. A namespaced Role cannot express it. Rules verbatim otherwise.
5. **DELETE is not one transaction.** `InstanceRepo.update_state` owns its connection, so
the CAS and the enqueue can't share one without reaching around the repo. Ordered for
the failure mode instead: CAS first, enqueue second — a crash between leaves `deleting`
with no task, which the reconciler sweeps up. The reverse would tear down a live service.
6. **Traceparent is `-03`, not the spec's `-01`.** Current SDKs set the random-trace-id bit
alongside sampled. The spec's acceptance line is stale.
7. **`make_redis` returns `Redis | None`.** "No Redis configured" has to be runnable.
## What was deliberately NOT built
Because Module 6 says so, and the restraint is the lesson: no `resize`, no `backup`/`restore`,
no `helm rollback` automation, no deprecation timers, no rollouts table, no pause/resume CLI.
A halted rollout is one column, cleared by hand with SQL.
RUNBOOK.md
+192
View File
@@ -0,0 +1,192 @@
# svcforge runbook
Four entries. Each starts from an alert firing and ends at either a fix or an escalation.
Every command is copy-pasteable; none of them require thinking at 3am, which is the point.
Set these first:
```bash
set -a; . ~/.config/svcforge/secrets.env; set +a # SVCFORGE_PG_DSN_SESSION for psql
alias sfsql='psql "$SVCFORGE_PG_DSN_SESSION"'
```
---
## Measured numbers
From `scripts/load.py` + in-process workers on a `FakeProvisioner` (`delay=0.05s`), 200 tasks
per run. **These came off local Postgres on the same box, with a sub-millisecond round trip.
Supabase's pooler is ~5ms away, so treat these as a ceiling the real thing will not reach** —
the shape is what transfers, not the absolute numbers.
| replicas | pool max_size | worker concurrency | connections used | drain (200 tasks) | throughput |
|---:|---:|---:|---:|---:|---:|
| 1 | 5 | 4 | 5 | 4.3s | 46.9 task/s |
| 2 | 5 | 4 | 10 | 4.2s | 47.6 task/s |
| 4 | 5 | 4 | 20 | 2.2s | 92.3 task/s |
| 2 | 20 | 16 | 40 | 2.1s | 97.1 task/s |
Three things this says:
1. **Going from 1 replica to 2 bought nothing** (46.9 → 47.6). Throughput here is bounded by
per-worker concurrency (the semaphore), not by replica count. Adding pods to a saturated
semaphore is the most common wrong fix for a slow queue.
2. **Concurrency is the knob that moved it** — 4 replicas (20 connections) and 2 replicas at
concurrency 16 (40 connections) land in the same place, ~92-97 task/s. The second buys the
same throughput for twice the connections, which on the free tier is the worse trade.
3. **The budget is `(api + worker replicas) x max_size`**, and it is spent whether or not the
connections are busy. The bottom row costs 40 connections for a 2% gain over the row above
it. On Supabase free tier, that arithmetic — not throughput — is what decides replica count.
Nothing failed at any setting, so the real connection ceiling was never hit locally. Finding
it against the actual pooler is the experiment worth running: raise `replicas x max_size`
until claims slow and `PoolTimeout` appears, and write the number here.
---
## Queue stuck
**Alert:** `SvcforgeQueueDepthRising`
**Diagnose.** Start here, always:
```bash
sfsql -c "select state, count(*) from tasks group by 1;"
```
Then split the three causes apart — they look identical from the alert and need opposite fixes:
```bash
# Stuck leases: rows 'running' with a locked_at that never advances.
sfsql -c "select kind, locked_by, locked_at, last_error from tasks
where state='running' order by locked_at limit 10;"
# No workers: is anything actually consuming?
kubectl get pods -l app=worker -o wide
kubectl logs -l app=worker --tail=20 --prefix
# run_after in the future: backoff has parked everything.
sfsql -c "select count(*) from tasks where state='queued' and run_after > now();"
```
| What you see | Cause | Fix |
|---|---|---|
| `running` rows, `locked_at` older than 5m, no worker pods hold those IDs | Workers died mid-task | None. The reconciler resets expired leases within 60s. If it does not, the reconciler is down — check it. |
| Zero worker pods, or all `CrashLoopBackOff` | No consumer | Fix the workers. `kubectl describe pod -l app=worker`. |
| Everything `queued` with `run_after` far in the future | Backoff, i.e. tasks are failing and retrying | This is not a queue problem. Go to **Provision failing**. |
| `queued` rows with `run_after <= now()` and healthy workers | Real: claim is not returning rows | Check pooler connection budget (see **Supabase full**). |
**Never** hand-edit `state='running'` back to `'queued'`. The lease does that, and doing it
by hand while the worker is actually alive gives you two workers on one task — the exact
thing the whole design prevents.
**Escalate** if workers are healthy, leases are fresh, and depth still grows: that is a
claim-query or pooler bug, not an ops problem.
---
## Provision failing
**Alert:** `SvcforgeTaskFailed` (tasks reaching the dead-letter state), or `SvcforgeProvisionSlow`
(they still succeed, but the p95 has drifted out — usually cluster capacity, diagnosed the same way).
**Diagnose:**
```bash
sfsql -c "select id, service_type, chart_version, error from instances where state='failed';"
sfsql -c "select id, kind, attempts, last_error from tasks where state='failed' order by id desc limit 10;"
NS=tenant-<team>
helm list -n "$NS"
kubectl get events -n "$NS" --sort-by=.lastTimestamp | tail -20
```
| `error` looks like | Cause | Fix |
|---|---|---|
| `chart "..." version "..." not found` | Bad pin in `catalog.yaml` | Correct the version, commit. The next `upgrade`/`provision` picks it up. |
| `timed out waiting for the condition` | Cluster capacity — the chart installed but pods never became ready | `kubectl describe pod -n $NS`. Usually `Insufficient cpu/memory` or a PVC pending on Longhorn. |
| `Error: ... forbidden: User "system:serviceaccount:svcforge:..."` | RBAC | The worker's ClusterRole is missing a verb. Chart change, not a manual `kubectl edit`. |
| `ImagePullBackOff` in events | Registry auth or a gated image | Prefer `bitnamilegacy/*` images, which pull anonymously. |
After fixing the cause, tasks that already dead-lettered do **not** retry themselves. Requeue
deliberately:
```bash
sfsql -c "update tasks set state='queued', attempts=0, run_after=now(), last_error=null
where id = <task_id>;"
```
**Escalate** if `error` is empty on a failed instance — that means the failure path itself
lost the message.
---
## Orphaned release
**Alert:** `SvcforgeReconcilerStale` — the reconciler has not completed a loop recently, so
drift is no longer being *detected* at all. Drift itself is reported in the reconciler's logs
and metrics rather than paged on, because it is usually benign and always needs a human to
judge. A stale reconciler is the real emergency: nothing is watching.
The control loop **never auto-deletes a release.** That is deliberate: a bug in the drift
check that deletes things is unrecoverable, and one that only reports is a Tuesday.
**Diagnose:**
```bash
helm list -A -o json | jq -r '.[].name' | sort > /tmp/real
sfsql -tAc "select release_name from instances where state in ('ready','provisioning');" | sort > /tmp/want
comm -23 /tmp/real /tmp/want # in the cluster, not in the DB -> orphan
comm -13 /tmp/real /tmp/want # in the DB, not in the cluster -> missing
```
| Direction | Meaning | Action |
|---|---|---|
| Orphan (cluster only) | A deprovision half-finished, or someone ran `helm install` by hand | Confirm the tenant is gone, then `helm uninstall <name> -n <ns>` **by hand**, and write down that you did. |
| Missing (DB only) | Someone deleted a release out from under us | Requeue a `provision` task for that instance. It is idempotent; it will rebuild. |
**Escalate** before uninstalling anything you did not personally trace to a deleted instance.
A wrong `helm uninstall` here deletes a tenant's data.
---
## Supabase full
**Alert:** none — and that is a gap, not a decision. The free tier is 0.5 GB and nothing pages
you before you hit it; you find out when writes start failing. Until someone adds a size rule,
this entry is driven by the calendar, not by an alert. Check it monthly:
```bash
sfsql -c "select pg_size_pretty(pg_database_size(current_database()));"
```
**Diagnose:**
```bash
sfsql -c "select pg_size_pretty(pg_database_size(current_database()));"
sfsql -c "select relname, pg_size_pretty(pg_total_relation_size(relid)) from pg_catalog.pg_statio_user_tables
order by pg_total_relation_size(relid) desc limit 5;"
sfsql -c "select count(*) from pg_stat_activity;"
```
It is almost always `tasks`. Every provision, upgrade and verify leaves a row forever.
```bash
sfsql -c "delete from tasks where state='done' and created_at < now() - interval '7 days';"
sfsql -c "vacuum (analyze) tasks;"
```
`vacuum` alone reclaims space **for reuse by Postgres**, but does not return it to the
filesystem — so `pg_database_size` may barely move. That is expected and fine; the space is
free for new rows. `vacuum full` does return it, takes an `ACCESS EXCLUSIVE` lock, and will
stall every worker for its duration. Only do it in a window, and only if you actually need
the bytes back.
If `count(*) from pg_stat_activity` is near the pooler's ceiling, the cause is arithmetic, not
load: `worker_replicas × pool_max_size + api_replicas × pool_max_size`. Lower `max_size` or
replicas. **Replica count is a database-capacity decision here**, which is unusual and worth
remembering.
**Escalate** if size is growing with `tasks` already pruned — that means `instances` is
growing, i.e. tenants are real, i.e. the free tier is the wrong tier.
catalog.yaml
+77
View File
@@ -0,0 +1,77 @@
# The product catalog: what svcforge will provision, and in which sizes.
# service_type -> chart / chart_version / security / sizes{name -> replicas + resources}
#
# security: true bypasses every tenant's maintenance window for this entry. Set it for a
# CVE with a public exploit; leave it false and the bump waits for 03:00 Sunday.
services:
elasticsearch:
chart: bitnamilegacy/elasticsearch
chart_version: 21.3.15
security: false
sizes:
small:
replicas: 1
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
medium:
replicas: 3
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 8Gi
redis:
chart: bitnamilegacy/redis
chart_version: 20.6.2
security: false
sizes:
small:
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
medium:
replicas: 3
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
postgres:
chart: bitnamilegacy/postgresql
chart_version: 16.4.5
security: false
sizes:
small:
replicas: 1
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
medium:
replicas: 2
resources:
requests:
cpu: "1"
memory: 2Gi
limits:
cpu: "2"
memory: 4Gi
deploy/argocd/app.yaml
+57
View File
@@ -0,0 +1,57 @@
# The other half of "CI does not deploy".
#
# CI's last act is a commit that changes image.*.digest in deploy/chart/values.yaml.
# ArgoCD notices that commit and syncs. There is no kubeconfig in CI, no `helm upgrade` in
# a pipeline step, and no human running kubectl. If you want to know what is running in the
# cluster, you read git — that is the whole property, and a single `kubectl apply` from a
# laptop is what destroys it (selfHeal below exists to undo exactly that).
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: svcforge
namespace: argocd
finalizers:
# Without this, deleting the Application orphans every resource it created.
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
repoURL: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge.git
targetRevision: master
path: deploy/chart
helm:
releaseName: svcforge
# No valueFiles override and no `parameters:` block. values.yaml in the repo is the
# single source of truth — a parameter here would be a second place the deployed
# digest could come from, invisible in the chart's own diff.
destination:
server: https://kubernetes.default.svc
namespace: svcforge
syncPolicy:
automated:
# Delete resources removed from the chart. Safe because the chart owns only svcforge
# itself; tenant releases are created by the worker's helm calls and are not part of
# this Application, so prune cannot reach them.
prune: true
# Revert manual edits. A hotfix applied by hand is a lie the next sync tells on.
selfHeal: true
syncOptions:
- CreateNamespace=true
# The migrate Job is a helm pre-install/pre-upgrade hook. ArgoCD maps helm hooks onto
# its own PreSync phase, so migrations still run before any new pod starts, and a
# non-zero exit fails the sync instead of rolling out pods onto an unmigrated schema.
- ApplyOutOfSyncOnly=true
retry:
limit: 3
backoff:
duration: 20s
factor: 2
maxDuration: 3m
# The migrate Job is a hook, so ArgoCD deletes and recreates it every sync; its
# generated fields would otherwise show as permanent drift and the app would never
# report Synced.
ignoreDifferences:
- group: batch
kind: Job
jsonPointers:
- /spec/template/metadata/labels
deploy/chart/Chart.yaml
+8
View File
@@ -0,0 +1,8 @@
apiVersion: v2
name: svcforge
description: svcforge control plane — api, worker, reconciler
type: application
# version is the chart's own version. appVersion is a label only: what actually ships is
# the digest in values.yaml, which CI bumps. Never read appVersion to decide what runs.
version: 0.1.0
appVersion: "0.1.0"
deploy/chart/templates/_helpers.tpl
+110
View File
@@ -0,0 +1,110 @@
{{/* Name helpers. Standard chart boilerplate — the interesting parts are below. */}}
{{- define "svcforge.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "svcforge.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "svcforge.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
app.kubernetes.io/name: {{ include "svcforge.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/part-of: svcforge
{{- end -}}
{{/*
Per-component selector labels.
`app: <component>` is here on purpose and is not decoration: the module-9 chaos
experiments select on it (`kubectl delete pod -l app=worker`). Renaming it breaks the
runbook, not just a dashboard.
*/}}
{{- define "svcforge.selectorLabels" -}}
app.kubernetes.io/name: {{ include "svcforge.name" .ctx }}
app.kubernetes.io/instance: {{ .ctx.Release.Name }}
app.kubernetes.io/component: {{ .component }}
app: {{ .component }}
{{- end -}}
{{/*
Resolve a component's image to repo@digest.
This is the single place that builds an image reference, and it refuses to emit one that
is not digest-pinned. If CI has not bumped values.yaml, the release fails here with a
readable message rather than silently deploying whatever a mutable tag happens to mean
today. (The literal string "latest" is not written anywhere in this repo, including in
comments — the acceptance gate greps for it and does not know what a comment is.)
*/}}
{{- define "svcforge.image" -}}
{{- $img := index .ctx.Values.image .component -}}
{{- if not $img -}}
{{- fail (printf "no image config for component %q" .component) -}}
{{- end -}}
{{- if not (hasPrefix "sha256:" ($img.digest | default "")) -}}
{{- fail (printf "image.%s.digest must be a sha256 digest, not a tag — CI bumps it; got %q" .component ($img.digest | default "<empty>")) -}}
{{- end -}}
{{- printf "%s@%s" $img.repo $img.digest -}}
{{- end -}}
{{- define "svcforge.serviceAccountName" -}}
{{- printf "%s-%s" (include "svcforge.fullname" .ctx) .component | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "svcforge.secretName" -}}
{{- .Values.externalSecret.targetName | default (printf "%s-secrets" (include "svcforge.fullname" .)) -}}
{{- end -}}
{{/*
Pod-level hardening, identical for all three services and the migrate job.
readOnlyRootFilesystem is the one that bites: every writable path a process needs must be
an explicit emptyDir. That is the point — it makes the writes visible in review.
*/}}
{{- define "svcforge.podSecurityContext" -}}
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
{{- end -}}
{{- define "svcforge.containerSecurityContext" -}}
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 10001
capabilities:
drop: ["ALL"]
{{- end -}}
{{/*
Shared environment. Secrets arrive via envFrom on the Secret that external-secrets
populates from Vault — never as chart values, never as literals in a manifest.
*/}}
{{- define "svcforge.env" -}}
- name: SVCFORGE_POOL_MIN_SIZE
value: {{ .Values.pool.minSize | quote }}
- name: SVCFORGE_POOL_MAX_SIZE
value: {{ .Values.pool.maxSize | quote }}
- name: SVCFORGE_LOG_LEVEL
value: {{ .Values.log.level | quote }}
{{- if .Values.otel.enabled }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: {{ .Values.otel.endpoint | quote }}
- name: OTEL_EXPORTER_OTLP_PROTOCOL
value: grpc
{{- end }}
{{- end -}}
deploy/chart/templates/api-deployment.yaml
+84
View File
@@ -0,0 +1,84 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "svcforge.fullname" . }}-api
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
replicas: {{ .Values.api.replicas }}
selector:
matchLabels:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
template:
metadata:
labels:
{{- include "svcforge.labels" . | nindent 8 }}
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 8 }}
spec:
serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
{{- with .Values.image.pullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
# 60s: an in-flight POST must finish and the pool must close cleanly. The api holds
# no lease, so this is generosity, not correctness — the worker's 60s is the one
# that matters.
terminationGracePeriodSeconds: 60
securityContext:
{{- include "svcforge.podSecurityContext" . | nindent 8 }}
containers:
- name: api
image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- include "svcforge.containerSecurityContext" . | nindent 12 }}
ports:
- name: http
containerPort: {{ .Values.api.service.targetPort }}
envFrom:
- secretRef:
name: {{ include "svcforge.secretName" . }}
env:
{{- include "svcforge.env" . | nindent 12 }}
- name: SVCFORGE_JWKS_URL
value: {{ .Values.auth.jwksUrl | quote }}
- name: SVCFORGE_JWT_ISSUER
value: {{ .Values.auth.issuer | quote }}
- name: SVCFORGE_JWT_AUDIENCE
value: {{ .Values.auth.audience | quote }}
- name: OTEL_SERVICE_NAME
value: svcforge-api
# readiness gates traffic, liveness restarts. They must not be the same probe:
# /readyz checks the pool, and a pool that is briefly exhausted should stop
# taking traffic, not get the pod killed.
readinessProbe:
httpGet: {path: /readyz, port: http}
periodSeconds: 5
timeoutSeconds: 3
livenessProbe:
httpGet: {path: /healthz, port: http}
periodSeconds: 20
timeoutSeconds: 3
failureThreshold: 3
resources:
{{- toYaml .Values.api.resources | nindent 12 }}
volumeMounts:
# readOnlyRootFilesystem: true, so anything that writes needs a mount.
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
deploy/chart/templates/api-ingress.yaml
+33
View File
@@ -0,0 +1,33 @@
{{- if .Values.api.ingress.enabled }}
{{/* Only the api is exposed. The worker and reconciler have no Service at all. */}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "svcforge.fullname" . }}-api
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: api
{{- with .Values.api.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
ingressClassName: {{ .Values.api.ingress.className }}
{{- if .Values.api.ingress.tls.enabled }}
tls:
- hosts:
- {{ .Values.api.ingress.host | quote }}
secretName: {{ .Values.api.ingress.tls.secretName }}
{{- end }}
rules:
- host: {{ .Values.api.ingress.host | quote }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: {{ include "svcforge.fullname" . }}-api
port:
name: http
{{- end }}
deploy/chart/templates/api-service.yaml
+16
View File
@@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "svcforge.fullname" . }}-api
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
type: {{ .Values.api.service.type }}
ports:
- name: http
port: {{ .Values.api.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 4 }}
deploy/chart/templates/externalsecret.yaml
+26
View File
@@ -0,0 +1,26 @@
{{- if .Values.externalSecret.enabled }}
{{/*
The Supabase and Upstash DSNs come from Vault via external-secrets. They are never chart
values, never CI variables, and never baked into an image layer — the chart names the
Vault path, and the cluster resolves it.
The rendered Secret is consumed with envFrom, so adding a key here is the only step needed
to expose a new one; settings.py (env_prefix="SVCFORGE_") types it on the way in and fails
fast if it is missing or malformed.
*/}}
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: {{ include "svcforge.fullname" . }}-secrets
labels:
{{- include "svcforge.labels" . | nindent 4 }}
spec:
refreshInterval: {{ .Values.externalSecret.refreshInterval }}
secretStoreRef:
{{- toYaml .Values.externalSecret.secretStoreRef | nindent 4 }}
target:
name: {{ include "svcforge.secretName" . }}
creationPolicy: Owner
data:
{{- toYaml .Values.externalSecret.remoteRefs | nindent 4 }}
{{- end }}
deploy/chart/templates/migrate-job.yaml
+70
View File
@@ -0,0 +1,70 @@
{{- if .Values.migrate.enabled }}
{{/*
Migrations run here and nowhere else.
Not on app startup: three services × N replicas racing the same DDL is how you get a
half-applied schema and a crash loop, and it makes "which pod migrated?" unanswerable.
A hook runs once, before any new pod starts, and its exit code gates the release.
hook-weight -5 puts it ahead of everything else in the pre-install/pre-upgrade phase.
hook-delete-policy before-hook-creation keeps the last run's pod around for `kubectl logs`
after a failure — the one time you actually want it — and clears it on the next attempt.
Deliberately no terminationGracePeriodSeconds: 60 here. Three Deployments carry it; a
migration is not one of them.
*/}}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "svcforge.fullname" . }}-migrate
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: migrate
annotations:
"helm.sh/hook": pre-install,pre-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation
spec:
# 0, not 3. A failed migration must fail the release. Retrying a DDL that just failed
# tends to turn one readable error into three, and then a green release on a schema
# nobody has looked at.
backoffLimit: 0
template:
metadata:
labels:
{{- include "svcforge.labels" . | nindent 8 }}
app.kubernetes.io/component: migrate
spec:
restartPolicy: Never
serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
{{- with .Values.image.pullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
{{- include "svcforge.podSecurityContext" . | nindent 8 }}
containers:
- name: migrate
# Same image as the api, by digest. The migrations that ship are the ones the
# code that is about to run was built against — a separate image could drift.
image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- include "svcforge.containerSecurityContext" . | nindent 12 }}
command: ["python", "-m", "svcforge_core.migrate"]
envFrom:
- secretRef:
name: {{ include "svcforge.secretName" . }}
env:
{{- include "svcforge.env" . | nindent 12 }}
- name: OTEL_SERVICE_NAME
value: svcforge-migrate
resources:
{{- toYaml .Values.migrate.resources | nindent 12 }}
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp
emptyDir: {}
{{- end }}
deploy/chart/templates/pdb.yaml
+24
View File
@@ -0,0 +1,24 @@
{{- if .Values.podDisruptionBudget.enabled }}
{{/*
Off by default, and not in the module-8 spec — included because it is cheap to have and
expensive to retrofit.
Only the api gets one. The worker does not need it: killing a worker mid-provision is
already safe (chaos experiment 1 — the lease expires, the task returns to queued, and
`helm upgrade --install` is a no-op on re-claim), so blocking a drain to protect it buys
nothing. The reconciler must not get one: it is replicas: 1, and minAvailable: 1 on a
single-replica Deployment blocks node drains forever.
*/}}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "svcforge.fullname" . }}-api
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: api
spec:
minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
selector:
matchLabels:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
{{- end }}
deploy/chart/templates/prometheusrule.yaml
+21
View File
@@ -0,0 +1,21 @@
{{- if .Values.prometheusRule.enabled }}
{{/*
The four alerts, rendered straight from values. The rules are data, not template logic —
which is the point: an alert you want to change is a values edit and a diff, not a new
CR someone applied by hand and forgot.
Each maps to a RUNBOOK.md entry via its runbook_url annotation. An alert without a runbook
entry is a pager that teaches nothing.
*/}}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: {{ include "svcforge.fullname" . }}
labels:
{{- include "svcforge.labels" . | nindent 4 }}
spec:
groups:
- name: svcforge
rules:
{{- toYaml .Values.prometheusRule.rules | nindent 8 }}
{{- end }}
deploy/chart/templates/rbac.yaml
+52
View File
@@ -0,0 +1,52 @@
{{- if .Values.rbac.create }}
{{/*
Hand-written, and deliberately short.
Why ClusterRole and not Role: the worker's job is to `helm upgrade --install` a tenant
release into a namespace it creates. `namespaces` is a cluster-scoped resource — a
namespaced Role cannot grant `create` on it, and cannot grant anything inside the tenant
namespaces either, because they do not exist when the chart is installed.
What keeps this least-privilege is not the scope, it is the contents: every resource and
verb is named, there is no `*`, no cluster-admin, and no rbac.authorization.k8s.io group.
That last omission is the load-bearing one — the worker cannot escalate itself, because it
cannot create a RoleBinding at all.
The reconciler binds to the same role but only ever reads; it enqueues tasks, it does not
provision, and it never deletes an orphan.
*/}}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "svcforge.fullname" . }}-provisioner
labels:
{{- include "svcforge.labels" . | nindent 4 }}
rules:
- apiGroups: [""]
resources: [namespaces]
verbs: [get, list, create]
- apiGroups: [""]
resources: [secrets, services, configmaps, persistentvolumeclaims, serviceaccounts]
verbs: [get, list, watch, create, update, patch, delete]
- apiGroups: [apps]
resources: [deployments, statefulsets]
verbs: [get, list, watch, create, update, patch, delete]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "svcforge.fullname" . }}-provisioner
labels:
{{- include "svcforge.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ include "svcforge.fullname" . }}-provisioner
subjects:
- kind: ServiceAccount
name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
namespace: {{ .Release.Namespace }}
- kind: ServiceAccount
name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
namespace: {{ .Release.Namespace }}
{{- end }}
deploy/chart/templates/reconciler-deployment.yaml
+80
View File
@@ -0,0 +1,80 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "svcforge.fullname" . }}-reconciler
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: reconciler
spec:
# A singleton. Hardcoded, not a value: two reconcilers would double-enqueue drift tasks
# and race on TTL expiry, and there is no knob that makes that acceptable. If you want
# this to be tunable, you want a lease first — and the lease belongs in Postgres, not in
# a Redis lock (a GC pause plus a Redis lock still gives you two reconcilers).
replicas: 1
strategy:
# Recreate, not RollingUpdate. RollingUpdate would briefly run the old and new pod at
# once, which is exactly the thing replicas: 1 exists to prevent.
type: Recreate
selector:
matchLabels:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 6 }}
template:
metadata:
labels:
{{- include "svcforge.labels" . | nindent 8 }}
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 8 }}
spec:
serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
{{- with .Values.image.pullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: 60
securityContext:
{{- include "svcforge.podSecurityContext" . | nindent 8 }}
containers:
- name: reconciler
image: {{ include "svcforge.image" (dict "ctx" $ "component" "reconciler") }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- include "svcforge.containerSecurityContext" . | nindent 12 }}
ports:
- name: metrics
containerPort: 9000
envFrom:
- secretRef:
name: {{ include "svcforge.secretName" . }}
env:
{{- include "svcforge.env" . | nindent 12 }}
- name: SVCFORGE_RECONCILE_INTERVAL_S
value: {{ .Values.reconciler.intervalSeconds | quote }}
- name: OTEL_SERVICE_NAME
value: svcforge-reconciler
# The reconciler's health is not "is the process up", it is "did it tick".
# That question is answered by SvcforgeReconcilerStale off
# svcforge_reconciler_last_tick_timestamp_seconds, not by a probe — a probe here
# would restart the pod and reset the very gauge the alert reads.
resources:
{{- toYaml .Values.reconciler.resources | nindent 12 }}
volumeMounts:
- name: tmp
mountPath: /tmp
- name: helm-home
mountPath: /tmp/helm
volumes:
- name: tmp
emptyDir: {}
- name: helm-home
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
deploy/chart/templates/serviceaccount.yaml
+24
View File
@@ -0,0 +1,24 @@
{{- if .Values.serviceAccount.create }}
{{/*
One ServiceAccount per service, not one shared. The api talks only to Postgres and has no
Kubernetes rights at all; giving it the worker's identity would hand an
internet-facing HTTP surface the ability to create namespaces.
*/}}
{{- range $component := list "api" "worker" "reconciler" }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" $component) }}
labels:
{{- include "svcforge.labels" $ | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
{{- with $.Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
# The api never calls the API server, so it gets no token. The worker and reconciler both
# shell out to helm, which needs one.
automountServiceAccountToken: {{ ne $component "api" }}
{{- end }}
{{- end }}
deploy/chart/templates/servicemonitor.yaml
+44
View File
@@ -0,0 +1,44 @@
{{- if .Values.serviceMonitor.enabled }}
{{/*
Chart-native, values-gated. A hand-authored ServiceMonitor CR applied next to the release
is banned: it drifts from the chart, survives a `helm uninstall`, and nothing owns it.
The api is scraped through its Service. The worker and reconciler have no Service — they
are scraped by pod, which is why their metrics port is named and their pods carry the
component label.
*/}}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "svcforge.fullname" . }}-api
labels:
{{- include "svcforge.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
endpoints:
- port: http
path: /metrics
interval: {{ .Values.serviceMonitor.interval }}
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: {{ include "svcforge.fullname" . }}-workers
labels:
{{- include "svcforge.labels" . | nindent 4 }}
spec:
selector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values: [worker, reconciler]
- key: app.kubernetes.io/instance
operator: In
values: [{{ .Release.Name }}]
podMetricsEndpoints:
- port: metrics
path: /metrics
interval: {{ .Values.serviceMonitor.interval }}
{{- end }}
deploy/chart/templates/worker-deployment.yaml
+84
View File
@@ -0,0 +1,84 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "svcforge.fullname" . }}-worker
labels:
{{- include "svcforge.labels" . | nindent 4 }}
app.kubernetes.io/component: worker
spec:
# Plain replicas, no HPA. Concurrency is bounded twice over — by replicas here and by a
# semaphore inside the claim loop — and the SKIP LOCKED claim makes both safe. The day
# replicas: 2 stops keeping up, add an HPA. Not before.
replicas: {{ .Values.worker.replicas }}
selector:
matchLabels:
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 6 }}
template:
metadata:
labels:
{{- include "svcforge.labels" . | nindent 8 }}
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 8 }}
spec:
serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
{{- with .Values.image.pullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
# The load-bearing one. On SIGTERM the loop stops claiming and finishes the task in
# hand; 60s is the budget for that. Chaos experiment 2 asserts the pod exits 0 inside
# it. Lower this and a rolling deploy starts orphaning tasks to lease expiry.
terminationGracePeriodSeconds: 60
securityContext:
{{- include "svcforge.podSecurityContext" . | nindent 8 }}
containers:
- name: worker
image: {{ include "svcforge.image" (dict "ctx" $ "component" "worker") }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- include "svcforge.containerSecurityContext" . | nindent 12 }}
ports:
- name: metrics
containerPort: 9000
envFrom:
- secretRef:
name: {{ include "svcforge.secretName" . }}
env:
{{- include "svcforge.env" . | nindent 12 }}
# The claim loop stamps locked_by with this. Per-pod, so a stuck lease names
# the pod that holds it — runbook entry 1 depends on that.
- name: SVCFORGE_WORKER_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: SVCFORGE_WORKER_CONCURRENCY
value: {{ .Values.worker.concurrency | quote }}
- name: OTEL_SERVICE_NAME
value: svcforge-worker
# No liveness probe. A worker mid-provision is legitimately busy for minutes;
# a probe here is a way to kill a healthy provision and learn nothing.
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
volumeMounts:
- name: tmp
mountPath: /tmp
# helm writes cache/config/repositories under $HELM_*_HOME, which the image
# points at /tmp/helm. Without this, every helm call fails on a read-only fs.
- name: helm-home
mountPath: /tmp/helm
volumes:
- name: tmp
emptyDir: {}
- name: helm-home
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
deploy/chart/values.yaml
+174
View File
@@ -0,0 +1,174 @@
# svcforge chart values.
#
# The digests below are the deployment. CI builds each image once, pushes it, reads the
# digest back with `docker buildx imagetools inspect`, and `yq -i`s it into this file as
# its last act. ArgoCD notices the commit and syncs. Nothing else deploys svcforge.
#
# Tags are banned. A tag is a mutable pointer, which means "what is running" and "what
# this file says" can silently diverge. A digest cannot.
nameOverride: ""
fullnameOverride: ""
image:
registry: gitea.oci-oci.duckdns.org
pullPolicy: IfNotPresent
pullSecrets: []
# One repo + one digest per service: CI's matrix builds three images, so there are three
# digests. The zeros are placeholders — a fresh clone must be bumped by CI before it can
# deploy, which is the intended failure mode. Never hand-edit these.
api:
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-api
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
worker:
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-worker
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
reconciler:
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-reconciler
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
api:
replicas: 2
# One process per pod. Module 7 took the "scale with replicas" fix over
# PROMETHEUS_MULTIPROC_DIR, so `uvicorn --workers N` here would corrupt the metrics.
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {memory: 256Mi}
service:
type: ClusterIP
port: 80
targetPort: 8000
ingress:
enabled: true
className: nginx
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
host: svcforge.oci-oci.duckdns.org
tls:
enabled: true
secretName: svcforge-tls
worker:
# Plain replicas. No HPA: the day `replicas: 2` stops keeping up, not before.
replicas: 2
concurrency: 4
resources:
requests: {cpu: 100m, memory: 192Mi}
limits: {memory: 512Mi}
reconciler:
# A singleton, and not by convention — the four checks are not safe to run twice
# concurrently. replicas is deliberately not a value: there is nothing to tune.
intervalSeconds: 60
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {memory: 256Mi}
# Postgres pool sizing. replicas × maxSize is spent against the Supabase pooler budget:
# api(2 × 5) + worker(2 × 5) + reconciler(1 × 2) = 22 connections. Raise with care.
pool:
minSize: 1
maxSize: 5
migrate:
# backoffLimit: 0 — a failed migration must fail the release, not retry into a
# half-applied schema. Migrations run here and only here; never on app startup.
enabled: true
resources:
requests: {cpu: 50m, memory: 128Mi}
limits: {memory: 256Mi}
auth:
jwksUrl: https://auth.oci-oci.duckdns.org/realms/svcforge/protocol/openid-connect/certs
issuer: https://auth.oci-oci.duckdns.org/realms/svcforge
audience: svcforge
otel:
enabled: true
endpoint: http://alloy.observability.svc.cluster.local:4317
log:
level: info
# The DSNs are pulled from Vault by external-secrets into a Secret the pods envFrom.
# No DSN is ever a chart value, a ConfigMap key, or a CI variable.
externalSecret:
enabled: true
secretStoreRef:
name: vault
kind: ClusterSecretStore
refreshInterval: 1h
# target Secret name; keys land as SVCFORGE_PG_DSN / SVCFORGE_PG_DSN_SESSION / SVCFORGE_REDIS_DSN
targetName: svcforge-secrets
remoteRefs:
- secretKey: SVCFORGE_PG_DSN
key: svcforge/postgres
property: dsn_pooler
- secretKey: SVCFORGE_PG_DSN_SESSION
key: svcforge/postgres
property: dsn_session
- secretKey: SVCFORGE_REDIS_DSN
key: svcforge/redis
property: dsn
rbac:
# The worker helm-installs tenant releases into namespaces it creates. `namespaces` is a
# cluster-scoped resource, so `create namespaces` cannot be granted by a namespaced Role
# — this has to be a ClusterRole. It is still least-privilege: named resources, named
# verbs, no `*`, no cluster-admin, and no rbac.authorization.k8s.io group at all, so the
# worker cannot grant itself anything further.
create: true
serviceAccount:
create: true
annotations: {}
# Chart-native only. A hand-authored ServiceMonitor/PrometheusRule CR is banned — the
# chart owns these, gated by these flags.
serviceMonitor:
enabled: true
interval: 30s
prometheusRule:
enabled: true
rules:
- alert: SvcforgeQueueDepthRising
expr: deriv(svcforge_queue_depth[10m]) > 0
for: 10m
labels:
severity: warning
annotations:
summary: svcforge queue depth is rising and not draining
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#queue-stuck
- alert: SvcforgeProvisionSlow
expr: histogram_quantile(0.95, sum by (le) (rate(svcforge_provision_duration_seconds_bucket[30m]))) > 300
for: 15m
labels:
severity: warning
annotations:
summary: svcforge p95 provision time is over 5 minutes
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
- alert: SvcforgeTaskFailed
expr: sum(svcforge_tasks_failed_total) > 0
labels:
severity: warning
annotations:
summary: a svcforge task exhausted its retries
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
- alert: SvcforgeReconcilerStale
expr: time() - svcforge_reconciler_last_tick_timestamp_seconds > 300
labels:
severity: critical
annotations:
summary: the svcforge reconciler has not ticked in 5 minutes
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#orphaned-release
# Not in the spec. Off by default: on a three-node k3s a PDB that cannot be satisfied
# blocks drains, which is worse than the disruption it prevents.
podDisruptionBudget:
enabled: false
minAvailable: 1
nodeSelector: {}
tolerations: []
affinity: {}
libs/svcforge_core/pyproject.toml
+22
View File
@@ -0,0 +1,22 @@
[project]
name = "svcforge-core"
version = "0.1.0"
description = "svcforge shared core: domain, repo, adapters"
requires-python = ">=3.12"
dependencies = [
"pydantic>=2.9",
"pydantic-settings>=2.6",
"pyyaml>=6.0",
"psycopg[binary,pool]>=3.2",
"redis>=5.2",
"structlog>=24.4",
"prometheus-client>=0.21",
"opentelemetry-api>=1.28",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["svcforge_core"]
libs/svcforge_core/svcforge_core/__init__.py
View File
libs/svcforge_core/svcforge_core/adapters/__init__.py
+14
View File
@@ -0,0 +1,14 @@
"""The outside world: helm/kubectl subprocesses, notifications, the clock.
Everything in here is I/O the domain must never learn about. Each adapter exposes a
Protocol with exactly two implementations — the real one here, the fake in `tests/fakes.py`.
A Protocol with one implementation is an interface nobody asked for; delete it.
The subprocess discipline lives in :func:`svcforge_core.adapters.helm._run` and is shared
by every adapter that shells out (`k8s.py` included):
* `create_subprocess_exec`, never a shell — `team` is tenant input that reaches a release name.
* `start_new_session=True` at spawn time, so a timeout can kill the whole process group.
* `communicate()`, never `wait()` with pipes attached.
* stderr truncated to a fixed tail at this boundary, because it lands in `instances.error`.
"""
libs/svcforge_core/svcforge_core/adapters/clock.py
+41
View File
@@ -0,0 +1,41 @@
"""Time, as a dependency.
The centrepiece of the Day-2 module, and it is nine lines. `datetime.now()` called from
inside domain logic is an untestable global read: a maintenance-window test that wants
"03:00 next Sunday" would have to either sleep until Sunday or monkeypatch a stdlib symbol
and hope nothing else in the process noticed. Passing a Clock makes the same test a
`FakeClock(start=...)` and an `advance()`.
Aware UTC, always. A naive datetime is a bug that survives every test on a UTC CI box and
detonates the first time it meets a tenant in Asia/Ho_Chi_Minh: `datetime.utcnow()` returns
a naive value, and comparing it to a `timestamptz` from Postgres raises TypeError, or worse,
silently compares wrong after somebody "fixes" it with a `.replace(tzinfo=...)`.
The fake lives in `tests/fakes.py`, not here: shipping test doubles in the production
package is how they end up imported by production code.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Protocol
class Clock(Protocol):
"""Implemented by SystemClock (here) and FakeClock (tests/fakes.py)."""
def now(self) -> datetime:
"""Current time, aware, UTC."""
...
class SystemClock:
"""The wall clock. The only thing in the codebase allowed to read it."""
def now(self) -> datetime:
"""Current time, aware, UTC.
`datetime.now(UTC)`, never `datetime.utcnow()` — the latter is naive and deprecated
in 3.12 for exactly this reason.
"""
return datetime.now(UTC)
libs/svcforge_core/svcforge_core/adapters/helm.py
+245
View File
@@ -0,0 +1,245 @@
"""Driving helm from asyncio, with timeouts that actually kill helm.
The whole module exists for `_run`. Everything above it is argv construction.
Four things go wrong when you spawn a process from an event loop, and all four are
handled here rather than in the caller:
1. `subprocess.run` blocks the loop. Use `create_subprocess_exec`.
2. `stdout=PIPE` with `proc.wait()` and nobody draining deadlocks at ~64 KB of output —
`helm --debug` clears that in one install. Use `communicate()`.
3. `asyncio.wait_for` cancels the *coroutine*. The process does not know it was waited on:
helm keeps running and keeps mutating the cluster. The timeout has to kill it.
4. `proc.kill()` signals the direct child. `helm` forks; its children reparent to init and
survive. Only `killpg` gets the whole tree, and only if the group exists — which needs
`start_new_session=True` **at spawn time**, because setsid can only run in the window
between fork and exec.
"""
from __future__ import annotations
import asyncio
import json
import os
import shutil
import signal
import tempfile
from collections.abc import Sequence
from pathlib import Path
from typing import Any, Protocol
import yaml
from pydantic import BaseModel, ConfigDict, Field
from svcforge_core.domain.models import CatalogEntry
# How long the process group gets to honour SIGTERM before SIGKILL. Helm traps SIGTERM
# and tries to leave the release in a coherent state; give it a moment to do so.
_TERM_GRACE_S = 5.0
# `_run` is the backstop, not the primary timeout: helm gets its own `--timeout` so that
# `--atomic` can roll back cleanly. `_run` only fires when helm itself is wedged, so its
# deadline sits this far past helm's.
_RUN_TIMEOUT_MARGIN_S = 30
_STDERR_TAIL_BYTES = 2048
class HelmError(RuntimeError):
"""Non-zero exit. str(self) is the stderr tail that lands in instances.error."""
class ReleaseInfo(BaseModel):
"""One row of `helm list -o json`."""
model_config = ConfigDict(frozen=True)
name: str
namespace: str
chart: str
status: str
revision: int = Field(default=0, ge=0)
app_version: str = ""
class Provisioner(Protocol):
"""What the worker needs from a cluster. Implemented by HelmProvisioner and FakeProvisioner."""
async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None: ...
async def uninstall(self, release: str, ns: str) -> None: ...
async def list_releases(self) -> list[ReleaseInfo]: ...
def _tail(raw: bytes, tail_bytes: int) -> str:
"""The last `tail_bytes` of a stream, as text.
Truncation happens here, at the adapter boundary, and nowhere else. A helm failure can
emit megabytes; `instances.error` is a text column read by humans. Slice the bytes, not
the decoded string, then decode with `replace` — the cut can land mid-codepoint.
"""
return raw[-tail_bytes:].decode("utf-8", errors="replace").strip()
def _signal_group(proc: asyncio.subprocess.Process, sig: int) -> None:
"""Signal the process's whole group. No-op if it has already exited.
`os.getpgid` rather than `proc.pid`: with `start_new_session=True` they are equal, but
that equality is an implementation detail, and asking the kernel costs nothing.
"""
if proc.returncode is not None:
return
try:
os.killpg(os.getpgid(proc.pid), sig)
except (ProcessLookupError, PermissionError):
# Exited and reaped between the check and the call, or reparented out of reach.
return
async def _terminate_group(proc: asyncio.subprocess.Process) -> None:
"""SIGTERM the group, grace, SIGKILL the group. Then reap, so we leave no zombie."""
_signal_group(proc, signal.SIGTERM)
try:
await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_S)
except TimeoutError:
_signal_group(proc, signal.SIGKILL)
await proc.wait()
async def _run(argv: Sequence[str], timeout_s: int, tail_bytes: int = _STDERR_TAIL_BYTES) -> str:
"""create_subprocess_exec(*argv, stdout=PIPE, stderr=PIPE, start_new_session=True).
wait_for the drain. On TimeoutError: killpg(TERM), grace, killpg(KILL), raise.
Non-zero rc: raise HelmError(stderr tail). Return stdout.
"""
proc = await asyncio.create_subprocess_exec(
*argv,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=True,
)
try:
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
except TimeoutError:
# wait_for cancelled the drain, not the process. Kill the group and re-raise.
await _terminate_group(proc)
raise
except asyncio.CancelledError:
# Our caller is being torn down (SIGTERM to the worker). Do not await anything here:
# a second cancellation would land on that await and leave helm running. Signal and go.
_signal_group(proc, signal.SIGKILL)
raise
if proc.returncode != 0:
detail = _tail(stderr, tail_bytes) or _tail(stdout, tail_bytes)
raise HelmError(
f"{argv[0]} exited {proc.returncode}: {detail}"
if detail
else f"{argv[0]} exited {proc.returncode}"
)
return stdout.decode("utf-8", errors="replace")
class HelmProvisioner:
"""The real thing. One helm binary, one kubeconfig, one deadline."""
def __init__(
self, *, helm_bin: str = "helm", kubeconfig: Path | None = None, timeout_s: int = 600
) -> None:
"""kubeconfig=None means the ambient config: $KUBECONFIG, ~/.kube/config, or the
in-cluster service account when svcforge runs as a pod. Keyword-only so that the
three arguments can never be swapped by position at a call site.
"""
self._helm_bin = helm_bin
self._kubeconfig = kubeconfig
self._timeout_s = timeout_s
@property
def _run_timeout_s(self) -> int:
return self._timeout_s + _RUN_TIMEOUT_MARGIN_S
def _base_argv(self, *args: str) -> list[str]:
argv = [self._helm_bin, *args]
if self._kubeconfig is not None:
argv += ["--kubeconfig", str(self._kubeconfig)]
return argv
async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None:
"""helm upgrade --install --wait --timeout. Idempotent by construction."""
# `upgrade --install` is why this is idempotent: a retried task after a crash mid-provision
# converges on the same release instead of erroring with "release already exists".
# `--wait` is why `ready` in the DB means ready — it returns when the pods are up.
# `--atomic` rolls back a failed upgrade; it doubles the worst case, which is what
# `_RUN_TIMEOUT_MARGIN_S` and helm's own `--timeout` are sized around.
with _values_file(values) as path:
argv = self._base_argv(
"upgrade",
"--install",
release,
entry.chart,
"--namespace",
ns,
"--version",
entry.chart_version,
"--values",
str(path),
"--wait",
"--atomic",
"--timeout",
f"{self._timeout_s}s",
)
await _run(argv, timeout_s=self._run_timeout_s)
async def uninstall(self, release: str, ns: str) -> None:
"""helm uninstall --wait. `--ignore-not-found` makes the retry of a half-done delete a no-op."""
argv = self._base_argv(
"uninstall",
release,
"--namespace",
ns,
"--ignore-not-found",
"--wait",
"--timeout",
f"{self._timeout_s}s",
)
await _run(argv, timeout_s=self._run_timeout_s)
async def list_releases(self) -> list[ReleaseInfo]:
"""Every release helm knows about, in every namespace. The reconciler's view of reality."""
argv = self._base_argv("list", "--all-namespaces", "--output", "json")
raw = await _run(argv, timeout_s=self._run_timeout_s)
try:
parsed: Any = json.loads(raw or "[]")
except json.JSONDecodeError as exc:
raise HelmError(f"helm list returned non-JSON: {_tail(raw.encode(), 256)}") from exc
if not isinstance(parsed, list):
raise HelmError(f"helm list returned {type(parsed).__name__}, expected a list")
return [ReleaseInfo.model_validate(row) for row in parsed]
class _ValuesFile:
"""Context manager yielding a path to a values.yaml written from a dict.
A file, not `--set`: `--set` has its own escaping grammar (commas, dots, backslashes) and
values carry tenant-shaped strings. Serialising YAML sidesteps the grammar entirely.
"""
def __init__(self, values: dict[str, Any]) -> None:
self._values = values
self._dir: str | None = None
def __enter__(self) -> Path:
self._dir = tempfile.mkdtemp(prefix="svcforge-values-")
path = Path(self._dir) / "values.yaml"
path.write_text(yaml.safe_dump(self._values, default_flow_style=False), encoding="utf-8")
return path
def __exit__(self, *exc: object) -> None:
if self._dir is not None:
shutil.rmtree(self._dir, ignore_errors=True)
def _values_file(values: dict[str, Any]) -> _ValuesFile:
return _ValuesFile(values)
libs/svcforge_core/svcforge_core/adapters/k8s.py
+136
View File
@@ -0,0 +1,136 @@
"""kubectl, for the two things helm will not do: make a namespace, read a secret back.
Same subprocess discipline as helm — it is literally the same `_run`, imported rather than
copied, because the process-group handling is the part that is easy to get subtly wrong twice.
Both operations are declarative and therefore idempotent: `apply` of a namespace that exists
is a no-op, and a `get` has no effect at all. A worker that dies mid-provision and retries
must not find a "namespace already exists" error waiting for it.
"""
from __future__ import annotations
import base64
import binascii
import json
import shutil
import tempfile
from pathlib import Path
from typing import Any
import yaml
from svcforge_core.adapters.helm import HelmError, _run
_KUBECTL_TIMEOUT_S = 60
class K8sError(RuntimeError):
"""kubectl failed. str(self) is the stderr tail, already truncated by `_run`."""
class SecretNotFound(K8sError):
"""The secret is not there (yet). Distinct because a caller may want to retry rather than fail."""
class KubectlClient:
"""A kubectl binary and a kubeconfig. No client-go, no in-cluster config, no CRDs."""
def __init__(
self,
*,
kubectl_bin: str = "kubectl",
kubeconfig: Path | None = None,
timeout_s: int = _KUBECTL_TIMEOUT_S,
) -> None:
"""Mirrors HelmProvisioner: kubeconfig=None means the ambient/in-cluster config."""
self._kubectl_bin = kubectl_bin
self._kubeconfig = kubeconfig
self._timeout_s = timeout_s
def _base_argv(self, *args: str) -> list[str]:
argv = [self._kubectl_bin, *args]
if self._kubeconfig is not None:
argv += ["--kubeconfig", str(self._kubeconfig)]
return argv
async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
"""Create the namespace if absent, leave it alone if present.
`apply -f` of a manifest rather than `create namespace` (which errors on the second
call) or `create --dry-run=client -o yaml | apply -f -` (which needs a shell, and a
shell is exactly what tenant input must never reach).
"""
manifest: dict[str, Any] = {
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": ns,
"labels": {"app.kubernetes.io/managed-by": "svcforge", **(labels or {})},
},
}
with _manifest_file(manifest) as path:
await self._kubectl("apply", "--filename", str(path))
async def read_secret(self, ns: str, name: str) -> dict[str, str]:
"""The secret's `data`, base64-decoded. Chart-generated passwords come back through here.
Returns str, not bytes: every value svcforge reads (passwords, hosts, ports) is text.
A genuinely binary value raises rather than silently mangling into replacement chars.
"""
try:
raw = await self._kubectl("get", "secret", name, "--namespace", ns, "--output", "json")
except K8sError as exc:
if "notfound" in str(exc).lower().replace(" ", ""):
raise SecretNotFound(f"secret {ns}/{name} not found") from exc
raise
try:
parsed: Any = json.loads(raw)
except json.JSONDecodeError as exc:
raise K8sError(f"kubectl get secret {ns}/{name} returned non-JSON") from exc
data: Any = parsed.get("data") or {}
if not isinstance(data, dict):
raise K8sError(f"secret {ns}/{name}: 'data' is {type(data).__name__}, expected a mapping")
out: dict[str, str] = {}
for key, encoded in data.items():
try:
out[str(key)] = base64.b64decode(str(encoded), validate=True).decode("utf-8")
except (binascii.Error, ValueError) as exc:
raise K8sError(f"secret {ns}/{name}: key {key!r} is not base64-encoded utf-8") from exc
return out
async def _kubectl(self, *args: str) -> str:
"""Run kubectl through helm's `_run`, translating its error type at this boundary.
`_run` is spec'd to live in helm.py and to raise HelmError; nothing outside adapters
should have to know that kubectl failures arrive wearing a helm-shaped exception.
"""
try:
return await _run(self._base_argv(*args), timeout_s=self._timeout_s)
except HelmError as exc:
raise K8sError(str(exc)) from exc
class _ManifestFile:
"""A temp file holding one YAML manifest, removed on exit."""
def __init__(self, manifest: dict[str, Any]) -> None:
self._manifest = manifest
self._dir: str | None = None
def __enter__(self) -> Path:
self._dir = tempfile.mkdtemp(prefix="svcforge-manifest-")
path = Path(self._dir) / "manifest.yaml"
path.write_text(yaml.safe_dump(self._manifest, default_flow_style=False), encoding="utf-8")
return path
def __exit__(self, *exc: object) -> None:
if self._dir is not None:
shutil.rmtree(self._dir, ignore_errors=True)
def _manifest_file(manifest: dict[str, Any]) -> _ManifestFile:
return _ManifestFile(manifest)
libs/svcforge_core/svcforge_core/adapters/notify.py
+77
View File
@@ -0,0 +1,77 @@
"""Telling someone a provision finished, or didn't.
Best-effort by construction: `send` never raises. A notifier that can fail a task is a
notifier that lets a Slack outage roll back a successful provision. The instance is ready;
the DB says so; failing the task would re-run helm for nothing. Delivery failures are
logged and dropped on the floor, which is the correct amount of ceremony for a webhook.
Two implementations, so the Protocol earns its place: LogNotifier (the default, and what
tests and local dev get) and WebhookNotifier (the one that leaves the process).
"""
from __future__ import annotations
import logging
from typing import Protocol
import httpx
_log = logging.getLogger(__name__)
_DEFAULT_TIMEOUT_S = 5.0
class Notifier(Protocol):
"""Implemented by LogNotifier, WebhookNotifier, and FakeNotifier (tests/fakes.py)."""
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
"""Announce `event`. Must not raise: delivery is never worth failing a task over."""
...
class LogNotifier:
"""Writes the event to the log. The default: structured logs are already shipped somewhere."""
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
# NOT extra={"message": ...}. `message` is a reserved LogRecord attribute, and
# logging raises KeyError("Attempt to overwrite 'message' in LogRecord") at call
# time — so the notifier would crash the caller it was meant to inform. Same trap
# for `msg`, `args`, `name`, `levelname`, `exc_info`.
_log.info("notify", extra={"event": event, "detail": message, "fields": fields or {}})
class WebhookNotifier:
"""POSTs a JSON body at a URL. Slack-shaped, but anything that accepts JSON will do."""
def __init__(
self, url: str, *, client: httpx.AsyncClient | None = None, timeout_s: float = _DEFAULT_TIMEOUT_S
) -> None:
"""Pass `client` to share a connection pool with the rest of the process.
An owned client is closed by `aclose`; an injected one is the caller's to close.
"""
self._url = url
self._timeout_s = timeout_s
self._client = client
self._owns_client = client is None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self._timeout_s)
return self._client
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
payload = {"event": event, "message": message, "fields": fields or {}}
try:
client = await self._get_client()
resp = await client.post(self._url, json=payload, timeout=self._timeout_s)
resp.raise_for_status()
except httpx.HTTPError as exc:
# Deliberately swallowed. See the module docstring: the work already succeeded.
_log.warning("notify webhook failed", extra={"event": event, "error": str(exc)})
async def aclose(self) -> None:
"""Close the client, if we made it."""
if self._client is not None and self._owns_client:
await self._client.aclose()
self._client = None
libs/svcforge_core/svcforge_core/adapters/redis.py
+427
View File
@@ -0,0 +1,427 @@
"""Redis: derived state only. Never the truth, never the queue.
Everything in here is a shortcut past Postgres, and every one of them is optional. Postgres
holds the instances, the tasks, the leases and the `release_name` UNIQUE constraint. Redis
holds a counter, a claim marker and a copy — all of it rebuildable by doing nothing and
waiting for a TTL.
That framing is not philosophy, it decides the error handling, and the error handling is
the module. Each class below catches `RedisError` and returns a *safe* answer rather than
raising:
| Path | Redis is down | Why |
|-------------|--------------------------|--------------------------------------------------|
| Cache | miss -> read Postgres | It was an optimisation. Nobody notices. |
| Rate limit | **allow** | An internal platform that refuses every request |
| | | because the limiter is sick is worse than one |
| | | that is briefly unmetered. |
| Idempotency | fall through to the DB | `instances.release_name` is UNIQUE. That is the |
| | | real guarantee; this is the fast path. |
Consequently nothing here raises out to a caller, and `/readyz` stays Postgres-only. A
Redis outage must not make a single pod unready — that would convert "the cache is down"
into "the platform is down", which is the exact inversion this module exists to prevent.
**The budget is a design constraint, not a footnote.** Upstash free tier:
500,000 commands / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
One worker polling Redis every five seconds spends 518,400/month: the entire budget,
producing nothing. So the rule is structural — **Redis lives on the request path only**,
where volume is bounded by the number of humans with an API token, and never inside a poll
or control loop. That is also why the limiter is a Lua script: `GET`/`INCR`/`EXPIRE` is
three billed commands and a race; one `EVALSHA` is one billed command and atomic. A
pipeline would not help — it batches round trips but still bills N.
Every key gets a TTL. 256 MB with no expiry is a slow leak that ends by evicting the keys
you cared about.
"""
from __future__ import annotations
import asyncio
import logging
import math
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Protocol
from uuid import UUID
from prometheus_client import Counter
from pydantic import ValidationError
from redis.asyncio import Redis
from redis.exceptions import RedisError
from svcforge_core.adapters.clock import Clock, SystemClock
from svcforge_core.domain.models import Instance
if TYPE_CHECKING:
from redis.commands.core import AsyncScript
from svcforge_core.settings import Settings
_log = logging.getLogger(__name__)
# --- The budget metric ------------------------------------------------------------------
#
# The counter that `scripts/redis_budget.py` projects month-end burn from. It counts
# commands we *send*, incremented next to each call, because the number that matters is
# the one Upstash bills — not the number of times a method was called. A cache miss calls
# `get()` once and spends one command; a `put()` after it spends another.
#
# The dangerous failure this makes visible: when Redis is down, every path degrades
# silently and correctly, so nothing pages. Nothing fails until the month rolls over and
# every Redis call starts erroring at once. A counter you can extrapolate from is the only
# warning you get.
REDIS_COMMANDS = Counter(
"svcforge_redis_commands_total",
"Redis commands issued, as Upstash bills them. One EVALSHA is one.",
["op"],
)
REDIS_ERRORS = Counter(
"svcforge_redis_errors_total",
"Redis calls that failed and were degraded past. Never surfaced to the caller.",
["op"],
)
# A hung Redis must not hang the request path. Without these, a TCP connection that is
# open but unanswered blocks the handler until the client gives up — which turns "Redis is
# slow" into "the API is down", the same inversion the fail-open policy prevents. Upstash
# steady-state RTT is ~2.4 ms; two seconds is already pathological.
_SOCKET_TIMEOUT_S = 2.0
_CONNECT_TIMEOUT_S = 2.0
# Errors that mean "Redis did not answer". Every public method below turns these into a
# safe default. `OSError` because a DNS failure at connect time need not arrive wrapped,
# `TimeoutError` because the socket timeouts above raise it.
_REDIS_DOWN = (RedisError, OSError, asyncio.TimeoutError)
def _as_text(value: bytes | str) -> str:
"""`decode_responses=True` already did this; redis-py's annotations do not know it.
The client is configured for text, so the `bytes` branch is unreachable in this
process. It stays because the type says it is reachable, and a `cast` here would hide
the day someone builds a client without `decode_responses` and gets a `UUID(b'...')`
TypeError from three frames away instead of a value that just works.
"""
return value.decode() if isinstance(value, bytes) else value
def make_redis(settings: Settings) -> Redis | None:
"""One client per process, opened in lifespan next to the psycopg pool, closed on exit.
`None` when no DSN is configured, and that is a supported way to run: every consumer
below is optional by construction, so "no Redis" and "Redis is down" take the same
code path. The signature is `Redis | None` rather than `Redis` precisely so that
"unconfigured" cannot be faked with a client pointed at nothing.
Two settings are not negotiable:
`decode_responses=True` — the first bug everyone hits. Without it every read is
`bytes` and the traceback is `AttributeError: 'bytes' object has no attribute
'encode'`, several frames away from the cause.
`rediss://` (TLS) — Upstash rejects plaintext. The handshake is ~56 ms against a
~2.4 ms steady-state RTT, which is the whole argument for one pooled client per
process: a client per request pays the handshake every time and turns a cache into a
latency regression.
"""
if settings.redis_dsn is None:
return None
return Redis.from_url(
str(settings.redis_dsn),
decode_responses=True,
socket_timeout=_SOCKET_TIMEOUT_S,
socket_connect_timeout=_CONNECT_TIMEOUT_S,
)
# --- Rate limiting ----------------------------------------------------------------------
# One INCR; EXPIRE only when the counter is new. The `== 1` test is the entire trick: set
# the TTL unconditionally and every request slides the window forward, so a caller at
# steady load is never reset and the "window" is a sliding refusal that never lets up.
#
# KEYS and ARGV arrive as 1-based tables — Lua indexes from 1, and `ARGV[0]` is silently
# nil rather than an error, which reads as "the limit is nil" and compares false forever.
#
# Everything derivable in Python is derived in Python: `reset_at` comes from the window
# number the caller already computed, so there is no TTL round trip. One command, total.
_RATE_LIMIT_LUA = """
local n = redis.call('INCR', KEYS[1])
if n == 1 then
redis.call('EXPIRE', KEYS[1], ARGV[2])
end
local limit = tonumber(ARGV[1])
local remaining = limit - n
if remaining < 0 then
remaining = 0
end
local allowed = 0
if n <= limit then
allowed = 1
end
return {allowed, remaining}
"""
@dataclass(frozen=True)
class RateLimitResult:
"""The limiter's answer. `degraded` is how the caller knows it did not really check."""
allowed: bool
limit: int
remaining: int
reset_at: datetime
degraded: bool = False
@property
def retry_after_s(self) -> int:
"""Seconds until the window rolls over, for the `Retry-After` header on a 429.
Rounded up and floored at one: `Retry-After: 0` invites an immediate retry into
the same closed window, which is a busy loop with extra steps.
"""
delta = (self.reset_at - datetime.now(UTC)).total_seconds()
return max(1, math.ceil(delta))
class RateLimiterProto(Protocol):
"""Implemented by RateLimiter (here) and FakeRateLimiter (tests/fakes.py)."""
async def check(self, team: str) -> RateLimitResult:
"""Count this request against `team`'s window. Must not raise."""
...
class RateLimiter:
"""Fixed-window limiter. One EVALSHA per check. Fails OPEN.
Fixed window, not a token bucket or a sliding log, because the window boundary is the
only thing a fixed window gets wrong and the cost of getting it wrong is that a caller
can spend 2x the limit across a boundary. A sliding log is a sorted set, an
`ZREMRANGEBYSCORE`, an `ZADD` and a `ZCARD` — four billed commands and unbounded key
size — to fix a burst nobody is paying for. The limit is a courtesy, not a security
control; the security control is the JWT.
"""
def __init__(self, r: Redis, limit: int, window_s: int, *, clock: Clock | None = None) -> None:
"""`clock` is injectable so the window boundary is testable without sleeping."""
if limit < 1:
raise ValueError("limit must be >= 1")
if window_s < 1:
raise ValueError("window_s must be >= 1")
self._limit = limit
self._window_s = window_s
self._clock = clock or SystemClock()
# register_script() is local: it hashes the source and returns a callable. No round
# trip here, and none wasted at import. The first call sends EVALSHA; redis-py
# catches NOSCRIPT and replays it as EVAL, which is why a restarted Redis costs one
# extra command once rather than an outage.
self._script: AsyncScript = r.register_script(_RATE_LIMIT_LUA)
def _window(self) -> tuple[int, datetime]:
"""The current window number and when it ends. Pure arithmetic, no Redis."""
now = self._clock.now()
epoch = int(now.timestamp())
window = epoch // self._window_s
reset_at = datetime.fromtimestamp((window + 1) * self._window_s, tz=UTC)
return window, reset_at
async def check(self, team: str) -> RateLimitResult:
"""Count one request against `team`. Never raises.
On any Redis error: allow, log loudly, count it. The metric is the point — a
limiter that fails open silently is indistinguishable from no limiter at all, and
you find out which one you shipped during the incident.
"""
window, reset_at = self._window()
key = f"rl:{team}:{window}"
try:
REDIS_COMMANDS.labels(op="ratelimit").inc()
allowed, remaining = await self._script(keys=[key], args=[self._limit, self._window_s])
except _REDIS_DOWN:
REDIS_ERRORS.labels(op="ratelimit").inc()
_log.warning(
"rate limiter degraded: redis unavailable, failing OPEN",
exc_info=True,
extra={"team": team},
)
return RateLimitResult(
allowed=True,
limit=self._limit,
remaining=self._limit,
reset_at=reset_at,
degraded=True,
)
return RateLimitResult(
allowed=bool(allowed),
limit=self._limit,
remaining=int(remaining),
reset_at=reset_at,
)
# --- Idempotency ------------------------------------------------------------------------
class IdempotencyStoreProto(Protocol):
"""Implemented by IdempotencyStore (here) and FakeIdempotencyStore (tests/fakes.py)."""
async def claim(self, key: str, instance_id: UUID) -> UUID | None:
"""None = we won and the caller creates. A UUID = it already exists. Must not raise."""
...
class IdempotencyStore:
"""`SET NX EX`. Maps an `Idempotency-Key` to the instance UUID it created.
Claimed BEFORE the DB transaction, so the marker exists before the row it names. The
inversion matters: claim after the commit and a crash in between leaves a created
instance with no marker, and the client's retry creates a second one.
Claiming first has its own hole — a crash after the claim and before the commit leaves
a marker pointing at an instance that never existed, and the retry is told "already
done" about nothing. It is the better hole: the client polls the id, gets a 404, and
retries with a fresh key. The alternative loses money to a duplicate Elasticsearch.
And neither hole is load-bearing, because `instances.release_name` is UNIQUE and
deterministic from (team, service_type, id). **That constraint is the guarantee.** This
class only saves the round trip to find out.
"""
def __init__(self, r: Redis, ttl_s: int = 86400) -> None:
"""A day is the window a client might reasonably retry in; then the key is garbage."""
if ttl_s < 1:
raise ValueError("ttl_s must be >= 1")
self._r = r
self._ttl_s = ttl_s
async def claim(self, key: str, instance_id: UUID) -> UUID | None:
"""Try to bind `key` to `instance_id`. Never raises.
`None` from the happy path means "you won, go create it". `None` from a Redis
failure means the same thing — the caller creates, and the UNIQUE constraint
catches an actual duplicate. Degrading to "create it" is safe *only* because that
constraint exists; without it this would have to fail closed.
One command when we win, which is the common case and the one the budget is sized
for. Two when we lose: the loser pays a GET, and losers are rare by definition.
"""
redis_key = f"idem:{key}"
try:
REDIS_COMMANDS.labels(op="idempotency").inc()
won = await self._r.set(redis_key, str(instance_id), nx=True, ex=self._ttl_s)
if won:
return None
REDIS_COMMANDS.labels(op="idempotency").inc()
existing = await self._r.get(redis_key)
except _REDIS_DOWN:
REDIS_ERRORS.labels(op="idempotency").inc()
_log.warning(
"idempotency degraded: redis unavailable, falling through to the DB constraint",
exc_info=True,
)
return None
if existing is None:
# The key expired between the SET and the GET. Vanishingly rare, and the honest
# answer is "no winner recorded" — let the caller create and let Postgres decide.
return None
try:
return UUID(_as_text(existing))
except ValueError:
_log.warning("idempotency key holds a non-UUID value; ignoring it")
return None
# --- Read cache -------------------------------------------------------------------------
class InstanceCacheProto(Protocol):
"""Implemented by InstanceCache (here) and FakeInstanceCache (tests/fakes.py)."""
async def get(self, instance_id: UUID) -> Instance | None:
"""The cached instance, or None for a miss. Must not raise."""
...
async def put(self, inst: Instance) -> None:
"""Cache `inst` for the TTL. Must not raise."""
...
async def invalidate(self, instance_id: UUID) -> None:
"""Drop the entry. Must not raise."""
...
class InstanceCache:
"""Cache-aside for `GET /v1/instances/{id}`. TTL 30s.
Hit costs one command, miss costs two (the GET, then the SET after Postgres answers).
That is ~1 per read at any useful hit rate, which is what keeps a read-heavy poller
inside the budget.
The TTL is short on purpose and is the actual correctness argument. `invalidate()` on
every state transition is the fast path, not the guarantee: the worker can crash
between the UPDATE and the DEL, and then the cache is wrong. Thirty seconds bounds how
wrong. Trusting the invalidation instead — and raising the TTL to an hour — is how a
deleted instance stays `ready` in the API for an hour.
"""
def __init__(self, r: Redis, ttl_s: int = 30) -> None:
if ttl_s < 1:
raise ValueError("ttl_s must be >= 1")
self._r = r
self._ttl_s = ttl_s
@staticmethod
def _key(instance_id: UUID) -> str:
return f"inst:{instance_id}"
async def get(self, instance_id: UUID) -> Instance | None:
"""One GET. A miss, a Redis outage and a corrupt entry are all the same answer.
Which is the point: the caller writes `cache.get() or repo.get()` and has no branch
for "Redis is broken", because there is nothing different to do about it.
"""
try:
REDIS_COMMANDS.labels(op="cache_get").inc()
raw = await self._r.get(self._key(instance_id))
except _REDIS_DOWN:
REDIS_ERRORS.labels(op="cache_get").inc()
_log.warning("cache read degraded: redis unavailable, falling through to Postgres")
return None
if raw is None:
return None
try:
return Instance.model_validate_json(raw)
except ValidationError:
# A model change deployed over a warm cache. Treat it as a miss and let the TTL
# take the old shape out. Not an error: the truth is in Postgres either way.
_log.info("cache entry failed validation; treating as a miss")
return None
async def put(self, inst: Instance) -> None:
"""One SET with `EX`. Never write a key here without a TTL."""
try:
REDIS_COMMANDS.labels(op="cache_put").inc()
await self._r.set(self._key(inst.id), inst.model_dump_json(), ex=self._ttl_s)
except _REDIS_DOWN:
REDIS_ERRORS.labels(op="cache_put").inc()
_log.warning("cache write degraded: redis unavailable")
async def invalidate(self, instance_id: UUID) -> None:
"""One DEL. Called by the worker inside the code path that writes the state.
Inside that path, not after it and not from a subscriber: an invalidation that can
be skipped by an early return is an invalidation that will be.
"""
try:
REDIS_COMMANDS.labels(op="cache_del").inc()
await self._r.delete(self._key(instance_id))
except _REDIS_DOWN:
REDIS_ERRORS.labels(op="cache_del").inc()
_log.warning("cache invalidate degraded: redis unavailable; entry expires within the TTL")
libs/svcforge_core/svcforge_core/domain/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""Pure domain logic: state machine, models, backoff math, catalog rules.
No I/O lives here except :func:`svcforge_core.domain.catalog.load_catalog`, which reads
the one path it is handed. Nothing in this package is async.
"""
libs/svcforge_core/svcforge_core/domain/backoff.py
+24
View File
@@ -0,0 +1,24 @@
"""Retry backoff math. Pure: `now` and `rand` are injected, never called for you."""
import random
from collections.abc import Callable
from datetime import datetime, timedelta
def next_attempt_at(
attempt: int,
*,
now: datetime,
base_s: float = 2.0,
cap_s: float = 300.0,
rand: Callable[[], float] = random.random,
) -> datetime:
"""Exponential backoff with full jitter, capped. Pure: takes `now` and `rand`, never calls them itself.
Delay is uniform in [0, min(cap_s, base_s * 2**attempt)]. attempt is 0-based; raise ValueError if < 0.
"""
if attempt < 0:
raise ValueError(f"attempt must be >= 0, got {attempt}")
ceiling = min(cap_s, base_s * 2.0**attempt)
delay_s = ceiling * rand()
return now + timedelta(seconds=delay_s)
libs/svcforge_core/svcforge_core/domain/catalog.py
+58
View File
@@ -0,0 +1,58 @@
"""Load and validate catalog.yaml into CatalogEntry objects.
The only I/O in `domain/`: reading the one path handed to :func:`load_catalog`.
"""
from pathlib import Path
from typing import Any
import yaml
from pydantic import ValidationError
from svcforge_core.domain.models import CatalogEntry
class CatalogError(Exception):
"""A catalog file could not be parsed or validated.
`key` names the offending service type, or None when the failure is file-level.
"""
def __init__(self, message: str, *, key: str | None = None) -> None:
self.key = key
super().__init__(message if key is None else f"{key}: {message}")
def load_catalog(path: Path) -> dict[str, CatalogEntry]:
"""Parse catalog.yaml -> {service_type: CatalogEntry}. yaml.safe_load, never yaml.load.
Raise CatalogError (typed, with the offending key) on a bad file.
"""
try:
raw: Any = yaml.safe_load(path.read_text())
except OSError as exc:
raise CatalogError(f"cannot read catalog at {path}: {exc}") from exc
except yaml.YAMLError as exc:
raise CatalogError(f"catalog at {path} is not valid YAML: {exc}") from exc
if not isinstance(raw, dict):
raise CatalogError(
f"catalog at {path} must be a mapping of service_type -> entry, got {type(raw).__name__}"
)
services: Any = raw.get("services", raw)
if not isinstance(services, dict):
raise CatalogError(f"catalog at {path}: 'services' must be a mapping, got {type(services).__name__}")
catalog: dict[str, CatalogEntry] = {}
for key, body in services.items():
service_type = str(key)
if not isinstance(body, dict):
raise CatalogError(f"entry must be a mapping, got {type(body).__name__}", key=service_type)
try:
catalog[service_type] = CatalogEntry(service_type=service_type, **body)
except ValidationError as exc:
raise CatalogError(f"invalid entry: {exc}", key=service_type) from exc
except TypeError as exc:
raise CatalogError(f"invalid entry: {exc}", key=service_type) from exc
return catalog
libs/svcforge_core/svcforge_core/domain/models.py
+101
View File
@@ -0,0 +1,101 @@
"""Domain models. Pydantic v2, all frozen.
`frozen=True` is not transitive: a `dict` field (e.g. `SizeSpec.resources`) stays mutable
in place. Treat those dicts as read-only by convention.
"""
from datetime import datetime
from enum import StrEnum
from typing import Any
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from svcforge_core.domain.states import InstanceState
TEAM_PATTERN = r"^[a-z0-9-]+$"
class TaskKind(StrEnum):
"""What a worker is being asked to do."""
PROVISION = "provision"
DEPROVISION = "deprovision"
UPGRADE = "upgrade"
VERIFY = "verify"
class TaskState(StrEnum):
"""Where a task is in the claim loop."""
QUEUED = "queued"
RUNNING = "running"
DONE = "done"
FAILED = "failed"
class SizeSpec(BaseModel):
"""One t-shirt size of a catalog entry."""
model_config = ConfigDict(frozen=True)
replicas: int = Field(ge=1)
resources: dict[str, Any]
class CatalogEntry(BaseModel):
"""One offerable service type and the sizes it comes in."""
model_config = ConfigDict(frozen=True)
service_type: str = Field(min_length=1)
chart: str = Field(min_length=1)
chart_version: str = Field(min_length=1)
sizes: dict[str, SizeSpec]
# Bypass tenant maintenance windows for this entry's upgrades. Defaults False: a
# normal version bump waits for 03:00 Sunday; a CVE with a public exploit does not.
security: bool = False
class Instance(BaseModel):
"""A provisioned (or in-flight) service instance owned by a team."""
model_config = ConfigDict(frozen=True)
id: UUID
team: str = Field(min_length=1, pattern=TEAM_PATTERN)
service_type: str
size: str
state: InstanceState
namespace: str
release_name: str
chart_version: str
endpoint: str | None = None
error: str | None = None
expires_at: datetime | None = None
created_at: datetime
updated_at: datetime
class Task(BaseModel):
"""A unit of work against an instance, claimed by exactly one worker at a time."""
model_config = ConfigDict(frozen=True)
id: int
instance_id: UUID
kind: TaskKind
state: TaskState
attempts: int = Field(ge=0)
run_after: datetime
locked_by: str | None = None
last_error: str | None = None
# The W3C trace context of whoever enqueued this. Null is normal, not an error: a task
# the reconciler raised on its own tick has no inbound request to belong to.
traceparent: str | None = None
# Denormalised from `instances` by the claim query. It is here so a worker can bind
# `team` onto its log context at claim time, before it has loaded anything — the point
# of structured logs is that the FIRST line already tells you whose tenant broke.
team: str | None = None
libs/svcforge_core/svcforge_core/domain/states.py
+37
View File
@@ -0,0 +1,37 @@
"""The instance lifecycle state machine, encoded as data."""
from enum import StrEnum
from typing import Final
class InstanceState(StrEnum):
"""Lifecycle of a provisioned service instance."""
REQUESTED = "requested"
PROVISIONING = "provisioning"
READY = "ready"
DELETING = "deleting"
DELETED = "deleted"
FAILED = "failed"
class IllegalTransition(Exception):
"""Raised by transition() when cur -> nxt is not in LEGAL."""
LEGAL: Final[dict[InstanceState, frozenset[InstanceState]]] = {
InstanceState.REQUESTED: frozenset({InstanceState.PROVISIONING, InstanceState.FAILED}),
InstanceState.PROVISIONING: frozenset({InstanceState.READY, InstanceState.FAILED}),
InstanceState.READY: frozenset({InstanceState.DELETING, InstanceState.FAILED}),
InstanceState.DELETING: frozenset({InstanceState.DELETED, InstanceState.FAILED}),
# `deleted` is terminal: an empty frozenset, not a missing key.
InstanceState.DELETED: frozenset(),
InstanceState.FAILED: frozenset({InstanceState.PROVISIONING, InstanceState.DELETING}),
}
def transition(cur: InstanceState, nxt: InstanceState) -> InstanceState:
"""Return nxt if LEGAL[cur] contains it, else raise IllegalTransition. Pure. No DB."""
if nxt not in LEGAL[cur]:
raise IllegalTransition(f"{cur} -> {nxt} is not a legal transition")
return nxt
libs/svcforge_core/svcforge_core/domain/windows.py
+111
View File
@@ -0,0 +1,111 @@
"""Maintenance windows: when a tenant will tolerate an upgrade.
Pure domain. No I/O, no `Clock`, no `datetime.now()` anywhere in this file — `now` arrives
as a parameter and every datetime crossing this module's boundary is aware and UTC. That
is the whole discipline: `datetime.now()` is naive and lies, comparing aware to naive
raises TypeError at 03:00 on a Sunday, and mypy will not catch it for you.
Local time is where the arithmetic has to happen, though. A window means "03:00 as the
tenant reads a clock", which is not a fixed UTC offset in any zone that observes DST. So
the cron is evaluated in the window's own zone and the result is converted back to UTC at
the door.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from croniter import croniter
CRON_FIELDS = 5
SEPARATOR = "|"
class BadWindow(ValueError):
"""A maintenance window spec is not a 5-field cron plus a known IANA zone."""
@dataclass(frozen=True)
class MaintenanceWindow:
"""When a tenant's instance may be upgraded."""
cron: str # standard 5-field cron
tz: str # IANA name, e.g. 'Asia/Ho_Chi_Minh'
def parse_window(spec: str | None) -> MaintenanceWindow | None:
"""'0 3 * * 0|Asia/Ho_Chi_Minh' -> MaintenanceWindow. None -> None (upgrade any time).
Raise BadWindow on an invalid cron expression or unknown IANA zone. Validation happens
here, once, on the way in — not at 03:00 in a worker, where the failure is a task that
dies inside someone else's maintenance window.
"""
if spec is None or not spec.strip():
return None
cron, sep, tz = spec.partition(SEPARATOR)
if not sep:
raise BadWindow(f"window {spec!r} is not 'CRON{SEPARATOR}IANA_ZONE'")
cron, tz = cron.strip(), tz.strip()
# croniter's is_valid() accepts a 6-field form (with seconds). The column is documented
# as 5-field, so a 6th field is a typo, not a feature.
if len(cron.split()) != CRON_FIELDS:
raise BadWindow(f"cron {cron!r} must have exactly {CRON_FIELDS} fields")
if not croniter.is_valid(cron):
raise BadWindow(f"cron {cron!r} is not a valid cron expression")
try:
ZoneInfo(tz)
except (ZoneInfoNotFoundError, ValueError) as exc:
# ZoneInfoNotFoundError in a slim image means tzdata is missing, not that the zone
# is fictional. Same exception, so the message names both possibilities.
raise BadWindow(f"timezone {tz!r} is not a known IANA zone (is tzdata installed?)") from exc
return MaintenanceWindow(cron=cron, tz=tz)
def next_window_open(window: MaintenanceWindow | None, now: datetime) -> datetime:
"""Next time the window opens, as an aware UTC datetime.
`now` must be aware; raise ValueError if it is naive. window=None -> return now.
Inclusive of `now`: if the window opens at this exact instant, that instant is the
answer. Otherwise an upgrade that became due precisely at 03:00 would be pushed a
whole week.
"""
if now.tzinfo is None or now.utcoffset() is None:
raise ValueError(f"now must be an aware datetime, got naive {now!r}")
if window is None:
return now.astimezone(UTC)
local_now = now.astimezone(ZoneInfo(window.tz))
# croniter.get_next() is strictly greater than its start. Backing the start off by one
# second makes a `now` that lands exactly on a cron minute return itself; a `now` at
# 03:00:30 still rolls to the next occurrence, because cron times are minute-aligned
# and 03:00:00 is already behind 03:00:29.
start = local_now - timedelta(seconds=1)
local_next: datetime = croniter(window.cron, start).get_next(datetime)
return local_next.astimezone(UTC)
def schedule_upgrade_at(window: MaintenanceWindow | None, security: bool, now: datetime) -> datetime:
"""run_after for an upgrade task. security=True bypasses the window -> now.
A security fix with a public exploit does not wait until Sunday. That is the entire
reason `security:` exists in the catalog.
`now` must be aware here too, bypass or not: the naive-datetime rule does not get a
hole punched in it by the branch that skips the window.
"""
if now.tzinfo is None or now.utcoffset() is None:
raise ValueError(f"now must be an aware datetime, got naive {now!r}")
if security:
return now.astimezone(UTC)
return next_window_open(window, now)
libs/svcforge_core/svcforge_core/migrate.py
+81
View File
@@ -0,0 +1,81 @@
"""Migration runner. Twenty lines of psycopg, not Alembic.
Three rules this file exists to enforce:
1. Never migrate on app startup. N replicas would race. This runs as a Helm
pre-upgrade/pre-install hook Job, once, before any new pod serves traffic.
2. Forward-only. There are no down scripts. A mistake is fixed by a new migration.
3. Expand/contract. Add nullable, backfill, switch reads, drop the old column in a
LATER release. A rename is three deploys, never one.
Run: python -m svcforge_core.migrate
"""
from __future__ import annotations
import sys
from pathlib import Path
import psycopg
from svcforge_core.settings import load_settings
MIGRATIONS_DIR = Path(__file__).resolve().parents[3] / "migrations"
# Advisory lock: two Jobs racing (a retried hook, a hand-run) must not both apply DDL.
# Session-scoped, so this needs the session pooler (5432), not pgbouncer (6543).
_LOCK_KEY = 0x5643_464F # "SVCFO"
_BOOTSTRAP = """
create table if not exists schema_migrations (
filename text primary key,
applied_at timestamptz not null default now()
);
"""
def _pending(conn: psycopg.Connection[tuple[str, ...]], files: list[Path]) -> list[Path]:
with conn.cursor() as cur:
cur.execute("select filename from schema_migrations")
done = {row[0] for row in cur.fetchall()}
return [f for f in files if f.name not in done]
def main() -> int:
settings = load_settings()
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
if not files:
print(f"no migrations found in {MIGRATIONS_DIR}", file=sys.stderr)
return 1
with psycopg.connect(settings.migration_dsn, autocommit=True) as conn:
with conn.cursor() as cur:
cur.execute("select pg_advisory_lock(%s)", (_LOCK_KEY,))
try:
with conn.cursor() as cur:
cur.execute(_BOOTSTRAP)
pending = _pending(conn, files)
if not pending:
print("up to date, nothing to apply")
return 0
for path in pending:
sql = path.read_text(encoding="utf-8")
# One transaction per file: a file applies completely or not at all.
with conn.transaction():
with conn.cursor() as cur:
cur.execute(sql)
cur.execute(
"insert into schema_migrations (filename) values (%s)",
(path.name,),
)
print(f"applied: {path.name}")
finally:
with conn.cursor() as cur:
cur.execute("select pg_advisory_unlock(%s)", (_LOCK_KEY,))
return 0
if __name__ == "__main__":
raise SystemExit(main())
libs/svcforge_core/svcforge_core/obs.py
+308
View File
@@ -0,0 +1,308 @@
"""Logs, traces, metrics. One setup() call, made once, before anything else.
Three libraries, one module, because the three are one decision. A log line without the
trace id it belongs to is a log line you cannot join to anything; a span without the
`instance_id` the request is about is a span you cannot search for. They are wired here
together so that no service can configure two of the three and ship.
The three things that make this module worth reading:
1. **Context does not cross a queue.** A trace is a chain of parent/child span contexts
passed in-process or over a wire header. `POST /v1/instances` inserts a row and
returns; the worker picks that row up ninety seconds later in a different pod. Nothing
carries the context across — unless we carry it ourselves. So: `inject_traceparent()`
at enqueue, a `traceparent` column, `context_from_traceparent()` at claim. Two
disconnected traces in Tempo is the symptom of skipping this.
2. **Histogram buckets are a domain decision.** prometheus_client's defaults top out at
10 seconds because they were chosen for HTTP handlers. A provision is `helm --wait` on
a StatefulSet: minutes. With the defaults every observation lands in `+Inf`,
`histogram_quantile` interpolates inside a bucket that spans 10s→infinity, and the p95
it prints is a number with no relationship to reality. The buckets below are sized for
what is being measured.
3. **One process per pod.** prometheus_client keeps its registry in process memory. Run
`uvicorn --workers 4` and Prometheus scrapes whichever of the four children the socket
happens to hand it, so counters appear to jump backwards. There are two fixes:
`PROMETHEUS_MULTIPROC_DIR` + `MultiProcessCollector` (a shared mmap directory, a
gauge-mode decision at every call site, and dead files to garbage-collect after every
crash), or one process per pod and scale with replicas. This repo takes the second.
`PROMETHEUS_MULTIPROC_DIR` is deliberately not set, and nothing here reads it.
"""
from __future__ import annotations
import logging
import sys
from typing import TYPE_CHECKING, Any
from uuid import UUID
import structlog
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from prometheus_client import Counter, Gauge, Histogram, start_http_server
if TYPE_CHECKING:
from svcforge_core.settings import Settings
# --- Metrics ----------------------------------------------------------------------------
#
# Module level, created exactly once at import. A second registration of the same name
# against the default registry raises ValueError, which is a feature: it turns "two modules
# each defined their own copy of this counter" into an ImportError at startup instead of a
# metric that silently reports half the truth.
TASKS_CLAIMED = Counter(
"svcforge_tasks_claimed_total",
"Tasks claimed off the queue by a worker.",
["kind"],
)
TASKS_FAILED = Counter(
"svcforge_tasks_failed_total",
"Tasks that exhausted their attempts and went to 'failed'.",
["kind"],
)
PROVISION_TIME = Histogram(
"svcforge_provision_duration_seconds",
"Wall time of a provision task, claim to terminal report.",
# NOT the defaults. See the module docstring: the defaults end at 10s and a provision
# takes minutes. The top finite bucket is 1800 because helm's own --timeout is 600 and
# a provision past thirty minutes is not slow, it is broken and belongs in +Inf.
buckets=(10, 30, 60, 120, 300, 600, 1800, float("inf")),
)
QUEUE_DEPTH = Gauge(
"svcforge_queue_depth",
"Runnable tasks waiting in the queue. Set by the reconciler each tick.",
)
INSTANCES = Gauge(
"svcforge_instances",
"Instances by lifecycle state. Set by the reconciler each tick.",
["state"],
)
RECONCILER_LAST_TICK = Gauge(
"svcforge_reconciler_last_tick_timestamp_seconds",
"Unix time of the reconciler's last completed tick. The liveness signal that matters.",
)
# --- Wiring -----------------------------------------------------------------------------
_TRACER_NAME = "svcforge"
# setup() is idempotent because it is called from three entrypoints and from tests, and
# because configuring structlog twice silently discards the first configuration while
# adding a second stdout handler to the root logger — every line then prints twice.
_configured = False
_propagator = TraceContextTextMapPropagator()
def _add_trace_ids(
_logger: Any, # noqa: ANN401 - structlog's Processor signature; the logger is untyped
_method: str,
event_dict: structlog.typing.EventDict,
) -> structlog.typing.EventDict:
"""Stamp the active trace/span id onto the line, if there is one.
This is the join key. Without it, "find the logs for this trace" is a full-text search
over a time window and a guess; with it, it is one query. Hex-formatted to the widths
the W3C spec uses, so the value pasted from Tempo matches the value in Loki.
"""
span = trace.get_current_span()
ctx = span.get_span_context()
if ctx.is_valid:
event_dict["trace_id"] = format(ctx.trace_id, "032x")
event_dict["span_id"] = format(ctx.span_id, "016x")
return event_dict
def setup(service_name: str, settings: Settings) -> None:
"""Configure structlog, the tracer provider, and the metric registry. Idempotent.
Called once from each service's entrypoint, before anything else — "before anything
else" because any logger bound before this runs keeps the default configuration
(`cache_logger_on_first_use`), and a module-level `log = structlog.get_logger()` in an
import that lands first will print unstructured text forever.
"""
global _configured # process-wide config is process-wide state
if _configured:
return
_setup_logging(service_name, settings)
_setup_tracing(service_name, settings)
_configured = True
def _setup_logging(service_name: str, settings: Settings) -> None:
"""structlog + stdlib logging, both rendering JSON to stdout through one handler.
The stdlib half is not optional. `psycopg`, `httpx`, `uvicorn` and the OTEL SDK all log
through `logging`; without the ProcessorFormatter bridge below, their lines arrive as
bare text on the same stdout and every one of them is a parse failure in the collector.
"""
level = getattr(logging, settings.log_level.upper(), logging.INFO)
shared: list[structlog.typing.Processor] = [
# FIRST. This is what puts instance_id/task_id/team on every line, including the
# lines written by code that has never heard of them.
structlog.contextvars.merge_contextvars,
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.processors.TimeStamper(fmt="iso", utc=True),
_add_trace_ids,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
]
structlog.configure(
processors=[
*shared,
# Hands off to the stdlib formatter below, which appends the renderer. This is
# what lets one handler render both structlog and foreign logs identically.
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# The renderer goes LAST and nothing follows it: it turns the event dict into a string,
# so any processor after it receives a str where it expects a dict and raises.
renderer: structlog.typing.Processor = (
structlog.processors.JSONRenderer() if settings.log_json else structlog.dev.ConsoleRenderer()
)
formatter = structlog.stdlib.ProcessorFormatter(
foreign_pre_chain=shared, # applied to records from logging.getLogger(...) callers
processors=[structlog.stdlib.ProcessorFormatter.remove_processors_meta, renderer],
)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
root = logging.getLogger()
# Replace rather than append: basicConfig may already have run, and two handlers means
# two copies of every line. stdout only — a container writes logs to stdout and the
# collector tails them from there. A log file inside a pod is deleted with the pod.
root.handlers = [handler]
root.setLevel(level)
structlog.contextvars.bind_contextvars(service=service_name)
def _setup_tracing(service_name: str, settings: Settings) -> None:
"""Set the global tracer provider, exporting over OTLP when an endpoint is configured.
Skipped entirely when something already set a provider: the API runs under
`opentelemetry-instrument`, whose auto-instrumentation installs one before our
`main()` is reached. Overwriting it drops the FastAPI and psycopg instrumentation's
spans on the floor, and the SDK only logs a warning about it.
"""
if isinstance(trace.get_tracer_provider(), TracerProvider):
return
provider = TracerProvider(resource=Resource.create({"service.name": service_name}))
if settings.otel_endpoint:
exporter = _otlp_exporter(settings.otel_endpoint)
if exporter is not None:
# Batch, not Simple: SimpleSpanProcessor exports inline on span end, so every
# helm span would block on a network round trip to the collector.
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)
def _otlp_exporter(endpoint: str) -> Any | None: # noqa: ANN401 - one of two exporter classes
"""The OTLP exporter, if the optional exporter package is installed.
Optional on purpose. In the cluster the API runs under `opentelemetry-instrument`,
which brings its own exporter and configures it from `OTEL_EXPORTER_OTLP_*`. Making it
a hard dependency of the shared library would mean every unit test imports gRPC.
"""
try:
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
except ImportError:
logging.getLogger(__name__).warning(
"otel_endpoint is set but no OTLP exporter is installed; traces stay in-process",
extra={"endpoint": endpoint},
)
return None
return OTLPSpanExporter(endpoint=endpoint)
def get_logger(name: str) -> structlog.stdlib.BoundLogger:
"""A bound logger. Call it inside a function, not at import time — see setup()."""
logger: structlog.stdlib.BoundLogger = structlog.get_logger(name)
return logger
def tracer() -> trace.Tracer:
"""The svcforge tracer. Manual spans wrap helm calls, and nothing else.
Everything else is auto-instrumented (FastAPI, psycopg). A hand-rolled span around a
function that the SDK already wraps is a duplicated span and a maintenance cost.
"""
return trace.get_tracer(_TRACER_NAME)
def start_metrics_server(port: int) -> None:
"""Expose /metrics on `port`, for the services with no HTTP server of their own.
The worker and reconciler have no Service; the chart's PodMonitor scrapes them by pod
on the port named `metrics`. The API does not use this — it mounts the same registry on
its own ASGI app.
"""
start_http_server(port)
# --- Context that has to cross a process boundary ---------------------------------------
def bind_task_context(instance_id: UUID, task_id: int, team: str) -> None:
"""Bind the three keys every log line in a task must carry. Called at claim time.
`clear_contextvars()` first, and this is the whole reason the function exists rather
than three `bind_contextvars` calls at the call site. A worker coroutine reuses its
context across loop iterations; without the clear, task 41's `instance_id` is still
bound when task 42 starts logging, and the log for the incident you are debugging
names the wrong tenant. Contextvars are per-task in asyncio, which makes this safe
under the concurrency semaphore: two handlers running at once do not see each other's.
"""
structlog.contextvars.clear_contextvars()
structlog.contextvars.bind_contextvars(
instance_id=str(instance_id),
task_id=task_id,
team=team,
)
def inject_traceparent() -> str | None:
"""Serialise the active span context to a W3C traceparent, for the tasks row.
None when there is no recording span — a task enqueued by the reconciler's own tick has
no inbound request to be part of. Nullable column, nullable return: an untraced task is
normal, not an error.
"""
carrier: dict[str, str] = {}
_propagator.inject(carrier)
return carrier.get("traceparent")
def context_from_traceparent(traceparent: str | None) -> Context:
"""Inverse of inject_traceparent. Used at claim to parent the worker span to the API's.
An empty Context for None or for a malformed value — `extract` does not raise on a
traceparent that fails to parse, it returns the carrier's context unchanged, and the
resulting span starts a new trace. A bad header must never fail a provision.
"""
if not traceparent:
return Context()
return _propagator.extract({"traceparent": traceparent})
libs/svcforge_core/svcforge_core/py.typed
View File
libs/svcforge_core/svcforge_core/repo/__init__.py
+1
View File
@@ -0,0 +1 @@
"""SQL. Rows in, domain objects out. Knows psycopg; knows nothing about HTTP."""
libs/svcforge_core/svcforge_core/repo/db.py
+57
View File
@@ -0,0 +1,57 @@
"""The connection pool.
Two settings here are load-bearing on Supabase's transaction pooler (port 6543) and
cost hours if you get them wrong. Both are documented at the call site below.
"""
from __future__ import annotations
from typing import Any
from psycopg import AsyncConnection
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
# The pool hands out dict-row connections because of `row_factory=dict_row` below. Say so
# in the type system too, or every `row["attempts"]` in this codebase is a mypy error
# against a bare `AsyncConnectionPool`, which resolves to tuple rows. The runtime was
# always right; without these aliases the annotations quietly disagree with it, and the
# fix people reach for is `# type: ignore`, which throws away the checking entirely.
type DictRow = dict[str, Any]
type DictConnection = AsyncConnection[DictRow]
type DictPool = AsyncConnectionPool[DictConnection]
def make_pool(dsn: str, min_size: int = 1, max_size: int = 5) -> DictPool:
"""Construct the pool. Does NOT open it — the caller owns open/close.
`open=False` is deliberate: the constructor does zero I/O, so building a pool at
import time and never opening it fails later as a PoolTimeout at first use, far
from the cause. The caller (a FastAPI lifespan, a worker main) opens and closes it.
kwargs are per-connection:
* `prepare_threshold=None` — REQUIRED through pgbouncer in transaction mode.
psycopg3 auto-prepares a statement after it sees it 5 times. pgbouncer may hand
the next execution to a different backend, which has never heard of that prepared
statement. Symptom: everything works for exactly five calls, then
`prepared statement "_pg3_0" does not exist` — intermittent, only under
concurrency, never in a unit test.
* `row_factory=dict_row` — rows arrive as dicts, so `Instance.model_validate(row)`
works directly instead of unpacking tuples by position.
Also gone on 6543: LISTEN/NOTIFY, session-level SET, cross-statement advisory locks.
`SELECT ... FOR UPDATE SKIP LOCKED` inside one transaction is unaffected — which is
exactly why the queue is built on it. Use the session pooler (5432) for migrations.
max_size is a database-capacity decision, not a throughput knob: the free tier has a
small connection budget, and replicas multiply this number.
"""
return AsyncConnectionPool(
conninfo=dsn,
min_size=min_size,
max_size=max_size,
open=False,
kwargs={"prepare_threshold": None, "row_factory": dict_row, "autocommit": False},
)
libs/svcforge_core/svcforge_core/repo/instances.py
+192
View File
@@ -0,0 +1,192 @@
"""Instance persistence.
AuthZ lives in the WHERE clause. `get(id, team)` filters by team in SQL rather than
fetching the row and comparing in Python: a wrong-team id must be indistinguishable
from a nonexistent one, and a check you can forget to write is a check you will forget
to write.
"""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from uuid import UUID
from psycopg import AsyncConnection
from psycopg.rows import dict_row
from svcforge_core.domain.models import Instance
from svcforge_core.domain.states import InstanceState
from svcforge_core.repo.db import DictPool
_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
endpoint, error, expires_at, created_at, updated_at"""
@dataclass(frozen=True)
class UpgradeCandidate:
"""One row of the day-2 work list: an instance, plus the raw window spec to schedule in.
`maintenance_window` rides alongside rather than inside `Instance` because `create()`
never writes it — folding it into the model would mean every freshly created Instance
reports `maintenance_window=None` whether or not the row has one. A column only the
work list reads is a column only the work list carries.
"""
instance: Instance
maintenance_window: str | None
class InstanceRepo:
"""Reads and writes `instances`."""
def __init__(self, pool: DictPool) -> None:
self._pool = pool
async def create(self, conn: AsyncConnection[dict[str, Any]], inst: Instance) -> Instance:
"""Insert one instance.
Takes a `conn` rather than using the pool so the caller can share a transaction
with TaskRepo.enqueue — the instance row and its provision task must commit
together or not at all. That single fact is why the queue is in Postgres.
"""
async with conn.cursor() as cur:
await cur.execute(
f"""insert into instances (id, team, service_type, size, state, namespace,
release_name, chart_version, endpoint, error, expires_at)
values (%(id)s, %(team)s, %(service_type)s, %(size)s, %(state)s, %(namespace)s,
%(release_name)s, %(chart_version)s, %(endpoint)s, %(error)s, %(expires_at)s)
returning {_COLUMNS}""", # noqa: S608 - _COLUMNS is a module constant, not input
{
"id": inst.id,
"team": inst.team,
"service_type": inst.service_type,
"size": inst.size,
"state": inst.state.value,
"namespace": inst.namespace,
"release_name": inst.release_name,
"chart_version": inst.chart_version,
"endpoint": inst.endpoint,
"error": inst.error,
"expires_at": inst.expires_at,
},
)
row = await cur.fetchone()
assert row is not None # noqa: S101 - `returning` always yields a row or raises
return Instance.model_validate(row)
async def get(self, id: UUID, team: str) -> Instance | None:
"""Fetch one instance owned by `team`. None if it does not exist OR is not theirs."""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
f"select {_COLUMNS} from instances where id = %s and team = %s", # noqa: S608
(id, team),
)
row = await cur.fetchone()
return Instance.model_validate(row) if row else None
async def list(self, team: str, limit: int = 50) -> list[Instance]:
"""The team's instances, newest first."""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
f"""select {_COLUMNS} from instances
where team = %s order by created_at desc limit %s""", # noqa: S608
(team, limit),
)
rows = await cur.fetchall()
return [Instance.model_validate(r) for r in rows]
async def update_state(
self,
id: UUID,
expect: InstanceState,
to: InstanceState,
error: str | None = None,
endpoint: str | None = None,
) -> bool:
"""Compare-and-set. False if the row moved under you.
`where id=%s and state=%s` is the whole trick: two workers racing to move the
same instance means exactly one UPDATE matches a row. The loser gets False and
must not treat it as an error — it means someone else already did the work.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"""update instances
set state = %(to)s,
error = %(error)s,
endpoint = coalesce(%(endpoint)s, endpoint),
updated_at = now()
where id = %(id)s and state = %(expect)s""",
{
"id": id,
"expect": expect.value,
"to": to.value,
"error": error,
"endpoint": endpoint,
},
)
return cur.rowcount == 1
async def set_error(self, id: UUID, error: str) -> None:
"""Record a terminal failure message. Used when a task exhausts its attempts."""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"update instances set error = %s, state = %s, updated_at = now() where id = %s",
(error[-2000:], InstanceState.FAILED.value, id),
)
async def list_upgradable(
self,
service_type: str,
catalog_version: str,
own_team: str,
max_in_flight: int = 1,
) -> Sequence[UpgradeCandidate]:
"""The day-2 work list: `ready` instances not yet on the catalog's pinned version.
The whole rollout is this query. Three clauses carry it:
`not exists (... rollout_state = 'halted')` — a failed `verify` writes one column
and this query goes empty for that service type. That is the stop button: no
rollouts table, no state machine, no pause/resume CLI. You clear it with SQL.
`order by team = %(own_team)s desc` — your own instances upgrade first, so you are
the tenant who finds out the chart is broken. Eating your own dog food is a
`order by`, not a policy document.
`limit %(max_in_flight)s` — a config value, not a scheduler. It stays at 1 until 1
is too slow, and 1 is what makes the halt meaningful: the fleet stops after the
first casualty rather than after all of them.
"""
# `row_factory=dict_row` is already the pool's per-connection default; naming it
# here is for mypy, which cannot see a row factory passed through a kwargs dict
# and would otherwise type these rows as tuples.
async with self._pool.connection() as conn, conn.cursor(row_factory=dict_row) as cur:
await cur.execute(
f"""select {_COLUMNS}, maintenance_window from instances
where state = 'ready'
and service_type = %(service_type)s
and chart_version <> %(catalog_version)s
and not exists (
select 1 from catalog_versions cv
where cv.service_type = %(service_type)s
and cv.rollout_state = 'halted')
order by team = %(own_team)s desc, created_at
limit %(max_in_flight)s""", # noqa: S608 - _COLUMNS is a module constant, not input
{
"service_type": service_type,
"catalog_version": catalog_version,
"own_team": own_team,
"max_in_flight": max_in_flight,
},
)
rows = await cur.fetchall()
return [
UpgradeCandidate(
instance=Instance.model_validate(r),
maintenance_window=r["maintenance_window"],
)
for r in rows
]
libs/svcforge_core/svcforge_core/repo/reconcile.py
+276
View File
@@ -0,0 +1,276 @@
"""The reconciler's SQL.
Why this file exists rather than the queries living in `services/reconciler/main.py`: the
layer rule says transport knows nothing about SQL, and the reconciler is transport — a CLI
entrypoint. It gets its own repo module rather than growing `InstanceRepo` and `TaskRepo`
because everything here is a *sweep*: it reads rows nobody asked about and it writes an
instance state and a task row in the same transaction. `InstanceRepo.update_state` owns its
own connection by design, so the reconciler cannot get atomicity from it without reaching
around the repo — which is the thing the layer rule exists to prevent.
The recurring shape below is: lock the row, re-check the condition under the lock, act.
The re-check is not paranoia about concurrency — the reconciler is a singleton. It is what
makes the sweep idempotent against *itself*: a tick that crashes after the insert and
before the commit must leave nothing behind, and the next tick must not double-enqueue.
"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from uuid import UUID
from psycopg import AsyncCursor
from svcforge_core.domain.models import Instance, TaskKind, TaskState
from svcforge_core.domain.states import InstanceState, transition
from svcforge_core.obs import inject_traceparent
from svcforge_core.repo.db import DictPool
_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
endpoint, error, expires_at, created_at, updated_at"""
# A task nobody will ever run again. The idempotency guard on every enqueue below asks
# "is one already outstanding?", and 'done'/'failed' are not outstanding: a failed
# deprovision that exhausted its attempts must be re-enqueueable by the next sweep, or a
# transient cluster outage would permanently strand the instance.
_UNFINISHED = (TaskState.QUEUED.value, TaskState.RUNNING.value)
class ReconcileRepo:
"""Fleet-wide sweeps. Read-mostly, and every write is one transaction."""
def __init__(self, pool: DictPool) -> None:
self._pool = pool
# --- Gauges -------------------------------------------------------------------------
async def queue_depth(self) -> int:
"""Tasks waiting to be claimed.
Counts every `queued` row, not just the runnable ones (`run_after <= now()`). The
alert on this gauge is `deriv(...) > 0` — "the backlog is growing" — and a backlog
of tasks parked on backoff is exactly the backlog you want to see growing.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select count(*) as n from tasks where state = %s", (TaskState.QUEUED.value,))
row = await cur.fetchone()
return int(row["n"]) if row else 0
async def instance_counts(self) -> dict[str, int]:
"""Instances per lifecycle state. States with no rows are absent, not zero."""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select state, count(*) as n from instances group by state")
rows = await cur.fetchall()
return {str(r["state"]): int(r["n"]) for r in rows}
# --- Drift --------------------------------------------------------------------------
async def ready_instances(self) -> list[Instance]:
"""Every instance the DB believes is running. The drift check's expectation."""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
f"select {_COLUMNS} from instances where state = %s", # noqa: S608 - module constant
(InstanceState.READY.value,),
)
rows = await cur.fetchall()
return [Instance.model_validate(r) for r in rows]
async def known_releases(self) -> set[tuple[str, str]]:
"""(release_name, namespace) for every instance row, in any state.
Any state, deliberately. An instance that is still `requested` has no release yet,
but a worker may be installing it *right now* — treating it as unknown would
report a healthy in-flight provision as an orphan on every tick.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select release_name, namespace from instances")
rows = await cur.fetchall()
return {(str(r["release_name"]), str(r["namespace"])) for r in rows}
async def enqueue_reprovision(self, instance_id: UUID, reason: str) -> int | None:
"""`ready` instance whose release vanished -> back to `provisioning`, with a task.
None if the row moved, or if a provision is already outstanding.
The two-hop state change is the interesting part, and it is not busywork:
* `LEGAL` has no `ready -> provisioning` edge. The tenant-visible lifecycle only
leaves `ready` through `deleting` or `failed`, and drift is a failure — the
service the tenant is paying for is gone. So: `ready -> failed -> provisioning`,
both edges legal, asserted below by the domain function rather than assumed.
* The row must land in `provisioning`, not `failed`, before the worker sees the
task. `handle_provision` CASes `requested -> provisioning` best-effort and then
CASes `provisioning -> ready` for real; hand it a `failed` row and helm runs, the
final CAS matches nothing, and the instance sits in `failed` forever with a
healthy release behind it.
Both hops and the insert are one transaction, so the row is never observable in the
intermediate `failed` state and a crash mid-sweep leaves nothing half-done.
"""
async with self._pool.connection() as conn:
async with conn.transaction(), conn.cursor() as cur:
await cur.execute(
"select state from instances where id = %s and state = %s for update",
(instance_id, InstanceState.READY.value),
)
if await cur.fetchone() is None:
return None
if await _has_unfinished(cur, instance_id, TaskKind.PROVISION):
return None
# Assert the path through the state machine instead of trusting the SQL.
# If someone edits LEGAL, this raises here rather than corrupting rows.
failed = transition(InstanceState.READY, InstanceState.FAILED)
provisioning = transition(failed, InstanceState.PROVISIONING)
await cur.execute(
"update instances set state = %s, error = %s, updated_at = now() where id = %s",
(provisioning.value, reason[-2000:], instance_id),
)
return await _insert_task(cur, instance_id, TaskKind.PROVISION)
# --- TTL ----------------------------------------------------------------------------
async def due_for_deprovision(self) -> list[Instance]:
"""Instances that should be torn down and have no deprovision task outstanding.
Two populations, one query:
* `ready` and past `expires_at` — the TTL sweep proper. The whole reason a
throwaway Elasticsearch does not become a permanent line on the cloud bill.
* `deleting` with nothing to do the deleting — the API CASes to `deleting` and then
enqueues in a second statement, and a crash between the two leaves exactly this.
That ordering is chosen *because* this sweep exists; the other order would leave
a deprovision task pointing at a `ready` instance, and a worker would tear down a
live service nobody asked to delete.
Note the parentheses around the OR. Without them, `and not exists (...)` binds to
the second branch alone and the query re-enqueues a deprovision for every deleting
instance on every tick, forever.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
f"""select {_COLUMNS} from instances i
where ((i.state = %(ready)s and i.expires_at < now()) or i.state = %(deleting)s)
and not exists (
select 1 from tasks t
where t.instance_id = i.id
and t.kind = %(kind)s
and t.state = any(%(unfinished)s))
order by i.created_at""", # noqa: S608 - module constant
{
"ready": InstanceState.READY.value,
"deleting": InstanceState.DELETING.value,
"kind": TaskKind.DEPROVISION.value,
"unfinished": list(_UNFINISHED),
},
)
rows = await cur.fetchall()
return [Instance.model_validate(r) for r in rows]
async def enqueue_deprovision(self, instance_id: UUID) -> int | None:
"""CAS to `deleting` if needed, and enqueue the task. One transaction. None if moot.
The instance must be in `deleting` before the worker claims the task, for the same
reason as `enqueue_reprovision`: `handle_deprovision` finishes with a
`deleting -> deleted` CAS, and a `ready` row would make helm uninstall the release
and the DB keep advertising an endpoint that no longer resolves.
"""
async with self._pool.connection() as conn:
async with conn.transaction(), conn.cursor() as cur:
await cur.execute(
"""select state from instances
where id = %(id)s
and ((state = %(ready)s and expires_at < now()) or state = %(deleting)s)
for update""",
{
"id": instance_id,
"ready": InstanceState.READY.value,
"deleting": InstanceState.DELETING.value,
},
)
row = await cur.fetchone()
if row is None:
return None
if await _has_unfinished(cur, instance_id, TaskKind.DEPROVISION):
return None
if row["state"] == InstanceState.READY.value:
deleting = transition(InstanceState.READY, InstanceState.DELETING)
await cur.execute(
"update instances set state = %s, updated_at = now() where id = %s",
(deleting.value, instance_id),
)
return await _insert_task(cur, instance_id, TaskKind.DEPROVISION)
# --- Version drift ------------------------------------------------------------------
async def enqueue_upgrade(self, instance_id: UUID, run_after: datetime) -> int | None:
"""Enqueue an upgrade unless one is already outstanding. None if it is.
The guard is what keeps the fleet at `max_in_flight`. The work list is a query over
`chart_version`, and that column is only written *after* helm reports success — so
an instance stays on the work list for the entire duration of its own upgrade, and
for the hours it spends parked waiting for its 03:00 window. Without this check the
sweep enqueues one more upgrade for the same instance every 60 seconds, and
`max_in_flight=1` becomes sixty tasks an hour against one release.
`verify` counts as outstanding too: an upgrade whose verify has not reported is an
upgrade still in progress, and re-enqueueing it would race the probe that decides
whether the whole rollout halts.
"""
async with self._pool.connection() as conn:
async with conn.transaction(), conn.cursor() as cur:
if await _has_unfinished(cur, instance_id, TaskKind.UPGRADE, TaskKind.VERIFY):
return None
return await _insert_task(cur, instance_id, TaskKind.UPGRADE, run_after)
async def _has_unfinished(
cur: AsyncCursor[dict[str, Any]],
instance_id: UUID,
*kinds: TaskKind,
) -> bool:
"""Is a task of any of these kinds queued or running for this instance?
Takes the caller's cursor on purpose: the answer is only true for as long as the
transaction that asked, and checking on a separate connection would be a check against
a different snapshot than the insert that follows it.
"""
await cur.execute(
"""select 1 from tasks
where instance_id = %(id)s and kind = any(%(kinds)s) and state = any(%(unfinished)s)
limit 1""",
{
"id": instance_id,
"kinds": [k.value for k in kinds],
"unfinished": list(_UNFINISHED),
},
)
return await cur.fetchone() is not None
async def _insert_task(
cur: AsyncCursor[dict[str, Any]],
instance_id: UUID,
kind: TaskKind,
run_after: datetime | None = None,
) -> int:
"""Insert one task in the caller's transaction, carrying the current trace context.
`traceparent` is written here rather than left to `TaskRepo.enqueue` because these rows
are inserted inside a transaction the reconciler owns. Nothing propagates a trace
through a table on its own — see `obs.inject_traceparent`. It is null when the sweep is
not itself inside a span, which is fine and expected: a nullable column for an untraced
task.
"""
await cur.execute(
"""insert into tasks (instance_id, kind, run_after, traceparent)
values (%s, %s, coalesce(%s, now()), %s)
returning id""",
(instance_id, kind.value, run_after, inject_traceparent()),
)
row = await cur.fetchone()
assert row is not None # noqa: S101 - `returning` always yields a row or raises
return int(row["id"])
libs/svcforge_core/svcforge_core/repo/tasks.py
+219
View File
@@ -0,0 +1,219 @@
"""The queue.
The queue is a Postgres table, not Redis. The reason is one sentence: a task and the
instance state it describes must commit atomically. Split them across two stores and you
own a distributed commit problem that has no winning move — the process can die between
the two writes, and whichever you write first is the one that lies.
Everything else here follows from that.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any, Final
from uuid import UUID
from psycopg import AsyncConnection
from svcforge_core.domain.backoff import next_attempt_at
from svcforge_core.domain.models import Task, TaskKind
from svcforge_core.domain.states import LEGAL, InstanceState
from svcforge_core.obs import inject_traceparent
from svcforge_core.repo.db import DictPool
# Which states may legally become `failed`, derived from the domain's own table rather
# than restated here. Without this guard the UPDATE below would happily move a `deleted`
# instance to `failed` — a transition domain.transition() explicitly forbids, performed
# by raw SQL that never asks it. The state machine has to be the same one everywhere, or
# it is decoration.
_CAN_FAIL: Final[tuple[str, ...]] = tuple(
state.value for state, allowed in LEGAL.items() if InstanceState.FAILED in allowed
)
# The claim query. Do not "simplify" this into two statements.
#
# Postgres has no `UPDATE ... LIMIT`, so the row is chosen by a subquery. That subquery
# takes a row lock (`for update`) and steps over rows other workers already hold
# (`skip locked`) instead of blocking behind them — which is what makes N workers scale
# instead of queueing single-file behind the oldest task.
#
# The whole thing is ONE statement on purpose. Select-then-update as two statements
# leaves a gap in which a second worker reads the same id, and both provision. The gap
# is small, which means you will not hit it in testing and will hit it in production.
#
# The `with claimed as (...)` wrapper is the ONLY addition to the canonical form, and it
# changes nothing about the locking: the UPDATE and its `for update skip locked` subquery
# are still one statement, executed once. The outer SELECT only joins `instances.team`
# onto the row that was already claimed, so the worker can bind `team` to its log context
# before it has loaded anything. A data-modifying CTE runs exactly once and cannot claim
# twice. The alternative — a second SELECT for the team — would be a second round trip per
# task to fetch a column the database already had in hand.
_CLAIM_SQL = """
with claimed as (
update tasks set state='running', attempts=attempts+1, locked_by=%(worker)s, locked_at=now()
where id = (
select id from tasks
where state='queued' and run_after <= now()
order by run_after
for update skip locked
limit 1
)
returning *
)
select claimed.*, instances.team
from claimed join instances on instances.id = claimed.instance_id;
"""
class TaskRepo:
"""Reads and writes `tasks`. Claiming is the only interesting part."""
def __init__(self, pool: DictPool) -> None:
self._pool = pool
async def enqueue(
self,
conn: AsyncConnection[dict[str, Any]],
instance_id: UUID,
kind: TaskKind,
run_after: datetime | None = None,
) -> Task:
"""Insert a task inside the CALLER's transaction.
Takes `conn` so the API can insert the instance and enqueue its provision task in
one transaction. Rolling back must lose both, or you get an orphan task pointing
at an instance that was never committed.
The `traceparent` is captured here, at enqueue time, because this is the last
moment the caller's span context still exists. Trace context does NOT survive a
queue on its own: the worker picks the row up in a different process, minutes
later, with no ambient context. Writing the W3C traceparent onto the row is the
thread that lets the worker re-parent its span to the POST that caused it — the
difference between one trace spanning API → queue → helm and two unrelated ones.
"""
async with conn.cursor() as cur:
await cur.execute(
"""insert into tasks (instance_id, kind, run_after, traceparent)
values (%s, %s, coalesce(%s, now()), %s)
returning id, instance_id, kind, state, attempts, run_after, locked_by, last_error""",
(instance_id, kind.value, run_after, inject_traceparent()),
)
row = await cur.fetchone()
assert row is not None # noqa: S101 - `returning` always yields a row or raises
return Task.model_validate(row)
async def enqueue_standalone(
self,
instance_id: UUID,
kind: TaskKind,
run_after: datetime | None = None,
) -> int:
"""Enqueue in its own transaction, returning the new task id.
For callers with nothing to commit alongside it — the reconciler, tests. The
module specs disagree about enqueue's shape (Module 2 passes a conn, Module 4
does not); rather than making `conn` optional and quietly hiding the transaction
question, both callers get an honest method name.
"""
async with self._pool.connection() as conn:
task = await self.enqueue(conn, instance_id, kind, run_after)
return task.id
async def claim(self, worker_id: str) -> Task | None:
"""Claim one runnable task, or None if nothing is runnable.
`attempts` increments HERE, at claim time, not on failure. A worker that dies
mid-task without reporting anything has still burned an attempt, so a task that
reliably kills its worker cannot retry forever.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(_CLAIM_SQL, {"worker": worker_id})
row = await cur.fetchone()
return Task.model_validate(row) if row else None
async def complete(
self,
task_id: int,
conn: AsyncConnection[dict[str, Any]] | None = None,
) -> None:
"""Mark done. Pass `conn` to commit alongside the caller's instance update.
Completing the task and recording what it accomplished belong in one transaction:
commit them separately and a crash in between leaves a task marked done whose
work never landed, or work that landed and will be redone.
"""
sql = "update tasks set state='done', locked_by=null where id = %s"
if conn is not None:
async with conn.cursor() as cur:
await cur.execute(sql, (task_id,))
return
async with self._pool.connection() as own, own.cursor() as cur:
await cur.execute(sql, (task_id,))
async def fail(self, task_id: int, err: str, max_attempts: int = 5) -> None:
"""Retry with backoff, or give up.
Under max_attempts: back to 'queued' with run_after pushed out by exponential
backoff with full jitter. Jitter matters — a cluster-wide outage fails every task
at once, and without it every worker retries in the same instant, forever.
At max_attempts: 'failed', and the error is copied onto the instance so the tenant
can see it. A dead-letter state, not an infinite retry: a task that cannot succeed
must stop and become someone's problem.
"""
now = datetime.now(UTC)
async with self._pool.connection() as conn:
async with conn.transaction(), conn.cursor() as cur:
await cur.execute(
"select attempts, instance_id from tasks where id = %s for update",
(task_id,),
)
row = await cur.fetchone()
if row is None:
return
attempts = int(row["attempts"])
instance_id = row["instance_id"]
if attempts < max_attempts:
await cur.execute(
"""update tasks
set state='queued', locked_by=null, locked_at=null,
last_error=%s, run_after=%s
where id = %s""",
(err[-2000:], next_attempt_at(attempts - 1, now=now), task_id),
)
return
await cur.execute(
"""update tasks
set state='failed', locked_by=null, locked_at=null, last_error=%s
where id = %s""",
(err[-2000:], task_id),
)
# `state = any(%s)` keeps this honest: a deprovision that exhausts its
# retries against an already-deleted instance records nothing rather than
# resurrecting it into `failed`.
await cur.execute(
"""update instances set error=%s, state=%s, updated_at=now()
where id=%s and state = any(%s)""",
(err[-2000:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
)
async def reset_expired_leases(self, lease_seconds: int) -> int:
"""Return tasks whose worker died back to the queue. Called by the reconciler.
No distributed lock survives a power cut. A worker that is SIGKILLed leaves
`state='running'` and `locked_by` set with nobody running it, and that row would
sit there forever. The lease is the only thing that recovers it: not a lock, a
timeout. This is why `locked_at` exists.
"""
async with self._pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"""update tasks
set state='queued', locked_by=null, locked_at=null
where state='running'
and locked_at < now() - make_interval(secs => %s)""",
(lease_seconds,),
)
return cur.rowcount
libs/svcforge_core/svcforge_core/settings.py
+91
View File
@@ -0,0 +1,91 @@
"""Typed configuration. Environment in, validated object out, fails fast at startup.
The whole point: a missing or malformed DSN kills the process on line one with a readable
error, instead of surfacing as a PoolTimeout twenty minutes into a provision.
"""
from __future__ import annotations
from pathlib import Path
from pydantic import Field, PostgresDsn, RedisDsn
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""Every knob svcforge has. Read once, at startup, and passed down explicitly."""
model_config = SettingsConfigDict(
env_prefix="SVCFORGE_",
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
frozen=True,
)
# --- Postgres -----------------------------------------------------------------
# Transaction pooler (6543 on Supabase). Everything the services do at runtime.
pg_dsn: PostgresDsn
# Session pooler (5432). Migrations and psql only: DDL and advisory locks need a
# session that outlives a single transaction.
pg_dsn_session: PostgresDsn | None = None
pool_min_size: int = Field(default=1, ge=0)
pool_max_size: int = Field(default=5, ge=1)
# --- Redis (derived state only; never the source of truth) ---------------------
redis_dsn: RedisDsn | None = None
# --- API ----------------------------------------------------------------------
jwks_url: str | None = None
jwt_audience: str = "svcforge"
jwt_issuer: str | None = None
# Dev escape hatch: skip JWT verification. Refused in prod by check_production().
auth_disabled: bool = False
# --- Worker -------------------------------------------------------------------
worker_id: str = Field(default="worker-local", min_length=1)
worker_concurrency: int = Field(default=4, ge=1)
poll_interval_s: float = Field(default=5.0, gt=0)
max_attempts: int = Field(default=5, ge=1)
lease_seconds: int = Field(default=300, ge=1)
# --- Reconciler ---------------------------------------------------------------
reconcile_interval_s: float = Field(default=60.0, gt=0)
# --- Catalog / helm -----------------------------------------------------------
catalog_path: Path = Path("catalog.yaml")
helm_bin: str = "helm"
kubectl_bin: str = "kubectl"
helm_timeout_s: float = Field(default=300.0, gt=0)
# --- CLI ----------------------------------------------------------------------
# The CLI is an API client and nothing more. It gets a URL and a token; it does not
# get a DSN, because the moment a human can reach the database directly, someone will
# "just fix one row" and the state machine stops being true.
api_url: str = "http://localhost:8000"
api_token: str | None = None
# --- Observability ------------------------------------------------------------
log_level: str = "info"
log_json: bool = True
otel_endpoint: str | None = None
service_name: str = "svcforge"
# The worker and reconciler have no HTTP server of their own, so they start a tiny one
# just for /metrics. 9000 matches the chart's PodMonitor; change both or neither.
metrics_port: int = Field(default=9000, ge=1, le=65535)
@property
def migration_dsn(self) -> str:
"""Migrations need a session-mode connection; fall back to the runtime DSN locally."""
return str(self.pg_dsn_session or self.pg_dsn)
def check_production(self) -> None:
"""Refuse the dev escape hatches when they would matter."""
if self.auth_disabled:
raise ValueError("SVCFORGE_AUTH_DISABLED=true is refused outside local development")
def load_settings() -> Settings:
"""Read the environment. Raises ValidationError — loudly — if anything is missing."""
return Settings() # type: ignore[call-arg] # pydantic-settings fills these from env
migrations/001_init.sql
+36
View File
@@ -0,0 +1,36 @@
-- 001_init.sql — instances and tasks.
--
-- Forward-only. There is no down script. If this is wrong, 002 fixes it.
create table instances (
id uuid primary key default gen_random_uuid(),
team text not null,
service_type text not null, -- 'elasticsearch' | 'redis' | ...
size text not null, -- 'small' | 'medium'
state text not null,
namespace text not null,
release_name text not null unique, -- helm release; the idempotency anchor
chart_version text not null, -- what's ACTUALLY deployed. day-2 hinges on this column.
endpoint text,
error text,
expires_at timestamptz,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table tasks (
id bigserial primary key,
instance_id uuid not null references instances(id) on delete cascade,
kind text not null, -- 'provision'|'deprovision'|'upgrade'|'verify'
state text not null default 'queued', -- queued|running|done|failed
attempts int not null default 0,
run_after timestamptz not null default now(), -- backoff lands here
locked_by text,
locked_at timestamptz,
last_error text,
created_at timestamptz not null default now()
);
-- Partial index: the claim query only ever looks at queued rows. Keeping the index
-- to that subset means it stays small no matter how much history `tasks` accumulates.
create index tasks_runnable on tasks (run_after) where state = 'queued';
migrations/003_day2.sql
+16
View File
@@ -0,0 +1,16 @@
-- 003_day2.sql — the whole of day 2: two columns and one query.
--
-- Forward-only, expand/contract. Both additions are nullable / defaulted, so the old
-- code keeps running against the new schema between the migrate and the deploy. That
-- ordering is not optional: migrate first, deploy second, and a column the running
-- code has never heard of must not break it.
alter table instances add column maintenance_window text; -- '0 3 * * 0|Asia/Ho_Chi_Minh', null = any time
-- One row per service type, one column that matters. `halted` is what stops a bad
-- chart after the first tenant instead of after all of them, and it is cleared by
-- hand with SQL — an automatic un-halt would just resume breaking things.
create table catalog_versions (
service_type text primary key,
rollout_state text not null default 'ok' -- 'ok' | 'halted'
);
migrations/004_traceparent.sql
+15
View File
@@ -0,0 +1,15 @@
-- 004_traceparent.sql — carry the trace across the queue.
--
-- A trace is a chain of span contexts. `POST /v1/instances` commits a row and returns;
-- a worker in another pod claims that row minutes later. Nothing propagates the context
-- across that gap, because the gap is a table. So the context rides in the table: the API
-- writes the W3C traceparent it is currently inside, and the worker extracts it at claim
-- and makes its span a child of the API's. Skip this and Tempo shows two unrelated traces
-- for one provision, which is worse than no tracing — it looks like it works.
--
-- Nullable, no default, no backfill: expand/contract done right. Old rows have no
-- traceparent and never will; a worker running the previous image ignores a column it has
-- never heard of. Migrate first, deploy second, and neither step needs the other to have
-- happened.
alter table tasks add column traceparent text; -- '00-<32 hex>-<16 hex>-01', null = untraced
pyproject.toml
+98
View File
@@ -0,0 +1,98 @@
[project]
name = "svcforge"
version = "0.1.0"
description = "X-as-a-Service control plane — reference implementation"
requires-python = ">=3.12"
dependencies = [
"svcforge-core",
"fastapi>=0.115",
"uvicorn[standard]>=0.32",
"psycopg[binary,pool]>=3.2",
"pyjwt[crypto]>=2.9",
"httpx>=0.27",
"croniter>=3.0",
"tzdata>=2024.2",
"structlog>=24.4",
"prometheus-client>=0.21",
"redis>=5.2",
"typer>=0.15",
"opentelemetry-api>=1.28",
"opentelemetry-sdk>=1.28",
"opentelemetry-instrumentation-fastapi>=0.49b0",
"opentelemetry-instrumentation-psycopg>=0.49b0",
]
[dependency-groups]
dev = [
"pytest>=8.3",
"pytest-asyncio>=0.24",
"pytest-cov>=6.0",
"mypy>=1.13",
"ruff>=0.8",
"hypothesis>=6.122",
"pre-commit>=4.0",
"testcontainers[postgres]>=4.9",
"types-pyyaml>=6.0.12.20260518",
]
[project.scripts]
svcforge = "services.cli.main:app"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["services"]
[tool.uv.sources]
# editable here is a DEVELOPMENT convenience: source edits are visible without a
# reinstall. The Dockerfiles deliberately override it with `uv sync --no-editable`,
# because an editable install in an image resolves imports to /app/libs and ships a
# path, not a package.
svcforge-core = { path = "libs/svcforge_core", editable = true }
[tool.ruff]
line-length = 110
target-version = "py312"
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "ANN", "S", "C4", "RUF"]
[tool.ruff.lint.isort]
# svcforge_core lives under libs/, so isort cannot infer it is ours.
known-first-party = ["svcforge_core", "services"]
[tool.ruff.lint.per-file-ignores]
# Tests may assert, and fixtures shadow names by design.
"tests/**" = ["S101"]
# The e2e tests drive the real `kubectl`/`helm`/`pgrep` off PATH — that is the whole point
# of them, and pinning absolute paths would make them pass on one machine only. Scoped to
# this directory so S603/S607 keep guarding the application code, where `team` is tenant
# input that reaches a helm release name.
"tests/e2e/**" = ["S101", "S603", "S607"]
[tool.mypy]
strict = true
python_version = "3.12"
warn_unreachable = true
[[tool.mypy.overrides]]
module = ["testcontainers.*", "croniter.*"]
ignore_missing_imports = true
# The OTLP exporter is an optional runtime dependency: in the cluster the API runs under
# `opentelemetry-instrument`, which brings its own. obs.py imports it inside a try/except
# and degrades to in-process traces without it, so mypy must not require it to be installed.
[[tool.mypy.overrides]]
module = ["opentelemetry.exporter.*"]
ignore_missing_imports = true
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-q --strict-markers"
asyncio_mode = "auto"
markers = [
"slow: >1s",
"e2e: needs a cluster and real helm",
]
scripts/__init__.py
View File
scripts/bump-digests.sh
Executable
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
#
# CI's last act.
#
# Resolves the digest each service's commit-SHA tag points at, writes those digests into
# deploy/chart/values.yaml, and commits. That commit is the deploy: ArgoCD is watching
# master and picks it up. This script does not, and must not, talk to the cluster.
#
# Called by .gitea/workflows/ci.yaml on master only. Runnable by hand for a re-bump:
# REGISTRY=gitea.oci-oci.duckdns.org IMAGE_NS=gitea_admin IMAGE_TAG=<sha> ./scripts/bump-digests.sh
#
# -e a failed inspect must not lead to committing a stale digest
# -u an unset REGISTRY would silently resolve the wrong image
# -o pipefail the digest comes out of a pipe; without this, a failing inspect that pipes
# into a successful grep exits 0 and writes garbage
set -euo pipefail
: "${REGISTRY:?REGISTRY must be set}"
: "${IMAGE_NS:?IMAGE_NS must be set}"
: "${IMAGE_TAG:?IMAGE_TAG must be set (the commit sha the images were built from)}"
SERVICES=(api worker reconciler)
CHART_VALUES="deploy/chart/values.yaml"
# yq, pinned by digest. Not python+pyyaml: a yaml round-trip strips every comment in
# values.yaml, and those comments are the only thing explaining why the digests are there.
# yq edits in place and leaves the rest of the file alone.
YQ_IMAGE="mikefarah/yq:4.44.6@sha256:b1d117c609ba990436ad1649299e2f6c378f62cb562caf30b6f2fb6144713422"
WORKDIR="$(mktemp -d)"
cleanup() {
rm -rf "${WORKDIR}"
}
trap cleanup EXIT
yq() {
docker run --rm -v "${PWD}:/work" -w /work -u "$(id -u):$(id -g)" "${YQ_IMAGE}" "$@"
}
echo "==> resolving digests for tag ${IMAGE_TAG}"
for svc in "${SERVICES[@]}"; do
image="${REGISTRY}/${IMAGE_NS}/svcforge-${svc}"
digest="$(docker buildx imagetools inspect "${image}:${IMAGE_TAG}" --format '{{.Manifest.Digest}}')"
# Defence against a silently empty inspect. Without this, `yq` would happily write an
# empty digest and the chart's own guard would fail the release later, further from
# the cause.
if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "!! ${svc}: refusing to write a non-digest: '${digest}'" >&2
exit 1
fi
echo " ${svc} -> ${digest}"
echo "${digest}" > "${WORKDIR}/${svc}.digest"
done
echo "==> writing ${CHART_VALUES}"
for svc in "${SERVICES[@]}"; do
digest="$(cat "${WORKDIR}/${svc}.digest")"
# env(...) rather than string interpolation: a digest is attacker-controlled only in
# theory, but yq expression injection is not a thing worth leaving open.
DIGEST="${digest}" yq -i ".image.${svc}.digest = strenv(DIGEST)" "${CHART_VALUES}"
done
if git diff --quiet -- "${CHART_VALUES}"; then
echo "==> no digest changed; nothing to commit"
exit 0
fi
echo "==> committing"
git config user.name "svcforge-ci"
git config user.email "ci@svcforge.invalid"
git add "${CHART_VALUES}"
git commit -m "ci: bump image digests to ${IMAGE_TAG}
Built and scanned by ${IMAGE_TAG}. ArgoCD syncs from this commit.
[skip ci]"
git push origin HEAD:master
echo "==> done. ArgoCD owns it from here."
scripts/load.py
+130
View File
@@ -0,0 +1,130 @@
"""Throwaway load generator. Enqueue N instances, watch the queue drain, print three numbers.
The point is not a benchmark. It is to find the ceiling on purpose, in a place where finding
it is free, so that the number in RUNBOOK.md comes from an observation instead of a guess.
The ceiling you are looking for is arithmetic, not mysterious:
total connections = (api_replicas + worker_replicas) x pool_max_size
Supabase's free-tier pooler has a small connection budget. Cross it and the failure does not
look like "too many connections" — it looks like slow claims, then PoolTimeout, then a queue
that grows while every worker looks idle. Once you have watched it once, you recognise it in
two seconds instead of an hour.
Usage:
python -m scripts.load --count 200 --watch
python -m scripts.load --count 200 --direct # skip the API, enqueue straight to the DB
"""
from __future__ import annotations
import argparse
import asyncio
import time
from datetime import UTC, datetime
from uuid import uuid4
import psycopg
from psycopg.rows import dict_row
from svcforge_core.settings import load_settings
async def _seed_direct(dsn: str, count: int) -> float:
"""Insert `count` instances + provision tasks. Returns seconds taken.
--direct exists to separate two questions that a single POST run conflates: "how fast can
the API accept work" and "how fast can workers drain it". Measure them apart or you will
tune the wrong one.
"""
started = time.monotonic()
async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
async with conn.transaction(), conn.cursor() as cur:
for _ in range(count):
iid = uuid4()
await cur.execute(
"""insert into instances (id, team, service_type, size, state, namespace,
release_name, chart_version)
values (%s, 'loadtest', 'elasticsearch', 'small', 'requested',
'tenant-loadtest', %s, '21.3.19')""",
(iid, f"loadtest-elasticsearch-{str(iid)[:8]}"),
)
await cur.execute(
"insert into tasks (instance_id, kind) values (%s, 'provision')",
(iid,),
)
return time.monotonic() - started
async def _depth(dsn: str) -> dict[str, int]:
async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
cur = await conn.execute("select state, count(*) as n from tasks group by 1")
return {str(r["state"]): int(r["n"]) for r in await cur.fetchall()}
async def _watch(dsn: str, timeout_s: float) -> None:
"""Print queue depth once a second until it drains. The slope is the number you want."""
started = time.monotonic()
peak = 0
print(f"{'t(s)':>6} {'queued':>7} {'running':>8} {'done':>6} {'failed':>7} slope/s")
prev_done, prev_t = 0, started
while time.monotonic() - started < timeout_s:
d = await _depth(dsn)
queued, running = d.get("queued", 0), d.get("running", 0)
done, failed = d.get("done", 0), d.get("failed", 0)
peak = max(peak, queued + running)
now = time.monotonic()
slope = (done - prev_done) / max(now - prev_t, 1e-9)
prev_done, prev_t = done, now
print(f"{now - started:6.1f} {queued:7d} {running:8d} {done:6d} {failed:7d} {slope:7.1f}")
if queued == 0 and running == 0:
elapsed = now - started
print(f"\ndrained in {elapsed:.1f}s peak depth {peak} throughput {done / elapsed:.1f} task/s")
if failed:
print(f"WARNING: {failed} tasks failed — the number above is not a clean drain")
return
await asyncio.sleep(1.0)
print(f"\nstill draining after {timeout_s}s — that IS the result. Record it.")
async def _amain() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--count", type=int, default=200)
ap.add_argument("--direct", action="store_true", help="enqueue via SQL instead of the API")
ap.add_argument("--watch", action="store_true", help="poll queue depth until drained")
ap.add_argument("--timeout", type=float, default=600.0)
ap.add_argument("--cleanup", action="store_true", help="delete loadtest rows and exit")
args = ap.parse_args()
settings = load_settings()
dsn = settings.pg_dsn.unicode_string()
if args.cleanup:
async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as conn:
await conn.execute("delete from instances where team = 'loadtest'") # tasks cascade
print("loadtest rows deleted")
return
if not args.direct:
raise SystemExit(
"POST mode needs a token; use --direct for the drain measurement, or drive the API "
"with k6 (one dependency, not two — do not add locust for this)."
)
print(f"seeding {args.count} instances at {datetime.now(UTC).isoformat()} ...")
took = await _seed_direct(dsn, args.count)
print(f"enqueued {args.count} in {took:.2f}s ({args.count / took:.0f}/s)\n")
if args.watch:
await _watch(dsn, args.timeout)
print("\nremember: `python -m scripts.load --cleanup` when you are done.")
if __name__ == "__main__":
asyncio.run(_amain())
scripts/redis_budget.py
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Project month-end Redis command burn from the live counter. Exit 1 if it blows the budget.
$ python3 scripts/redis_budget.py
$ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000
Upstash's free tier is 500,000 commands/month, which sounds enormous and is not:
500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
0.19 commands per second is the entire engineering constraint. One worker polling Redis
every five seconds spends 518,400/month — the whole budget, to learn nothing. That is why
Redis is only ever on the request path here, and why this script exists: the rule is easy
to state and invisible to violate. A `cache.get()` added inside the reconciler's
per-instance loop is one line in review and 2,160,000 commands/month in production.
**Why a projection and not an alarm on the counter.** Exhausting the budget is a slow,
silent failure with a cliff at the end: nothing degrades, nothing pages, every call
succeeds, and then the month rolls over and every Redis call starts erroring at once. By
then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible
on day two, which is the only time it is cheap to fix.
Stdlib only, on purpose — this is meant to run from CI, from a laptop, or from inside a
pod that has nothing but python3, without an environment to activate first.
"""
from __future__ import annotations
import argparse
import sys
import time
import urllib.request
from urllib.parse import urlparse
# A 30-day month. Upstash bills on a calendar month; 30 days is the honest rounding and
# errs slightly pessimistic on the long ones, which is the correct direction for a budget.
_MONTH_S = 30 * 24 * 60 * 60
_FREE_TIER_BUDGET = 500_000
_COMMANDS_METRIC = "svcforge_redis_commands_total"
_START_TIME_METRIC = "process_start_time_seconds"
class BudgetError(RuntimeError):
"""The metrics endpoint did not give us enough to project from."""
def scrape(url: str, timeout_s: float = 5.0) -> str:
"""GET the Prometheus text exposition. http/https only."""
if urlparse(url).scheme not in ("http", "https"):
raise BudgetError(f"refusing to fetch a non-http(s) url: {url}")
# S310 is satisfied by the scheme check above: this cannot open file:// or ftp://.
with urllib.request.urlopen(url, timeout=timeout_s) as resp: # noqa: S310
body: str = resp.read().decode("utf-8")
return body
def _parse_sample(line: str) -> tuple[str, dict[str, str], float] | None:
"""One exposition line -> (name, labels, value). None for comments and blanks.
A deliberately small parser rather than prometheus_client's: importing the library
would mean this script only runs where the app's venv is already active, which is
exactly where you least need to check the budget.
"""
line = line.strip()
if not line or line.startswith("#"):
return None
head, _, raw_value = line.rpartition(" ")
if not head:
return None
try:
value = float(raw_value)
except ValueError:
return None
name, brace, rest = head.partition("{")
labels: dict[str, str] = {}
if brace:
for pair in rest.rstrip("}").split(","):
key, eq, val = pair.partition("=")
if eq:
labels[key.strip()] = val.strip().strip('"')
return name.strip(), labels, value
def collect(text: str) -> tuple[dict[str, float], float]:
"""Extract per-op command totals and the process start time from a scrape."""
per_op: dict[str, float] = {}
started_at: float | None = None
for line in text.splitlines():
parsed = _parse_sample(line)
if parsed is None:
continue
name, labels, value = parsed
# prometheus_client exposes counters with a `_total` suffix already; tolerate both
# spellings so this keeps working if the client library changes its mind.
if name in (_COMMANDS_METRIC, _COMMANDS_METRIC.removesuffix("_total")):
per_op[labels.get("op", "unknown")] = value
elif name == _START_TIME_METRIC:
started_at = value
if not per_op:
raise BudgetError(
f"{_COMMANDS_METRIC} is not exposed. Either the API never imported "
f"svcforge_core.adapters.redis, or you are scraping the wrong process."
)
if started_at is None:
raise BudgetError(
f"{_START_TIME_METRIC} is missing, so there is no window to project over. "
f"It comes from prometheus_client's default collector."
)
return per_op, started_at
def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int:
"""Print the projection. Returns the process exit code."""
elapsed_s = max(1.0, now - started_at)
total = sum(per_op.values())
rate = total / elapsed_s
projected = rate * _MONTH_S
print(f"window {elapsed_s / 3600:.2f} h since process start")
print(f"commands {total:,.0f}")
for op, value in sorted(per_op.items(), key=lambda kv: -kv[1]):
share = (value / total * 100) if total else 0.0
print(f" {op:<14}{value:>12,.0f} ({share:.1f}%)")
print(f"rate {rate:.4f} /s (budget allows {budget / _MONTH_S:.4f} /s sustained)")
print(f"projected {projected:,.0f} / month")
print(f"budget {budget:,} / month")
if total < 100:
# Extrapolating a month from a handful of commands is astrology. Say so rather than
# printing a confident number derived from six samples.
print("verdict INCONCLUSIVE — fewer than 100 commands; let it run longer")
return 0
if projected >= budget:
headroom = projected / budget
print(f"verdict OVER BUDGET — {headroom:.1f}x. Find the Redis call in a loop.")
return 1
share = projected / budget * 100
print(f"verdict OK — {share:.1f}% of budget, {budget / projected:.1f}x headroom")
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint")
parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month")
args = parser.parse_args(argv)
try:
per_op, started_at = collect(scrape(args.url))
except BudgetError as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
except OSError as exc:
print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr)
return 2
return report(per_op, started_at, args.budget, time.time())
if __name__ == "__main__":
raise SystemExit(main())
services/__init__.py
+1
View File
@@ -0,0 +1 @@
"""svcforge services: api, worker, reconciler."""
services/api/Dockerfile
+55
View File
@@ -0,0 +1,55 @@
# syntax=docker/dockerfile:1.10
#
# svcforge api. Build from the REPO ROOT:
# docker buildx build -f services/api/Dockerfile -t svcforge/api:dev .
# `COPY ../..` is illegal, so the context must be the root. There is no other option.
#
# Two syncs, not one: deps change rarely and our own code changes every commit, so the
# expensive layer (third-party wheels) must land before the cheap one (our source).
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
WORKDIR /app
# --- layer 1: third-party dependencies only -------------------------------------------
# --no-install-project skips the root; --no-install-package skips our path dependency.
# Without the latter, uv would try to build svcforge-core here, where its source is not
# yet in the context, and the build would fail.
COPY pyproject.toml uv.lock ./
COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable \
--no-install-project --no-install-package svcforge-core
# --- layer 2: our code ----------------------------------------------------------------
COPY libs/ libs/
COPY services/api/ services/api/
COPY catalog.yaml ./
# --no-editable is what turns svcforge-core into a real wheel in site-packages.
# pyproject.toml declares it `editable = true` for local dev; an editable install in an
# image points at /app/libs, which is a source tree that need not survive the final stage.
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable && \
/app/.venv/bin/python -c 'import svcforge_core, sys; \
p = svcforge_core.__file__; \
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
# --- runtime --------------------------------------------------------------------------
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
ARG BUILD_SHA=unknown
LABEL org.opencontainers.image.title="svcforge-api" \
org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
org.opencontainers.image.revision="${BUILD_SHA}"
RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
WORKDIR /app
COPY --from=builder --chown=10001:10001 /app /app
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1
USER 10001
ENTRYPOINT ["python", "-m", "services.api"]
services/api/__init__.py
+1
View File
@@ -0,0 +1 @@
"""The api service: HTTP transport over the domain."""
services/api/__main__.py
+29
View File
@@ -0,0 +1,29 @@
"""`python -m services.api` — what the container's ENTRYPOINT runs.
One uvicorn worker per container, on purpose. Replicas are Kubernetes' job: `--workers N`
forks processes the orchestrator cannot see, size, or drain, and it breaks the in-process
Prometheus registry that /metrics depends on.
"""
from __future__ import annotations
import uvicorn
from svcforge_core.settings import load_settings
def main() -> None:
"""Load settings (fail fast if the env is wrong), then serve."""
settings = load_settings()
uvicorn.run(
"services.api.main:app",
factory=True,
host="0.0.0.0", # noqa: S104 - a container binds all interfaces; the pod is the boundary
port=8000,
log_level=settings.log_level,
access_log=False, # structlog owns request logging (Module 7); two sources would double it
)
if __name__ == "__main__":
main()
services/api/deps.py
+171
View File
@@ -0,0 +1,171 @@
"""Dependency injection: how a handler gets a pool, a repo, a catalog, and a team.
Everything expensive — the pool, the JWKS client, the parsed catalog — is built once in
`lifespan` and parked on `app.state`. These functions only hand it out. A `Depends` that
does I/O per request is a `Depends` that does that I/O on every request forever.
"""
from __future__ import annotations
import asyncio
from typing import Annotated, Any
import jwt
from fastapi import Depends, HTTPException, Request, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from jwt import PyJWKClient
from svcforge_core.domain.models import CatalogEntry
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings
# The algorithm allow-list is the whole point of naming algorithms explicitly.
# `jwt.decode(..., algorithms=...)` without it accepts whatever the *token* claims in its
# own header — including `none`, and including HS256 verified with the RSA public key as
# an HMAC secret. Both are forgery. The list is not configuration.
ALLOWED_ALGORITHMS = ["RS256"]
# What `auth_disabled` returns. Settings.check_production() refuses that flag in prod.
DEV_TEAM = "platform"
TEAM_CLAIM = "team"
# auto_error=False is load-bearing. HTTPBearer(auto_error=True) answers a *missing*
# Authorization header with 403, not 401 — an old FastAPI wart. The spec (and every
# client that knows what to do about it) wants 401, so the error is raised here.
_bearer = HTTPBearer(auto_error=False)
def _unauthorized() -> HTTPException:
"""One shape for every auth failure.
Expired, wrong issuer, wrong audience, bad signature, malformed, no header: all the
same 401 with the same body. Telling a caller *which* one turns the endpoint into an
oracle they can tune a forgery against.
"""
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail={"code": "unauthorized", "message": "invalid or missing credentials"},
headers={"WWW-Authenticate": "Bearer"},
)
def get_settings(request: Request) -> Settings:
"""The Settings that create_app() was handed."""
settings: Settings = request.app.state.settings
return settings
async def get_pool(request: Request) -> DictPool:
"""Return the pool that lifespan put on app.state."""
pool: DictPool = request.app.state.pool
return pool
def get_catalog(request: Request) -> dict[str, CatalogEntry]:
"""The catalog, parsed once at startup.
Read from disk per request and a mid-flight edit to catalog.yaml changes the answer
between two requests of the same deploy. Load it at startup; a change is a restart.
"""
catalog: dict[str, CatalogEntry] = request.app.state.catalog
return catalog
def get_instance_repo(pool: Annotated[DictPool, Depends(get_pool)]) -> InstanceRepo:
"""An InstanceRepo bound to the app's pool. Cheap: it is a handle, not a connection."""
return InstanceRepo(pool)
def get_task_repo(pool: Annotated[DictPool, Depends(get_pool)]) -> TaskRepo:
"""A TaskRepo bound to the app's pool."""
return TaskRepo(pool)
async def get_current_team(
request: Request,
creds: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)],
settings: Annotated[Settings, Depends(get_settings)],
) -> str:
"""Verify the JWT against the cached JWKS. Check aud/iss/exp and the alg allow-list.
Returns the team claim. Raises HTTPException(401) on any failure — never leaks why.
"""
if settings.auth_disabled:
return DEV_TEAM
if creds is None or not creds.credentials:
raise _unauthorized()
jwks_client: PyJWKClient | None = getattr(request.app.state, "jwks_client", None)
if jwks_client is None:
# Auth is on but there is no key source. Fail closed. Answering 500 here would be
# honest about the cause and would also let a misconfigured deploy be told apart
# from a bad token; 401 is the same answer a forger gets.
raise _unauthorized()
try:
# PyJWKClient keeps its own TTL cache, so this is a dict lookup on the hot path.
# It is only blocking on a cache MISS (key rotation) — hence to_thread, which
# costs a thread hop we take a handful of times a day rather than an event loop
# stalled on someone else's HTTP call once per rotation.
signing_key = await _signing_key(jwks_client, creds.credentials)
claims: dict[str, Any] = jwt.decode(
creds.credentials,
signing_key.key,
algorithms=ALLOWED_ALGORITHMS,
audience=settings.jwt_audience,
issuer=settings.jwt_issuer,
options={
"require": ["exp", "aud", "iss"],
"verify_exp": True,
"verify_aud": True,
"verify_iss": settings.jwt_issuer is not None,
"verify_signature": True,
},
)
except Exception as exc: # deliberate catch-all: every failure becomes one opaque 401
raise _unauthorized() from exc
team = claims.get(TEAM_CLAIM)
if not isinstance(team, str) or not team:
raise _unauthorized()
return team
async def _signing_key(client: PyJWKClient, token: str) -> jwt.PyJWK:
"""Fetch the signing key off the event loop.
PyJWKClient.get_signing_key_from_jwt() does a synchronous urlopen on a cache miss.
Called directly from `async def`, that blocks the loop — every other in-flight request
on this worker stops until the identity provider answers, and if it hangs, so does the
pod, and /readyz keeps saying it is fine.
"""
return await asyncio.to_thread(client.get_signing_key_from_jwt, token)
async def rate_limit(team: Annotated[str, Depends(get_current_team)]) -> None:
"""Per-team rate limiting. Seam only — Module 10 fills this in (Redis, Lua, token bucket).
It exists now, wired into the routes, so that turning it on is an edit to one function
body rather than a change to every handler signature.
"""
return None
async def idempotency_key(request: Request) -> str | None:
"""`Idempotency-Key` handling. Seam only — Module 10 fills this in (Redis store).
Until then the real idempotency anchor is `instances.release_name`, which is unique in
the schema and deterministic from (team, service_type, id).
"""
return request.headers.get("Idempotency-Key")
PoolDep = Annotated[DictPool, Depends(get_pool)]
TeamDep = Annotated[str, Depends(get_current_team)]
InstanceRepoDep = Annotated[InstanceRepo, Depends(get_instance_repo)]
TaskRepoDep = Annotated[TaskRepo, Depends(get_task_repo)]
CatalogDep = Annotated[dict[str, CatalogEntry], Depends(get_catalog)]
services/api/main.py
+121
View File
@@ -0,0 +1,121 @@
"""The app factory and its lifespan.
`create_app(settings)` is a factory, not a module-level `app = FastAPI()`, for one reason:
a test needs an app pointed at a throwaway Postgres, and an import-time app reads the real
environment at import time — before any fixture can say otherwise.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from jwt import PyJWKClient
from services.api.models import ErrorBody
from services.api.routes import health, instances
from svcforge_core.domain.catalog import load_catalog
from svcforge_core.repo.db import make_pool
from svcforge_core.settings import Settings, load_settings
log = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Open the pool, yield, close the pool.
A lifespan context, not the deprecated startup/shutdown event decorators: those cannot
express "this resource lives for exactly as long as the app", and give you no place to
put the teardown next to the setup. Closing the pool matters — an unclosed pool means
connections linger server-side after SIGTERM, and on a pooled Postgres with a small
connection budget a few rolling deploys exhaust it.
(The old decorator's name is spelled nowhere in this package on purpose: CI greps for
the literal string, and a comment quoting it fails the gate just as loudly as a call.)
"""
settings: Settings = app.state.settings
app.state.catalog = load_catalog(settings.catalog_path)
pool = make_pool(str(settings.pg_dsn), settings.pool_min_size, settings.pool_max_size)
# wait=True fails NOW, loudly, if the DSN is wrong — instead of at the first request,
# as a PoolTimeout, in front of a user.
await pool.open(wait=True)
app.state.pool = pool
# The pool is open from here on, so everything below is inside the try: an exception
# in JWKS setup must still close it, or a crash-looping pod leaks a connection per
# restart until the database refuses new ones.
try:
if settings.jwks_url and not settings.auth_disabled:
client = PyJWKClient(settings.jwks_url, cache_keys=True, lifespan=300)
app.state.jwks_client = client
# Warm the cache off the loop so the first authenticated request does not pay
# a blocking urlopen. Best-effort: a slow identity provider must not stop the
# pod from starting — a cache miss later just costs one to_thread hop.
try:
await asyncio.to_thread(client.get_signing_keys)
except Exception: # deliberate catch-all: startup must not hinge on the IdP being up
log.warning("JWKS warm-up failed; keys will be fetched on first use", exc_info=True)
else:
app.state.jwks_client = None
yield
finally:
await pool.close()
async def _http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Render HTTPException bodies as ErrorBody, so every error has one shape.
Handlers raise `detail={"code": ..., "message": ...}`; FastAPI's default would nest
that under `{"detail": {...}}`. Plain-string details (raised by FastAPI itself, e.g.
a 405) are wrapped so clients never have to branch on the body's type.
"""
assert isinstance(exc, HTTPException) # noqa: S101 - registered only for HTTPException
# Widened to object deliberately. Starlette types `detail` as str, but FastAPI passes
# through whatever a handler raised — our handlers raise dicts. Narrowing off the
# declared type would make mypy call the dict branch unreachable and delete it.
detail: object = exc.detail
if isinstance(detail, dict) and "code" in detail and "message" in detail:
body = ErrorBody(code=str(detail["code"]), message=str(detail["message"]))
else:
body = ErrorBody(code=f"http_{exc.status_code}", message=str(detail))
return JSONResponse(status_code=exc.status_code, content=body.model_dump(), headers=exc.headers)
def create_app(settings: Settings | None = None) -> FastAPI:
"""App factory: lifespan, routers, exception handler, /metrics."""
settings = settings or load_settings()
app = FastAPI(
title="svcforge",
version="0.1.0",
summary="X-as-a-Service control plane",
lifespan=lifespan,
)
app.state.settings = settings
# /metrics is a normal route on health.router, not an app.mount — see health.metrics
# for why the mount does not actually serve a bare /metrics.
app.include_router(health.router)
app.include_router(instances.router)
app.add_exception_handler(HTTPException, _http_exception_handler)
return app
def app() -> FastAPI:
"""Entry point for `uvicorn services.api.main:app --factory`."""
return create_app()
if __name__ == "__main__": # pragma: no cover
import uvicorn
uvicorn.run("services.api.main:app", factory=True, host="0.0.0.0", port=8000) # noqa: S104
services/api/models.py
+52
View File
@@ -0,0 +1,52 @@
"""Wire types.
These are deliberately NOT the domain models. `Instance` carries `team`, `namespace` and
`release_name` — placement details a tenant has no business seeing and no business
setting. The response model is the allow-list that keeps them off the wire, which is why
it is written out by hand instead of derived from `Instance`.
"""
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from svcforge_core.domain.states import InstanceState
class CreateInstanceRequest(BaseModel):
"""What a tenant may ask for.
`service_type` and `size` are plain strings, not enums: the catalog is data loaded at
runtime, so baking its keys into a type would mean a redeploy to add a service type,
and a 422 (schema) where the spec wants a 404 (unknown resource). They are validated
against the catalog in the handler.
"""
model_config = ConfigDict(extra="forbid")
service_type: str = Field(min_length=1)
size: str
ttl_days: int | None = Field(default=None, ge=1, le=30)
class InstanceResponse(BaseModel):
"""What a tenant gets back. A subset of Instance, on purpose."""
model_config = ConfigDict(from_attributes=True)
id: UUID
state: InstanceState
service_type: str
size: str
endpoint: str | None
chart_version: str
error: str | None
class ErrorBody(BaseModel):
"""Every non-2xx body. `code` is for machines, `message` is for humans."""
code: str
message: str
services/api/routes/__init__.py
+1
View File
@@ -0,0 +1 @@
"""HTTP routers."""
services/api/routes/health.py
+81
View File
@@ -0,0 +1,81 @@
"""Liveness, readiness, metrics.
The distinction between the first two is not pedantry, it is the difference between a
30-second blip and a fleet-wide outage:
* `/healthz` (liveness) answers "is this process wedged?" A failure here gets the
container KILLED. It must therefore touch NOTHING external. Wire it to the DB and a
20-second Postgres failover restarts every pod at once; they come back, find the DB
still down, and CrashLoopBackOff with exponential restart delays — so the fleet is now
down for minutes after the database recovered.
* `/readyz` (readiness) answers "should this pod get traffic?" A failure here only removes
it from the Service endpoints. It is allowed to check dependencies, and it recovers by
itself the moment the check passes.
"""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, HTTPException, Request, Response, status
from prometheus_client import REGISTRY
from prometheus_client.exposition import choose_encoder
from services.api.deps import PoolDep
from services.api.models import ErrorBody
router = APIRouter(tags=["ops"])
# No PROMETHEUS_MULTIPROC_DIR here, deliberately: it exists for prefork servers where each
# worker process holds a slice of the counters. One uvicorn process per container means
# the default in-process registry is already correct, and multiproc mode would add a
# shared temp dir, a cleanup obligation, and a class of stale-file bugs for nothing.
@router.get("/healthz", status_code=status.HTTP_200_OK)
async def healthz() -> dict[str, str]:
"""Liveness. No I/O. If the event loop can run this, the process is alive."""
return {"status": "ok"}
@router.get(
"/readyz",
responses={503: {"model": ErrorBody, "description": "A dependency is unavailable"}},
)
async def readyz(pool: PoolDep) -> dict[str, str]:
"""Readiness. Postgres only.
Postgres-only is the rule, and Redis is the temptation. Redis holds derived state —
rate-limit buckets, caches — and everything degrades gracefully without it. Put it in
this check and an Upstash hiccup marks every pod unready, Kubernetes empties the
Service, and a cache outage becomes a total API outage.
"""
try:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select 1")
row: Any = await cur.fetchone()
if row is None:
raise RuntimeError("select 1 returned no row")
except Exception as exc: # closed pool, timeout, dead DB — all mean the same 'not ready'
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail={"code": "not_ready", "message": "database unavailable"},
) from exc
return {"status": "ready"}
@router.get("/metrics", response_class=Response)
async def metrics(request: Request) -> Response:
"""The Prometheus scrape endpoint.
A route rather than `app.mount("/metrics", make_asgi_app())`, for two reasons. A
Starlette `Mount` compiles to `^/metrics(?P<path>/.*)$`, which does not match a bare
`/metrics` — the exact URL every scrape config uses — and a `Mount` is invisible to
OpenAPI, while the deliverable asks for `/metrics` in `openapi.json`.
The encoding is still prometheus_client's: `choose_encoder` reads the Accept header and
picks the exposition format (Prometheus text vs OpenMetrics) with its matching content
type. Hand-rolling either is how you end up serving text/plain that a scraper rejects.
"""
encoder, content_type = choose_encoder(request.headers.get("Accept", ""))
return Response(content=encoder(REGISTRY), media_type=content_type)
services/api/routes/instances.py
+218
View File
@@ -0,0 +1,218 @@
"""The tenant-facing API.
Two rules run through every handler here:
* **AuthZ is the WHERE clause.** No handler ever compares `inst.team` to the caller's
team, because the repo never returns another team's row to compare. A wrong-team id is
a 404. 403 would confirm the id exists, which is the leak.
* **The instance and its task commit together.** A committed instance with no task is an
instance that never provisions and that nothing will ever retry.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
from services.api.deps import (
CatalogDep,
InstanceRepoDep,
PoolDep,
TaskRepoDep,
TeamDep,
idempotency_key,
rate_limit,
)
from services.api.models import CreateInstanceRequest, ErrorBody, InstanceResponse
from svcforge_core.domain.models import CatalogEntry, Instance, TaskKind
from svcforge_core.domain.states import IllegalTransition, InstanceState, transition
# Declared on the router so every error shape lands in openapi.json under ErrorBody.
# The exception handler already renders this at runtime; without declaring it, generated
# clients see the contract for 2xx only and invent their own guess for the rest.
ERROR_RESPONSES: dict[int | str, dict[str, Any]] = {
401: {"model": ErrorBody, "description": "Missing or invalid credentials"},
404: {"model": ErrorBody, "description": "No such instance, or not this team's"},
409: {"model": ErrorBody, "description": "Instance is not in a state that allows this"},
422: {"model": ErrorBody, "description": "The body is well-formed but cannot be processed"},
}
router = APIRouter(prefix="/v1/instances", tags=["instances"], responses=ERROR_RESPONSES)
def release_name_for(team: str, service_type: str, instance_id: UUID) -> str:
"""The helm release name. Deterministic, and `unique` in the schema.
This is the idempotency anchor. A worker that dies after `helm install` but before it
marks the task done will retry, compute the same name, and `helm upgrade --install`
onto the same release instead of creating a second one. Derive it from anything that
is not already durable — a timestamp, a random suffix, the retry count — and a retry
provisions a duplicate.
Truncated to the uuid's first 8 chars to stay inside the 53-char limit helm imposes
on release names (Kubernetes label values, minus room for chart-generated suffixes).
"""
return f"{team}-{service_type}-{str(instance_id)[:8]}"
def namespace_for(team: str) -> str:
"""One namespace per tenant. The blast radius of a bad chart is one team."""
return f"tenant-{team}"
def _resolve(catalog: dict[str, CatalogEntry], service_type: str, size: str) -> CatalogEntry:
"""Look up service_type + size, or raise the right 4xx.
The two failures are different HTTP problems and the spec asks for different codes:
an unknown service_type is a resource that does not exist (404); an unknown size for a
real service_type is a body the server understood and cannot process (422).
"""
entry = catalog.get(service_type)
if entry is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={
"code": "unknown_service_type",
"message": f"no such service_type: {service_type}",
},
)
if size not in entry.sizes:
raise HTTPException(
status_code=422, # starlette renamed the 422 constant; the number never moved
detail={
"code": "unknown_size",
"message": f"{service_type} has no size {size!r}; available: {sorted(entry.sizes)}",
},
)
return entry
@router.post(
"",
status_code=status.HTTP_202_ACCEPTED,
response_model=InstanceResponse,
dependencies=[Depends(rate_limit), Depends(idempotency_key)],
)
async def create_instance(
body: CreateInstanceRequest,
response: Response,
team: TeamDep,
pool: PoolDep,
instances: InstanceRepoDep,
tasks: TaskRepoDep,
catalog: CatalogDep,
) -> Instance:
"""Accept a provisioning request. 202, never 201.
Nothing is provisioned when this returns. The row exists and a task is queued; a
worker will do the work seconds or minutes from now. 201 Created would be a lie about
a resource that does not exist yet, and clients would stop polling.
"""
entry = _resolve(catalog, body.service_type, body.size)
instance_id = uuid4()
now = datetime.now(UTC)
inst = Instance(
id=instance_id,
team=team,
service_type=body.service_type,
size=body.size,
state=InstanceState.REQUESTED,
namespace=namespace_for(team),
release_name=release_name_for(team, body.service_type, instance_id),
# Pinned from the catalog AT CREATION TIME, not read from the catalog later.
# This column records what is actually deployed; bumping catalog.yaml must show up
# as drift the reconciler can see, not silently rewrite history.
chart_version=entry.chart_version,
expires_at=now + timedelta(days=body.ttl_days) if body.ttl_days is not None else None,
created_at=now,
updated_at=now,
)
# The transaction. Both writes go through THIS conn, or the atomicity is theatre.
async with pool.connection() as conn, conn.transaction():
created = await instances.create(conn, inst)
await tasks.enqueue(conn, created.id, TaskKind.PROVISION)
response.headers["Location"] = f"/v1/instances/{created.id}"
return created
@router.get("/{instance_id}", response_model=InstanceResponse)
async def get_instance(
instance_id: UUID,
team: TeamDep,
instances: InstanceRepoDep,
) -> Instance:
"""404 if the repo returns None. A wrong-team id is a 404, not a 403."""
inst = await instances.get(instance_id, team)
if inst is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
)
return inst
@router.get("", response_model=list[InstanceResponse])
async def list_instances(
team: TeamDep,
instances: InstanceRepoDep,
limit: int = Query(default=50, ge=1, le=200),
) -> list[Instance]:
"""The caller's instances, newest first. Bounded: no endpoint returns 'all rows'."""
return await instances.list(team, limit=limit)
@router.delete(
"/{instance_id}",
status_code=status.HTTP_202_ACCEPTED,
response_model=InstanceResponse,
dependencies=[Depends(rate_limit)],
)
async def delete_instance(
instance_id: UUID,
team: TeamDep,
instances: InstanceRepoDep,
tasks: TaskRepoDep,
) -> Instance:
"""state -> deleting, enqueue deprovision. 202: the helm uninstall has not happened yet.
Ordering note. `InstanceRepo.update_state` owns its own connection, so the CAS and the
enqueue cannot share one transaction without reaching around the repo. Given two
statements, the order is chosen for its failure mode: CAS first, enqueue second. A
crash in between leaves an instance in `deleting` with no task, which the reconciler's
sweep re-enqueues. The other order leaves a deprovision task pointing at a `ready`
instance, and a worker would tear down a live service nobody asked to delete.
"""
inst = await instances.get(instance_id, team)
if inst is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"code": "not_found", "message": f"no such instance: {instance_id}"},
)
try:
target = transition(inst.state, InstanceState.DELETING)
except IllegalTransition as exc:
# Already deleting or already deleted. Not an error the tenant can fix.
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"code": "illegal_transition",
"message": f"instance is {inst.state}; delete is not a legal transition",
},
) from exc
if not await instances.update_state(inst.id, expect=inst.state, to=target):
# Someone moved the row between the read and the CAS.
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={"code": "conflict", "message": "instance changed concurrently; retry"},
)
await tasks.enqueue_standalone(inst.id, TaskKind.DEPROVISION)
return inst.model_copy(update={"state": target})
services/cli/__init__.py
View File
services/cli/main.py
+170
View File
@@ -0,0 +1,170 @@
"""svcforge — the control plane client.
This talks to the API over HTTP and never touches the database. That restraint is the
whole design: if the CLI could write to Postgres, every invariant the API enforces
(the state machine, the one-transaction create, AuthZ in the WHERE clause) would have a
back door, and the first 3am incident would go through it.
"""
from __future__ import annotations
import sys
import time
import uuid
from enum import StrEnum
from typing import Annotated, Any
import httpx
import typer
from svcforge_core.domain.states import InstanceState
from svcforge_core.settings import load_settings
app = typer.Typer(help="svcforge control plane client", no_args_is_help=True)
_TERMINAL = {InstanceState.READY, InstanceState.FAILED, InstanceState.DELETED}
class ServiceType(StrEnum):
"""What the catalog offers. Kept as an enum so typer can complete and validate it."""
ELASTICSEARCH = "elasticsearch"
REDIS = "redis"
POSTGRES = "postgres"
class Size(StrEnum):
SMALL = "small"
MEDIUM = "medium"
def _client() -> httpx.Client:
settings = load_settings()
headers = {"authorization": f"Bearer {settings.api_token}"} if settings.api_token else {}
return httpx.Client(base_url=settings.api_url, headers=headers, timeout=10.0)
def _die(msg: str) -> None:
typer.secho(msg, fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
def _check(resp: httpx.Response) -> Any: # noqa: ANN401 - a decoded JSON body genuinely is Any
if resp.status_code == 401:
_die("401 unauthorized: check SVCFORGE_API_TOKEN")
if resp.status_code == 404:
_die("404 not found")
if resp.status_code == 429:
_die("429 rate limited: slow down")
if resp.status_code >= 400:
_die(f"{resp.status_code}: {resp.text[:400]}")
return resp.json()
def _parse_ttl(ttl: str | None) -> int | None:
"""'7d' -> 7. Only days, because the API only takes days."""
if ttl is None:
return None
if not ttl.endswith("d") or not ttl[:-1].isdigit():
_die(f"bad --ttl {ttl!r}: expected something like '7d'")
return int(ttl[:-1])
def _print_table(rows: list[dict[str, Any]]) -> None:
if not rows:
typer.echo("(none)")
return
cols = ["id", "service_type", "size", "state", "chart_version", "endpoint"]
widths = {c: max(len(c), *(len(str(r.get(c) or "-")) for r in rows)) for c in cols}
typer.echo(" ".join(c.ljust(widths[c]) for c in cols))
typer.echo(" ".join("-" * widths[c] for c in cols))
for r in rows:
typer.echo(" ".join(str(r.get(c) or "-").ljust(widths[c]) for c in cols))
@app.command()
def create(
service_type: Annotated[ServiceType, typer.Argument(help="what to provision")],
size: Annotated[Size, typer.Option()] = Size.SMALL,
ttl: Annotated[str | None, typer.Option(help="e.g. 7d")] = None,
wait: Annotated[bool, typer.Option(help="poll until ready or failed")] = False,
) -> None:
"""Request an instance. Returns as soon as the API accepts it (202)."""
body: dict[str, Any] = {"service_type": service_type.value, "size": size.value}
ttl_days = _parse_ttl(ttl)
if ttl_days is not None:
body["ttl_days"] = ttl_days
with _client() as c:
data = _check(c.post("/v1/instances", json=body))
typer.echo(f"{data['id']} {data['state']}")
if not wait:
return
# 202 means "accepted", not "done". Polling is the client's job precisely because
# the API refused to block on a helm install that takes four minutes.
instance_id = data["id"]
deadline = time.monotonic() + 600
state = data["state"]
while time.monotonic() < deadline:
time.sleep(2)
data = _check(c.get(f"/v1/instances/{instance_id}"))
if data["state"] != state:
state = data["state"]
typer.echo(f" -> {state}")
if state in _TERMINAL:
break
else:
_die("timed out waiting; the task may still be running — check `svcforge status`")
if state == InstanceState.FAILED:
_die(f"failed: {data.get('error') or 'no error recorded'}")
typer.echo(f"endpoint: {data.get('endpoint') or '-'}")
@app.command("list")
def list_instances(
state: Annotated[InstanceState | None, typer.Option(help="filter by state")] = None,
) -> None:
"""List your team's instances."""
with _client() as c:
rows = _check(c.get("/v1/instances"))
if state is not None:
rows = [r for r in rows if r["state"] == state.value]
_print_table(rows)
@app.command()
def status(instance_id: Annotated[uuid.UUID, typer.Argument()]) -> None:
"""Show one instance. Exits non-zero if it is failed, so scripts can branch on it."""
with _client() as c:
data = _check(c.get(f"/v1/instances/{instance_id}"))
_print_table([data])
if data["state"] == InstanceState.FAILED:
typer.secho(f"error: {data.get('error')}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
@app.command()
def delete(
instance_id: Annotated[uuid.UUID, typer.Argument()],
yes: Annotated[bool, typer.Option("--yes", "-y", help="skip the confirmation")] = False,
) -> None:
"""Deprovision an instance."""
if not yes and not typer.confirm(f"delete {instance_id}?"):
raise typer.Abort
with _client() as c:
data = _check(c.delete(f"/v1/instances/{instance_id}"))
typer.echo(f"{data['id']} {data['state']}")
def main() -> None: # pragma: no cover - console-script entrypoint
try:
app()
except httpx.ConnectError:
typer.secho("cannot reach the API: check SVCFORGE_API_URL", fg=typer.colors.RED, err=True)
sys.exit(1)
if __name__ == "__main__": # pragma: no cover
main()
services/reconciler/Dockerfile
+53
View File
@@ -0,0 +1,53 @@
# syntax=docker/dockerfile:1.10
#
# svcforge reconciler. Build from the REPO ROOT:
# docker buildx build -f services/reconciler/Dockerfile -t svcforge/reconciler:dev .
#
# Reads helm state to detect drift, so it carries helm — but never kubectl, and it never
# writes: the four checks enqueue tasks, they do not provision. Orphans are logged, never
# deleted.
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable \
--no-install-project --no-install-package svcforge-core
COPY libs/ libs/
COPY services/reconciler/ services/reconciler/
COPY catalog.yaml ./
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable && \
/app/.venv/bin/python -c 'import svcforge_core, sys; \
p = svcforge_core.__file__; \
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
# --- runtime --------------------------------------------------------------------------
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
ARG BUILD_SHA=unknown
LABEL org.opencontainers.image.title="svcforge-reconciler" \
org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
org.opencontainers.image.revision="${BUILD_SHA}"
RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
WORKDIR /app
COPY --from=builder --chown=10001:10001 /app /app
COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
HELM_CACHE_HOME=/tmp/helm/cache \
HELM_CONFIG_HOME=/tmp/helm/config \
HELM_DATA_HOME=/tmp/helm/data
USER 10001
ENTRYPOINT ["python", "-m", "services.reconciler.main"]
services/reconciler/__init__.py
+1
View File
@@ -0,0 +1 @@
"""The reconciler service: one singleton loop that makes the world match the database."""
services/reconciler/main.py
+410
View File
@@ -0,0 +1,410 @@
"""The control loop.
Every other service in svcforge is edge-triggered: a tenant POSTs, a row appears, a worker
claims it. Edge-triggered systems are correct exactly as long as nothing is ever missed —
and things are missed. A worker is SIGKILLed holding a lease. An operator runs
`helm uninstall` by hand. A pod dies between the CAS and the enqueue. Nobody sends an event
for any of that, because the thing that would have sent it is the thing that died.
So: level-triggered. Every 60 seconds, compare the world to the database and enqueue what
is missing. The four checks below do not know or care what went wrong, or whether anything
did; they are the same code on the happy path and after an outage. That property is the
entire reason this service exists, and it is why each check is written as a *query for
work*, never as a reaction to an event.
Three rules hold the design together:
* **Singleton.** `replicas: 1`, `strategy: Recreate` in the chart. Two reconcilers
double-enqueue drift and race on TTL. There is no leader election here on purpose: the
correct lease for that lives in Postgres next to the data, not in a Redis lock, and
until there is a second replica to elect between, an election is a subsystem that can
only fail. One pod, and the `SvcforgeReconcilerStale` alert is what notices it is gone.
* **Each check is independent.** One failing check must not skip the other three. A helm
binary that cannot reach the API server must not stop TTLs from expiring.
* **Enqueue, never act.** The reconciler diagnoses; workers treat. It writes task rows and
instance states, and never calls `helm install`. The one exception is reading — the drift
check runs `helm list`, because seeing reality is the job.
"""
from __future__ import annotations
import asyncio
import contextlib
import signal
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
import typer
from svcforge_core.adapters.clock import Clock, SystemClock
from svcforge_core.adapters.helm import HelmProvisioner, Provisioner
from svcforge_core.adapters.notify import LogNotifier, Notifier
from svcforge_core.domain.catalog import load_catalog
from svcforge_core.domain.models import CatalogEntry
from svcforge_core.domain.windows import BadWindow, parse_window, schedule_upgrade_at
from svcforge_core.obs import (
INSTANCES,
QUEUE_DEPTH,
RECONCILER_LAST_TICK,
get_logger,
setup,
start_metrics_server,
tracer,
)
from svcforge_core.repo.db import DictPool, make_pool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.reconcile import ReconcileRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings, load_settings
log = get_logger("svcforge.reconciler")
# The chart's PodMonitor scrapes the port named `metrics` on 9000. Keep them in step.
DEFAULT_METRICS_PORT = 9000
@dataclass(frozen=True)
class ReconcilerDeps:
"""Everything a check is allowed to touch. Built once in main(), passed down.
Same shape as `WorkerDeps` for the same reason: the checks take `deps` instead of
reaching for globals, so the integration tests below run every check against a real
Postgres and a `FakeProvisioner` without a cluster anywhere in sight.
"""
pool: DictPool
instances: InstanceRepo
tasks: TaskRepo
reconcile: ReconcileRepo
provisioner: Provisioner
notifier: Notifier
clock: Clock
catalog: dict[str, CatalogEntry]
settings: Settings
# Whose instances go first in the day-2 work list. Eating your own dog food is an
# `order by`, not a policy document — see InstanceRepo.list_upgradable.
own_team: str
# A config value, not a scheduler. Leave it at 1 until 1 is too slow.
max_in_flight: int
# --- The four checks ---------------------------------------------------------------------
async def check_drift(deps: ReconcilerDeps) -> None:
"""`helm list -A -o json` versus what the database believes.
This is the only check that looks outside Postgres, and the only one that can catch the
failure nothing else can: someone ran `helm uninstall` by hand, or a node was drained
and the release never came back. The DB still says `ready` and still hands the tenant an
endpoint that resolves to nothing.
Two directions, two very different answers:
* **Release gone, DB says `ready`** -> re-enqueue provision. Safe, because provisioning
is `helm upgrade --install` against a deterministic release name: converging on
desired state, not a blind re-install.
* **Release exists, DB knows nothing** -> log at error with release and namespace, and
stop. **Never delete in v1.** The reconciler's view of "the DB knows nothing" is one
query against one database; the release might belong to another team, another tool,
or a migration half-finished. Deleting on that evidence is how an automated system
takes down production faster than any human could. A human reads the log and decides.
"""
with tracer().start_as_current_span("helm.list"):
releases = await deps.provisioner.list_releases()
live = {(r.name, r.namespace) for r in releases}
for inst in await deps.reconcile.ready_instances():
if (inst.release_name, inst.namespace) in live:
continue
reason = f"drift: helm release {inst.release_name} missing from namespace {inst.namespace}"
task_id = await deps.reconcile.enqueue_reprovision(inst.id, reason)
if task_id is None:
continue # already being dealt with, or the row moved under us
log.warning(
"drift.release_missing",
instance_id=str(inst.id),
team=inst.team,
release=inst.release_name,
namespace=inst.namespace,
task_id=task_id,
)
try:
await deps.notifier.send(
"drift.release_missing",
f"re-provisioning {inst.id}: release {inst.release_name} vanished",
{"instance_id": str(inst.id), "team": inst.team},
)
except Exception:
# The task is already committed; the notification is a courtesy. A webhook
# timing out must not abandon the rest of the sweep — the instances after this
# one in the loop have the same problem and nobody else is coming to find them.
log.exception("notify.failed", instance_id=str(inst.id))
known = await deps.reconcile.known_releases()
for name, namespace in sorted(live - known):
# error, not warning: this is a resource nobody is billing for and nobody owns.
# It will sit here every 60s until a human deletes it or adopts it. That is the
# intended pressure.
log.error("drift.orphan_release", release=name, namespace=namespace, action="none (v1 never deletes)")
async def check_lease_expiry(deps: ReconcilerDeps) -> None:
"""Tasks whose worker died -> back to `queued`.
A lease, not a lock. No lock survives a power cut: a worker SIGKILLed mid-provision
leaves `state='running'` with `locked_by` set and nobody running it, and no amount of
cleanup code in the worker helps, because the worker is the part that died. `locked_at`
plus a timeout is the only thing that recovers the row, which is why `locked_at` exists.
The 5-minute default is not arbitrary: it must exceed the longest a healthy task can
hold a lease, or the reconciler hands a still-running provision to a second worker.
Handlers are idempotent, so that is survivable rather than fatal — but survivable is
not free, and `lease_seconds` sits above helm's `--timeout` for that reason.
"""
freed = await deps.tasks.reset_expired_leases(deps.settings.lease_seconds)
if freed:
log.warning("lease.expired", tasks_freed=freed, lease_seconds=deps.settings.lease_seconds)
async def check_ttl(deps: ReconcilerDeps) -> None:
"""Expired instances -> `deleting`, plus a deprovision task.
The line item that stops a demo cluster from becoming a permanent cloud bill. Also the
sweep the API's DELETE route depends on: it CASes to `deleting` and enqueues in two
statements, and a crash in between lands here on the next tick.
Idempotent by construction — the work list excludes anything that already has a queued
or running deprovision, and the CAS and the insert share one transaction. Without that
guard, a deprovision that takes longer than 60 seconds gets a second task on the next
tick, and a third on the tick after.
"""
for inst in await deps.reconcile.due_for_deprovision():
task_id = await deps.reconcile.enqueue_deprovision(inst.id)
if task_id is None:
continue
log.info(
"ttl.expired",
instance_id=str(inst.id),
team=inst.team,
expires_at=inst.expires_at.isoformat() if inst.expires_at else None,
previous_state=inst.state.value,
task_id=task_id,
)
async def check_version_drift(deps: ReconcilerDeps) -> None:
"""The day-2 rollout: the work-list query, one service type at a time.
Everything that makes this safe is somewhere else, which is the point:
* `list_upgradable` limits to `max_in_flight` and returns nothing while
`rollout_state='halted'`, so a bad chart stops after one tenant.
* `schedule_upgrade_at` turns the tenant's maintenance window into a `run_after`; the
queue does the waiting, in `where run_after <= now()`. There is no scheduler here and
there must not be one — a task parked in Postgres until 03:00 Sunday survives a
reconciler restart, and an in-memory timer does not.
* `security: true` in the catalog bypasses the window. A CVE with a public exploit does
not wait until Sunday.
A bad window spec is this instance's problem, not the fleet's: log it and move to the
next one. Failing the whole check would let one tenant's typo freeze everyone's
security rollout.
"""
now = deps.clock.now()
for service_type, entry in deps.catalog.items():
candidates = await deps.instances.list_upgradable(
service_type=service_type,
catalog_version=entry.chart_version,
own_team=deps.own_team,
max_in_flight=deps.max_in_flight,
)
for candidate in candidates:
inst = candidate.instance
try:
window = parse_window(candidate.maintenance_window)
except BadWindow:
log.exception(
"upgrade.bad_window",
instance_id=str(inst.id),
team=inst.team,
maintenance_window=candidate.maintenance_window,
)
continue
run_after = schedule_upgrade_at(window, security=entry.security, now=now)
task_id = await deps.reconcile.enqueue_upgrade(inst.id, run_after)
if task_id is None:
continue # already queued or running; this is the max_in_flight guard
log.info(
"upgrade.scheduled",
instance_id=str(inst.id),
team=inst.team,
service_type=service_type,
from_version=inst.chart_version,
to_version=entry.chart_version,
run_after=run_after.isoformat(),
security=entry.security,
task_id=task_id,
)
CHECKS: dict[str, Callable[[ReconcilerDeps], Awaitable[None]]] = {
"drift": check_drift,
"lease_expiry": check_lease_expiry,
"ttl": check_ttl,
"version_drift": check_version_drift,
}
# --- The tick ----------------------------------------------------------------------------
async def tick(deps: ReconcilerDeps) -> None:
"""One pass: all four checks, then the gauges, then the heartbeat.
Checks first, gauges second: `svcforge_queue_depth` is read straight after the checks
that add to the queue, so the value scraped is the value the tick left behind rather
than one from before its own work.
The heartbeat is set unconditionally, and that is deliberate. It answers "is the loop
running", not "is everything fine" — the checks have their own alerts. Gating it on
success would make `SvcforgeReconcilerStale` fire for a helm blip and mean two things
at once, and an alert that means two things gets muted.
The whole tick runs inside one span, which is a considered exception to "manual spans go
around helm calls only". That rule exists so the API does not hand-roll spans that
`opentelemetry-instrument` already creates for it. Nothing auto-instruments the
reconciler: without a span here it emits no traces at all, and — because
`inject_traceparent` serialises the *active* context — every task it enqueues would be
written with a null `traceparent` and be unjoinable to the tick that decided to create
it. One span per tick is what makes "why was this instance re-provisioned at 03:00?" a
question the traces can answer.
"""
with tracer().start_as_current_span("reconciler.tick"):
await _run_checks(deps)
RECONCILER_LAST_TICK.set(deps.clock.now().timestamp())
log.info("tick.done")
async def _run_checks(deps: ReconcilerDeps) -> None:
"""The four checks and the gauges. Split out so `tick` reads as span + heartbeat."""
for name, check in CHECKS.items():
try:
await check(deps)
except Exception: # the tick is the error boundary
# The swallow is the design. These four checks share nothing but a database
# handle, and the value of a level-triggered loop is that it keeps running: an
# unreachable cluster must not stop TTLs from expiring, and one tenant's broken
# window spec must not stop drift detection. This means "this check achieved
# nothing for 60 seconds", which the log says out loud. It never means "the
# reconciler stops".
log.exception("check.failed", check=name)
try:
QUEUE_DEPTH.set(await deps.reconcile.queue_depth())
counts = await deps.reconcile.instance_counts()
for state, n in counts.items():
INSTANCES.labels(state=state).set(n)
except Exception: # gauges are diagnostics; a failed read is not a failed tick
log.exception("gauges.failed")
async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
"""Sleep, but wake immediately on SIGTERM. A 60s nap must not cost 60s of shutdown."""
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(stop.wait(), timeout=seconds)
async def run_reconciler(deps: ReconcilerDeps, stop: asyncio.Event) -> None:
"""Tick, sleep, repeat, until told to stop.
Tick first, then sleep: a pod that has just been restarted should reconcile now, not in
sixty seconds. Fixed interval rather than a fixed period — a tick that overruns simply
delays the next one, instead of stacking a second tick on top of the first, which for a
singleton would be exactly the concurrent reconciler `replicas: 1` exists to prevent.
"""
while not stop.is_set():
await tick(deps)
await _sleep_or_stop(stop, deps.settings.reconcile_interval_s)
def build_deps(
pool: DictPool,
settings: Settings,
own_team: str,
max_in_flight: int,
) -> ReconcilerDeps:
"""Wire the real collaborators. The only place that names concrete classes."""
return ReconcilerDeps(
pool=pool,
instances=InstanceRepo(pool),
tasks=TaskRepo(pool),
reconcile=ReconcileRepo(pool),
provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
notifier=LogNotifier(),
clock=SystemClock(),
catalog=load_catalog(settings.catalog_path),
settings=settings,
own_team=own_team,
max_in_flight=max_in_flight,
)
async def _amain(once: bool, metrics_port: int, own_team: str, max_in_flight: int) -> None:
settings = load_settings()
setup("svcforge-reconciler", settings)
pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
await pool.open(wait=True)
deps = build_deps(pool, settings, own_team, max_in_flight)
try:
if once:
# One pass and exit: the acceptance path, and how you drive a reconcile by hand
# from a shell. No metrics server — nothing would ever scrape it.
await tick(deps)
return
start_metrics_server(metrics_port)
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
# add_signal_handler, NOT signal.signal. signal.signal fires the handler at an
# arbitrary bytecode boundary on the main thread and the loop does not notice
# until its next timer — which here is up to a full 60s tick away. This one is
# scheduled as an ordinary loop callback, so the `stop.wait()` above returns
# immediately.
loop.add_signal_handler(sig, stop.set)
await run_reconciler(deps, stop)
finally:
await pool.close()
app = typer.Typer(add_completion=False, help="svcforge reconciler: the control loop.")
@app.command()
def main(
once: bool = typer.Option(False, "--once", help="Run one tick and exit."),
metrics_port: int = typer.Option(
DEFAULT_METRICS_PORT, envvar="SVCFORGE_METRICS_PORT", help="Port for /metrics."
),
own_team: str = typer.Option(
"platform", envvar="SVCFORGE_OWN_TEAM", help="Team whose instances upgrade first."
),
max_in_flight: int = typer.Option(
1, envvar="SVCFORGE_MAX_IN_FLIGHT", min=1, help="Concurrent upgrades across the fleet."
),
) -> None:
"""Run the reconciler."""
# One asyncio.run, at the top, never nested. Everything below it is already async.
asyncio.run(_amain(once, metrics_port, own_team, max_in_flight))
if __name__ == "__main__":
app()
services/worker/Dockerfile
+54
View File
@@ -0,0 +1,54 @@
# syntax=docker/dockerfile:1.10
#
# svcforge worker. Build from the REPO ROOT:
# docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
#
# The only service that shells out to helm/kubectl, so the only one carrying those two
# binaries. They are copied from pinned images rather than curl'd, so the version is a
# reviewable line in a Dockerfile instead of a network call at build time.
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
WORKDIR /app
COPY pyproject.toml uv.lock ./
COPY libs/svcforge_core/pyproject.toml libs/svcforge_core/
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable \
--no-install-project --no-install-package svcforge-core
COPY libs/ libs/
COPY services/worker/ services/worker/
COPY catalog.yaml ./
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-dev --no-editable && \
/app/.venv/bin/python -c 'import svcforge_core, sys; \
p = svcforge_core.__file__; \
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
# --- runtime --------------------------------------------------------------------------
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
ARG BUILD_SHA=unknown
LABEL org.opencontainers.image.title="svcforge-worker" \
org.opencontainers.image.source="https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge" \
org.opencontainers.image.revision="${BUILD_SHA}"
RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
WORKDIR /app
COPY --from=builder --chown=10001:10001 /app /app
COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
COPY --from=bitnamilegacy/kubectl:1.31.2@sha256:0eab9ec8f5e0f75271277467ebb7513b36dea0122bc68615d18c47fece4fd82c /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
HELM_CACHE_HOME=/tmp/helm/cache \
HELM_CONFIG_HOME=/tmp/helm/config \
HELM_DATA_HOME=/tmp/helm/data
USER 10001
ENTRYPOINT ["python", "-m", "services.worker.main"]
services/worker/__init__.py
View File
services/worker/deps.py
+33
View File
@@ -0,0 +1,33 @@
"""What a worker needs to do its job.
One frozen bag of collaborators, constructed once in main() and passed down. Handlers
take `deps` rather than reaching for globals, which is the entire reason the worker tests
run in milliseconds against a FakeProvisioner instead of needing a cluster.
"""
from __future__ import annotations
from dataclasses import dataclass
from svcforge_core.adapters.clock import Clock
from svcforge_core.adapters.helm import Provisioner
from svcforge_core.adapters.notify import Notifier
from svcforge_core.domain.models import CatalogEntry
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings
@dataclass(frozen=True)
class WorkerDeps:
"""Everything a handler is allowed to touch."""
pool: DictPool
instances: InstanceRepo
tasks: TaskRepo
provisioner: Provisioner
notifier: Notifier
clock: Clock
catalog: dict[str, CatalogEntry]
settings: Settings
services/worker/handlers.py
+161
View File
@@ -0,0 +1,161 @@
"""Task handlers.
Every handler here obeys one rule: running it twice must equal running it once.
That is not a nicety. A worker can be SIGKILLed after helm has installed the release but
before the DB row says so; the lease expires; another worker claims the same task and runs
this function again. If the handler is not idempotent, the tenant gets two Elasticsearches
and you get a bill. Idempotency is what makes the crash safe, and it is bought in two
places: a deterministic `release_name`, and adapters that state desired state
(`helm upgrade --install`) instead of issuing imperative commands.
"""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from services.worker.deps import WorkerDeps
from svcforge_core.domain.models import CatalogEntry, Instance, Task, TaskKind
from svcforge_core.domain.states import InstanceState
class HandlerError(RuntimeError):
"""A task failed in a way worth retrying. The message lands in tasks.last_error."""
async def _load_instance(task: Task, deps: WorkerDeps) -> Instance:
async with deps.pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"""select id, team, service_type, size, state, namespace, release_name,
chart_version, endpoint, error, expires_at, created_at, updated_at
from instances where id = %s""",
(task.instance_id,),
)
row = await cur.fetchone()
if row is None:
raise HandlerError(f"instance {task.instance_id} vanished")
return Instance.model_validate(row)
def _values_for(inst: Instance, entry: CatalogEntry) -> dict[str, Any]:
"""Turn a catalog size into helm values."""
size = entry.sizes.get(inst.size)
if size is None:
raise HandlerError(f"size {inst.size!r} not in catalog for {inst.service_type!r}")
return {"replicaCount": size.replicas, "resources": size.resources}
async def handle_provision(task: Task, deps: WorkerDeps) -> None:
"""Install the release and mark the instance ready. Idempotent."""
inst = await _load_instance(task, deps)
if inst.state is InstanceState.READY:
# A previous attempt already finished; the crash was after the work, before the
# bookkeeping. Nothing to do — and re-installing would be the bug.
return
entry = deps.catalog.get(inst.service_type)
if entry is None:
raise HandlerError(f"unknown service_type {inst.service_type!r}")
# Best-effort CAS. It returning False means someone else moved the row; the helm call
# below is idempotent either way, so this is bookkeeping, not a lock.
await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
await deps.provisioner.install(
release=inst.release_name,
ns=inst.namespace,
entry=entry,
values=_values_for(inst, entry),
)
endpoint = f"http://{inst.release_name}.{inst.namespace}.svc.cluster.local"
ok = await deps.instances.update_state(
inst.id, InstanceState.PROVISIONING, InstanceState.READY, endpoint=endpoint
)
if ok:
await deps.notifier.send(
"instance.ready",
f"instance {inst.id} is ready at {endpoint}",
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
)
async def handle_deprovision(task: Task, deps: WorkerDeps) -> None:
"""Remove the release and mark the instance deleted. Idempotent."""
inst = await _load_instance(task, deps)
if inst.state is InstanceState.DELETED:
return
# `helm uninstall` of an already-gone release is not an error to us: the adapter
# swallows not-found, because the desired state — no release — is already true.
await deps.provisioner.uninstall(release=inst.release_name, ns=inst.namespace)
await deps.instances.update_state(inst.id, InstanceState.DELETING, InstanceState.DELETED)
async def handle_upgrade(task: Task, deps: WorkerDeps) -> None:
"""Upgrade the release to the catalog's pinned version, then record it.
`instances.chart_version` is written only AFTER helm reports success. That column is
what the day-2 work-list query compares against, so writing it optimistically would
make the fleet look upgraded while it isn't.
"""
inst = await _load_instance(task, deps)
entry = deps.catalog.get(inst.service_type)
if entry is None:
raise HandlerError(f"unknown service_type {inst.service_type!r}")
if inst.chart_version == entry.chart_version:
return # already there
await deps.provisioner.install(
release=inst.release_name,
ns=inst.namespace,
entry=entry,
values=_values_for(inst, entry),
)
async with deps.pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"update instances set chart_version = %s, updated_at = now() where id = %s",
(entry.chart_version, inst.id),
)
async def handle_verify(task: Task, deps: WorkerDeps) -> None:
"""Post-upgrade health probe. On failure, halt the whole rollout for this service type.
One column decides whether the fleet keeps rolling. The work-list query returns nothing
while `rollout_state='halted'`, so a bad chart stops after the first tenant instead of
after all of them. You clear it with SQL, deliberately: an automatic un-halt would just
resume breaking things.
"""
inst = await _load_instance(task, deps)
releases = {r.name for r in await deps.provisioner.list_releases()}
if inst.release_name in releases:
return
async with deps.pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"""insert into catalog_versions (service_type, rollout_state)
values (%s, 'halted')
on conflict (service_type) do update set rollout_state = 'halted'""",
(inst.service_type,),
)
await deps.notifier.send(
"rollout.halted",
f"rollout halted for {inst.service_type}: {inst.release_name} failed verify",
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
)
raise HandlerError(f"verify failed for {inst.release_name}; rollout halted")
HANDLERS: dict[TaskKind, Callable[[Task, WorkerDeps], Awaitable[None]]] = {
TaskKind.PROVISION: handle_provision,
TaskKind.DEPROVISION: handle_deprovision,
TaskKind.UPGRADE: handle_upgrade,
TaskKind.VERIFY: handle_verify,
}
services/worker/main.py
+175
View File
@@ -0,0 +1,175 @@
"""The claim loop.
Poll every 5 seconds. Claim while a semaphore slot is free. Run the handler. Report.
That is the whole design, and the restraint is the point: LISTEN/NOTIFY would shave the
latency, is fire-and-forget so it can never replace the poll anyway, is strictly extra
code, and does not exist on pgbouncer's transaction pooler. The poll is not a placeholder
for something better.
"""
from __future__ import annotations
import asyncio
import contextlib
import signal
import time
from opentelemetry import trace
from services.worker.deps import WorkerDeps
from services.worker.handlers import HANDLERS
from svcforge_core import obs
from svcforge_core.adapters.clock import SystemClock
from svcforge_core.adapters.helm import HelmProvisioner
from svcforge_core.adapters.notify import LogNotifier
from svcforge_core.domain.catalog import load_catalog
from svcforge_core.domain.models import Task
from svcforge_core.repo.db import make_pool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings, load_settings
log = obs.get_logger("svcforge.worker")
async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
"""Sleep, but wake immediately on shutdown.
`await asyncio.sleep(5)` would make every SIGTERM cost up to five seconds of
Kubernetes waiting on terminationGracePeriod for no reason.
"""
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(stop.wait(), timeout=seconds)
async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None:
"""Run one task to a terminal report. Never lets an exception escape the TaskGroup."""
try:
# Every log line from here carries instance_id/task_id/team. Bound once, at claim,
# rather than passed down: the alternative is threading three arguments through
# every function that might log, and the first one anyone forgets is the one you
# need at 3am.
obs.bind_task_context(task.instance_id, task.id, team=task.team or "unknown")
log.info("task claimed", kind=task.kind.value, attempt=task.attempts)
obs.TASKS_CLAIMED.labels(kind=task.kind.value).inc()
handler = HANDLERS.get(task.kind)
if handler is None:
await deps.tasks.fail(task.id, f"no handler for {task.kind}", max_attempts=1)
return
# Re-parent to the span that enqueued this task. Without the stored traceparent
# the worker's span starts a brand-new trace, and the POST that caused the work
# is in a different trace to the helm call that did it.
ctx = obs.context_from_traceparent(task.traceparent)
started = time.monotonic()
with obs.tracer().start_as_current_span(
f"task.{task.kind.value}",
context=ctx,
kind=trace.SpanKind.CONSUMER,
) as span:
span.set_attribute("task.id", task.id)
span.set_attribute("task.kind", task.kind.value)
span.set_attribute("instance.id", str(task.instance_id))
try:
await handler(task, deps)
except asyncio.CancelledError:
# CancelledError inherits from BaseException, so `except Exception` below
# would never see it. Catch it only to release the claim, then ALWAYS
# re-raise: swallowing it breaks cancellation for everyone above us.
await deps.tasks.fail(task.id, "cancelled", max_attempts=deps.settings.max_attempts)
raise
except Exception as exc:
log.exception("task failed", kind=task.kind.value, error=str(exc))
span.record_exception(exc)
span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc)))
obs.TASKS_FAILED.labels(kind=task.kind.value).inc()
await deps.tasks.fail(task.id, str(exc), max_attempts=deps.settings.max_attempts)
else:
obs.PROVISION_TIME.observe(time.monotonic() - started)
await deps.tasks.complete(task.id)
finally:
sem.release()
async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
"""Claim and run until told to stop, then drain what is in flight.
Draining is what makes a rolling deploy invisible. Exiting the `async with` block
awaits every in-flight handler, so a pod that is being replaced finishes the provision
it already started instead of abandoning it half-done for the lease to clean up
five minutes later.
"""
sem = asyncio.Semaphore(deps.settings.worker_concurrency)
worker_id = deps.settings.worker_id
async with asyncio.TaskGroup() as tg:
while not stop.is_set():
await sem.acquire()
if stop.is_set():
sem.release()
break
try:
task = await deps.tasks.claim(worker_id)
except Exception:
# A DB blip must not kill the worker; back off and try again.
log.exception("claim failed")
sem.release()
await _sleep_or_stop(stop, deps.settings.poll_interval_s)
continue
if task is None:
sem.release()
await _sleep_or_stop(stop, deps.settings.poll_interval_s)
continue
tg.create_task(_run_one(deps, task, sem))
# TaskGroup.__aexit__ awaited the in-flight handlers. Now it is safe to exit 0.
async def _amain() -> None:
settings: Settings = load_settings()
# Before anything else: nothing logged above this line is structured, and the metrics
# the SvcforgeTaskFailed / SvcforgeProvisionSlow alerts query do not exist until the
# registry is up.
obs.setup("svcforge-worker", settings)
obs.start_metrics_server(settings.metrics_port)
pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
await pool.open(wait=True)
deps = WorkerDeps(
pool=pool,
instances=InstanceRepo(pool),
tasks=TaskRepo(pool),
provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
notifier=LogNotifier(),
clock=SystemClock(),
catalog=load_catalog(settings.catalog_path),
settings=settings,
)
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
# add_signal_handler, NOT signal.signal. signal.signal runs the handler at an
# arbitrary bytecode boundary on whatever thread the C-level handler lands on,
# and the event loop will not notice until its next timer fires. This one is
# loop-safe: the callback runs as a normal loop callback.
loop.add_signal_handler(sig, stop.set)
try:
await run_worker(deps, stop)
finally:
await pool.close()
def main() -> None:
"""One asyncio.run, at the top, never nested."""
asyncio.run(_amain())
if __name__ == "__main__":
main()
tests/__init__.py
View File
tests/e2e/__init__.py
View File
tests/e2e/test_provision_real.py
+133
View File
@@ -0,0 +1,133 @@
"""The real thing: a real helm, against a real cluster, installing a real chart.
Everything else in this suite runs against a FakeProvisioner, which is what makes the
worker tests take milliseconds. That trade has one cost, and this file is the payment: the
fake proves the *worker* is correct, and cannot prove the *adapter* is. Argv order, the
`--wait` flag, chart resolution, RBAC, whether `upgrade --install` is genuinely idempotent
against a live release — none of it is exercised by a fake that returns None.
So one test does it for real, exactly once. It is marked `e2e` and excluded from `make test`
and from every pre-commit run; CI runs `-m e2e` in a job that has a cluster.
kind create cluster --name svcforge
uv run pytest -m e2e -q
It skips (does not fail) with no cluster, because a laptop without kubectl is not a
regression.
"""
from __future__ import annotations
import asyncio
import shutil
import subprocess
import uuid
import pytest
from svcforge_core.adapters.helm import HelmProvisioner
from svcforge_core.domain.models import CatalogEntry, SizeSpec
pytestmark = [pytest.mark.e2e, pytest.mark.slow]
NAMESPACE = "svcforge-e2e"
# A chart with no dependencies, no PVCs and no image pulls worth waiting on. The point is
# to exercise the adapter, not to wait four minutes for Elasticsearch.
ENTRY = CatalogEntry(
service_type="podinfo",
chart="oci://ghcr.io/stefanprodan/charts/podinfo",
chart_version="6.7.1",
sizes={"small": SizeSpec(replicas=1, resources={})},
)
def _cluster_reachable() -> bool:
if not shutil.which("helm") or not shutil.which("kubectl"):
return False
out = subprocess.run(
["kubectl", "cluster-info"],
capture_output=True,
timeout=15,
)
return out.returncode == 0
requires_cluster = pytest.mark.skipif(
not _cluster_reachable(),
reason="no reachable cluster: `kind create cluster --name svcforge`",
)
@pytest.fixture(scope="module")
def namespace() -> str:
subprocess.run(
["kubectl", "create", "namespace", NAMESPACE],
capture_output=True,
check=False, # already exists is fine — the whole system is idempotent or it is broken
)
return NAMESPACE
@requires_cluster
async def test_install_is_idempotent_against_a_real_cluster(namespace: str) -> None:
"""Install twice. Get one release. This is the claim the fake cannot make for us."""
release = f"e2e-podinfo-{uuid.uuid4().hex[:8]}"
prov = HelmProvisioner(timeout_s=300)
try:
await prov.install(release=release, ns=namespace, entry=ENTRY, values={"replicaCount": 1})
releases = [r for r in await prov.list_releases() if r.name == release]
assert len(releases) == 1, f"expected exactly one release, got {releases}"
# The redelivery, for real: same task, same deterministic release name, run again.
# `helm upgrade --install` must converge, not duplicate and not error.
await prov.install(release=release, ns=namespace, entry=ENTRY, values={"replicaCount": 1})
releases = [r for r in await prov.list_releases() if r.name == release]
assert len(releases) == 1, "second install created a second release: not idempotent"
# --wait means the DB writing `ready` is telling the truth.
out = subprocess.run(
[
"kubectl",
"get",
"deploy",
"-n",
namespace,
"-l",
f"app.kubernetes.io/instance={release}",
"-o",
"jsonpath={.items[*].status.readyReplicas}",
],
capture_output=True,
text=True,
timeout=30,
)
assert out.stdout.strip() == "1", f"--wait returned before the pod was ready: {out.stdout!r}"
finally:
await prov.uninstall(release=release, ns=namespace)
@requires_cluster
async def test_uninstall_of_a_missing_release_is_not_an_error(namespace: str) -> None:
"""The desired state — no release — is already true. That is success, not failure."""
prov = HelmProvisioner(timeout_s=120)
await prov.uninstall(release=f"never-existed-{uuid.uuid4().hex[:8]}", ns=namespace)
@requires_cluster
async def test_real_helm_timeout_leaves_no_helm_behind(namespace: str) -> None:
"""A deadline that does not kill helm is a deadline that lets helm keep mutating."""
prov = HelmProvisioner(timeout_s=1) # cannot possibly finish
release = f"e2e-timeout-{uuid.uuid4().hex[:8]}"
with pytest.raises((TimeoutError, Exception)):
await prov.install(release=release, ns=namespace, entry=ENTRY, values={})
await asyncio.sleep(0.5)
out = subprocess.run(["pgrep", "-f", f"helm.*{release}"], capture_output=True, text=True)
assert out.stdout.strip() == "", "helm survived its own timeout and is still touching the cluster"
await prov.uninstall(release=release, ns=namespace)
tests/fakes.py
+210
View File
@@ -0,0 +1,210 @@
"""Test doubles. The second implementation that makes each Protocol worth having.
These are fakes, not mocks: they have working behaviour (a dict of releases, a monotonic
counter) and are asserted against by their *state*, not by "was this method called with
these arguments". A mock proves your mock does what you told it to.
They live here, outside `svcforge_core`, so that no production import path can reach them.
"""
from __future__ import annotations
import asyncio
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID
from svcforge_core.adapters.clock import Clock
from svcforge_core.adapters.helm import HelmError, ReleaseInfo
from svcforge_core.adapters.redis import RateLimitResult
from svcforge_core.domain.models import CatalogEntry, Instance
class FakeProvisioner:
"""Dict of release -> ReleaseInfo. Same signatures. Optional fail_on / delay for tests.
This is what makes the worker's tests run in milliseconds with no cluster.
"""
def __init__(self, *, delay: float = 0.0, fail_on: set[str] | None = None) -> None:
"""`delay`: seconds `install` sleeps, to hold a task in flight (SIGTERM drain tests).
`fail_on`: substring match against the release name, not equality — release names
carry a random suffix (`platform-elasticsearch-1a2b3c4d`), so a test that wants "every
elasticsearch install fails" cannot name the release up front.
"""
self.delay = delay
self.fail_on = fail_on or set()
self.releases: dict[str, ReleaseInfo] = {}
self.installed: list[str] = []
self.uninstalled: list[str] = []
# Concurrency accounting, so a test can assert the worker's semaphore actually caps.
self.in_flight = 0
self.max_concurrent = 0
def _should_fail(self, release: str) -> bool:
return any(needle in release for needle in self.fail_on)
async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None:
self.in_flight += 1
self.max_concurrent = max(self.max_concurrent, self.in_flight)
try:
if self.delay:
await asyncio.sleep(self.delay)
if self._should_fail(release):
# Same type the real adapter raises, so handlers cannot pass here and fail in prod.
raise HelmError(f"fake: install of {release} failed on purpose")
self.releases[release] = ReleaseInfo(
name=release,
namespace=ns,
chart=f"{entry.chart}-{entry.chart_version}",
status="deployed",
revision=self.releases[release].revision + 1 if release in self.releases else 1,
app_version=entry.chart_version,
)
# Appended only on success: a failed install must not look installed.
self.installed.append(release)
finally:
self.in_flight -= 1
async def uninstall(self, release: str, ns: str) -> None:
"""Absent release is not an error: the desired state is already true."""
self.releases.pop(release, None)
self.uninstalled.append(release)
async def list_releases(self) -> list[ReleaseInfo]:
return list(self.releases.values())
class FakeClock:
"""Time under test control. No sleeping, no monkeypatching the stdlib."""
def __init__(self, start: datetime) -> None:
if start.tzinfo is None:
raise ValueError("FakeClock needs an aware datetime; a naive start defeats the point")
self._now = start
def now(self) -> datetime:
return self._now
def advance(self, delta: timedelta) -> None:
"""Move forward. Only forward — a clock that goes backwards is a different bug entirely."""
if delta < timedelta(0):
raise ValueError("FakeClock cannot go backwards")
self._now += delta
class FakeNotifier:
"""Records what it was asked to send. Never leaves the process."""
def __init__(self) -> None:
self.sent: list[tuple[str, str, dict[str, str]]] = []
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
self.sent.append((event, message, fields or {}))
def events(self) -> list[str]:
return [event for event, _, _ in self.sent]
# --- Redis (Module 10) ------------------------------------------------------------------
#
# Three fakes, and each one is the second implementation that earns its Protocol. They also
# earn their keep against the budget: Upstash's free tier is 500K commands/month, so a test
# that wants a thousand rate-limit checks runs them here and spends nothing. Real Upstash
# is reserved for the handful of `@pytest.mark.slow` tests that prove the wire protocol,
# the Lua, and the command count.
#
# `down=True` is the interesting knob. Every real class degrades internally rather than
# raising, so these degrade the same way — a fake that raises when the real one returns a
# safe default would let a caller ship a `try/except` that production never exercises.
class FakeRateLimiter:
"""In-memory fixed window. Same semantics as the Lua, none of the network.
Counts `commands` so a test can assert the one-command-per-check budget without a
server, and takes a Clock so a window rollover is an `advance()` rather than a sleep.
"""
def __init__(self, limit: int, window_s: int, clock: Clock, *, down: bool = False) -> None:
self.limit = limit
self.window_s = window_s
self.clock = clock
self.down = down
self.commands = 0
self.counts: dict[str, int] = {}
async def check(self, team: str) -> RateLimitResult:
window = int(self.clock.now().timestamp()) // self.window_s
reset_at = datetime.fromtimestamp((window + 1) * self.window_s, tz=UTC)
self.commands += 1
if self.down:
# Fails OPEN, exactly like the real one. A limiter that refused here would make
# "Redis is down" indistinguishable from "you are over quota".
return RateLimitResult(
allowed=True, limit=self.limit, remaining=self.limit, reset_at=reset_at, degraded=True
)
key = f"rl:{team}:{window}"
n = self.counts.get(key, 0) + 1
self.counts[key] = n
return RateLimitResult(
allowed=n <= self.limit,
limit=self.limit,
remaining=max(0, self.limit - n),
reset_at=reset_at,
)
class FakeIdempotencyStore:
"""A dict with SET-NX semantics. No TTL: no test outlives one."""
def __init__(self, *, down: bool = False) -> None:
self.down = down
self.claims: dict[str, UUID] = {}
async def claim(self, key: str, instance_id: UUID) -> UUID | None:
if self.down:
# Falls through to the DB, where `instances.release_name` is UNIQUE. The real
# guarantee was never here.
return None
existing = self.claims.get(key)
if existing is not None:
return existing
self.claims[key] = instance_id
return None
class FakeInstanceCache:
"""A dict, plus hit/miss accounting so a test can assert the second read never hits the DB."""
def __init__(self, *, down: bool = False) -> None:
self.down = down
self.entries: dict[UUID, Instance] = {}
self.hits = 0
self.misses = 0
self.invalidations: list[UUID] = []
async def get(self, instance_id: UUID) -> Instance | None:
if self.down:
self.misses += 1
return None
inst = self.entries.get(instance_id)
if inst is None:
self.misses += 1
return None
self.hits += 1
return inst
async def put(self, inst: Instance) -> None:
if self.down:
return
self.entries[inst.id] = inst
async def invalidate(self, instance_id: UUID) -> None:
self.invalidations.append(instance_id)
if self.down:
return
self.entries.pop(instance_id, None)
tests/integration/__init__.py
View File
tests/integration/conftest.py
+115
View File
@@ -0,0 +1,115 @@
"""Integration fixtures: a real Postgres, real SQL, no mocks.
The DB is never mocked. A mocked database proves your mock returns what you told it to.
Every bug worth catching here — SKIP LOCKED semantics, CAS rowcounts, transaction
rollback, `timestamptz` round-tripping — lives in the part a mock replaces.
Two ways to get a database, in priority order:
1. `SVCFORGE_TEST_DSN` in the environment — an already-running Postgres. This is the
path on a host that has no Docker daemon (for example one whose containerd belongs
to a Kubernetes kubelet, where installing Docker would evict the runtime).
2. testcontainers, which starts `postgres:16-alpine` and throws it away after. This is
the CI path.
Same tests either way.
"""
from __future__ import annotations
import os
from collections.abc import AsyncIterator, Iterator
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
import psycopg
import pytest
import pytest_asyncio
from svcforge_core.repo.db import DictPool, make_pool
MIGRATIONS = Path(__file__).resolve().parents[2] / "migrations"
def _apply_migrations(dsn: str) -> None:
"""Run every migration in lexical order, one transaction each.
No `create table if not exists` and no reset: every caller hands this a database that
was created moments ago (a per-process clone, or a fresh container), so the schema is
always empty and the migrations always apply cleanly from zero.
"""
with psycopg.connect(dsn, autocommit=True) as conn:
for path in sorted(MIGRATIONS.glob("*.sql")):
with conn.transaction(), conn.cursor() as cur:
cur.execute(path.read_text(encoding="utf-8"))
def _private_database(admin_dsn: str) -> Iterator[str]:
"""Clone a scratch database for THIS pytest process only, and drop it after.
The `pool` fixture truncates between tests. That is correct within one process and
catastrophic across several: two pytest runs sharing a database truncate each other's
rows mid-test, and the failures look like real bugs in the code under test rather than
like the harness eating itself. Isolating per process makes concurrent runs
(several agents, or pytest-xdist -n auto) simply work.
"""
name = f"svcforge_test_{os.getpid()}"
parsed = urlsplit(admin_dsn)
with psycopg.connect(admin_dsn, autocommit=True) as conn:
conn.execute(f'drop database if exists "{name}"')
conn.execute(f'create database "{name}"')
dsn = urlunsplit(parsed._replace(path=f"/{name}"))
try:
_apply_migrations(dsn)
yield dsn
finally:
with psycopg.connect(admin_dsn, autocommit=True) as conn:
# Boot any lingering connections, or the drop blocks forever.
conn.execute(
"select pg_terminate_backend(pid) from pg_stat_activity where datname = %s",
(name,),
)
conn.execute(f'drop database if exists "{name}"')
@pytest.fixture(scope="session")
def pg_dsn() -> Iterator[str]:
"""A migrated Postgres, private to this process, from the environment or a container."""
env_dsn = os.getenv("SVCFORGE_TEST_DSN")
if env_dsn:
yield from _private_database(env_dsn)
return
try:
from testcontainers.postgres import PostgresContainer
except ImportError: # pragma: no cover - CI always has it
pytest.skip("set SVCFORGE_TEST_DSN or install testcontainers")
# A container is already private to this process; no need to clone inside it.
with PostgresContainer("postgres:16-alpine", driver=None) as pg:
dsn = pg.get_connection_url()
_apply_migrations(dsn)
yield dsn
@pytest_asyncio.fixture
async def pool(pg_dsn: str) -> AsyncIterator[DictPool]:
"""A clean database and an open pool, per test.
max_size=60 is not a performance choice. test_skip_locked_claims_each_task_exactly_once
races 50 concurrent claims; a pool smaller than that serialises them at the pool
instead of at the database, and the test passes while proving nothing.
"""
async with await psycopg.AsyncConnection.connect(pg_dsn, autocommit=True) as conn:
# catalog_versions joins the list because a halted rollout is sticky by design:
# leave it behind and every later test in the session sees an empty work list.
await conn.execute("truncate tasks, instances, catalog_versions restart identity cascade")
p = make_pool(pg_dsn, min_size=1, max_size=60)
await p.open(wait=True)
try:
yield p
finally:
await p.close()
tests/integration/helpers.py
+44
View File
@@ -0,0 +1,44 @@
"""Builders for integration tests. Keeps the tests about the behaviour under test."""
from __future__ import annotations
from datetime import UTC, datetime
from uuid import UUID, uuid4
from svcforge_core.domain.models import Instance
from svcforge_core.domain.states import InstanceState
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
def build_instance(
team: str = "platform",
service_type: str = "elasticsearch",
size: str = "small",
state: InstanceState = InstanceState.REQUESTED,
chart_version: str = "21.3.19",
) -> Instance:
"""An Instance with a deterministic release_name, as the domain requires."""
iid = uuid4()
now = datetime.now(UTC)
return Instance(
id=iid,
team=team,
service_type=service_type,
size=size,
state=state,
namespace=f"tenant-{team}",
release_name=f"{team}-{service_type}-{str(iid)[:8]}",
chart_version=chart_version,
created_at=now,
updated_at=now,
)
async def make_instance(pool: DictPool, **kwargs: object) -> UUID:
"""Insert an instance and return its id."""
inst = build_instance(**kwargs) # type: ignore[arg-type]
repo = InstanceRepo(pool)
async with pool.connection() as conn:
await repo.create(conn, inst)
return inst.id
tests/integration/test_api.py
+582
View File
@@ -0,0 +1,582 @@
"""API integration tests: real app, real Postgres, real JWTs. No mocks.
`httpx.ASGITransport` calls the app in-process — no uvicorn, no socket, no port to race
over. It exercises the same routing, dependency resolution and lifespan a real request
would; only the TCP hop is gone.
The JWTs here are real RS256 tokens signed by a key generated in-fixture and served
through a PyJWKClient whose cache is pre-seeded. That is deliberate: overriding
`get_current_team` would leave the audience check, the issuer check and the algorithm
allow-list — the parts worth having — completely untested.
"""
from __future__ import annotations
import json
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import UUID, uuid4
import httpx
import jwt
import pytest
import pytest_asyncio
from cryptography.hazmat.primitives.asymmetric import rsa
from fastapi import FastAPI
from services.api.main import create_app
from services.api.routes.instances import release_name_for
from svcforge_core.repo.db import DictPool
from svcforge_core.settings import Settings
CATALOG = Path(__file__).resolve().parents[2] / "catalog.yaml"
ISSUER = "https://issuer.test/realms/svcforge"
AUDIENCE = "svcforge"
KID = "test-key-1"
# --------------------------------------------------------------------------- key material
@pytest.fixture(scope="session")
def rsa_key() -> rsa.RSAPrivateKey:
"""One 2048-bit key for the whole session. Generating it per test costs ~100ms each."""
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
@pytest.fixture(scope="session")
def jwks(rsa_key: rsa.RSAPrivateKey) -> dict[str, Any]:
"""The public half, as a JWKS document — exactly what Keycloak/Zitadel would serve."""
algo = jwt.algorithms.RSAAlgorithm
key_dict: dict[str, Any] = json.loads(algo.to_jwk(rsa_key.public_key()))
key_dict.update({"kid": KID, "alg": "RS256", "use": "sig"})
return {"keys": [key_dict]}
def make_token(
rsa_key: rsa.RSAPrivateKey,
team: str = "platform",
*,
audience: str = AUDIENCE,
issuer: str = ISSUER,
expires_in: timedelta = timedelta(minutes=5),
algorithm: str = "RS256",
) -> str:
"""Sign a token. Defaults are valid; every argument exists so a test can invalidate one."""
now = datetime.now(UTC)
claims: dict[str, Any] = {
"sub": f"user@{team}",
"team": team,
"aud": audience,
"iss": issuer,
"iat": now,
"exp": now + expires_in,
}
key = rsa_key if algorithm == "RS256" else "x" * 32 # HS256 needs >=32 bytes to sign quietly
return jwt.encode(claims, key, algorithm=algorithm, headers={"kid": KID})
def auth(token: str) -> dict[str, str]:
"""An Authorization header."""
return {"Authorization": f"Bearer {token}"}
# --------------------------------------------------------------------------- the app
@pytest.fixture
def settings(pg_dsn: str) -> Settings:
"""Settings pointed at the throwaway database. The whole reason create_app is a factory.
`jwks_url=None` so lifespan builds no real client and attempts no warm-up: the `app`
fixture injects a pre-seeded one straight after. Set it to a fake URL and every test
pays a DNS timeout resolving a host that does not exist — ~4s each, ~2min a run. The
warm-up path itself is covered by test_lifespan_survives_an_unreachable_jwks.
"""
return Settings(
pg_dsn=pg_dsn, # type: ignore[arg-type]
jwks_url=None,
jwt_audience=AUDIENCE,
jwt_issuer=ISSUER,
catalog_path=CATALOG,
pool_min_size=1,
pool_max_size=5,
)
class _FrozenJWKClient:
"""A PyJWKClient with its cache pre-seeded and its network path removed.
Stands in for the real client at the seam `get_current_team` uses. It resolves a kid to
a key exactly as PyJWKClient does; it just cannot reach out to an identity provider
that does not exist in a test run.
"""
def __init__(self, jwks: dict[str, Any]) -> None:
self._keys = jwt.PyJWKSet.from_dict(jwks)
def get_signing_key_from_jwt(self, token: str) -> jwt.PyJWK:
"""Resolve the token's `kid` against the key set. Raises if it is unknown."""
kid = jwt.get_unverified_header(token)["kid"]
for key in self._keys.keys:
if key.key_id == kid:
return key
raise jwt.exceptions.PyJWKClientError(f"unable to find key {kid}")
def get_signing_keys(self) -> list[jwt.PyJWK]:
"""Warm-up hook, called by lifespan."""
return list(self._keys.keys)
@pytest_asyncio.fixture
async def app(settings: Settings, pool: DictPool, jwks: dict[str, Any]) -> AsyncIterator[FastAPI]:
"""A live app with its lifespan run.
Depends on `pool` only for its truncate-per-test side effect; the app opens its own.
"""
application = create_app(settings)
# ASGITransport does NOT run lifespan — it only speaks the `http` scope. Drive the
# lifespan by hand rather than reaching for asgi-lifespan: without this the pool is
# never opened, app.state.pool does not exist, and every DB test dies on AttributeError.
async with application.router.lifespan_context(application):
# Swap the real (network-bound) JWKS client for one whose cache is pre-seeded.
application.state.jwks_client = _FrozenJWKClient(jwks)
yield application
@pytest_asyncio.fixture
async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]:
"""An httpx client wired straight into the ASGI app."""
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
yield c
@pytest.fixture
def token(rsa_key: rsa.RSAPrivateKey) -> str:
"""A valid token for team `platform`."""
return make_token(rsa_key, "platform")
async def _fetch_all(pool: DictPool, sql: str, args: tuple[Any, ...] = ()) -> list[Any]:
"""Read rows back out of the database, bypassing the API entirely."""
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(sql, args)
return list(await cur.fetchall())
# --------------------------------------------------------------------------- create
async def test_post_returns_202_and_location(client: httpx.AsyncClient, token: str) -> None:
"""202 Accepted, not 201: nothing is provisioned yet. Location points at the poll target."""
resp = await client.post(
"/v1/instances",
headers=auth(token),
json={"service_type": "elasticsearch", "size": "small"},
)
assert resp.status_code == 202, resp.text
body = resp.json()
assert resp.headers["location"] == f"/v1/instances/{body['id']}"
assert body["state"] == "requested"
assert body["service_type"] == "elasticsearch"
# Pinned from catalog.yaml at creation time, not echoed from the request.
assert body["chart_version"] == "21.3.15"
assert body["endpoint"] is None
# The response model is an allow-list: placement details stay off the wire.
assert "team" not in body and "namespace" not in body and "release_name" not in body
async def test_post_commits_instance_and_task_together(
client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
"""The point of the whole module: both rows exist, or neither does."""
resp = await client.post(
"/v1/instances",
headers=auth(token),
json={"service_type": "redis", "size": "small"},
)
assert resp.status_code == 202
iid = UUID(resp.json()["id"])
rows = await _fetch_all(pool, "select * from instances where id = %s", (iid,))
assert len(rows) == 1
assert rows[0]["state"] == "requested"
assert rows[0]["team"] == "platform"
assert rows[0]["namespace"] == "tenant-platform"
# The idempotency anchor, and it is unique in the schema.
assert rows[0]["release_name"] == release_name_for("platform", "redis", iid)
tasks = await _fetch_all(pool, "select * from tasks where instance_id = %s", (iid,))
assert len(tasks) == 1
assert tasks[0]["kind"] == "provision"
assert tasks[0]["state"] == "queued"
assert tasks[0]["attempts"] == 0
async def test_release_name_is_deterministic_and_unique(
client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
"""Two requests for the same service_type get distinct releases; the name is pure."""
ids: list[UUID] = []
for _ in range(2):
resp = await client.post(
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
)
assert resp.status_code == 202
ids.append(UUID(resp.json()["id"]))
names = await _fetch_all(pool, "select release_name from instances order by created_at")
assert len({r["release_name"] for r in names}) == 2
# Pure function of (team, service_type, id): recomputing on a worker retry gives the
# same answer, so `helm upgrade --install` lands on the same release.
assert release_name_for("platform", "redis", ids[0]) == release_name_for("platform", "redis", ids[0])
async def test_ttl_days_sets_expires_at(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
"""ttl_days is the tenant-facing knob; expires_at is what the reconciler sweeps."""
resp = await client.post(
"/v1/instances",
headers=auth(token),
json={"service_type": "redis", "size": "small", "ttl_days": 7},
)
assert resp.status_code == 202
rows = await _fetch_all(
pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
)
delta = rows[0]["expires_at"] - datetime.now(UTC)
assert timedelta(days=6, hours=23) < delta <= timedelta(days=7)
async def test_no_expires_at_without_ttl(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
"""No TTL means no expiry. A default TTL would delete someone's database by surprise."""
resp = await client.post(
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
)
rows = await _fetch_all(
pool, "select expires_at from instances where id = %s", (UUID(resp.json()["id"]),)
)
assert rows[0]["expires_at"] is None
# --------------------------------------------------------------------------- validation
async def test_unknown_size_is_rejected(client: httpx.AsyncClient, pool: DictPool, token: str) -> None:
"""size='enormous' is not in the catalog. Nothing is written."""
resp = await client.post(
"/v1/instances",
headers=auth(token),
json={"service_type": "elasticsearch", "size": "enormous"},
)
# The spec says 422 for an unknown size and 404 for an unknown service_type; its
# summary line collapses both into "404 if unknown". Either is defensible; what is not
# defensible is writing the row. Assert the contract that matters and allow both codes.
assert resp.status_code in (404, 422), resp.text
assert await _fetch_all(pool, "select 1 from instances") == []
assert await _fetch_all(pool, "select 1 from tasks") == []
async def test_unknown_service_type_is_404(client: httpx.AsyncClient, token: str) -> None:
"""A service_type the catalog has never heard of is a resource that does not exist."""
resp = await client.post(
"/v1/instances", headers=auth(token), json={"service_type": "mongodb", "size": "small"}
)
assert resp.status_code == 404
assert resp.json()["code"] == "unknown_service_type"
async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) -> None:
"""ttl_days has a schema bound (1..30); pydantic rejects it before the handler runs."""
resp = await client.post(
"/v1/instances",
headers=auth(token),
json={"service_type": "redis", "size": "small", "ttl_days": 999},
)
assert resp.status_code == 422
# --------------------------------------------------------------------------- authn
async def test_missing_authorization_is_401(client: httpx.AsyncClient) -> None:
"""No header -> 401, NOT 403.
This is the HTTPBearer(auto_error=True) trap: FastAPI's default answers a missing
header with 403, which tells the client "you are known and forbidden" when the truth
is "you never said who you are". deps.py passes auto_error=False for exactly this.
"""
resp = await client.post("/v1/instances", json={"service_type": "redis", "size": "small"})
assert resp.status_code == 401
assert resp.headers.get("www-authenticate") == "Bearer"
@pytest.mark.parametrize(
("kwargs", "case"),
[
({"audience": "some-other-api"}, "wrong audience: a token minted for another service"),
({"issuer": "https://evil.test/"}, "wrong issuer"),
({"expires_in": timedelta(minutes=-5)}, "expired"),
({"algorithm": "HS256"}, "algorithm confusion: signed with HMAC, not RS256"),
],
)
async def test_bad_tokens_are_401(
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey, kwargs: dict[str, Any], case: str
) -> None:
"""Every way a token can be wrong produces the same opaque 401.
Identical bodies matter: a caller who can tell "expired" from "bad signature" from
"wrong audience" has an oracle to tune a forgery against.
"""
resp = await client.get("/v1/instances", headers=auth(make_token(rsa_key, **kwargs)))
assert resp.status_code == 401, f"{case} should be rejected"
assert resp.json() == {"code": "unauthorized", "message": "invalid or missing credentials"}
async def test_garbage_token_is_401(client: httpx.AsyncClient) -> None:
"""Not even a JWT. Same 401, no stack trace, no 500."""
resp = await client.get("/v1/instances", headers=auth("not-a-jwt"))
assert resp.status_code == 401
# --------------------------------------------------------------------------- authz
async def test_team_a_cannot_get_team_b_instance(
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
"""404, not 403. AuthZ is the WHERE clause.
403 would confirm the id exists — an enumeration oracle. 404 is the same answer a
made-up uuid gets, so the two are indistinguishable, which is the point.
"""
a_token = make_token(rsa_key, "team-a")
b_token = make_token(rsa_key, "team-b")
created = await client.post(
"/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
)
assert created.status_code == 202
iid = created.json()["id"]
assert (await client.get(f"/v1/instances/{iid}", headers=auth(a_token))).status_code == 200
stolen = await client.get(f"/v1/instances/{iid}", headers=auth(b_token))
assert stolen.status_code == 404
# Byte-identical to a genuinely nonexistent id? The message embeds the id, so compare
# the code: the shape a client can branch on must not distinguish the two cases.
nonexistent = await client.get(f"/v1/instances/{uuid4()}", headers=auth(b_token))
assert nonexistent.status_code == 404
assert stolen.json()["code"] == nonexistent.json()["code"] == "not_found"
async def test_list_is_scoped_to_the_callers_team(
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
"""A list endpoint is where cross-tenant leaks show up first."""
a_token = make_token(rsa_key, "team-a")
b_token = make_token(rsa_key, "team-b")
for tok in (a_token, a_token, b_token):
await client.post("/v1/instances", headers=auth(tok), json={"service_type": "redis", "size": "small"})
a_list = await client.get("/v1/instances", headers=auth(a_token))
assert a_list.status_code == 200
assert len(a_list.json()) == 2
b_list = await client.get("/v1/instances", headers=auth(b_token))
assert len(b_list.json()) == 1
# --------------------------------------------------------------------------- get / delete
async def test_get_unknown_id_is_404(client: httpx.AsyncClient, token: str) -> None:
"""A well-formed uuid that is not a row."""
resp = await client.get(f"/v1/instances/{uuid4()}", headers=auth(token))
assert resp.status_code == 404
async def test_get_malformed_id_is_422(client: httpx.AsyncClient, token: str) -> None:
"""Not a uuid at all: a path-schema failure, caught before the handler."""
resp = await client.get("/v1/instances/not-a-uuid", headers=auth(token))
assert resp.status_code == 422
async def test_delete_moves_to_deleting_and_enqueues_deprovision(
client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
"""202: the helm uninstall has not happened yet. Only the intent is durable."""
created = await client.post(
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
)
iid = UUID(created.json()["id"])
# requested -> deleting is not legal; the state machine only allows it from ready.
async with pool.connection() as conn:
await conn.execute("update instances set state='ready' where id = %s", (iid,))
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
assert resp.status_code == 202, resp.text
assert resp.json()["state"] == "deleting"
rows = await _fetch_all(pool, "select state from instances where id = %s", (iid,))
assert rows[0]["state"] == "deleting"
kinds = await _fetch_all(pool, "select kind from tasks where instance_id = %s order by id", (iid,))
assert [r["kind"] for r in kinds] == ["provision", "deprovision"]
async def test_delete_other_teams_instance_is_404(
client: httpx.AsyncClient, rsa_key: rsa.RSAPrivateKey
) -> None:
"""The destructive endpoint gets the same WHERE-clause treatment as the read."""
a_token = make_token(rsa_key, "team-a")
created = await client.post(
"/v1/instances", headers=auth(a_token), json={"service_type": "redis", "size": "small"}
)
iid = created.json()["id"]
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(make_token(rsa_key, "team-b")))
assert resp.status_code == 404
async def test_delete_from_illegal_state_is_409(
client: httpx.AsyncClient, pool: DictPool, token: str
) -> None:
"""`deleted` is terminal. The state machine says no, and the API does not overrule it."""
created = await client.post(
"/v1/instances", headers=auth(token), json={"service_type": "redis", "size": "small"}
)
iid = UUID(created.json()["id"])
async with pool.connection() as conn:
await conn.execute("update instances set state='deleted' where id = %s", (iid,))
resp = await client.delete(f"/v1/instances/{iid}", headers=auth(token))
assert resp.status_code == 409
assert resp.json()["code"] == "illegal_transition"
async def test_delete_requires_auth(client: httpx.AsyncClient) -> None:
"""No token, no teardown."""
assert (await client.delete(f"/v1/instances/{uuid4()}")).status_code == 401
# --------------------------------------------------------------------------- ops endpoints
async def test_healthz_needs_no_database(settings: Settings) -> None:
"""Liveness does no I/O, so it answers 200 with no database anywhere near it.
Built by hand against a nonsense DSN and with NO lifespan: if /healthz touched the
pool, there is no pool to touch and this would fail. That is the assertion — a DB blip
must never get the whole fleet killed and CrashLoopBackOff'd.
"""
broken = create_app(settings.model_copy(update={"pg_dsn": "postgresql://nobody@127.0.0.1:1/nothing"}))
transport = httpx.ASGITransport(app=broken)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
resp = await c.get("/healthz") # no `async with transport` => lifespan never ran
assert resp.status_code == 200
assert resp.json() == {"status": "ok"}
async def test_healthz_needs_no_auth(client: httpx.AsyncClient) -> None:
"""The kubelet has no bearer token."""
assert (await client.get("/healthz")).status_code == 200
async def test_readyz_is_200_when_the_pool_is_open(client: httpx.AsyncClient) -> None:
"""`select 1` against a live database."""
resp = await client.get("/readyz")
assert resp.status_code == 200
assert resp.json() == {"status": "ready"}
async def test_readyz_is_503_when_the_pool_is_closed(client: httpx.AsyncClient, app: FastAPI) -> None:
"""Readiness fails closed. 503 pulls the pod out of the Service; it does not kill it."""
await app.state.pool.close()
resp = await client.get("/readyz")
assert resp.status_code == 503
assert resp.json()["code"] == "not_ready"
async def test_metrics_serves_the_prometheus_exposition_format(client: httpx.AsyncClient) -> None:
"""A bare /metrics (what every scrape config requests) returns the exposition format."""
resp = await client.get("/metrics")
assert resp.status_code == 200
assert "text/plain" in resp.headers["content-type"]
assert "python_gc_objects_collected_total" in resp.text
# --------------------------------------------------------------------------- contract
async def test_openapi_lists_the_seven_endpoints(client: httpx.AsyncClient) -> None:
"""The deliverable, asserted: /docs renders these paths."""
paths = (await client.get("/openapi.json")).json()["paths"]
assert sorted(paths) == [
"/healthz",
"/metrics",
"/readyz",
"/v1/instances",
"/v1/instances/{instance_id}",
]
assert sorted(paths["/v1/instances"]) == ["get", "post"]
assert sorted(paths["/v1/instances/{instance_id}"]) == ["delete", "get"]
async def test_errors_are_documented_as_errorbody(client: httpx.AsyncClient) -> None:
"""Every error the API returns has one declared shape, and clients can generate against it."""
schema = (await client.get("/openapi.json")).json()
assert sorted(schema["components"]["schemas"]["ErrorBody"]["properties"]) == ["code", "message"]
post = schema["paths"]["/v1/instances"]["post"]["responses"]
for code in ("401", "404", "409", "422"):
ref = post[code]["content"]["application/json"]["schema"]["$ref"]
assert ref.endswith("/ErrorBody"), f"{code} is not documented as ErrorBody"
async def test_202_is_on_the_decorator_not_the_response_object(client: httpx.AsyncClient) -> None:
"""The schema must say 202, not just the runtime.
Setting response.status_code in a handler body changes the response and leaves the
OpenAPI document claiming 200 — so generated clients treat a 202 as an error.
"""
schema = (await client.get("/openapi.json")).json()
assert "202" in schema["paths"]["/v1/instances"]["post"]["responses"]
assert "202" in schema["paths"]["/v1/instances/{instance_id}"]["delete"]["responses"]
def test_auth_disabled_short_circuits(settings: Settings) -> None:
"""The dev escape hatch exists, and Settings.check_production() refuses it in prod."""
dev = settings.model_copy(update={"auth_disabled": True})
with pytest.raises(ValueError, match="refused outside local development"):
dev.check_production()
@pytest.mark.slow
async def test_lifespan_survives_an_unreachable_jwks(settings: Settings) -> None:
"""A down identity provider must not stop the pod from starting.
The warm-up is best-effort on purpose: fail startup on it and an IdP blip means no pod
in the fleet can start, so the outage outlives the blip. A cache miss later costs one
to_thread hop. Requests still 401 until the keys arrive — fail closed, stay up.
"""
unreachable = settings.model_copy(
update={"jwks_url": "https://nonexistent.invalid/protocol/openid-connect/certs"}
)
booted = create_app(unreachable)
async with booted.router.lifespan_context(booted):
transport = httpx.ASGITransport(app=booted)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
assert (await c.get("/healthz")).status_code == 200
# Keys never loaded, so auth fails closed rather than falling open.
assert (await c.get("/v1/instances", headers=auth("whatever"))).status_code == 401
async def test_auth_disabled_accepts_an_unauthenticated_request(settings: Settings) -> None:
"""With the hatch open, no header is needed and a fixed team is used."""
dev_app = create_app(settings.model_copy(update={"auth_disabled": True}))
async with dev_app.router.lifespan_context(dev_app):
transport = httpx.ASGITransport(app=dev_app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
resp = await c.get("/v1/instances")
assert resp.status_code == 200
tests/integration/test_claim.py
+43
View File
@@ -0,0 +1,43 @@
"""The claim race. If only one test in this repo survives, it should be this one."""
from __future__ import annotations
import asyncio
from svcforge_core.domain.models import TaskKind
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.tasks import TaskRepo
from tests.integration.helpers import make_instance
async def test_skip_locked_claims_each_task_exactly_once(pool: DictPool) -> None:
"""50 workers, 50 tasks, one claim each — no double-claims, no lost tasks.
This is the test that fails if you split the claim into select-then-update.
"""
repo = TaskRepo(pool)
inst = await make_instance(pool)
ids = {await repo.enqueue_standalone(inst, TaskKind.PROVISION) for _ in range(50)}
async with asyncio.TaskGroup() as tg:
claims = [tg.create_task(repo.claim(f"w{i}")) for i in range(50)]
got = [c.result() for c in claims]
assert all(t is not None for t in got)
assert sorted(t.id for t in got if t is not None) == sorted(ids) # each exactly once
assert all(t.attempts == 1 for t in got if t is not None)
async def test_more_workers_than_tasks_get_none_not_a_duplicate(pool: DictPool) -> None:
"""Contention must produce None for the losers, never a second claim on one row."""
repo = TaskRepo(pool)
inst = await make_instance(pool)
await repo.enqueue_standalone(inst, TaskKind.PROVISION)
async with asyncio.TaskGroup() as tg:
claims = [tg.create_task(repo.claim(f"w{i}")) for i in range(10)]
got = [c.result() for c in claims]
won = [t for t in got if t is not None]
assert len(won) == 1
assert len([t for t in got if t is None]) == 9
tests/integration/test_day2.py
+157
View File
@@ -0,0 +1,157 @@
"""Day 2 against a real Postgres: the work list, the halt, and the window.
The work-list query is the entire rollout, so it is tested where it runs. `order by
team = %s desc` and `not exists (... halted)` are SQL semantics — a fake repo asserting
them would only prove the fake agrees with itself.
"""
from __future__ import annotations
from datetime import UTC, datetime
from uuid import UUID
import pytest
from svcforge_core.domain.models import TaskKind
from svcforge_core.domain.states import InstanceState
from svcforge_core.domain.windows import parse_window, schedule_upgrade_at
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from tests.integration.helpers import make_instance
OLD = "21.3.19" # what is deployed
PINNED = "21.3.20" # what catalog.yaml now says
OWN_TEAM = "platform"
async def _set_window(pool: DictPool, instance_id: UUID, spec: str | None) -> None:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"update instances set maintenance_window = %s where id = %s",
(spec, instance_id),
)
async def _halt(pool: DictPool, service_type: str) -> None:
"""What `handle_verify` does on a failed probe, and what you undo by hand with SQL."""
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"""insert into catalog_versions (service_type, rollout_state) values (%s, 'halted')
on conflict (service_type) do update set rollout_state = 'halted'""",
(service_type,),
)
@pytest.fixture
async def fleet(pool: DictPool) -> list[UUID]:
"""Three ready instances on the old version. The own-team one is created LAST.
Created last on purpose: `created_at` is the tiebreak, so if the `team = %s desc` sort
were dropped this fixture makes the test fail instead of passing by luck.
"""
ids = [
await make_instance(pool, team="tenant-a", state=InstanceState.READY, chart_version=OLD),
await make_instance(pool, team="tenant-b", state=InstanceState.READY, chart_version=OLD),
await make_instance(pool, team=OWN_TEAM, state=InstanceState.READY, chart_version=OLD),
]
return ids
async def test_work_list_returns_one_row_and_it_is_the_own_team_row(
pool: DictPool, fleet: list[UUID]
) -> None:
"""max_in_flight=1 means one instance moves at a time, and yours is the guinea pig."""
repo = InstanceRepo(pool)
rows = await repo.list_upgradable(
service_type="elasticsearch",
catalog_version=PINNED,
own_team=OWN_TEAM,
max_in_flight=1,
)
assert len(rows) == 1
assert rows[0].instance.team == OWN_TEAM
assert rows[0].instance.id == fleet[2]
assert rows[0].instance.chart_version == OLD
async def test_work_list_skips_instances_already_on_the_pinned_version(
pool: DictPool, fleet: list[UUID]
) -> None:
"""The query is the progress bar: as instances land on PINNED, the list drains to empty."""
repo = InstanceRepo(pool)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("update instances set chart_version = %s", (PINNED,))
rows = await repo.list_upgradable(
service_type="elasticsearch", catalog_version=PINNED, own_team=OWN_TEAM, max_in_flight=10
)
assert rows == []
async def test_halted_rollout_returns_zero_rows(pool: DictPool, fleet: list[UUID]) -> None:
"""One column stops the fleet. This is the whole stop button."""
repo = InstanceRepo(pool)
await _halt(pool, "elasticsearch")
rows = await repo.list_upgradable(
service_type="elasticsearch",
catalog_version=PINNED,
own_team=OWN_TEAM,
max_in_flight=10, # generous on purpose: it is the halt returning 0, not the limit
)
assert rows == []
async def test_halt_is_scoped_to_one_service_type(pool: DictPool) -> None:
"""A broken redis chart must not freeze elasticsearch upgrades."""
repo = InstanceRepo(pool)
await make_instance(
pool, team=OWN_TEAM, service_type="redis", state=InstanceState.READY, chart_version=OLD
)
await make_instance(
pool, team=OWN_TEAM, service_type="elasticsearch", state=InstanceState.READY, chart_version=OLD
)
await _halt(pool, "redis")
assert await repo.list_upgradable("redis", PINNED, OWN_TEAM, 10) == []
assert len(await repo.list_upgradable("elasticsearch", PINNED, OWN_TEAM, 10)) == 1
async def test_windowed_upgrade_is_scheduled_in_the_future_and_security_bypasses_it(
pool: DictPool,
) -> None:
"""The window lands in `tasks.run_after`, and `security: true` ignores it.
Both paths go through the real enqueue, so this also pins the `timestamptz` round-trip:
an aware UTC datetime must come back out of Postgres still aware and still that instant.
"""
repo = InstanceRepo(pool)
tasks = TaskRepo(pool)
iid = await make_instance(pool, team=OWN_TEAM, state=InstanceState.READY, chart_version=OLD)
await _set_window(pool, iid, "0 3 * * 0|Asia/Ho_Chi_Minh")
(candidate,) = await repo.list_upgradable("elasticsearch", PINNED, OWN_TEAM, 1)
window = parse_window(candidate.maintenance_window)
assert window is not None
now = datetime.now(UTC)
routine = await tasks.enqueue_standalone(
iid, TaskKind.UPGRADE, schedule_upgrade_at(window, security=False, now=now)
)
urgent = await tasks.enqueue_standalone(
iid, TaskKind.UPGRADE, schedule_upgrade_at(window, security=True, now=now)
)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select id, run_after from tasks where id = any(%s)", ([routine, urgent],))
run_after = {r["id"]: r["run_after"] for r in await cur.fetchall()}
assert run_after[routine] > now # waits for 03:00 Sunday, Vietnam time
assert run_after[urgent] <= now # a public exploit does not wait
assert run_after[routine].tzinfo is not None
tests/integration/test_helm_timeout.py
+29
View File
@@ -0,0 +1,29 @@
"""The one test that proves the timeout is real.
`bash -c "sleep 300 & sleep 300"` is a miniature helm: a process that forks a child and
waits on another. Kill the direct child only and the backgrounded `sleep` reparents to init
and keeps running — which, when the process is helm, means a timed-out task retries while
the original helm is still mutating the same release.
This test needs a real process tree, so it lives in integration/. It needs no database:
the `pool` fixture in conftest is not autouse.
"""
from __future__ import annotations
import asyncio
import subprocess
import pytest
from svcforge_core.adapters.helm import _run
@pytest.mark.asyncio
async def test_timeout_kills_the_whole_process_group() -> None:
argv = ["bash", "-c", "sleep 300 & sleep 300"] # child forks a grandchild
with pytest.raises(TimeoutError):
await _run(argv, timeout_s=1)
await asyncio.sleep(0.5)
out = subprocess.run(["pgrep", "-f", "sleep 300"], capture_output=True, text=True) # noqa: S607
assert out.stdout.strip() == "", "grandchild survived: you killed the child, not the group"
tests/integration/test_instances.py
+104
View File
@@ -0,0 +1,104 @@
"""InstanceRepo against real SQL."""
from __future__ import annotations
import pytest
from svcforge_core.domain.models import TaskKind
from svcforge_core.domain.states import InstanceState
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from tests.integration.helpers import build_instance
async def test_create_then_get_round_trips(pool: DictPool) -> None:
repo = InstanceRepo(pool)
inst = build_instance()
async with pool.connection() as conn:
created = await repo.create(conn, inst)
assert created.id == inst.id
assert created.release_name == inst.release_name
got = await repo.get(inst.id, team="platform")
assert got is not None
assert got.service_type == "elasticsearch"
assert got.state is InstanceState.REQUESTED
# timestamptz round-trips as aware, or every later comparison raises TypeError.
assert got.created_at.tzinfo is not None
async def test_get_by_other_team_is_none_not_403(pool: DictPool) -> None:
"""A wrong-team id is indistinguishable from a missing one."""
repo = InstanceRepo(pool)
inst = build_instance(team="platform")
async with pool.connection() as conn:
await repo.create(conn, inst)
assert await repo.get(inst.id, team="quant") is None
async def test_list_is_filtered_by_team(pool: DictPool) -> None:
repo = InstanceRepo(pool)
async with pool.connection() as conn:
await repo.create(conn, build_instance(team="platform"))
await repo.create(conn, build_instance(team="quant"))
mine = await repo.list(team="platform")
assert len(mine) == 1
assert all(i.team == "platform" for i in mine)
async def test_update_state_cas_rejects_stale_expectation(pool: DictPool) -> None:
repo = InstanceRepo(pool)
inst = build_instance()
async with pool.connection() as conn:
await repo.create(conn, inst)
ok = await repo.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
assert ok is True
# The row already moved: the second caller must lose, and must not raise.
lost = await repo.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
assert lost is False
async def test_update_state_sets_endpoint(pool: DictPool) -> None:
repo = InstanceRepo(pool)
inst = build_instance()
async with pool.connection() as conn:
await repo.create(conn, inst)
await repo.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
ok = await repo.update_state(
inst.id, InstanceState.PROVISIONING, InstanceState.READY, endpoint="http://es:9200"
)
assert ok is True
got = await repo.get(inst.id, team="platform")
assert got is not None
assert got.endpoint == "http://es:9200"
assert got.state is InstanceState.READY
@pytest.mark.parametrize("explode", [True])
async def test_instance_and_task_roll_back_together(pool: DictPool, explode: bool) -> None:
"""The reason the queue is in Postgres, as an executable claim.
If the transaction aborts, BOTH the instance and its provision task must vanish.
An instance with no task never gets built; a task with no instance is an orphan.
"""
instances, tasks = InstanceRepo(pool), TaskRepo(pool)
inst = build_instance()
with pytest.raises(RuntimeError):
async with pool.connection() as conn, conn.transaction():
await instances.create(conn, inst)
await tasks.enqueue(conn, inst.id, TaskKind.PROVISION)
if explode:
raise RuntimeError("boom, mid-transaction")
assert await instances.get(inst.id, team="platform") is None
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select count(*) as n from tasks")
row = await cur.fetchone()
assert row is not None
assert row["n"] == 0
tests/integration/test_reconciler.py
+556
View File
@@ -0,0 +1,556 @@
"""The four checks, against a real Postgres and a fake cluster.
Integration, not unit, because there is nothing to unit test: each check is a query and a
transaction. The behaviour worth asserting — that the CAS and the insert commit together,
that the idempotency guard is a real `not exists`, that a second tick does not double-
enqueue — lives entirely in the part a mock would replace.
The cluster is faked; the database is not. FakeProvisioner is a dict of releases, which is
all the drift check needs: drift is "helm says X, the DB says Y", and a dict says X.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID
import pytest
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from services.reconciler.main import (
ReconcilerDeps,
check_drift,
check_lease_expiry,
check_ttl,
check_version_drift,
tick,
)
from svcforge_core.domain.models import CatalogEntry, SizeSpec, TaskKind, TaskState
from svcforge_core.domain.states import InstanceState
from svcforge_core.obs import RECONCILER_LAST_TICK
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.reconcile import ReconcileRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings
from tests.fakes import FakeClock, FakeNotifier, FakeProvisioner
from tests.integration.helpers import build_instance
NOW = datetime(2026, 7, 17, 12, 0, tzinfo=UTC) # a Friday
OLD_VERSION = "21.3.19"
NEW_VERSION = "21.3.20"
# '0 3 * * 0' is 03:00 Sunday. From a Friday noon that is always in the future, which is
# the whole assertion of the window test.
SUNDAY_0300_HCM = "0 3 * * 0|Asia/Ho_Chi_Minh"
def _entry(*, version: str = NEW_VERSION, security: bool = False) -> CatalogEntry:
return CatalogEntry(
service_type="elasticsearch",
chart="bitnamilegacy/elasticsearch",
chart_version=version,
security=security,
sizes={"small": SizeSpec(replicas=1, resources={})},
)
def _settings(**over: object) -> Settings:
return Settings(pg_dsn="postgresql://u:p@localhost:5432/db", **over) # type: ignore[arg-type]
def _deps(
pool: DictPool,
provisioner: FakeProvisioner,
*,
catalog: dict[str, CatalogEntry] | None = None,
max_in_flight: int = 1,
**settings_over: object,
) -> ReconcilerDeps:
return ReconcilerDeps(
pool=pool,
instances=InstanceRepo(pool),
tasks=TaskRepo(pool),
reconcile=ReconcileRepo(pool),
provisioner=provisioner,
notifier=FakeNotifier(),
clock=FakeClock(NOW),
catalog=catalog if catalog is not None else {"elasticsearch": _entry()},
settings=_settings(**settings_over),
own_team="platform",
max_in_flight=max_in_flight,
)
async def _seed(
pool: DictPool,
*,
state: InstanceState = InstanceState.READY,
team: str = "platform",
chart_version: str = OLD_VERSION,
expires_at: datetime | None = None,
maintenance_window: str | None = None,
) -> tuple[UUID, str, str]:
"""Insert one instance. Returns (id, release_name, namespace).
`expires_at` and `maintenance_window` go in with SQL rather than through
`InstanceRepo.create`: create() does not write `maintenance_window` at all (nothing but
the day-2 work list reads it), and that is a fact about the repo, not a gap in it.
"""
inst = build_instance(team=team, state=state, chart_version=chart_version)
repo = InstanceRepo(pool)
async with pool.connection() as conn:
await repo.create(conn, inst)
async with conn.cursor() as cur:
await cur.execute(
"update instances set expires_at = %s, maintenance_window = %s where id = %s",
(expires_at, maintenance_window, inst.id),
)
return inst.id, inst.release_name, inst.namespace
async def _tasks_for(pool: DictPool, instance_id: UUID) -> list[dict[str, Any]]:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"select id, kind, state, run_after, traceparent from tasks where instance_id = %s order by id",
(instance_id,),
)
return list(await cur.fetchall())
async def _state_of(pool: DictPool, instance_id: UUID) -> str:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select state from instances where id = %s", (instance_id,))
row = await cur.fetchone()
assert row is not None
return str(row["state"])
# --- Check 1: drift ------------------------------------------------------------------------
async def test_drift_reprovisions_a_ready_instance_whose_release_vanished(
pool: DictPool,
) -> None:
"""`helm uninstall` by hand. Nobody sends an event; the next tick notices anyway.
This is the acceptance path from Module 7:
helm uninstall <release> -n <ns> && python -m services.reconciler.main --once
select kind, state from tasks order by id desc limit 1 -> provision | queued
"""
instance_id, _, _ = await _seed(pool)
deps = _deps(pool, FakeProvisioner()) # empty cluster: the release is gone
await check_drift(deps)
tasks = await _tasks_for(pool, instance_id)
assert [(t["kind"], t["state"]) for t in tasks] == [(TaskKind.PROVISION.value, TaskState.QUEUED.value)]
async def test_drift_leaves_the_instance_in_provisioning_not_failed(pool: DictPool) -> None:
"""The two-hop state change, and why it matters.
`handle_provision` returns early on a `ready` row and ends with a
`provisioning -> ready` CAS. Hand it anything else and helm runs but the bookkeeping
lands nowhere. So the reconciler must leave the row in `provisioning` — via `failed`,
because LEGAL has no `ready -> provisioning` edge — before the worker can claim it.
"""
instance_id, _, _ = await _seed(pool)
await check_drift(_deps(pool, FakeProvisioner()))
assert await _state_of(pool, instance_id) == InstanceState.PROVISIONING.value
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select error from instances where id = %s", (instance_id,))
row = await cur.fetchone()
assert row is not None
assert "drift" in row["error"] # the tenant gets told why, not just that
async def test_drift_ignores_an_instance_whose_release_is_present(pool: DictPool) -> None:
"""The happy path is the same code. It must enqueue nothing at all."""
instance_id, release, namespace = await _seed(pool)
provisioner = FakeProvisioner()
await provisioner.install(release, namespace, _entry(), {})
await check_drift(_deps(pool, provisioner))
assert await _tasks_for(pool, instance_id) == []
assert await _state_of(pool, instance_id) == InstanceState.READY.value
async def test_drift_is_idempotent_across_ticks(pool: DictPool) -> None:
"""Two ticks, one task. A provision takes minutes; ticks are 60 seconds apart.
Without the guard the second tick sees a `provisioning` row — no longer `ready`, so the
drift branch skips it. The guard is what covers the case where it is `ready` again
before the task is done.
"""
instance_id, _, _ = await _seed(pool)
deps = _deps(pool, FakeProvisioner())
await check_drift(deps)
await check_drift(deps)
assert len(await _tasks_for(pool, instance_id)) == 1
async def test_drift_never_deletes_an_orphan_release(pool: DictPool) -> None:
"""A release the DB has never heard of. Log it, bill nobody, delete nothing.
v1 policy, and it is a policy about evidence: "no row in this table" is not proof the
release is unowned. It might belong to another tool, another team, or a migration that
is half done. An operator deletes it after reading the log.
"""
provisioner = FakeProvisioner()
await provisioner.install("someone-elses-redis", "other-ns", _entry(), {})
await check_drift(_deps(pool, provisioner))
assert "someone-elses-redis" in provisioner.releases
assert provisioner.uninstalled == []
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select count(*) as n from tasks")
row = await cur.fetchone()
assert row is not None
assert row["n"] == 0
async def test_drift_does_not_call_an_in_flight_provision_an_orphan(pool: DictPool) -> None:
"""A `requested` instance a worker is installing right now is not an orphan.
`known_releases` covers every row in any state for exactly this reason. Scope it to
`ready` and every provision in progress gets reported as an orphan on every tick, which
trains everyone to ignore the orphan log.
"""
_, release, namespace = await _seed(pool, state=InstanceState.REQUESTED)
provisioner = FakeProvisioner()
await provisioner.install(release, namespace, _entry(), {})
await check_drift(_deps(pool, provisioner))
assert provisioner.uninstalled == []
# --- Check 2: lease expiry -----------------------------------------------------------------
async def test_lease_expiry_returns_a_dead_workers_task_to_the_queue(pool: DictPool) -> None:
"""SIGKILL leaves `running` with `locked_by` set and nobody running it.
No cleanup code in the worker can fix this, because the worker is the part that died.
The lease is the only thing that recovers the row.
"""
instance_id, _, _ = await _seed(pool)
tasks = TaskRepo(pool)
await tasks.enqueue_standalone(instance_id, TaskKind.PROVISION)
claimed = await tasks.claim("worker-that-is-about-to-die")
assert claimed is not None
# The worker died six minutes ago and never reported.
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"update tasks set locked_at = now() - interval '6 minutes' where id = %s",
(claimed.id,),
)
await check_lease_expiry(_deps(pool, FakeProvisioner(), lease_seconds=300))
rows = await _tasks_for(pool, instance_id)
assert rows[0]["state"] == TaskState.QUEUED.value
async def test_lease_expiry_leaves_a_live_worker_alone(pool: DictPool) -> None:
"""A task claimed a second ago is not a dead worker. Reclaiming it would double-provision.
Handlers are idempotent, so a wrongly-freed lease is survivable — but survivable is not
free, and this is why lease_seconds sits above helm's own --timeout.
"""
instance_id, _, _ = await _seed(pool)
tasks = TaskRepo(pool)
await tasks.enqueue_standalone(instance_id, TaskKind.PROVISION)
assert await tasks.claim("worker-1") is not None
await check_lease_expiry(_deps(pool, FakeProvisioner(), lease_seconds=300))
rows = await _tasks_for(pool, instance_id)
assert rows[0]["state"] == TaskState.RUNNING.value
# --- Check 3: TTL --------------------------------------------------------------------------
async def test_ttl_expired_instance_goes_to_deleting_with_a_deprovision_task(
pool: DictPool,
) -> None:
"""The check that stops a demo cluster from becoming a permanent line on the bill."""
instance_id, _, _ = await _seed(pool, expires_at=datetime.now(UTC) - timedelta(minutes=1))
await check_ttl(_deps(pool, FakeProvisioner()))
tasks = await _tasks_for(pool, instance_id)
assert [(t["kind"], t["state"]) for t in tasks] == [(TaskKind.DEPROVISION.value, TaskState.QUEUED.value)]
# `deleting` before the worker claims it: handle_deprovision ends with a
# `deleting -> deleted` CAS, and a `ready` row would leave the DB advertising an
# endpoint for a release helm has already removed.
assert await _state_of(pool, instance_id) == InstanceState.DELETING.value
async def test_ttl_ignores_an_instance_that_has_not_expired(pool: DictPool) -> None:
"""And ignores one with no expires_at at all: null means no TTL, not expired."""
live, _, _ = await _seed(pool, expires_at=datetime.now(UTC) + timedelta(hours=1))
forever, _, _ = await _seed(pool, expires_at=None)
await check_ttl(_deps(pool, FakeProvisioner()))
assert await _tasks_for(pool, live) == []
assert await _tasks_for(pool, forever) == []
async def test_ttl_is_idempotent_across_ticks(pool: DictPool) -> None:
"""A deprovision takes minutes and ticks are 60s apart. One task, not four."""
instance_id, _, _ = await _seed(pool, expires_at=datetime.now(UTC) - timedelta(minutes=1))
deps = _deps(pool, FakeProvisioner())
await check_ttl(deps)
await check_ttl(deps)
await check_ttl(deps)
assert len(await _tasks_for(pool, instance_id)) == 1
async def test_ttl_recovers_a_deleting_instance_whose_task_was_never_enqueued(
pool: DictPool,
) -> None:
"""The API's DELETE crashed between the CAS and the enqueue. This is the sweep it relies on.
That statement order is chosen *because* this check exists. The other order leaves a
deprovision task pointing at a `ready` instance, and a worker tears down a live service
nobody asked to delete.
"""
instance_id, _, _ = await _seed(pool, state=InstanceState.DELETING)
await check_ttl(_deps(pool, FakeProvisioner()))
tasks = await _tasks_for(pool, instance_id)
assert [t["kind"] for t in tasks] == [TaskKind.DEPROVISION.value]
assert await _state_of(pool, instance_id) == InstanceState.DELETING.value
async def test_ttl_re_enqueues_after_a_deprovision_exhausted_its_attempts(
pool: DictPool,
) -> None:
"""`done` and `failed` are not outstanding. A transient outage must not strand the row.
The guard asks "is one queued or running", not "has one ever existed" — otherwise a
deprovision that burned its five attempts during a cluster outage would leave the
instance billing forever with nothing left to retry it.
"""
instance_id, _, _ = await _seed(pool, state=InstanceState.DELETING)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"insert into tasks (instance_id, kind, state) values (%s, %s, %s)",
(instance_id, TaskKind.DEPROVISION.value, TaskState.FAILED.value),
)
await check_ttl(_deps(pool, FakeProvisioner()))
states = [t["state"] for t in await _tasks_for(pool, instance_id)]
assert TaskState.QUEUED.value in states
# --- Check 4: version drift ----------------------------------------------------------------
async def test_version_drift_enqueues_one_upgrade_for_the_own_team_instance_first(
pool: DictPool,
) -> None:
"""max_in_flight=1 across three stale instances, and it picks ours.
Eating your own dog food is an `order by`: we are the tenant who finds out the chart is
broken, and the halt stops the other two before they ever hear about it.
"""
await _seed(pool, team="payments")
await _seed(pool, team="search")
ours, _, _ = await _seed(pool, team="platform")
await check_version_drift(_deps(pool, FakeProvisioner()))
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select instance_id, kind from tasks")
rows = list(await cur.fetchall())
assert len(rows) == 1
assert rows[0]["instance_id"] == ours
assert rows[0]["kind"] == TaskKind.UPGRADE.value
async def test_version_drift_enqueues_nothing_while_the_rollout_is_halted(
pool: DictPool,
) -> None:
"""One column stops the fleet. A failed verify writes it; a human clears it with SQL."""
instance_id, _, _ = await _seed(pool)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute(
"insert into catalog_versions (service_type, rollout_state) values ('elasticsearch', 'halted')"
)
await check_version_drift(_deps(pool, FakeProvisioner()))
assert await _tasks_for(pool, instance_id) == []
async def test_version_drift_ignores_an_instance_already_on_the_catalog_version(
pool: DictPool,
) -> None:
"""`chart_version` is written only after helm succeeds, which is what makes this the query."""
instance_id, _, _ = await _seed(pool, chart_version=NEW_VERSION)
await check_version_drift(_deps(pool, FakeProvisioner()))
assert await _tasks_for(pool, instance_id) == []
async def test_version_drift_parks_the_upgrade_until_the_maintenance_window(
pool: DictPool,
) -> None:
"""The queue does the waiting, in `where run_after <= now()`.
Not a scheduler and not an in-memory timer: a task parked in Postgres until 03:00 Sunday
survives a reconciler restart. That is the whole reason `run_after` exists.
"""
instance_id, _, _ = await _seed(pool, maintenance_window=SUNDAY_0300_HCM)
await check_version_drift(_deps(pool, FakeProvisioner()))
tasks = await _tasks_for(pool, instance_id)
assert len(tasks) == 1
assert tasks[0]["run_after"] > NOW # a Friday; the next 03:00 Sunday is days away
async def test_version_drift_bypasses_the_window_for_a_security_bump(pool: DictPool) -> None:
"""A CVE with a public exploit does not wait until Sunday. That is what `security:` is for."""
instance_id, _, _ = await _seed(pool, maintenance_window=SUNDAY_0300_HCM)
catalog = {"elasticsearch": _entry(security=True)}
await check_version_drift(_deps(pool, FakeProvisioner(), catalog=catalog))
tasks = await _tasks_for(pool, instance_id)
assert len(tasks) == 1
assert tasks[0]["run_after"] <= NOW
async def test_version_drift_is_idempotent_across_ticks(pool: DictPool) -> None:
"""The guard that makes max_in_flight mean anything.
The instance stays on the work list for the whole duration of its own upgrade —
`chart_version` is only written on success — and for the hours it spends parked waiting
for 03:00. Without the guard, `max_in_flight=1` is sixty tasks an hour against one
release.
"""
instance_id, _, _ = await _seed(pool, maintenance_window=SUNDAY_0300_HCM)
deps = _deps(pool, FakeProvisioner())
for _ in range(3):
await check_version_drift(deps)
assert len(await _tasks_for(pool, instance_id)) == 1
async def test_version_drift_skips_one_bad_window_and_keeps_going(pool: DictPool) -> None:
"""One tenant's typo must not freeze everyone else's security rollout."""
broken, _, _ = await _seed(pool, team="payments", maintenance_window="not a cron|Asia/Ho_Chi_Minh")
deps = _deps(pool, FakeProvisioner(), max_in_flight=5)
await check_version_drift(deps) # must not raise
assert await _tasks_for(pool, broken) == []
# --- The tick ------------------------------------------------------------------------------
class _AngryProvisioner(FakeProvisioner):
"""A cluster that cannot be reached. The drift check's worst day."""
async def list_releases(self) -> list[Any]:
raise RuntimeError("dial tcp: i/o timeout")
async def test_tick_runs_the_other_three_checks_when_one_blows_up(pool: DictPool) -> None:
"""A helm binary that cannot reach the API server must not stop TTLs from expiring.
This is the entire argument for wrapping each check independently, and it is asserted
rather than assumed because the failure mode — a tick that dies on check one — looks
exactly like a tick that found nothing to do.
"""
expired, _, _ = await _seed(pool, expires_at=datetime.now(UTC) - timedelta(minutes=1))
deps = _deps(pool, _AngryProvisioner())
await tick(deps) # must not raise
assert [t["kind"] for t in await _tasks_for(pool, expired)] == [TaskKind.DEPROVISION.value]
async def test_tick_sets_the_gauges_and_the_heartbeat(pool: DictPool) -> None:
"""queue_depth after the checks, not before, and the heartbeat unconditionally.
The heartbeat is what `SvcforgeReconcilerStale` reads. It answers "is the loop running",
not "is everything fine" — the checks have their own alerts, and an alert that means two
things gets muted.
"""
from prometheus_client import REGISTRY
await _seed(pool, expires_at=datetime.now(UTC) - timedelta(minutes=1))
deps = _deps(pool, _AngryProvisioner()) # one check fails; the heartbeat still ticks
await tick(deps)
assert REGISTRY.get_sample_value("svcforge_queue_depth") == 1.0
assert REGISTRY.get_sample_value("svcforge_instances", {"state": "deleting"}) == 1.0
assert RECONCILER_LAST_TICK._value.get() == pytest.approx(NOW.timestamp())
@pytest.fixture(scope="session")
def tracing() -> InMemorySpanExporter:
"""A real tracer provider for the process, collecting spans in memory.
Global because OTEL's is: `trace.set_tracer_provider` takes once per process, and
`obs.tracer()` resolves it at call time. Session-scoped so the second call never
happens.
"""
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
trace.set_tracer_provider(provider)
return exporter
async def test_a_task_the_tick_enqueues_carries_the_ticks_traceparent(
pool: DictPool,
tracing: InMemorySpanExporter,
) -> None:
"""Nothing propagates a trace through a table. The column is written at insert or never.
Driven through `tick`, not through `check_drift` with a span wrapped around it by the
test — that version passed while the real entrypoint wrote null on every row, because
the only span in the production path (`helm.list`) had already closed by the time the
insert ran. A test that supplies the context under test proves the propagator works and
nothing about this service.
"""
instance_id, _, _ = await _seed(pool)
tracing.clear()
await tick(_deps(pool, FakeProvisioner()))
tasks = await _tasks_for(pool, instance_id)
traceparent = tasks[0]["traceparent"]
assert traceparent is not None, "the reconciler's own tasks are unjoinable to its tick"
# Same trace as the tick's span, which is the entire point of storing the column.
tick_spans = [s for s in tracing.get_finished_spans() if s.name == "reconciler.tick"]
assert len(tick_spans) == 1
assert traceparent.split("-")[1] == format(tick_spans[0].context.trace_id, "032x")
tests/integration/test_redis.py
+494
View File
@@ -0,0 +1,494 @@
"""Module 10 acceptance: the limiter, the idempotency store, the cache, and the budget.
Six checks, in the spec's order:
1. Rate limit 10/min — the 11th is refused.
2. A check costs exactly ONE Redis command.
3. Idempotency — the same key twice yields the same UUID.
4. Cache — the second read costs one command and no DB query.
5. Redis DOWN = the platform stays UP.
6. The budget metric exists, and `scripts/redis_budget.py` reads it correctly.
**Two tiers, and the split is the budget.** Upstash's free tier is 500K commands/month;
a test suite that hammers it is itself the bug this module is about. So the semantics are
proved against `tests/fakes.py` (free, deterministic, runs on every commit), and only the
things a fake cannot prove — that the Lua is valid Lua, that `KEYS`/`ARGV` are 1-based,
that redis-py bills one EVALSHA, that `decode_responses=True` is set — are proved against
real Upstash under `@pytest.mark.slow`. That tier spends roughly thirty commands per run,
and `pytest -m "not slow"` skips it entirely.
Check 5 needs no server at all: a closed port is a more faithful outage than a mock.
"""
from __future__ import annotations
import importlib.util
import os
import time
from collections.abc import AsyncIterator, Callable
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
from typing import Any
from uuid import UUID, uuid4
import pytest
import pytest_asyncio
from prometheus_client import REGISTRY, generate_latest
from redis.asyncio import Redis
from svcforge_core.adapters.redis import (
IdempotencyStore,
InstanceCache,
RateLimiter,
RateLimitResult,
make_redis,
)
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.settings import Settings
from tests.fakes import FakeClock, FakeIdempotencyStore, FakeInstanceCache, FakeRateLimiter
from tests.integration.helpers import build_instance
# Any DSN that parses. These tests never open a Postgres connection through Settings; the
# `pool` fixture owns the real database.
_DUMMY_PG_DSN = "postgresql://unused:unused@127.0.0.1:5432/unused"
# A port nothing listens on. `make_redis` will build a client, every command will get
# ECONNREFUSED, and that is the point of check 5.
_DEAD_REDIS_DSN = "redis://127.0.0.1:1/0"
_T0 = datetime(2026, 7, 17, 12, 0, 0, tzinfo=UTC)
def _metric(op: str) -> float:
"""The budget counter for one op. Absent labels read as 0, not as an error."""
value = REGISTRY.get_sample_value("svcforge_redis_commands_total", {"op": op})
return value or 0.0
def _errors(op: str) -> float:
value = REGISTRY.get_sample_value("svcforge_redis_errors_total", {"op": op})
return value or 0.0
# --- Fixtures ---------------------------------------------------------------------------
@pytest_asyncio.fixture
async def upstash() -> AsyncIterator[Redis]:
"""The real thing, from `~/.config/svcforge/secrets.env`. Skipped when unset.
Built through `make_redis` rather than `Redis.from_url` directly, so that
`decode_responses=True` is covered by these tests instead of being a comment. Forget it
and every assertion below dies on `bytes != str`, which is the whole reason it is the
first bug everyone hits.
"""
dsn = os.getenv("SVCFORGE_REDIS_DSN")
if not dsn:
pytest.skip("SVCFORGE_REDIS_DSN unset; real-Upstash checks skipped")
client = make_redis(Settings(pg_dsn=_DUMMY_PG_DSN, redis_dsn=dsn)) # type: ignore[arg-type] # pydantic coerces str -> *Dsn
assert client is not None
try:
yield client
finally:
await client.aclose()
@pytest_asyncio.fixture
async def dead_redis() -> AsyncIterator[Redis]:
"""A client pointed at a closed port. No server was harmed, no commands were billed."""
client = make_redis(Settings(pg_dsn=_DUMMY_PG_DSN, redis_dsn=_DEAD_REDIS_DSN)) # type: ignore[arg-type] # pydantic coerces str -> *Dsn
assert client is not None
try:
yield client
finally:
await client.aclose()
class _CommandCounter:
"""Counts round trips by wrapping `execute_command` on one client instance.
This counts what Upstash bills, which is the only definition that matters here. The
metric counter is our own bookkeeping and could be wrong in the same direction as the
code it measures; this one cannot.
"""
def __init__(self, r: Redis) -> None:
self.count = 0
self._inner: Callable[..., Any] = r.execute_command
r.execute_command = self._counting # type: ignore[method-assign]
async def _counting(self, *args: Any, **kwargs: Any) -> Any: # noqa: ANN401
self.count += 1
return await self._inner(*args, **kwargs)
def _budget_script() -> ModuleType:
"""Import `scripts/redis_budget.py` by path — `scripts/` is not a package, deliberately."""
path = Path(__file__).resolve().parents[2] / "scripts" / "redis_budget.py"
spec = importlib.util.spec_from_file_location("redis_budget", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
# --- 1. Rate limit: 10/min, the 11th is refused ------------------------------------------
async def test_eleventh_request_in_the_window_is_refused() -> None:
"""`200 x10` then `429`. The fake, so this runs on every commit for free."""
limiter = FakeRateLimiter(limit=10, window_s=60, clock=FakeClock(start=_T0))
results = [await limiter.check("acme") for _ in range(11)]
assert [r.allowed for r in results] == [True] * 10 + [False]
assert results[9].remaining == 0
assert results[10].remaining == 0
# What the handler puts in `Retry-After` on the 429. Never 0: a client told to retry
# immediately retries into the same closed window.
assert results[10].retry_after_s >= 1
async def test_the_window_rolls_over_and_the_caller_is_allowed_again() -> None:
"""A fixed window resets on a boundary, not on a sleep. Hence the injected clock."""
clock = FakeClock(start=_T0)
limiter = FakeRateLimiter(limit=2, window_s=60, clock=clock)
assert (await limiter.check("acme")).allowed
assert (await limiter.check("acme")).allowed
assert not (await limiter.check("acme")).allowed
clock.advance(datetime(2026, 7, 17, 12, 1, 0, tzinfo=UTC) - _T0)
assert (await limiter.check("acme")).allowed
async def test_teams_do_not_share_a_window() -> None:
"""`rl:{team}:{window}` — a noisy tenant must not refuse a quiet one."""
limiter = FakeRateLimiter(limit=1, window_s=60, clock=FakeClock(start=_T0))
assert (await limiter.check("acme")).allowed
assert not (await limiter.check("acme")).allowed
assert (await limiter.check("globex")).allowed
@pytest.mark.slow
async def test_real_lua_refuses_the_eleventh(upstash: Redis) -> None:
"""The same assertion against real Upstash. ~11 commands.
This is what a fake cannot prove: that the script is valid Lua, that `KEYS[1]` and
`ARGV[1]` are 1-based (0-based indexing would read nil and compare false forever), and
that `INCR`-then-conditional-`EXPIRE` actually holds a window open.
"""
team = f"test-{uuid4().hex[:8]}"
limiter = RateLimiter(upstash, limit=10, window_s=60)
results = [await limiter.check(team) for _ in range(11)]
assert [r.allowed for r in results] == [True] * 10 + [False]
assert not any(r.degraded for r in results)
assert results[0].remaining == 9
# The EXPIRE fired on the first INCR only, so the key is not immortal. `Every key gets
# a TTL` is a rule with no enforcement other than checking.
window = int(time.time()) // 60
keys = [f"rl:{team}:{window}", f"rl:{team}:{window - 1}"]
ttls = [await upstash.ttl(k) for k in keys]
assert any(0 < ttl <= 60 for ttl in ttls)
await upstash.delete(*keys)
# --- 2. It costs ONE command per check ---------------------------------------------------
@pytest.mark.slow
async def test_a_check_costs_exactly_one_redis_command(upstash: Redis) -> None:
"""One EVALSHA. Not GET+INCR+EXPIRE, which is three billed commands and a race.
At 500K/month the difference is not academic: three commands per request caps the
platform at 166K requests/month instead of 500K, for a limiter that is also wrong.
The first call is excluded from the count on purpose. redis-py sends EVALSHA, Upstash
answers NOSCRIPT because it has never seen the hash, and redis-py replays it as EVAL —
two commands, once per Redis restart, and irrelevant to the steady state this measures.
"""
team = f"test-{uuid4().hex[:8]}"
limiter = RateLimiter(upstash, limit=100, window_s=60)
await limiter.check(team) # warm the script cache
counter = _CommandCounter(upstash)
before = _metric("ratelimit")
result = await limiter.check(team)
assert counter.count == 1, "a rate limit check must be one round trip and one billed command"
assert _metric("ratelimit") - before == 1, "the budget counter must agree with the wire"
assert result.allowed
window = int(time.time()) // 60
await upstash.delete(f"rl:{team}:{window}", f"rl:{team}:{window - 1}")
# --- 3. Idempotency: the same key twice is one instance ----------------------------------
async def test_same_idempotency_key_returns_the_first_instance_id() -> None:
"""The first caller wins; the second is told who won and must not create anything."""
store = FakeIdempotencyStore()
key = str(uuid4())
first, second = uuid4(), uuid4()
assert await store.claim(key, first) is None, "None means 'you won, go create it'"
assert await store.claim(key, second) == first, "the loser gets the winner's id, not its own"
async def test_different_idempotency_keys_do_not_collide() -> None:
store = FakeIdempotencyStore()
a, b = uuid4(), uuid4()
assert await store.claim(str(uuid4()), a) is None
assert await store.claim(str(uuid4()), b) is None
@pytest.mark.slow
async def test_real_set_nx_ex_claims_once(upstash: Redis) -> None:
"""SET NX EX against Upstash. ~3 commands.
Also asserts the TTL, because a claim marker without one is a permanent record of a
request from last March, and 256 MB of those ends by evicting the keys you cared about.
"""
key = f"test-{uuid4()}"
first, second = uuid4(), uuid4()
store = IdempotencyStore(upstash, ttl_s=60)
assert await store.claim(key, first) is None
assert await store.claim(key, second) == first
assert 0 < await upstash.ttl(f"idem:{key}") <= 60
await upstash.delete(f"idem:{key}")
# --- 4. Cache: the second read costs one command and no DB query -------------------------
class _CountingRepo:
"""Wraps InstanceRepo and counts reads. The DB query count is the assertion."""
def __init__(self, repo: InstanceRepo) -> None:
self._repo = repo
self.gets = 0
async def get(self, id: UUID, team: str) -> Any: # noqa: ANN401
self.gets += 1
return await self._repo.get(id, team)
async def _read_through(
instance_id: UUID,
team: str,
cache: FakeInstanceCache | InstanceCache,
repo: _CountingRepo,
) -> Any: # noqa: ANN401
"""Cache-aside, as `GET /v1/instances/{id}` performs it. Hit = 1 command, miss = 2."""
cached = await cache.get(instance_id)
if cached is not None:
return cached
inst = await repo.get(instance_id, team)
if inst is not None:
await cache.put(inst)
return inst
async def test_second_read_is_served_from_cache_without_touching_postgres(
pool: DictPool,
) -> None:
"""Miss, then hit. The DB is read exactly once for two reads."""
repo = InstanceRepo(pool)
# `create` returns the row as Postgres stored it. Compare against that, never against
# the model that went in: `created_at`/`updated_at` are DB defaults, and a timestamptz
# comes back tagged `Etc/UTC` rather than `timezone.utc`. Same instant, different repr.
async with pool.connection() as conn:
inst = await repo.create(conn, build_instance(team="platform"))
counting = _CountingRepo(repo)
cache = FakeInstanceCache()
first = await _read_through(inst.id, inst.team, cache, counting)
second = await _read_through(inst.id, inst.team, cache, counting)
assert first == second == inst
assert counting.gets == 1, "the second read must not reach Postgres"
assert (cache.misses, cache.hits) == (1, 1)
async def test_invalidate_sends_the_next_read_back_to_postgres() -> None:
"""The worker calls this inside the code path that writes the state, not after it."""
cache = FakeInstanceCache()
inst = build_instance()
await cache.put(inst)
assert await cache.get(inst.id) == inst
await cache.invalidate(inst.id)
assert await cache.get(inst.id) is None
@pytest.mark.slow
async def test_real_cache_hit_costs_one_command_and_no_db_query(pool: DictPool, upstash: Redis) -> None:
"""Real Redis, real Postgres. ~4 commands.
The round-trip count is the acceptance criterion: a hit that costs two commands is a
cache that has doubled the bill it was added to reduce.
"""
repo = InstanceRepo(pool)
async with pool.connection() as conn:
inst = await repo.create(conn, build_instance(team="platform"))
counting = _CountingRepo(repo)
cache = InstanceCache(upstash, ttl_s=30)
miss = await _read_through(inst.id, inst.team, cache, counting)
assert counting.gets == 1
counter = _CommandCounter(upstash)
before = _metric("cache_get")
hit = await _read_through(inst.id, inst.team, cache, counting)
assert counter.count == 1, "a cache hit is one GET"
assert _metric("cache_get") - before == 1
assert counting.gets == 1, "the second read must not reach Postgres"
# Round-tripped through JSON and back: `decode_responses=True` and the datetime/UUID
# serialisation both have to be right for this to compare equal.
assert hit == miss == inst
assert 0 < await upstash.ttl(f"inst:{inst.id}") <= 30
await cache.invalidate(inst.id)
assert await cache.get(inst.id) is None
# --- 5. Redis down = the platform stays up -----------------------------------------------
async def test_rate_limiter_fails_open_when_redis_is_down(dead_redis: Redis) -> None:
"""The load-bearing one. A limiter that fails closed turns a cache outage into an outage.
`degraded=True` and the error counter are what stop this from being invisible: fail
open silently and you cannot tell a working limiter from one that has been allowing
everything for a month.
"""
limiter = RateLimiter(dead_redis, limit=1, window_s=60)
before = _errors("ratelimit")
results = [await limiter.check("acme") for _ in range(3)]
assert all(r.allowed for r in results), "Redis being down must never refuse legitimate traffic"
assert all(r.degraded for r in results)
assert _errors("ratelimit") - before == 3
async def test_idempotency_falls_through_to_the_db_when_redis_is_down(dead_redis: Redis) -> None:
"""None means "create it". Safe only because `instances.release_name` is UNIQUE."""
store = IdempotencyStore(dead_redis)
key = str(uuid4())
assert await store.claim(key, uuid4()) is None
assert await store.claim(key, uuid4()) is None
async def test_cache_misses_instead_of_raising_when_redis_is_down(dead_redis: Redis) -> None:
"""A miss falls through to Postgres. `put` and `invalidate` swallow it too."""
cache = InstanceCache(dead_redis)
inst = build_instance()
assert await cache.get(inst.id) is None
await cache.put(inst) # must not raise
await cache.invalidate(inst.id) # must not raise
async def test_platform_serves_reads_from_postgres_with_redis_down(pool: DictPool, dead_redis: Redis) -> None:
"""The acceptance check: `docker compose stop redis` then GET -> 200.
Every Redis path degrades and the read still returns the row. Nothing here consults
Redis for readiness, which is the other half of the rule — `/readyz` is Postgres-only,
so a Redis outage cannot make a single pod unready.
"""
repo = InstanceRepo(pool)
async with pool.connection() as conn:
inst = await repo.create(conn, build_instance(team="platform"))
limiter = RateLimiter(dead_redis, limit=10, window_s=60)
cache = InstanceCache(dead_redis)
store = IdempotencyStore(dead_redis)
assert (await limiter.check(inst.team)).allowed
assert await store.claim(str(uuid4()), inst.id) is None
assert await cache.get(inst.id) is None
assert await repo.get(inst.id, inst.team) == inst # the 200
# --- 6. The budget metric exists and is sane ---------------------------------------------
async def test_the_budget_metric_is_exposed_and_labelled_by_op() -> None:
"""`curl -s localhost:8000/metrics | grep svcforge_redis_commands_total`."""
limiter = FakeRateLimiter(limit=10, window_s=60, clock=FakeClock(start=_T0))
await limiter.check("acme") # the fake does not touch the real counter
RateLimitResult(allowed=True, limit=10, remaining=9, reset_at=_T0)
text = generate_latest(REGISTRY).decode()
assert "svcforge_redis_commands_total" in text
assert "svcforge_redis_errors_total" in text
# Per-op labels, because "you are over budget" is useless without "on cache_get".
assert _metric("ratelimit") >= 0
def test_budget_script_projects_a_five_second_poller_over_budget() -> None:
"""The spec's headline number, reproduced: one worker polling every 5s = 518,400/month.
Exactly the free tier, spent doing nothing. This is the case the script exists to
catch, so it is the case that is asserted rather than left to a comment.
"""
budget = _budget_script()
now = time.time()
# One hour at one command every five seconds.
per_op, started_at = budget.collect(
f'svcforge_redis_commands_total{{op="poll"}} 720.0\nprocess_start_time_seconds {now - 3600}\n'
)
assert per_op == {"poll": 720.0}
assert budget.report(per_op, started_at, budget._FREE_TIER_BUDGET, now) == 1
def test_budget_script_passes_a_request_path_workload() -> None:
"""Request-path volume is bounded by humans, and humans are slow. That is the whole rule."""
budget = _budget_script()
now = time.time()
per_op, started_at = budget.collect(
f'svcforge_redis_commands_total{{op="ratelimit"}} 100.0\n'
f'svcforge_redis_commands_total{{op="cache_get"}} 40.0\n'
f"process_start_time_seconds {now - 3600}\n"
)
assert sum(per_op.values()) == 140.0
assert budget.report(per_op, started_at, budget._FREE_TIER_BUDGET, now) == 0
def test_budget_script_refuses_to_guess_without_the_metric() -> None:
"""No counter means the process never imported the adapter — say so, do not print a 0."""
budget = _budget_script()
with pytest.raises(budget.BudgetError, match="not exposed"):
budget.collect("process_start_time_seconds 1.0\n")
with pytest.raises(budget.BudgetError, match="process_start_time_seconds"):
budget.collect('svcforge_redis_commands_total{op="ratelimit"} 5.0\n')
def test_budget_script_refuses_a_non_http_url() -> None:
"""`--url file:///etc/passwd` is not a metrics endpoint."""
budget = _budget_script()
with pytest.raises(budget.BudgetError, match="non-http"):
budget.scrape("file:///etc/passwd")
tests/integration/test_tasks.py
+155
View File
@@ -0,0 +1,155 @@
"""TaskRepo: enqueue, complete, fail, backoff, leases."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from svcforge_core.domain.models import TaskKind, TaskState
from svcforge_core.domain.states import InstanceState
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from tests.integration.helpers import make_instance
async def _task_row(pool: DictPool, task_id: int) -> dict[str, object]:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select * from tasks where id = %s", (task_id,))
row = await cur.fetchone()
assert row is not None
return dict(row)
async def test_enqueue_defaults_to_runnable_now(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
row = await _task_row(pool, tid)
assert row["state"] == "queued"
assert row["attempts"] == 0
async def test_complete_marks_done_and_releases_lock(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
claimed = await repo.claim("w1")
assert claimed is not None
await repo.complete(claimed.id)
row = await _task_row(pool, tid)
assert row["state"] == "done"
assert row["locked_by"] is None
async def test_fail_under_max_attempts_requeues_with_future_run_after(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
claimed = await repo.claim("w1")
assert claimed is not None
assert claimed.attempts == 1 # attempts increments at CLAIM time, not on failure
await repo.fail(tid, "helm exploded", max_attempts=5)
row = await _task_row(pool, tid)
assert row["state"] == "queued"
assert row["last_error"] == "helm exploded"
assert row["locked_by"] is None
# Backoff pushed it out; it must not be immediately runnable again.
run_after = row["run_after"]
assert isinstance(run_after, datetime)
assert run_after >= datetime.now(UTC) - timedelta(seconds=1)
async def test_fail_at_max_attempts_dead_letters_and_marks_instance(pool: DictPool) -> None:
"""A dead-letter state, not an infinite retry."""
iid = await make_instance(pool)
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
tid = await tasks.enqueue_standalone(iid, TaskKind.PROVISION)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
await tasks.fail(tid, "chart not found", max_attempts=5)
row = await _task_row(pool, tid)
assert row["state"] == "failed"
inst = await instances.get(iid, team="platform")
assert inst is not None
assert inst.state is InstanceState.FAILED
assert inst.error == "chart not found"
async def test_fail_does_not_resurrect_a_deleted_instance(pool: DictPool) -> None:
"""Raw SQL must obey the same state machine domain.transition() enforces.
LEGAL[DELETED] is empty — deleted is terminal. A deprovision task that exhausts its
retries after the instance is already gone must record nothing on it, not drag it
back to 'failed'.
"""
iid = await make_instance(pool, state=InstanceState.DELETED)
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
tid = await tasks.enqueue_standalone(iid, TaskKind.DEPROVISION)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
await tasks.fail(tid, "helm uninstall kept failing", max_attempts=5)
# The task still dead-letters — that part is unconditional.
row = await _task_row(pool, tid)
assert row["state"] == "failed"
# But the instance stays deleted.
inst = await instances.get(iid, team="platform")
assert inst is not None
assert inst.state is InstanceState.DELETED, "raw SQL bypassed the state machine"
assert inst.error is None
async def test_fail_truncates_error_to_2kb(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
await repo.claim("w1")
await repo.fail(tid, "x" * 9000, max_attempts=5)
row = await _task_row(pool, tid)
assert isinstance(row["last_error"], str)
assert len(row["last_error"]) == 2000
async def test_run_after_in_the_future_is_not_claimable(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
future = datetime.now(UTC) + timedelta(hours=1)
await repo.enqueue_standalone(iid, TaskKind.PROVISION, run_after=future)
assert await repo.claim("w1") is None
async def test_reset_expired_leases_recovers_a_dead_workers_task(pool: DictPool) -> None:
"""No distributed lock survives a power cut. Only the lease recovers this row."""
iid = await make_instance(pool)
repo = TaskRepo(pool)
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
claimed = await repo.claim("worker-that-will-die")
assert claimed is not None
# Simulate: the worker was SIGKILLed 10 minutes ago and never reported.
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("update tasks set locked_at = now() - interval '10 minutes' where id = %s", (tid,))
n = await repo.reset_expired_leases(lease_seconds=300)
assert n == 1
row = await _task_row(pool, tid)
assert row["state"] == TaskState.QUEUED.value
assert row["locked_by"] is None
# And it is claimable again.
assert await repo.claim("w2") is not None
async def test_fresh_lease_is_not_reset(pool: DictPool) -> None:
iid = await make_instance(pool)
repo = TaskRepo(pool)
await repo.enqueue_standalone(iid, TaskKind.PROVISION)
await repo.claim("w1")
assert await repo.reset_expired_leases(lease_seconds=300) == 0
tests/integration/test_worker.py
+203
View File
@@ -0,0 +1,203 @@
"""The worker loop: drains on SIGTERM, retries, stays idempotent.
These run in milliseconds against a FakeProvisioner. That is the payoff for putting helm
behind a Protocol in Module 5: the crash-safety properties are testable without a cluster.
"""
from __future__ import annotations
import asyncio
import time
from pathlib import Path
from uuid import UUID
import pytest
from services.worker.deps import WorkerDeps
from services.worker.main import run_worker
from svcforge_core.adapters.clock import SystemClock
from svcforge_core.domain.catalog import load_catalog
from svcforge_core.domain.models import TaskKind
from svcforge_core.domain.states import InstanceState
from svcforge_core.repo.db import DictPool
from svcforge_core.repo.instances import InstanceRepo
from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings
from tests.fakes import FakeNotifier, FakeProvisioner
from tests.integration.helpers import build_instance
CATALOG = load_catalog(Path(__file__).resolve().parents[2] / "catalog.yaml")
def _settings(**over: object) -> Settings:
base: dict[str, object] = {
"pg_dsn": "postgresql://x:x@127.0.0.1:5432/x",
"worker_id": "w-test",
"worker_concurrency": 4,
"poll_interval_s": 0.05,
"max_attempts": 3,
}
base.update(over)
return Settings(**base) # type: ignore[arg-type]
def _deps(
pool: DictPool,
prov: FakeProvisioner,
notifier: FakeNotifier | None = None,
**over: object,
) -> WorkerDeps:
return WorkerDeps(
pool=pool,
instances=InstanceRepo(pool),
tasks=TaskRepo(pool),
provisioner=prov,
notifier=notifier or FakeNotifier(),
clock=SystemClock(),
catalog=CATALOG,
settings=_settings(**over),
)
async def _seed(pool: DictPool, **kw: object) -> tuple[str, int]:
inst = build_instance(**kw) # type: ignore[arg-type]
async with pool.connection() as conn:
await InstanceRepo(pool).create(conn, inst)
tid = await TaskRepo(pool).enqueue_standalone(inst.id, TaskKind.PROVISION)
return str(inst.id), tid
async def _state_of(pool: DictPool, tid: int) -> str:
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select state from tasks where id = %s", (tid,))
row = await cur.fetchone()
assert row is not None
return str(row["state"])
async def test_worker_provisions_and_marks_ready(pool: DictPool) -> None:
iid, tid = await _seed(pool)
prov = FakeProvisioner()
notifier = FakeNotifier()
stop = asyncio.Event()
worker = asyncio.create_task(run_worker(_deps(pool, prov, notifier), stop))
await asyncio.sleep(0.5)
stop.set()
await asyncio.wait_for(worker, timeout=5)
assert await _state_of(pool, tid) == "done"
inst = await InstanceRepo(pool).get(UUID(iid), team="platform")
assert inst is not None
assert inst.state is InstanceState.READY
assert inst.endpoint is not None
assert len(prov.installed) == 1
# Assert the notification fired, and that it happened on the FIRST attempt.
#
# Without this the handler could raise after marking the instance ready — the task
# requeues, the retry hits the idempotency early-return, and everything above still
# passes while the worker is quietly crashing on every provision. Idempotency is
# supposed to make crashes survivable, not invisible; asserting attempts==1 is what
# keeps a masked crash from reading as success.
assert notifier.events() == ["instance.ready"]
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select attempts from tasks where id = %s", (tid,))
row = await cur.fetchone()
assert row is not None
assert row["attempts"] == 1, "task was retried: the handler raised after doing the work"
async def test_sigterm_drains_in_flight(pool: DictPool) -> None:
"""Stop is requested mid-provision: the worker must FINISH the task, then exit.
Abandoning it would not lose the task — the lease would recover it — but only after
five minutes of a tenant watching 'provisioning'. Draining costs two seconds.
"""
_, tid = await _seed(pool)
prov = FakeProvisioner(delay=2.0)
stop = asyncio.Event()
started = time.monotonic()
worker = asyncio.create_task(run_worker(_deps(pool, prov), stop))
await asyncio.sleep(0.5)
stop.set() # mid-flight: the handler is still inside its 2s install
await asyncio.wait_for(worker, timeout=5)
elapsed = time.monotonic() - started
assert await _state_of(pool, tid) == "done", "worker abandoned an in-flight task"
assert elapsed >= 2.0, "worker returned before the in-flight task finished"
assert elapsed < 5.0
async def test_idle_worker_stops_promptly(pool: DictPool) -> None:
"""Nothing queued: stop must wake the poll sleep, not wait it out."""
stop = asyncio.Event()
worker = asyncio.create_task(run_worker(_deps(pool, FakeProvisioner(), poll_interval_s=5.0), stop))
await asyncio.sleep(0.2)
started = time.monotonic()
stop.set()
await asyncio.wait_for(worker, timeout=2)
assert time.monotonic() - started < 1.0, "stop did not interrupt the poll sleep"
async def test_failed_task_is_requeued_with_backoff(pool: DictPool) -> None:
_, tid = await _seed(pool)
prov = FakeProvisioner(fail_on={"platform-elasticsearch"})
stop = asyncio.Event()
worker = asyncio.create_task(run_worker(_deps(pool, prov), stop))
await asyncio.sleep(0.6)
stop.set()
await asyncio.wait_for(worker, timeout=5)
async with pool.connection() as conn, conn.cursor() as cur:
await cur.execute("select state, attempts, last_error from tasks where id = %s", (tid,))
row = await cur.fetchone()
assert row is not None
assert row["state"] == "queued" # requeued, not failed — attempts remain
assert row["attempts"] >= 1
assert row["last_error"]
async def test_provision_twice_installs_once(pool: DictPool) -> None:
"""The idempotency claim, executed.
Simulates the crash window: the release is installed and the instance is READY, but
the task got re-queued (worker died before reporting). Re-running must not re-install.
"""
from services.worker.handlers import handle_provision
inst = build_instance(state=InstanceState.REQUESTED)
async with pool.connection() as conn:
await InstanceRepo(pool).create(conn, inst)
tid = await TaskRepo(pool).enqueue_standalone(inst.id, TaskKind.PROVISION)
task = await TaskRepo(pool).claim("w1")
assert task is not None
prov = FakeProvisioner()
deps = _deps(pool, prov)
await handle_provision(task, deps)
await handle_provision(task, deps) # the redelivery
assert len(prov.installed) == 1, "second run re-installed: handler is not idempotent"
assert tid == task.id
@pytest.mark.parametrize("concurrency", [1, 4])
async def test_concurrency_cap_is_respected(pool: DictPool, concurrency: int) -> None:
"""The semaphore is what stops one worker from starting 200 helm processes."""
for _ in range(6):
await _seed(pool)
prov = FakeProvisioner(delay=0.2)
stop = asyncio.Event()
worker = asyncio.create_task(run_worker(_deps(pool, prov, worker_concurrency=concurrency), stop))
await asyncio.sleep(0.5)
stop.set()
await asyncio.wait_for(worker, timeout=10)
assert prov.max_concurrent <= concurrency, f"ran {prov.max_concurrent} at once, cap was {concurrency}"
tests/unit/__init__.py
View File
tests/unit/test_backoff.py
+40
View File
@@ -0,0 +1,40 @@
"""Unit tests for retry backoff math."""
from datetime import UTC, datetime, timedelta
import pytest
from hypothesis import given
from hypothesis import strategies as st
from svcforge_core.domain.backoff import next_attempt_at
T = datetime(2026, 7, 17, 12, 0, 0, tzinfo=UTC)
def test_attempt_zero_full_jitter_is_base() -> None:
assert next_attempt_at(0, now=T, rand=lambda: 1.0) == T + timedelta(seconds=2)
def test_zero_jitter_returns_now() -> None:
assert next_attempt_at(0, now=T, rand=lambda: 0.0) == T
def test_negative_attempt_raises_value_error() -> None:
with pytest.raises(ValueError, match="attempt"):
next_attempt_at(-1, now=T)
@given(attempt=st.integers(min_value=0, max_value=64), r=st.floats(min_value=0.0, max_value=1.0))
def test_delay_is_always_within_zero_and_cap(attempt: int, r: float) -> None:
cap_s = 300.0
got = next_attempt_at(attempt, now=T, cap_s=cap_s, rand=lambda: r)
delay = (got - T).total_seconds()
assert 0.0 <= delay <= cap_s
@given(attempt=st.integers(min_value=0, max_value=63))
def test_delay_is_non_decreasing_in_attempt(attempt: int) -> None:
def ceiling_of(a: int) -> float:
return (next_attempt_at(a, now=T, rand=lambda: 1.0) - T).total_seconds()
assert ceiling_of(attempt) <= ceiling_of(attempt + 1)
tests/unit/test_catalog.py
+155
View File
@@ -0,0 +1,155 @@
"""Unit tests for catalog loading and validation."""
import textwrap
from pathlib import Path
import pytest
from svcforge_core.domain.catalog import CatalogError, load_catalog
from svcforge_core.domain.models import CatalogEntry
VALID_YAML = textwrap.dedent("""
services:
redis:
chart: bitnamilegacy/redis
chart_version: 20.6.2
sizes:
small:
replicas: 1
resources:
requests: {cpu: 100m, memory: 256Mi}
medium:
replicas: 3
resources:
requests: {cpu: 500m, memory: 1Gi}
postgres:
chart: bitnamilegacy/postgresql
chart_version: 16.4.5
sizes:
small:
replicas: 1
resources:
requests: {cpu: 250m, memory: 512Mi}
""")
MISSING_CHART_VERSION_YAML = textwrap.dedent("""
services:
redis:
chart: bitnamilegacy/redis
sizes:
small:
replicas: 1
resources: {}
""")
ZERO_REPLICAS_YAML = textwrap.dedent("""
services:
redis:
chart: bitnamilegacy/redis
chart_version: 20.6.2
sizes:
small:
replicas: 0
resources: {}
""")
def _write(tmp_path: Path, body: str) -> Path:
path = tmp_path / "catalog.yaml"
path.write_text(body)
return path
def test_valid_yaml_loads_to_catalog_entries(tmp_path: Path) -> None:
catalog = load_catalog(_write(tmp_path, VALID_YAML))
assert set(catalog) == {"redis", "postgres"}
assert all(isinstance(entry, CatalogEntry) for entry in catalog.values())
redis = catalog["redis"]
assert redis.service_type == "redis"
assert redis.chart_version == "20.6.2"
assert set(redis.sizes) == {"small", "medium"}
assert redis.sizes["medium"].replicas == 3
def test_missing_chart_version_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError) as excinfo:
load_catalog(_write(tmp_path, MISSING_CHART_VERSION_YAML))
assert excinfo.value.key == "redis"
assert "redis" in str(excinfo.value)
def test_zero_replicas_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError) as excinfo:
load_catalog(_write(tmp_path, ZERO_REPLICAS_YAML))
assert excinfo.value.key == "redis"
def test_missing_file_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError, match="cannot read catalog"):
load_catalog(tmp_path / "nope.yaml")
def test_unparseable_yaml_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError, match="not valid YAML"):
load_catalog(_write(tmp_path, "services: [unclosed\n"))
def test_scalar_root_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError, match="must be a mapping"):
load_catalog(_write(tmp_path, "just-a-string\n"))
def test_non_mapping_services_raises_catalog_error(tmp_path: Path) -> None:
with pytest.raises(CatalogError, match="'services' must be a mapping"):
load_catalog(_write(tmp_path, "services:\n - redis\n"))
def test_non_mapping_entry_raises_catalog_error_naming_the_key(tmp_path: Path) -> None:
with pytest.raises(CatalogError) as excinfo:
load_catalog(_write(tmp_path, "services:\n redis: just-a-string\n"))
assert excinfo.value.key == "redis"
assert "must be a mapping" in str(excinfo.value)
def test_non_string_field_key_raises_catalog_error_naming_the_key(tmp_path: Path) -> None:
"""A non-string YAML key inside an entry breaks `**body`; it must surface as CatalogError."""
body = textwrap.dedent("""
services:
redis:
1: oops
chart: bitnamilegacy/redis
chart_version: 20.6.2
sizes: {}
""")
with pytest.raises(CatalogError) as excinfo:
load_catalog(_write(tmp_path, body))
assert excinfo.value.key == "redis"
def test_bare_mapping_without_services_key_is_accepted(tmp_path: Path) -> None:
"""The top-level `services:` wrapper is optional; a bare service_type mapping also loads."""
body = textwrap.dedent("""
redis:
chart: bitnamilegacy/redis
chart_version: 20.6.2
sizes:
small:
replicas: 1
resources: {}
""")
catalog = load_catalog(_write(tmp_path, body))
assert set(catalog) == {"redis"}
def test_repo_catalog_yaml_is_valid() -> None:
catalog = load_catalog(Path(__file__).parents[2] / "catalog.yaml")
assert set(catalog) == {"elasticsearch", "redis", "postgres"}
for entry in catalog.values():
assert set(entry.sizes) == {"small", "medium"}
tests/unit/test_obs.py
+272
View File
@@ -0,0 +1,272 @@
"""obs.py: the three things that are wrong by default.
Not tested here: that structlog logs, that prometheus counts, that OTEL traces. Those are
the libraries' tests. What is tested is every place where the default is a bug — the
histogram buckets, the context that does not cross a queue, and the contextvars that leak
between tasks.
"""
from __future__ import annotations
import io
import json
import logging
import re
from collections.abc import Iterator
from typing import Any
from uuid import uuid4
import pytest
import structlog
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from prometheus_client import REGISTRY
from svcforge_core import obs
from svcforge_core.settings import Settings
# W3C traceparent: version-traceid-spanid-flags.
#
# The flags byte is matched loosely and the sampled bit checked separately, on purpose.
# Module 7's acceptance line says `00-<32 hex>-<16 hex>-01`, and current SDKs emit `-03`:
# bit 0x01 is `sampled`, and bit 0x02 is `random-trace-id` from trace-context level 2. An
# assertion pinned to `-01` fails on a spec revision that changed nothing we care about.
# What we care about is the trace being sampled, which is bit 0x01 and nothing else.
TRACEPARENT_RE = re.compile(
r"^00-(?P<trace_id>[0-9a-f]{32})-(?P<span_id>[0-9a-f]{16})-(?P<flags>[0-9a-f]{2})$"
)
SAMPLED_BIT = 0x01
# A provision takes minutes. prometheus_client's default buckets end at 10 seconds.
DEFAULT_TOP_BUCKET = 10.0
def _settings(**over: object) -> Settings:
return Settings(pg_dsn="postgresql://u:p@localhost:5432/db", **over) # type: ignore[arg-type]
@pytest.fixture
def logs() -> Iterator[io.StringIO]:
"""setup() against a captured stream, restoring the process-wide config afterwards.
setup() is deliberately global and deliberately once-only, which makes it deliberately
awkward to test. Reaching into the module flag is the honest price of that; the
alternative is a seam that exists only for tests.
"""
stream = io.StringIO()
saved_handlers = logging.getLogger().handlers[:]
saved_level = logging.getLogger().level
saved_config = structlog.get_config()
obs._configured = False
obs.setup("test-service", _settings())
# Re-point the handler setup() installed at our buffer; everything else it configured —
# processors, formatter, the JSON renderer last — is exactly what production gets.
handler = logging.getLogger().handlers[0]
assert isinstance(handler, logging.StreamHandler)
handler.setStream(stream)
try:
yield stream
finally:
structlog.contextvars.clear_contextvars()
structlog.configure(**saved_config)
logging.getLogger().handlers = saved_handlers
logging.getLogger().setLevel(saved_level)
obs._configured = False
def _lines(stream: io.StringIO) -> list[dict[str, Any]]:
return [json.loads(line) for line in stream.getvalue().splitlines() if line.strip()]
# --- Buckets ------------------------------------------------------------------------------
def test_provision_histogram_has_a_bucket_for_a_thirty_minute_provision() -> None:
"""The acceptance check, as a unit test: `le="1800"` exists.
With the library defaults the top finite bucket is 10s, every provision lands in +Inf,
and histogram_quantile interpolates across a bucket spanning 10s to infinity. The p95
it returns is not slow or fast, it is meaningless — and it is meaningless silently,
which is why this is asserted rather than eyeballed on a dashboard.
"""
bounds = obs.PROVISION_TIME._upper_bounds
assert 1800.0 in bounds
assert bounds[-1] == float("inf")
assert max(b for b in bounds if b != float("inf")) > DEFAULT_TOP_BUCKET
def test_provision_histogram_exports_the_1800_bucket() -> None:
"""Same claim, checked through the exposition format the scrape actually reads."""
obs.PROVISION_TIME.observe(42.0)
bucket = REGISTRY.get_sample_value("svcforge_provision_duration_seconds_bucket", {"le": "1800.0"})
assert bucket is not None
def test_metric_names_are_the_ones_the_alerts_query() -> None:
"""The four alerts are PromQL strings in values.yaml; nothing type-checks them.
A rename here is a silent alert that never fires again. This test is the link between
the chart's rules and the code.
"""
for name in (
"svcforge_tasks_claimed_total",
"svcforge_tasks_failed_total",
"svcforge_provision_duration_seconds",
"svcforge_queue_depth",
"svcforge_instances",
"svcforge_reconciler_last_tick_timestamp_seconds",
):
assert REGISTRY._names_to_collectors.get(name) is not None, f"{name} is queried by an alert"
# --- Trace context across the queue -------------------------------------------------------
def test_traceparent_round_trips_through_a_string() -> None:
"""inject -> a W3C string -> extract -> the same trace. This is the queue crossing."""
provider = TracerProvider()
tracer = provider.get_tracer("test")
with tracer.start_as_current_span("api.post") as span:
traceparent = obs.inject_traceparent()
api_trace_id = span.get_span_context().trace_id
assert traceparent is not None
match = TRACEPARENT_RE.match(traceparent)
assert match is not None, traceparent
assert match["trace_id"] == format(api_trace_id, "032x")
assert int(match["flags"], 16) & SAMPLED_BIT, "unsampled: the worker's span would be dropped"
# The worker's side: a fresh context, minutes later, in another process.
ctx = obs.context_from_traceparent(traceparent)
with tracer.start_as_current_span("worker.claim", context=ctx) as worker_span:
assert worker_span.get_span_context().trace_id == api_trace_id
def test_inject_returns_none_without_a_span() -> None:
"""A task the reconciler enqueued has no inbound request. Null column, not an error."""
assert obs.inject_traceparent() is None
def test_context_from_traceparent_survives_none_and_garbage() -> None:
"""A malformed traceparent must start a new trace, never fail a provision.
Extract does not raise on bad input — it returns a context with no span. Asserted here
because the alternative would be a tenant's provision failing over a telemetry header.
"""
for bad in (None, "", "not-a-traceparent", "00-tooshort-01"):
ctx = obs.context_from_traceparent(bad)
assert not trace.get_current_span(ctx).get_span_context().is_valid
# --- Contextvars --------------------------------------------------------------------------
def test_every_line_carries_instance_id_task_id_and_team(logs: io.StringIO) -> None:
"""The acceptance check: bind once at claim, and the keys ride on every line after."""
instance_id = uuid4()
obs.bind_task_context(instance_id, 42, "platform")
obs.get_logger("t").info("provision.started")
obs.get_logger("t").warning("helm.slow")
for line in _lines(logs):
assert line["instance_id"] == str(instance_id)
assert line["task_id"] == 42
assert line["team"] == "platform"
def test_foreign_stdlib_logs_are_json_and_carry_the_context(logs: io.StringIO) -> None:
"""psycopg and uvicorn log through stdlib `logging`, and their lines must parse too.
Without the ProcessorFormatter bridge these arrive as bare text on the same stdout, and
every one of them is a parse failure in the collector.
"""
obs.bind_task_context(uuid4(), 7, "payments")
logging.getLogger("some.library").warning("connection reset")
line = _lines(logs)[-1]
assert line["event"] == "connection reset"
assert line["task_id"] == 7
assert line["team"] == "payments"
def test_bind_task_context_clears_the_previous_task(logs: io.StringIO) -> None:
"""The bug this prevents: task 2's log line naming task 1's tenant.
A worker coroutine reuses its context across claim-loop iterations. Bind without
clearing and the stale instance_id survives into the next task, which means the log for
the incident you are debugging points at the wrong customer.
"""
obs.bind_task_context(uuid4(), 1, "team-a")
second = uuid4()
obs.bind_task_context(second, 2, "team-b")
obs.get_logger("t").info("claimed")
line = _lines(logs)[-1]
assert line["instance_id"] == str(second)
assert line["task_id"] == 2
assert line["team"] == "team-b"
def test_json_renderer_is_last_and_output_is_one_object_per_line(logs: io.StringIO) -> None:
"""JSONRenderer last in the chain, and nothing after it.
A processor appended after the renderer receives a `str` where it expects a dict and
raises. The symptom is not a crash — structlog's default is to fail the log call, so
the line simply never appears.
"""
obs.get_logger("t").info("hello", extra_key="value")
lines = _lines(logs)
assert len(lines) == 1
assert lines[0]["event"] == "hello"
assert lines[0]["extra_key"] == "value"
assert lines[0]["level"] == "info"
assert lines[0]["service"] == "test-service"
assert "timestamp" in lines[0]
def test_setup_is_idempotent(logs: io.StringIO) -> None:
"""Called twice, one handler, one line. Not a nicety: two handlers is two of every line.
Each service calls setup() from its entrypoint, and an entrypoint that imports another
entrypoint (the CLI shelling into the reconciler) calls it twice.
"""
obs.setup("test-service", _settings())
obs.setup("test-service", _settings())
# Count ours, not pytest's — its capture handler is on the root logger too.
ours = [
h
for h in logging.getLogger().handlers
if isinstance(h.formatter, structlog.stdlib.ProcessorFormatter)
]
assert len(ours) == 1
obs.get_logger("t").info("once")
assert len(_lines(logs)) == 1
def test_metrics_are_single_process_in_memory() -> None:
"""One process per pod, scale with replicas. Multiprocess mode is not in use.
`ValueClass` is prometheus_client's fork in the road, chosen at import from
PROMETHEUS_MULTIPROC_DIR: MutexValue keeps counters in memory, MultiProcessValue mmaps
them into a shared directory. This repo takes the other fix for `uvicorn --workers 4`
corrupting counters — one process per pod — so MutexValue is the correct answer, and a
stray PROMETHEUS_MULTIPROC_DIR in a Deployment's env would silently change it to the
other one along with the meaning of every gauge.
"""
from prometheus_client import values
assert values.ValueClass is values.MutexValue
# And the registry the services expose is the default in-memory one, not a
# MultiProcessCollector reading files off disk.
obs.QUEUE_DEPTH.set(3)
assert REGISTRY.get_sample_value("svcforge_queue_depth") == 3.0
tests/unit/test_protocol_conformance.py
+109
View File
@@ -0,0 +1,109 @@
"""Every Protocol, with both of its implementations, checked by mypy.
These functions have no assertions and cannot fail at runtime — the check happens under
`mypy --strict`. If FakeProvisioner drifts from Provisioner (a renamed parameter, a changed
return type), the type-check fails here rather than the fake quietly diverging from the real
adapter and the worker's fast tests proving nothing about production.
The test bodies exist so pytest runs the imports too: a Protocol satisfied statically but
broken at import time is still broken.
"""
from __future__ import annotations
from pathlib import Path
from svcforge_core.adapters.clock import Clock, SystemClock
from svcforge_core.adapters.helm import HelmProvisioner, Provisioner
from svcforge_core.adapters.notify import LogNotifier, Notifier, WebhookNotifier
from svcforge_core.adapters.redis import (
IdempotencyStore,
IdempotencyStoreProto,
InstanceCache,
InstanceCacheProto,
RateLimiter,
RateLimiterProto,
make_redis,
)
from svcforge_core.settings import Settings
from tests.fakes import (
FakeClock,
FakeIdempotencyStore,
FakeInstanceCache,
FakeNotifier,
FakeProvisioner,
FakeRateLimiter,
)
def take(p: Provisioner) -> None:
"""Accepts anything structurally a Provisioner. The whole assertion is the annotation."""
def take_clock(c: Clock) -> None: ...
def take_notifier(n: Notifier) -> None: ...
def test_provisioner_implementations_conform() -> None:
take(HelmProvisioner(kubeconfig=Path("/dev/null")))
take(FakeProvisioner())
print("ok")
def test_clock_implementations_conform() -> None:
from datetime import UTC, datetime
take_clock(SystemClock())
take_clock(FakeClock(start=datetime(2026, 1, 1, tzinfo=UTC)))
print("ok")
def test_notifier_implementations_conform() -> None:
take_notifier(LogNotifier())
take_notifier(WebhookNotifier(url="https://example.invalid/hook"))
take_notifier(FakeNotifier())
print("ok")
def take_limiter(rl: RateLimiterProto) -> None: ...
def take_idempotency(store: IdempotencyStoreProto) -> None: ...
def take_cache(c: InstanceCacheProto) -> None: ...
def test_redis_implementations_conform() -> None:
"""A client at a closed port is still a Redis. Nothing here connects — no commands billed.
That is not a trick to keep the test fast; it is the module's thesis restated as a
fixture. Every one of these classes has to be constructible and callable with Redis
unreachable, because that is the state they are designed for.
"""
from datetime import UTC, datetime
settings = Settings(
# pydantic parses these strings into PostgresDsn/RedisDsn at runtime; the
# annotation names the parsed type, so the ignores sit on the arguments.
pg_dsn="postgresql://unused:unused@127.0.0.1:5432/unused", # type: ignore[arg-type]
redis_dsn="redis://127.0.0.1:1/0", # type: ignore[arg-type]
)
r = make_redis(settings)
assert r is not None
take_limiter(RateLimiter(r, limit=10, window_s=60))
take_limiter(FakeRateLimiter(10, 60, FakeClock(start=datetime(2026, 1, 1, tzinfo=UTC))))
take_idempotency(IdempotencyStore(r))
take_idempotency(FakeIdempotencyStore())
take_cache(InstanceCache(r))
take_cache(FakeInstanceCache())
# redis_dsn=None EXPLICITLY. Omitting it does not mean "unset": pydantic-settings
# reads SVCFORGE_REDIS_DSN from the environment, so on any machine that has the real
# DSN exported this assertion sees a live Upstash client and fails — a green test that
# depends on your shell being empty is not a test.
assert make_redis(Settings(pg_dsn=settings.pg_dsn, redis_dsn=None)) is None
print("ok")
tests/unit/test_smoke.py
+4
View File
@@ -0,0 +1,4 @@
def test_import_core() -> None:
import svcforge_core
assert svcforge_core is not None
tests/unit/test_states.py
+42
View File
@@ -0,0 +1,42 @@
"""Unit tests for the instance state machine."""
import pytest
from svcforge_core.domain.states import LEGAL, IllegalTransition, InstanceState, transition
def test_requested_to_provisioning_is_legal() -> None:
assert transition(InstanceState.REQUESTED, InstanceState.PROVISIONING) is InstanceState.PROVISIONING
def test_deleted_to_ready_raises() -> None:
with pytest.raises(IllegalTransition):
transition(InstanceState.DELETED, InstanceState.READY)
def test_failed_to_provisioning_is_legal_retry() -> None:
assert transition(InstanceState.FAILED, InstanceState.PROVISIONING) is InstanceState.PROVISIONING
@pytest.mark.parametrize("state", list(InstanceState))
def test_every_state_has_a_legal_entry(state: InstanceState) -> None:
"""A new state with no LEGAL entry must fail the suite, not KeyError at runtime."""
assert state in LEGAL
assert isinstance(LEGAL[state], frozenset)
@pytest.mark.parametrize("state", list(InstanceState))
def test_every_legal_target_is_an_instance_state(state: InstanceState) -> None:
for target in LEGAL[state]:
assert isinstance(target, InstanceState)
def test_deleted_is_terminal_with_an_empty_frozenset() -> None:
assert LEGAL[InstanceState.DELETED] == frozenset()
def test_strenum_compares_equal_to_its_value() -> None:
# mypy calls this non-overlapping by declared type. That is exactly what is being
# tested: StrEnum members ARE their values at runtime, which is why psycopg can
# adapt them straight to text and model_validate round-trips them for free.
assert (InstanceState.READY == "ready") is True # type: ignore[comparison-overlap]

Some files were not shown because too many files have changed in this diff Show More