Compare commits
63 Commits
851f8919a8
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
| fca9a8b187 | |||
| 7a3dfb06db | |||
| d40bc682ce | |||
| cbd0709281 | |||
| cd61eca12e | |||
| 1b56cda231 | |||
| 173acc8612 | |||
| 60ee0f1cbf | |||
| 9249051ee2 | |||
| c691f4f4aa | |||
| a0215ef63f | |||
| b2cdcecdc1 | |||
| e7496562b3 | |||
| 3b957a8566 | |||
| 08a529fa63 | |||
| 58ffb9c2e0 | |||
| 95a894d816 | |||
| c53734d2bc | |||
| 7918dc2b37 | |||
| 64b18d2823 | |||
| 51a1bb62c0 | |||
| 6974b3620f | |||
| bb8b14ef03 | |||
| 72296ace84 | |||
| e971e04d75 | |||
| 9c8d10ce1f | |||
| 7079d6340f | |||
| 66eb6cb0ee | |||
| d64c3c9f39 | |||
| d6c1b64512 | |||
| 76cadca8e3 | |||
| 66336d3648 | |||
| a843494627 | |||
| 4193a18cae | |||
| 2356ac4ef3 | |||
| c2a27952d1 | |||
| 0dbb5af1d3 | |||
| d4ac3801a3 | |||
| 6093c52160 | |||
| 31fa9165ff | |||
| 89ad9625f3 | |||
| e8b6116a58 | |||
| 5f18f9eeeb | |||
| 1a36f43c69 | |||
| 37b297bf5c | |||
| 5d7f46483e | |||
| b02d4e85c6 | |||
| d70a7b622c | |||
| 2098443f31 | |||
| b35c551160 | |||
| f87d8d4d78 | |||
| c537073c21 | |||
| 4544765ec5 | |||
| f2b159ef7e | |||
| ca21b6e70d | |||
| dc51bd1d9d | |||
| 78a0a6d500 | |||
| 4a426dbe50 | |||
| a59d3848ae | |||
| a51227d952 | |||
| c76154aeaa | |||
| 77d560ddae | |||
| c9d0176bb3 |
+310
-43
@@ -27,15 +27,28 @@ concurrency:
|
||||
env:
|
||||
REGISTRY: gitea.oci-oci.duckdns.org
|
||||
IMAGE_NS: gitea_admin
|
||||
UV_VERSION: "0.5.11"
|
||||
UV_VERSION: "0.11.29"
|
||||
# Every uv job logged this, five times a run:
|
||||
#
|
||||
# warning: Failed to hardlink files; falling back to full copy. This may lead to
|
||||
# degraded performance.
|
||||
#
|
||||
# The uv cache (the runner's cache volume) and the target venv are on different
|
||||
# filesystems here, so hardlinking cannot work and uv copies 86 packages anyway. Saying
|
||||
# `copy` up front does not make it slower — it is already copying — it just stops the
|
||||
# warning from being noise that trains people to skim CI logs. The Dockerfiles set the
|
||||
# same variable for the same reason.
|
||||
UV_LINK_MODE: copy
|
||||
GITLEAKS_VERSION: "8.30.1"
|
||||
GITLEAKS_SHA256: "e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080"
|
||||
|
||||
jobs:
|
||||
# --- stage 1: lint -- fast, fails first ---------------------------------------------
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -50,8 +63,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -65,8 +78,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -90,7 +103,7 @@ jobs:
|
||||
needs: [unit]
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20
|
||||
image: mirror.gcr.io/library/postgres:18@sha256:32ca0af8e77bfb8c6610c488e4691f83f972a3e9e64d3b02facf3ab111ad5500
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: svcforge
|
||||
@@ -112,8 +125,8 @@ jobs:
|
||||
# to the one `migrate` just ran against.
|
||||
SVCFORGE_TEST_DSN: postgresql://postgres:postgres@postgres:5432/svcforge
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -133,12 +146,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
# gitleaks scans history, not just the tip. A secret committed and then reverted
|
||||
# is still a leaked secret, and a shallow clone cannot see it.
|
||||
fetch-depth: 0
|
||||
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0
|
||||
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: true
|
||||
@@ -155,12 +168,37 @@ jobs:
|
||||
run: uv run --with 'bandit[toml]' bandit -c pyproject.toml -r libs services -ll
|
||||
|
||||
- name: gitleaks (secret scan)
|
||||
# Pinned by digest and run directly, so the command is the documented one rather
|
||||
# than a marketplace action's opinion of it.
|
||||
# A downloaded binary, not `docker run -v "$PWD:/repo"`.
|
||||
#
|
||||
# The container form was scanning NOTHING and passing. `-v "$PWD:/repo"` is
|
||||
# interpreted by the dind sidecar's daemon, which cannot see the job container's
|
||||
# checkout, so gitleaks got an empty mount and logged:
|
||||
#
|
||||
# ERR [git] fatal: not a git repository (or any parent up to mount point /)
|
||||
# ERR failed to scan Git repository error="stderr is not empty"
|
||||
# INF scan completed in 35.2ms
|
||||
# INF no leaks found <- exit 0
|
||||
#
|
||||
# It reported success without looking, which is worse than having no scanner: the
|
||||
# gate was green and meaningless. The 35ms runtime was the tell — a real history
|
||||
# scan of this repo takes seconds.
|
||||
#
|
||||
# The `--log-opts` + commit-count assertion below is the guard against that class
|
||||
# of failure returning. A scanner that cannot fail is not a gate.
|
||||
run: |
|
||||
docker run --rm -v "$PWD:/repo" -w /repo \
|
||||
ghcr.io/gitleaks/gitleaks:v8.21.2@sha256:0e99e8821643ea5b235718642b93bb32486af9c8162c8b8731f7cbdc951a7f46 \
|
||||
detect --no-banner --source /repo
|
||||
set -euo pipefail
|
||||
curl -fsSL -o /tmp/gitleaks.tgz \
|
||||
"https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_arm64.tar.gz"
|
||||
echo "${GITLEAKS_SHA256} /tmp/gitleaks.tgz" | sha256sum -c -
|
||||
tar -xzf /tmp/gitleaks.tgz -C /tmp gitleaks
|
||||
install -m 0755 /tmp/gitleaks /usr/local/bin/gitleaks
|
||||
|
||||
# Prove there is a repository to scan before trusting the verdict.
|
||||
commits=$(git rev-list --count HEAD)
|
||||
echo "scanning $commits commits"
|
||||
test "$commits" -gt 0
|
||||
|
||||
gitleaks detect --no-banner --source . --redact --exit-code 1
|
||||
|
||||
- name: pip-audit (dependency CVEs)
|
||||
# --strict fails on an audit error rather than shrugging and reporting clean.
|
||||
@@ -182,24 +220,118 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: hadolint
|
||||
uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0
|
||||
uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
|
||||
with:
|
||||
recursive: true
|
||||
dockerfile: "services/*/Dockerfile"
|
||||
failure-threshold: warning
|
||||
|
||||
# --- stage 9b: the chart must render --------------------------------------------------
|
||||
# Without this, a chart that does not template reaches ArgoCD and fails in the cluster,
|
||||
# where the error surfaces as a sync failure with no PR attached to it. `helm template`
|
||||
# is the real gate: it is what ArgoCD does, and _helpers.tpl's image helper calls `fail`
|
||||
# on anything that is not a full sha256 digest.
|
||||
chart:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
env:
|
||||
# The same helm version the worker and reconciler images carry, so CI renders with
|
||||
# the version that ships.
|
||||
HELM_VERSION: "3.21.3"
|
||||
# sha256 of helm-v3.21.3-linux-arm64.tar.gz, from https://get.helm.sh/*.sha256sum.
|
||||
# Pinned for the same reason every image here is pinned by digest: a tarball fetched
|
||||
# over HTTPS is still a tarball whoever controls the bucket chose to serve.
|
||||
HELM_SHA256: "67f58155079ff9ffab98ba5c88daff0ed9b542f3a4732f5dd426dde7dd0f5244"
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: install helm
|
||||
# A downloaded binary rather than `docker run alpine/helm`. The container form
|
||||
# cannot see the checkout: `docker run -v "$PWD:/repo"` is interpreted by the dind
|
||||
# SIDECAR's daemon, and the chart path resolved to nothing inside it —
|
||||
# "stat deploy/chart/Chart.yaml: no such file or directory" while the file plainly
|
||||
# exists in the job. A binary on PATH has no such boundary to cross.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL -o /tmp/helm.tgz \
|
||||
"https://get.helm.sh/helm-v${HELM_VERSION}-linux-arm64.tar.gz"
|
||||
echo "${HELM_SHA256} /tmp/helm.tgz" | sha256sum -c -
|
||||
tar -xzf /tmp/helm.tgz -C /tmp
|
||||
install -m 0755 /tmp/linux-arm64/helm /usr/local/bin/helm
|
||||
helm version --short
|
||||
|
||||
- name: helm lint
|
||||
run: |
|
||||
helm lint deploy/chart
|
||||
|
||||
- name: helm template (the digest guard rejects bad digests)
|
||||
# Feed the guard bad digests explicitly with --set. The earlier version of this
|
||||
# step ran a bare `helm template` and asserted it FAILED, on the assumption that
|
||||
# values.yaml always holds all-zeros placeholders. That assumption dies the first
|
||||
# time the `bump` job runs: bump commits real digests into values.yaml, so the bare
|
||||
# render then succeeds and the assertion reports "the guard is not guarding" about
|
||||
# a guard that is fine. A test whose expected result flips depending on whether CI
|
||||
# has run before is not a test.
|
||||
#
|
||||
# Each case below is a distinct way to get a digest wrong, and each must be
|
||||
# rejected on its own.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ZEROS="sha256:$(printf '0%.0s' $(seq 64))"
|
||||
for bad_desc in \
|
||||
"all-zeros placeholder|${ZEROS}" \
|
||||
"not a digest at all|latest" \
|
||||
"right prefix, wrong length|sha256:abc123" \
|
||||
"empty|"; do
|
||||
desc="${bad_desc%%|*}"; bad="${bad_desc#*|}"
|
||||
if helm template svcforge deploy/chart \
|
||||
--set image.api.digest="${bad}" >/dev/null 2>&1; then
|
||||
echo "FAIL: the chart rendered with a ${desc} digest (${bad@Q})." >&2
|
||||
echo "The digest guard in _helpers.tpl is not guarding." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "ok: rejected ${desc}"
|
||||
done
|
||||
|
||||
- name: helm template (renders with real digests)
|
||||
# Dummy but well-formed digests: this checks the templates themselves render, with
|
||||
# the two values-gated monitoring blocks explicitly on so they are covered too.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
A="sha256:$(printf 'a%.0s' $(seq 64))"
|
||||
B="sha256:$(printf 'b%.0s' $(seq 64))"
|
||||
C="sha256:$(printf 'c%.0s' $(seq 64))"
|
||||
helm template svcforge deploy/chart \
|
||||
--set image.api.digest="$A" \
|
||||
--set image.worker.digest="$B" \
|
||||
--set image.reconciler.digest="$C" \
|
||||
--set serviceMonitor.enabled=true \
|
||||
--set prometheusRule.enabled=true \
|
||||
>/dev/null
|
||||
echo "ok: chart renders"
|
||||
|
||||
# --- stage 10: build -> trivy -> push by digest --------------------------------------
|
||||
image:
|
||||
runs-on: ubuntu-latest
|
||||
# Every gate above is required. An image is not built until all of them are green,
|
||||
# which is what makes "the digest CI pushed is a digest that passed everything" true.
|
||||
needs: [types, unit, integration, security, dockerfile]
|
||||
needs: [types, unit, integration, security, dockerfile, chart]
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Kept for correctness on runners that honour it, but do NOT rely on it here:
|
||||
# act_runner IGNORES strategy.max-parallel. Measured in run #14 with this set to 1 —
|
||||
# the api leg ran 04:23:03-04:24:04, then the worker leg started 04:25:47 while
|
||||
# reconciler was still building. Two concurrent, exactly what it was meant to prevent.
|
||||
#
|
||||
# The lever that actually binds is the runner's own `capacity`, set in oci-k8s
|
||||
# (k8s/roles/addons/tasks/main.yml) and now 1. Contention there is what produced both
|
||||
# `DeadlineExceeded: no active session` in the build and `Failed to connect to
|
||||
# gitea-http:3000` in checkout.
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
svc: [api, worker, reconciler]
|
||||
# Deliberately no `outputs:` here. Matrix legs share one outputs map and clobber each
|
||||
@@ -207,12 +339,14 @@ jobs:
|
||||
# arrive empty or stale, intermittently. The bump job resolves the digests from the
|
||||
# registry instead, which is a read, not a rebuild.
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: registry login
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
# Not gated to master any more: the build step now reads AND writes the layer cache
|
||||
# in the registry, so every run needs credentials. Pushing the release image is
|
||||
# still master-only — that gate lives on the `push by digest` step, where it belongs.
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USER }}
|
||||
@@ -226,27 +360,108 @@ jobs:
|
||||
# Loaded locally, not pushed. Trivy scans this exact image next; only then does it
|
||||
# get pushed. The alternative — push, scan, and hope nobody pulled meanwhile — is
|
||||
# how a CRITICAL ends up in the registry with a green checkmark next to it.
|
||||
#
|
||||
# The layer cache is `type=registry`, NOT `type=gha`. Two reasons, both hard:
|
||||
#
|
||||
# 1. act_runner's cache server is backed by a 1Gi PVC that also holds `.runner`,
|
||||
# the runner's own registration file. `mode=max` stores every intermediate layer
|
||||
# of three images — several GB. Filling that volume does not merely lose the
|
||||
# cache: the runner cannot write its state and has to be re-registered by hand.
|
||||
# Trading "slow CI" for "broken CI" is not a trade.
|
||||
# 2. act_runner evicts by AGE, with no size cap in its config. It will fill whatever
|
||||
# it is given and then wedge. The registry has no such limit and already holds
|
||||
# the images anyway.
|
||||
#
|
||||
# The uv/pip cache still uses the runner's cache service — that one is a few hundred
|
||||
# MB and fits.
|
||||
run: |
|
||||
docker buildx build \
|
||||
-f services/${{ matrix.svc }}/Dockerfile \
|
||||
--build-arg BUILD_SHA=${{ github.sha }} \
|
||||
--cache-from type=gha,scope=${{ matrix.svc }} \
|
||||
--cache-to type=gha,mode=max,scope=${{ matrix.svc }} \
|
||||
--cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \
|
||||
--cache-to type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache,mode=max \
|
||||
--load \
|
||||
-t svcforge/${{ matrix.svc }}:ci \
|
||||
.
|
||||
|
||||
- name: import smoke test
|
||||
# Import the service's entrypoint module INSIDE the built image, which every unit and
|
||||
# integration test that passes cannot do: they import from the source tree, where
|
||||
# every file exists. The image is a different filesystem — each Dockerfile copies only
|
||||
# its own `services/<svc>/`, so a shared module added at the `services/` root, or any
|
||||
# dependency the Dockerfile forgets, is present in the tests and absent in the image.
|
||||
#
|
||||
# That gap shipped a reconciler that crashed on boot with
|
||||
# `ModuleNotFoundError: No module named 'services._runtime'` while every gate was
|
||||
# green. Importing main here loads the whole transitive graph and fails the build
|
||||
# before the digest is pushed, instead of after ArgoCD has rolled it out.
|
||||
run: |
|
||||
docker run --rm --entrypoint python svcforge/${{ matrix.svc }}:ci \
|
||||
-c "import services.${{ matrix.svc }}.main"
|
||||
|
||||
- name: trivy
|
||||
uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
|
||||
with:
|
||||
image-ref: svcforge/${{ matrix.svc }}:ci
|
||||
severity: HIGH,CRITICAL
|
||||
# A CVE with no fix available is not something this PR can act on; failing on it
|
||||
# only teaches people to add ignore entries. Rebuilding on a new base image picks
|
||||
# the fix up the day it exists.
|
||||
ignore-unfixed: true
|
||||
exit-code: "1"
|
||||
format: table
|
||||
# Run trivy directly rather than via aquasecurity/trivy-action, for the same reason
|
||||
# gitleaks is run directly above: the command is the documented one, pinned by
|
||||
# digest, with no nested action resolution.
|
||||
#
|
||||
# It is also the only thing that works here. trivy-action internally does
|
||||
# `uses: aquasecurity/setup-trivy@v0.2.2`, and that tag no longer exists upstream —
|
||||
# the earliest published tag today is v0.2.6. The runner clones it and fails with
|
||||
# "Unable to resolve v0.2.2: reference not found". A third-party action pinned by
|
||||
# SHA still resolves ITS OWN dependencies by mutable tag, so pinning the outer
|
||||
# action bought nothing.
|
||||
#
|
||||
# --ignore-unfixed: a CVE with no fix available is not something this PR can act
|
||||
# on, and failing on it only teaches people to add ignore entries. Rebuilding on a
|
||||
# new base image picks the fix up the day it exists.
|
||||
#
|
||||
# The vuln DB cache is a NAMED VOLUME, not `-v "$PWD/.trivycache:..."`.
|
||||
#
|
||||
# The bind form was the same bug that made gitleaks scan nothing: `$PWD` is a path
|
||||
# in the job container, but the -v is resolved by the daemon in the dind sidecar,
|
||||
# which has no such directory and silently creates an empty one. Every run then
|
||||
# logged `[vulndb] Need to update DB` and spent ~35s re-downloading it, and the
|
||||
# cache it wrote went into a throwaway directory inside dind.
|
||||
#
|
||||
# A named volume lives in the dind daemon's own storage, which is the one thing on
|
||||
# this runner both sides agree on. It survives across matrix legs and across runs.
|
||||
#
|
||||
# Measured, not assumed: cold pass logs the three download lines and leaves 1.1G
|
||||
# in the volume; warm pass logs nothing. 1.1G is far more than the 50MB an older
|
||||
# comment here claimed, so the retention story matters. Trivy replaces the DB in
|
||||
# place rather than accumulating versions, so the volume stays at roughly one DB,
|
||||
# and the weekly prune CronJob reclaims it at the cost of one re-download.
|
||||
env:
|
||||
TRIVY: ghcr.io/aquasecurity/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f
|
||||
run: |
|
||||
# Pull by digest, then give it a local tag, and run the TAG.
|
||||
#
|
||||
# An image pulled by digest carries no tag, which makes it dangling the moment its
|
||||
# container exits — so the reclaim step below deleted it at the end of every matrix
|
||||
# leg and the next leg paid a 178MB re-pull. Three legs, three pulls, every run.
|
||||
#
|
||||
# The `until=168h` filter there does not save it. That filter reads the image's
|
||||
# CREATED timestamp, not when it was pulled, and this image was built months ago,
|
||||
# so it matches the age filter immediately. A tag is what actually takes an image
|
||||
# out of `docker image prune`'s scope. Verified in run #68: "Unable to find image
|
||||
# ... locally" in all three legs, and "untagged: aquasec/trivy@sha256:cffe..." in
|
||||
# each prune.
|
||||
#
|
||||
# The digest is still the pin — it is enforced here, at pull time. `trivy:pinned`
|
||||
# is a local alias for an image whose content was already verified.
|
||||
docker pull "${TRIVY}"
|
||||
docker tag "${TRIVY}" trivy:pinned
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v svcforge-trivy-db:/root/.cache/trivy \
|
||||
trivy:pinned \
|
||||
image \
|
||||
--severity HIGH,CRITICAL \
|
||||
--ignore-unfixed \
|
||||
--exit-code 1 \
|
||||
--format table \
|
||||
--no-progress \
|
||||
svcforge/${{ matrix.svc }}:ci
|
||||
|
||||
- name: push by digest
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
@@ -262,34 +477,86 @@ jobs:
|
||||
docker buildx build \
|
||||
-f services/${{ matrix.svc }}/Dockerfile \
|
||||
--build-arg BUILD_SHA=${{ github.sha }} \
|
||||
--cache-from type=gha,scope=${{ matrix.svc }} \
|
||||
--cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \
|
||||
--push \
|
||||
-t "${IMAGE}:${GITHUB_SHA}" \
|
||||
.
|
||||
docker buildx imagetools inspect "${IMAGE}:${GITHUB_SHA}" \
|
||||
--format '{{.Manifest.Digest}}'
|
||||
|
||||
- name: reclaim dind disk
|
||||
# dind's /var/lib/docker is a hostPath on node0 (see oci-k8s addons role), so
|
||||
# nothing reclaims it automatically — kubelet's image GC does not manage a nested
|
||||
# daemon's store. Left alone it grows every run until node0 hits disk pressure and
|
||||
# starts evicting pods, which looks like a cluster problem rather than a CI one.
|
||||
#
|
||||
# `always()`: a failed build still leaves layers behind, and that is exactly when
|
||||
# the disk is most likely to be the reason it failed.
|
||||
#
|
||||
# Deliberately narrow. `docker image prune` WITHOUT -a removes dangling images
|
||||
# only; with -a it would delete the act runner image, which no container references
|
||||
# between jobs, and buy back a 1.6GB re-pull on the very next run.
|
||||
#
|
||||
# Named volumes are never pruned here: that is where the trivy vuln DB lives.
|
||||
#
|
||||
# NO age filter on the image prune, and that is the whole point of this comment.
|
||||
# `--filter until=168h` reads an image's CREATED time, so it got both cases exactly
|
||||
# backwards: it deleted trivy every leg (a released tool image is always older than
|
||||
# any window — run #68, 178MB re-pulled three times) while protecting the dangling
|
||||
# build layers it was added to remove (they are minutes old). Measured on node0
|
||||
# afterwards: 21 dangling images, 5.96GB, 19 of them created inside 25 hours, none
|
||||
# of them reclaimable while the filter was there. Tagging trivy is what protects
|
||||
# trivy; nothing needs to protect a dangling layer from the current build, because
|
||||
# `always()` runs this after that build has already been pushed.
|
||||
#
|
||||
# The buildx cache keeps its age filter: a week is recent enough that `--cache-from`
|
||||
# still hits on normal traffic, and that cache really does grow by age.
|
||||
#
|
||||
# The act runner image survives only because this step runs inside an act container,
|
||||
# so the image is in use exactly while the prune runs. That is luck, not design; if
|
||||
# it ever starts disappearing, tag it the same way.
|
||||
if: always()
|
||||
run: |
|
||||
docker image prune -f
|
||||
docker buildx prune -af --filter until=168h
|
||||
echo "--- dind disk after prune ---"
|
||||
docker system df
|
||||
|
||||
# --- stage 11: bump the chart's digests. CI's last act. ------------------------------
|
||||
bump:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [image]
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
# The master+push guard lives on the STEPS, not on the job.
|
||||
#
|
||||
# A job-level `if` here evaluated false and skipped `bump` at 0s — before `image` had
|
||||
# even started — on a genuine push to master (`event: push`, `head_branch: master`).
|
||||
# The identical expression on the `push by digest` STEP inside the matrix job evaluates
|
||||
# true and runs. Gitea appears not to resolve a job-level `if` correctly when `needs`
|
||||
# points at a matrix job.
|
||||
#
|
||||
# `needs: [image]` still does the ordering and still gates on all three legs passing.
|
||||
# The steps below carry the branch guard, in the form this runner is known to evaluate
|
||||
# correctly. On a PR the job starts and every step no-ops, which costs a few seconds
|
||||
# and is the price of a guard that actually fires.
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
with:
|
||||
# A bot token with contents:write on this repo and nothing else. Note what is
|
||||
# absent: no kubeconfig, no cluster credential, no ArgoCD API token. CI's maximum
|
||||
# blast radius is a bad commit, which is revertable.
|
||||
# A bot token with contents:write on this repo and nothing else: no kubeconfig,
|
||||
# no cluster credential, no ArgoCD API token. CI's maximum blast radius is a bad
|
||||
# commit, which is revertable.
|
||||
token: ${{ secrets.CI_BOT_TOKEN }}
|
||||
ref: master
|
||||
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
- uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
- name: bump image digests in the chart
|
||||
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
|
||||
env:
|
||||
REGISTRY: ${{ env.REGISTRY }}
|
||||
IMAGE_NS: ${{ env.IMAGE_NS }}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.8.4
|
||||
rev: v0.15.22
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix]
|
||||
- id: ruff-format
|
||||
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.21.2
|
||||
rev: v8.30.1
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
3.12
|
||||
3.14
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Accepted misconfiguration findings, each with a reason and an expiry date.
|
||||
#
|
||||
# The expiry is the point. An ignore without one is a permanent hole that nobody revisits;
|
||||
# trivy stops honouring these entries after the date, the gate goes red, and someone has to
|
||||
# look again. Re-dating an entry is a decision. Letting it lapse silently is not possible.
|
||||
#
|
||||
# Nothing here is suppressed because it was inconvenient. Both entries are inherent to what
|
||||
# a control plane that installs arbitrary Helm charts *is*, and both are documented in
|
||||
# ARCHITECTURE.md under "The provisioner is privileged".
|
||||
|
||||
misconfigurations:
|
||||
- id: KSV-0041
|
||||
# "ClusterRole shouldn't have access to manage resource 'secrets'"
|
||||
#
|
||||
# This is true and it is the design. Helm stores release state as Secrets in the
|
||||
# release's namespace, so anything that runs `helm upgrade --install` must be able to
|
||||
# create and read Secrets there. The worker installs into a namespace per tenant, and
|
||||
# those namespaces are created at provision time, so the grant cannot be enumerated in
|
||||
# advance and ends up cluster-scoped.
|
||||
#
|
||||
# The consequence, stated plainly: svcforge can read any Secret in the cluster,
|
||||
# including ServiceAccount tokens, and can therefore impersonate any workload. It is a
|
||||
# privileged component. Treat compromise of the worker as compromise of the cluster.
|
||||
#
|
||||
# The real fix is per-namespace Roles bound at provision time, which needs RBAC write
|
||||
# permission — itself an escalation path unless carefully constrained. That is a
|
||||
# larger piece of work than this reference implementation takes on, and pretending
|
||||
# otherwise by hiding the finding would be worse than recording it.
|
||||
statement: "helm stores release state in Secrets; namespaces are created per tenant at runtime"
|
||||
expired_at: 2026-10-01
|
||||
|
||||
- id: KSV-0056
|
||||
# "ClusterRole should not have create/update/delete on services, endpoints, ..."
|
||||
#
|
||||
# Same root cause. A chart that installs Elasticsearch creates a Service; the
|
||||
# provisioner has to be able to create it. The interception risk the rule describes is
|
||||
# real and follows from the same privileged position as KSV-0041.
|
||||
statement: "installing a chart necessarily creates the Services that chart defines"
|
||||
expired_at: 2026-10-01
|
||||
@@ -0,0 +1,153 @@
|
||||
# AGENTS.md — working in svcforge-reference
|
||||
|
||||
Rules for any AI agent (Claude Code, Cursor, Codex, Copilot, Gemini CLI) editing this repo.
|
||||
Read this before your first edit.
|
||||
|
||||
---
|
||||
|
||||
## What this repo is, and the one rule that follows from it
|
||||
|
||||
This is a **reference implementation**: a complete, verified build of the system that
|
||||
`../learn-python/` teaches Phuc to build himself. It exists to be *compared against*, not
|
||||
copied from.
|
||||
|
||||
**THE RULE: never propose copying `domain/`, `repo/`, `services/worker/`, or the claim
|
||||
query into `learn-python/`.** Those four are the course. If asked to "help with Module 4",
|
||||
help him find his own bug; do not paste `repo/tasks.py`. Scaffolding — `pyproject.toml`,
|
||||
Dockerfiles, CI, the chart — is fair game to copy, because struggling with hatchling
|
||||
teaches nothing.
|
||||
|
||||
`learn-python/AGENTS.md` has its own, stricter rules. They still apply over there.
|
||||
|
||||
---
|
||||
|
||||
## Before you change anything
|
||||
|
||||
Read [ARCHITECTURE.md](ARCHITECTURE.md). Every design choice here was paid for by a
|
||||
specific failure mode, and the comments say which. A change that removes a comment's
|
||||
subject usually removes a guard.
|
||||
|
||||
Four things that look like over-engineering and are not. Do not "simplify" them:
|
||||
|
||||
1. **The claim query is one statement.** `repo/tasks.py::_CLAIM_SQL`. Splitting the
|
||||
`UPDATE ... WHERE id = (SELECT ... FOR UPDATE SKIP LOCKED)` into select-then-update
|
||||
creates a double-provision race that testing will not catch.
|
||||
2. **`complete()` and `fail()` check `locked_by`.** That check is a repro-confirmed bug
|
||||
fix, with regression tests in `tests/integration/test_tasks.py`. Removing it lets a
|
||||
worker whose lease expired clobber a task another worker now owns.
|
||||
3. **`domain/` has no I/O and no `async`.** CI greps for both.
|
||||
4. **Migrations never run at app startup.** They are a Helm `pre-upgrade,pre-install` hook.
|
||||
|
||||
---
|
||||
|
||||
## Conventions
|
||||
|
||||
**Layers.** `transport → domain ← repo/adapters`. `domain/` imports nothing from the other
|
||||
three. `repo/` knows SQL and not HTTP. `adapters/` owns every subprocess and network call.
|
||||
|
||||
**Python 3.14** at runtime, but ruff's `target-version` is pinned to `py313` so the
|
||||
formatter never emits 3.14-only syntax. Full annotations everywhere including tests.
|
||||
`X | None`, `StrEnum`, `Final`, PEP 695 `type` aliases. Keep `except (A, B):`
|
||||
parenthesized — PEP 758's unparenthesized form reads exactly like Python 2's
|
||||
`except E, name:`, which means something else.
|
||||
|
||||
**Errors.** `SvcforgeError` in `svcforge_core/errors.py` is the base. New exception types
|
||||
inherit from it, plus a stdlib base when the behaviour matters (`HelmError(SvcforgeError,
|
||||
RuntimeError)`).
|
||||
|
||||
**Logging.** `obs.get_logger(__name__)`, structlog kwargs. Never stdlib `logging` directly,
|
||||
and never `extra={...}` — the ProcessorFormatter bridge silently discards it. Never use
|
||||
`message` or `event` as a field name; both are reserved and raise.
|
||||
|
||||
**SQL.** Raw psycopg3, no ORM. Parameters, never f-strings. Rows come back as dicts
|
||||
(`DictPool` in `repo/db.py`); annotate with that alias rather than a bare
|
||||
`AsyncConnectionPool`, which mypy resolves to tuple rows.
|
||||
|
||||
**Subprocess.** `create_subprocess_exec` with `start_new_session=True`. Never `shell=True` —
|
||||
`team` is tenant input that reaches a helm release name. Timeouts kill the process *group*.
|
||||
|
||||
**Protocols.** A `Protocol` needs two implementations (a real one and a fake) or it should
|
||||
be a concrete class.
|
||||
|
||||
**Datetimes.** Always aware, always UTC. Domain functions take `now` as a parameter;
|
||||
only adapters hold a `Clock`.
|
||||
|
||||
**Pinning.** Everything by digest: base images, tool images, GitHub Actions by SHA.
|
||||
A SHA-pinned third-party action still resolves *its own* dependencies by mutable tag —
|
||||
that is how `trivy-action` broke. Prefer running tools directly from a digest-pinned image.
|
||||
|
||||
---
|
||||
|
||||
## Writing style
|
||||
|
||||
Plain declaratives. **No back-and-forth phrasing**, which means none of:
|
||||
|
||||
- Rhetorical setup-knockdowns: "That is not a nicety. It is the whole design."
|
||||
- not-X-but-Y pivots used for rhythm: "It is not a lock, but a lease."
|
||||
- Throat-clearing: "It is worth noting that", "The key insight here", "Here's the thing".
|
||||
- Rhetorical questions you then answer.
|
||||
- Dramatic fragments for rhythm: "Not slow. Not fast. Meaningless."
|
||||
|
||||
Write the claim directly. Keep the *why* — it is the most valuable content in this repo —
|
||||
and drop the scaffolding around it. Factual contrasts that carry information stay
|
||||
("LEFT join, not inner"; "202, never 201").
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- **Never mock the database.** Integration tests run against real Postgres. Each pytest
|
||||
process clones its own database (`tests/integration/conftest.py`).
|
||||
- **Fakes, not mocks**, for helm/kubectl/Redis/clock. `tests/fakes.py`.
|
||||
- **A test that cannot fail is worse than no test.** Before adding one, ask what change
|
||||
would make it red. Bound assertions on both sides where it matters: `max_concurrent
|
||||
<= cap` alone passes on a fully serial worker.
|
||||
- Markers: `slow` (>1s or hits real Upstash), `e2e` (needs a cluster). `make test` runs
|
||||
neither.
|
||||
|
||||
```bash
|
||||
export SVCFORGE_TEST_DSN="postgresql://svcforge:svcforge@127.0.0.1:5432/svcforge_test"
|
||||
make lint # ruff + ruff format + mypy --strict
|
||||
make test # unit
|
||||
make integration # needs the DSN above
|
||||
make ci # every gate CI runs, same commands
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Environment facts that will bite you
|
||||
|
||||
- **This host has no Docker daemon, and must not get one.** Its `containerd` belongs to a
|
||||
Kubernetes kubelet; installing `docker.io` would evict the runtime and take down every
|
||||
pod. You cannot `docker build` here. Verify Dockerfiles by inspection and `hadolint`.
|
||||
- **Postgres is local** (apt, v16) for tests. testcontainers is the CI path.
|
||||
- **Redis is real Upstash on a free tier: 500K commands/month.** Redis-hitting tests are
|
||||
marked `slow`. Do not write loops that spend thousands of commands; use the fakes.
|
||||
- **Secrets live in `~/.config/svcforge/secrets.env`**, chmod 600, outside every repo.
|
||||
Never echo, log, commit, or paste a DSN or token. `.env` is gitignored.
|
||||
- **Infra changes go through Terraform/Ansible**, never `kubectl edit`. The Gitea runner's
|
||||
config lives in `oci-k8s/k8s/roles/addons/tasks/main.yml`.
|
||||
|
||||
---
|
||||
|
||||
## CI/CD
|
||||
|
||||
`.gitea/workflows/ci.yaml`. Gates in order: ruff → mypy → unit+coverage → migrate+integration
|
||||
→ bandit/gitleaks/pip-audit → hadolint → helm lint/template → build → trivy → push by digest
|
||||
→ bump chart digests → ArgoCD syncs.
|
||||
|
||||
- **CI holds no kubeconfig and must never hold one.** Its last act is a git commit.
|
||||
- **Build once, promote by digest.** The image is scanned before it is pushed.
|
||||
- **Every gate blocks.** If a gate is failing, fix the cause. Do not add an ignore, lower a
|
||||
threshold, or mark it advisory. A CVE finding from trivy is real: bump the pinned version.
|
||||
|
||||
`RUNBOOK.md` has the setup steps and the traps that cost real time.
|
||||
|
||||
---
|
||||
|
||||
## When you are unsure
|
||||
|
||||
Say so, and check. This repo's value is that its claims were executed, not asserted — the
|
||||
race conditions have reproductions, the timeout kill has a negative control, the coverage
|
||||
gate was caught measuring 0%. Guessing and sounding confident is the one failure mode that
|
||||
makes it worthless.
|
||||
+484
@@ -0,0 +1,484 @@
|
||||
# svcforge — how it works
|
||||
|
||||
A control plane for internal service provisioning. A team asks for an Elasticsearch; a
|
||||
worker installs one; a control loop keeps reality matching the database.
|
||||
|
||||
This document explains the design and **why each piece is the way it is**. Every "why" here
|
||||
was paid for by a specific failure mode.
|
||||
|
||||
---
|
||||
|
||||
## The one-paragraph version
|
||||
|
||||
`POST /v1/instances` writes two rows in one transaction: the instance, and a task to build
|
||||
it. It returns `202` immediately. A worker claims that task with a single `SELECT ... FOR
|
||||
UPDATE SKIP LOCKED` statement, runs `helm upgrade --install`, and marks the instance ready.
|
||||
A reconciler sweeps every 60 seconds for the things that go wrong when a process dies at
|
||||
the wrong moment. The queue is a Postgres table, not Redis, and that single decision
|
||||
determines most of the rest of the design.
|
||||
|
||||
---
|
||||
|
||||
## The shape
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph tenant["Tenant"]
|
||||
CLI["svcforge CLI<br/><i>HTTP only, never the DB</i>"]
|
||||
end
|
||||
|
||||
subgraph plane["svcforge control plane"]
|
||||
API["<b>api</b><br/>FastAPI, N replicas<br/>validates, authenticates,<br/>enqueues"]
|
||||
WORKER["<b>worker</b><br/>N replicas<br/>claims tasks, runs helm"]
|
||||
RECON["<b>reconciler</b><br/><i>exactly 1</i><br/>4 checks every 60s"]
|
||||
end
|
||||
|
||||
subgraph state["State"]
|
||||
PG[("<b>Postgres</b><br/>instances + tasks<br/><i>the truth</i>")]
|
||||
REDIS[("<b>Redis</b><br/>rate limit, cache<br/><i>derived only</i>")]
|
||||
end
|
||||
|
||||
K8S["<b>Kubernetes</b><br/>helm releases<br/><i>the real world</i>"]
|
||||
|
||||
CLI -->|"POST /v1/instances"| API
|
||||
API -->|"one transaction:<br/>instance + task"| PG
|
||||
API -.->|"best effort"| REDIS
|
||||
WORKER -->|"claim<br/>SKIP LOCKED"| PG
|
||||
WORKER -->|"helm upgrade --install"| K8S
|
||||
RECON -->|"drift, leases,<br/>TTL, versions"| PG
|
||||
RECON -->|"list releases"| K8S
|
||||
|
||||
classDef truth fill:#2d4a22,stroke:#5a8f3d,color:#fff
|
||||
classDef derived fill:#4a3222,stroke:#8f6a3d,color:#fff
|
||||
class PG truth
|
||||
class REDIS derived
|
||||
```
|
||||
|
||||
Three services, four layers, two stores. The layer rule is one line:
|
||||
|
||||
```
|
||||
transport/ HTTP handlers, CLI entrypoints. Knows FastAPI. Knows nothing about SQL.
|
||||
domain/ Pure logic: state machine, catalog, backoff, windows. No I/O. No async.
|
||||
repo/ SQL. Rows in, domain objects out. Knows psycopg. Knows nothing about HTTP.
|
||||
adapters/ The outside world: helm, kubectl, webhooks, Redis, the clock.
|
||||
```
|
||||
|
||||
Dependencies point inward: `transport → domain ← repo/adapters`. `domain/` imports nothing
|
||||
from the other three, which is why its tests need no mocks and run in milliseconds.
|
||||
|
||||
---
|
||||
|
||||
## The decision everything else follows from
|
||||
|
||||
**The queue is a Postgres table.**
|
||||
|
||||
A task and the instance state it describes must commit atomically. Split them across two
|
||||
stores and you own a distributed commit problem with no winning move: the process can die
|
||||
between the two writes, and whichever you write first is the one that lies.
|
||||
|
||||
- Task first, then instance → an orphan task pointing at an instance that never existed.
|
||||
- Instance first, then task → an instance nobody will ever build.
|
||||
|
||||
In one table, in one transaction, neither is possible:
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
INSERT INTO instances (...); -- state='requested'
|
||||
INSERT INTO tasks (...); -- kind='provision'
|
||||
COMMIT; -- both, or neither
|
||||
```
|
||||
|
||||
Redis cannot do this, so Redis is not the queue. It holds derived state only — things that
|
||||
can be recomputed and whose loss is an inconvenience, never a corruption.
|
||||
|
||||
| | Postgres | Redis |
|
||||
|---|---|---|
|
||||
| Holds | instances, tasks | rate limits, idempotency keys, cache |
|
||||
| If it is down | the platform is down | the platform is fine |
|
||||
| If it disagrees with reality | reality wins, reconciler fixes it | discard it |
|
||||
| Lives on the | control loop **and** request path | request path only |
|
||||
|
||||
That last row is a budget constraint. Upstash's free tier is 500K
|
||||
commands/month = **0.19 commands/second sustained**. One worker polling Redis every 5
|
||||
seconds is 518,400/month — the entire budget, spent by one pod doing nothing.
|
||||
|
||||
---
|
||||
|
||||
## Provisioning, end to end
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
autonumber
|
||||
participant T as Tenant
|
||||
participant A as api
|
||||
participant P as Postgres
|
||||
participant W as worker
|
||||
participant K as Kubernetes
|
||||
|
||||
T->>A: POST /v1/instances {elasticsearch, small}
|
||||
A->>A: verify JWT, look up catalog
|
||||
rect rgb(45, 74, 34)
|
||||
A->>P: BEGIN
|
||||
A->>P: INSERT instance (state=requested)
|
||||
A->>P: INSERT task (kind=provision, traceparent)
|
||||
A->>P: COMMIT
|
||||
end
|
||||
A-->>T: 202 Accepted + Location
|
||||
|
||||
Note over W,P: every 5s, while a semaphore slot is free
|
||||
W->>P: UPDATE ... FOR UPDATE SKIP LOCKED
|
||||
P-->>W: task (attempts now 1, locked_by=me)
|
||||
W->>K: helm upgrade --install --create-namespace --wait
|
||||
K-->>W: release ready
|
||||
W->>P: CAS provisioning -> ready, set endpoint
|
||||
W->>P: complete(task, worker_id)
|
||||
|
||||
T->>A: GET /v1/instances/{id}
|
||||
A-->>T: {state: ready, endpoint: ...}
|
||||
```
|
||||
|
||||
**Why 202 and not 201.** A provision is `helm --wait` on a StatefulSet: minutes. Holding
|
||||
an HTTP connection open for that is a request that dies to any proxy timeout, and a client
|
||||
that cannot tell "still working" from "lost". The queue absorbs the API's output, which is
|
||||
the property that lets 50 simultaneous POSTs all return instantly.
|
||||
|
||||
---
|
||||
|
||||
## The claim query
|
||||
|
||||
This is the heart of the system. It is one statement, and it must stay one statement.
|
||||
|
||||
```sql
|
||||
WITH claimed AS (
|
||||
UPDATE tasks SET state='running', attempts=attempts+1,
|
||||
locked_by=%(worker)s, locked_at=now()
|
||||
WHERE id = (
|
||||
SELECT id FROM tasks
|
||||
WHERE state='queued' AND run_after <= now()
|
||||
ORDER BY run_after
|
||||
FOR UPDATE SKIP LOCKED -- step over rows other workers hold, do not queue behind them
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING *
|
||||
)
|
||||
SELECT claimed.*, instances.team
|
||||
FROM claimed LEFT JOIN instances ON instances.id = claimed.instance_id;
|
||||
```
|
||||
|
||||
- **`FOR UPDATE SKIP LOCKED`** is what makes N workers scale. Without `SKIP LOCKED` they
|
||||
queue single-file behind whoever holds the oldest row.
|
||||
- **The subquery exists because Postgres has no `UPDATE ... LIMIT`.**
|
||||
- **Select-then-update as two statements is the bug this prevents.** Between the `SELECT`
|
||||
and the `UPDATE`, a second worker reads the same id and both provision. The window is
|
||||
small, which means you will not hit it in testing and you will hit it in production.
|
||||
- **`LEFT` join, not inner.** The `UPDATE` has already taken effect when the outer select
|
||||
runs. An inner join matching nothing would return no row, so `claim()` would report
|
||||
"queue empty" for a task it had just marked `running` — stranding it until the lease
|
||||
expires, having silently burned an attempt.
|
||||
- **`attempts` increments at claim time, not on failure.** A worker that dies without
|
||||
reporting has still burned an attempt, so a task that reliably kills its worker cannot
|
||||
retry forever.
|
||||
|
||||
---
|
||||
|
||||
## The instance lifecycle
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> requested: POST /v1/instances
|
||||
requested --> provisioning: worker claims
|
||||
provisioning --> ready: helm --wait succeeded
|
||||
ready --> deleting: DELETE, or TTL expired
|
||||
deleting --> deleted: helm uninstall succeeded
|
||||
|
||||
requested --> failed: provision attempts exhausted
|
||||
provisioning --> failed: provision attempts exhausted
|
||||
ready --> failed: drift — the release vanished
|
||||
failed --> provisioning: retry
|
||||
failed --> deleting: give up, tear it down
|
||||
deleting --> deleting: deprovision retried, never failed
|
||||
|
||||
deleted --> [*]: terminal
|
||||
```
|
||||
|
||||
`LEGAL` is a `dict[InstanceState, frozenset[InstanceState]]` in `domain/states.py`, not a
|
||||
chain of `if`s. `deleted` maps to an **empty frozenset** rather than being absent, so
|
||||
"terminal" is stated rather than implied by a missing key.
|
||||
|
||||
**Dead-lettering the task does not fail the instance, except for provision.** When a task
|
||||
exhausts its retries `TaskRepo.fail` marks the *task* `failed` for every kind. It moves the
|
||||
*instance* to `failed` only for `provision`, because that is the only kind where a dead
|
||||
letter means the instance is broken. For the others the instance is still healthy and
|
||||
something else owns its recovery:
|
||||
|
||||
| kind | instance state on dead-letter | why |
|
||||
|------|-------------------------------|-----|
|
||||
| provision | `failed` | it never came up; a human re-provisions |
|
||||
| deprovision | stays `deleting` | so `due_for_deprovision` re-enqueues it; `failed` would strand it and leak the release |
|
||||
| upgrade | stays `ready` | `helm --atomic` rolled back; it still serves the old version |
|
||||
| verify | stays `ready` | `handle_verify` already halted the rollout |
|
||||
|
||||
The provision write is still guarded by the state machine, derived from the same `LEGAL`
|
||||
table rather than restated:
|
||||
|
||||
```python
|
||||
_CAN_FAIL = tuple(s.value for s, allowed in LEGAL.items() if InstanceState.FAILED in allowed)
|
||||
...
|
||||
if kind == 'provision':
|
||||
UPDATE instances SET state='failed' WHERE id=%s AND state = ANY(%s)
|
||||
```
|
||||
|
||||
The `SvcforgeTaskDeadLettered` alert fires for every kind, so leaving the instance alone
|
||||
loses no operator visibility.
|
||||
|
||||
---
|
||||
|
||||
## What happens when things die
|
||||
|
||||
This is the part that matters. Every guarantee below has a test.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
START["worker claims task<br/>state=running, locked_by=me"] --> WORK["helm upgrade --install"]
|
||||
WORK -->|success| REPORT["complete(task, worker_id)"]
|
||||
WORK -->|"raises"| FAIL["fail(task, err, worker_id)"]
|
||||
WORK -->|"pod SIGKILLed"| DEAD["nothing reported<br/>row stuck at 'running'"]
|
||||
|
||||
REPORT --> CAS{"still locked_by me?"}
|
||||
CAS -->|yes| DONE["state=done"]
|
||||
CAS -->|"no — lease was stolen"| DROP["log and drop.<br/>the new owner reports"]
|
||||
|
||||
FAIL --> ATT{"attempts < max?"}
|
||||
ATT -->|yes| REQUEUE["state=queued<br/>run_after += backoff+jitter"]
|
||||
ATT -->|no| DEADLETTER["state=failed<br/>error copied to instance"]
|
||||
|
||||
DEAD --> LEASE["reconciler: locked_at older<br/>than lease_seconds"]
|
||||
LEASE --> REQUEUE
|
||||
|
||||
REQUEUE --> START
|
||||
|
||||
classDef bad fill:#4a2222,stroke:#8f3d3d,color:#fff
|
||||
classDef good fill:#2d4a22,stroke:#5a8f3d,color:#fff
|
||||
class DEAD,DEADLETTER bad
|
||||
class DONE,DROP good
|
||||
```
|
||||
|
||||
**Recovery is by lease.** No distributed lock survives a power cut. A SIGKILLed worker leaves
|
||||
`state='running'` with `locked_by` set and nobody running it; that row would sit there
|
||||
forever. The lease is the only thing that recovers it, which is why `locked_at` exists.
|
||||
|
||||
**Ownership is checked on report.** A worker that hangs past its lease has its task
|
||||
requeued and re-claimed by someone else. When it finally returns, `complete()` and `fail()`
|
||||
both check `AND state='running' AND locked_by=%s`. Without that check, the stale worker
|
||||
marks the task done while the new owner is still running it — and if the new owner then
|
||||
fails, a *third* worker provisions the same instance. That is the double-provision the
|
||||
claim query exists to prevent, arriving through the back door.
|
||||
|
||||
**Idempotency is what makes all of this safe.** It is bought in two places:
|
||||
|
||||
1. A deterministic release name: `f"{team}-{service_type}-{id[:8]}"`, `UNIQUE` in the schema.
|
||||
2. Adapters that state desired state — `helm upgrade --install`, `kubectl apply` — instead
|
||||
of issuing imperative commands.
|
||||
|
||||
Running a handler twice equals running it once, so redelivery is boring.
|
||||
|
||||
---
|
||||
|
||||
## The reconciler
|
||||
|
||||
One replica. Four checks. Every 60 seconds.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
TICK(("tick<br/>every 60s")) --> D["<b>drift</b><br/>live releases vs DB"]
|
||||
TICK --> L["<b>lease expiry</b><br/>running + locked_at old"]
|
||||
TICK --> T["<b>TTL</b><br/>ready + expires_at passed"]
|
||||
TICK --> V["<b>version drift</b><br/>chart_version ≠ catalog"]
|
||||
|
||||
D --> D1["release missing → re-enqueue provision"]
|
||||
D --> D2["release unknown → <b>log only</b>"]
|
||||
L --> L1["→ queued, locked_by=null"]
|
||||
T --> T1["→ deleting + deprovision task"]
|
||||
V --> V1["→ upgrade task, inside<br/>the maintenance window"]
|
||||
```
|
||||
|
||||
**The drift check never auto-deletes.** A bug in a delete path is unrecoverable; a bug in a
|
||||
report path is a Tuesday. Unknown releases are logged for a human.
|
||||
|
||||
**It is a singleton because it sweeps.** Two reconcilers double-enqueue and race on TTL.
|
||||
`replicas: 1` and `strategy: Recreate` in the chart, plus per-check idempotency guards that
|
||||
re-verify under `FOR UPDATE`.
|
||||
|
||||
---
|
||||
|
||||
## Day 2: upgrading the fleet
|
||||
|
||||
The team that runs this also patches it. That is the whole module, and it is **two columns
|
||||
and one query**.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
EDIT["edit catalog.yaml<br/>21.3.15 → 21.3.16"] --> Q
|
||||
|
||||
Q["<b>the work list</b><br/>state=ready AND chart_version ≠ catalog<br/>AND NOT halted<br/>ORDER BY own_team DESC<br/>LIMIT max_in_flight"]
|
||||
|
||||
Q --> U["upgrade task<br/>run_after = next maintenance window"]
|
||||
U --> H["helm upgrade --install"]
|
||||
H --> W["write instances.chart_version<br/><i>only after helm succeeds</i>"]
|
||||
W --> VER["verify task"]
|
||||
VER -->|healthy| NEXT["next instance"]
|
||||
VER -->|"failed"| HALT["catalog_versions.rollout_state='halted'<br/><i>work list goes empty</i>"]
|
||||
|
||||
NEXT --> Q
|
||||
HALT --> HUMAN["cleared by hand, with SQL"]
|
||||
|
||||
classDef stop fill:#4a2222,stroke:#8f3d3d,color:#fff
|
||||
class HALT,HUMAN stop
|
||||
```
|
||||
|
||||
- **`instances.chart_version` is written only after helm succeeds.** Write it optimistically
|
||||
and the fleet looks upgraded while it is not.
|
||||
- **`ORDER BY team = own_team DESC`** puts your own instances first, so you are the tenant
|
||||
who discovers the chart is broken. Eating your own dog food is enforced by an `ORDER BY`
|
||||
rather than left to policy.
|
||||
- **`max_in_flight` starts at 1** — a config value, not a scheduler. One at a time is what
|
||||
makes the halt meaningful: the fleet stops after the first casualty, not after all of them.
|
||||
- **The halt is one column, cleared by hand.** An automatic un-halt would just resume
|
||||
breaking things.
|
||||
|
||||
Deliberately **not** built: `resize`, `backup`/`restore`, `helm rollback` automation,
|
||||
deprecation timers, a rollouts table with history, a pause/resume CLI. Backup is a whole
|
||||
subsystem and an untested restore is a rumour.
|
||||
|
||||
---
|
||||
|
||||
## Observability: one trace across the queue
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant A as api
|
||||
participant P as tasks table
|
||||
participant W as worker
|
||||
|
||||
A->>A: span "POST /v1/instances" (trace abc123)
|
||||
A->>P: INSERT ... traceparent='00-abc123-...'
|
||||
Note over P: minutes pass. different pod.
|
||||
W->>P: claim → row carries traceparent
|
||||
W->>W: span "task.provision", parent=abc123
|
||||
Note over A,W: one trace: POST → queue → helm
|
||||
```
|
||||
|
||||
Trace context does **not** survive a queue on its own. The worker picks the row up in
|
||||
another process with no ambient context. So the traceparent rides in the table. Skip this
|
||||
and Tempo shows two unrelated traces for one provision, which is worse than no tracing
|
||||
because it looks like it works.
|
||||
|
||||
Logs carry `instance_id` / `task_id` / `team` on every line via contextvars, bound once at
|
||||
claim. Metrics are deliberately few, and two of them are named to prevent a specific
|
||||
mistake: `svcforge_task_attempts_failed_total` counts *attempts* that raised, while
|
||||
`svcforge_tasks_dead_lettered_total` counts tasks that gave up. Alerting on the first
|
||||
pages you for ordinary retries that later succeed.
|
||||
|
||||
---
|
||||
|
||||
## Delivery: CI never touches the cluster
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
PUSH["push to master"] --> GATES
|
||||
|
||||
subgraph GATES["gates — all required, none advisory"]
|
||||
direction TB
|
||||
G1["ruff"] --> G2["mypy --strict"] --> G3["pytest unit + coverage"]
|
||||
G3 --> G4["migrate + pytest integration"]
|
||||
G4 --> G5["bandit / gitleaks / pip-audit"]
|
||||
G5 --> G6["hadolint + helm lint/template"]
|
||||
end
|
||||
|
||||
GATES --> BUILD["build image (--load)"]
|
||||
BUILD --> SCAN["trivy HIGH,CRITICAL"]
|
||||
SCAN -->|clean| PUSHIMG["push by digest"]
|
||||
SCAN -->|"CVE"| STOP["pipeline fails"]
|
||||
PUSHIMG --> BUMP["commit digest to values.yaml<br/>[skip ci]"]
|
||||
BUMP --> ARGO["ArgoCD notices the commit"]
|
||||
ARGO --> CLUSTER["cluster"]
|
||||
|
||||
classDef stop fill:#4a2222,stroke:#8f3d3d,color:#fff
|
||||
class STOP stop
|
||||
```
|
||||
|
||||
**CI holds no kubeconfig, and must never hold one.** Its last act is a git commit; ArgoCD
|
||||
pulls. The maximum blast radius of a compromised pipeline is a bad commit, which is
|
||||
revertable.
|
||||
|
||||
**Build once, promote the artifact.** The image is loaded locally, scanned, and only then
|
||||
pushed — push-then-scan means a CRITICAL sits in the registry behind a green checkmark.
|
||||
Deploys are **by digest**, never a mutable tag.
|
||||
|
||||
**Migrations never run on app startup.** N replicas would race. They run as a Helm
|
||||
`pre-upgrade,pre-install` hook Job, once, before any new pod serves traffic. Forward-only,
|
||||
expand/contract: a rename is three deploys.
|
||||
|
||||
### The chart works with or without ArgoCD
|
||||
|
||||
The chart is **pure Helm**. It contains no `argocd.argoproj.io/*` annotations, no sync
|
||||
waves, and no ArgoCD-specific ordering. Both paths are supported and both are verified:
|
||||
|
||||
```bash
|
||||
# Path 1: plain Helm, no ArgoCD anywhere
|
||||
helm upgrade --install svcforge deploy/chart -n svcforge \
|
||||
--set image.api.digest=sha256:... --set image.worker.digest=sha256:... \
|
||||
--set image.reconciler.digest=sha256:...
|
||||
|
||||
# Path 2: GitOps. ArgoCD watches master and applies the same chart.
|
||||
kubectl apply -f deploy/argocd/app.yaml
|
||||
```
|
||||
|
||||
Ordering survives both because **ArgoCD translates Helm hooks into its own sync phases**
|
||||
rather than ignoring them:
|
||||
|
||||
| Annotation | Plain Helm | ArgoCD |
|
||||
|---|---|---|
|
||||
| `helm.sh/hook: pre-install,pre-upgrade` | runs before the release, aborts it on failure | mapped to the **PreSync** phase |
|
||||
| `helm.sh/hook-weight: "-5"` | orders hooks within the phase | mapped to hook ordering |
|
||||
| `helm.sh/hook-delete-policy: before-hook-creation` | deletes the previous Job first | mapped to `BeforeHookCreation` |
|
||||
|
||||
Using `argocd.argoproj.io/hook` instead would have been the trap: plain `helm install`
|
||||
does not understand that annotation, so it would create the migration Job as an ordinary
|
||||
resource with no ordering guarantee — the migration and the new pods would start together,
|
||||
and the failure would appear only in whichever path nobody tested.
|
||||
|
||||
Verified with `helm install --dry-run=server` against a real cluster, which validates every
|
||||
manifest through the API server rather than only rendering the templates locally.
|
||||
|
||||
---
|
||||
|
||||
## User stories, and what each one exercises
|
||||
|
||||
| As a… | I want… | So that… | Exercised by |
|
||||
|---|---|---|---|
|
||||
| tenant team | to request an Elasticsearch without filing a ticket | I am unblocked in minutes | `POST /v1/instances` → 202 |
|
||||
| tenant team | to see why my instance failed | I can fix my own request | `instances.error`, `svcforge status` |
|
||||
| tenant team | a throwaway instance to clean itself up | I do not pay for what I forgot | `ttl_days` → reconciler TTL sweep |
|
||||
| platform team | a worker pod to be killable at any instant | a rolling deploy is not an outage | SIGTERM drain + lease recovery |
|
||||
| platform team | to patch a CVE across every tenant | one edit, not N | `catalog.yaml` bump → work list |
|
||||
| platform team | a bad chart to stop after the first casualty | I do not break 40 tenants | `verify` → `rollout_state='halted'` |
|
||||
| platform team | to know the queue is stuck before a tenant tells me | I look competent | `SvcforgeQueueDepthRising` → RUNBOOK |
|
||||
| on-call | a copy-pasteable diagnosis at 3am | I do not have to think | [RUNBOOK.md](RUNBOOK.md) |
|
||||
|
||||
---
|
||||
|
||||
## Where to read the code
|
||||
|
||||
| To understand | Read | Then |
|
||||
|---|---|---|
|
||||
| the data model | `domain/models.py`, `migrations/001_init.sql` | `domain/states.py` |
|
||||
| **the queue** | `repo/tasks.py` — read `_CLAIM_SQL` twice | `services/worker/main.py` |
|
||||
| crash safety | `services/worker/handlers.py` | `tests/integration/test_worker.py` |
|
||||
| the API contract | `services/api/routes/instances.py` | `tests/integration/test_api.py` |
|
||||
| subprocess discipline | `adapters/helm.py::_run` | `tests/integration/test_helm_timeout.py` |
|
||||
| day 2 | `domain/windows.py`, `InstanceRepo.list_upgradable` | `services/reconciler/main.py` |
|
||||
| how it ships | `.gitea/workflows/ci.yaml`, `deploy/chart/` | [RUNBOOK.md](RUNBOOK.md) |
|
||||
|
||||
Related: [README.md](README.md) for how to read this repo without spoiling the course,
|
||||
[RUNBOOK.md](RUNBOOK.md) for operating it.
|
||||
@@ -0,0 +1,10 @@
|
||||
# CLAUDE.md
|
||||
|
||||
The rules for this repo live in [AGENTS.md](AGENTS.md) — one file, so every tool reads the
|
||||
same thing. Read it before your first edit.
|
||||
|
||||
The two that matter most:
|
||||
|
||||
1. This is a **reference implementation**. Never propose copying `domain/`, `repo/`,
|
||||
`services/worker/`, or the claim query into `../learn-python/`. Those are the course.
|
||||
2. **No back-and-forth writing style** in prose, comments, or docstrings. Plain declaratives.
|
||||
@@ -27,21 +27,45 @@ run-reconciler:
|
||||
# Every gate CI runs, except the image build — that needs a docker daemon and three
|
||||
# minutes, and the point of this target is to find out you failed before pushing.
|
||||
# Same commands as .gitea/workflows/ci.yaml, deliberately: a gate that only exists in CI
|
||||
# is a gate you debug through a web UI.
|
||||
# is a gate you debug through a web UI. Each recipe line below is the command from the
|
||||
# corresponding job, character for character. If you change one, change both.
|
||||
ci:
|
||||
uv run ruff check .
|
||||
uv run ruff format --check .
|
||||
uv run ruff check . && uv run ruff format --check .
|
||||
uv run mypy --strict .
|
||||
uv run pytest tests/unit --cov=libs/svcforge_core/domain --cov-fail-under=90
|
||||
uv run pytest tests/unit --cov=svcforge_core.domain --cov-fail-under=90
|
||||
uv run python -m svcforge_core.migrate
|
||||
uv run pytest tests/integration
|
||||
uv run --with bandit bandit -r libs services -ll
|
||||
uv run --with pip-audit pip-audit --strict
|
||||
uv run --with 'bandit[toml]' bandit -c pyproject.toml -r libs services -ll
|
||||
uv export --frozen --no-dev \
|
||||
--no-emit-project --no-emit-package svcforge-core \
|
||||
-o /tmp/requirements-audit.txt
|
||||
uv run --with pip-audit pip-audit --strict -r /tmp/requirements-audit.txt
|
||||
hadolint services/*/Dockerfile
|
||||
helm lint deploy/chart
|
||||
helm template svcforge deploy/chart >/dev/null
|
||||
$(MAKE) chart
|
||||
|
||||
# The chart's own gates. `helm template` is not a formality here: the image helper calls
|
||||
# `fail` on any digest that is not a sha256, so this catches an unbumped values.yaml.
|
||||
# The chart's own gates, mirroring the `chart` job in ci.yaml.
|
||||
#
|
||||
# `helm template` against plain values.yaml MUST FAIL: the digests there are all-zeros
|
||||
# placeholders and _helpers.tpl refuses to build an image reference from one. So this
|
||||
# asserts the failure rather than running the command bare — a bare `helm template` here
|
||||
# would report the guard working as a broken build, and the previous version of this
|
||||
# target did exactly that.
|
||||
#
|
||||
# The second render uses well-formed dummy digests to prove the templates themselves are
|
||||
# valid, with both values-gated monitoring blocks turned on.
|
||||
chart:
|
||||
helm lint deploy/chart
|
||||
helm template svcforge deploy/chart
|
||||
@if helm template svcforge deploy/chart >/dev/null 2>&1; then \
|
||||
echo "FAIL: chart rendered against the placeholder digests in values.yaml."; \
|
||||
echo "The digest guard in _helpers.tpl is not guarding."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "ok: placeholder digests rejected"
|
||||
helm template svcforge deploy/chart \
|
||||
--set image.api.digest=sha256:$(shell printf 'a%.0s' $$(seq 64)) \
|
||||
--set image.worker.digest=sha256:$(shell printf 'b%.0s' $$(seq 64)) \
|
||||
--set image.reconciler.digest=sha256:$(shell printf 'c%.0s' $$(seq 64)) \
|
||||
--set serviceMonitor.enabled=true \
|
||||
--set prometheusRule.enabled=true \
|
||||
>/dev/null
|
||||
@echo "ok: chart renders"
|
||||
|
||||
@@ -11,11 +11,11 @@ A complete, working, verified build of the system `../learn-python/` teaches you
|
||||
exactly that code. You asked for it deliberately, and the rule allows you to overrule it —
|
||||
but the reason for the rule did not go away when you did.
|
||||
|
||||
Module 4 is five evenings of getting the claim query wrong. That is not a tax on the way to
|
||||
the answer; it **is** the answer. The evening you spend watching two workers grab the same
|
||||
task is the evening `FOR UPDATE SKIP LOCKED` stops being a phrase and starts being a thing
|
||||
you understand. Reading `repo/tasks.py` here takes ninety seconds and teaches you close to
|
||||
nothing, while feeling exactly like learning. That feeling is the trap.
|
||||
Module 4 is five evenings of getting the claim query wrong, and those five evenings are the
|
||||
learning. The evening you spend watching two workers grab the same task is the evening
|
||||
`FOR UPDATE SKIP LOCKED` stops being a phrase and becomes something you understand.
|
||||
Reading `repo/tasks.py` here takes ninety seconds, teaches you close to nothing, and feels
|
||||
exactly like learning. That feeling is the trap.
|
||||
|
||||
So:
|
||||
|
||||
@@ -78,6 +78,24 @@ uv run pytest -q -m slow
|
||||
uv run python -m scripts.redis_budget # projects month-end burn, exits 1 if over
|
||||
```
|
||||
|
||||
## Using the API
|
||||
|
||||
**[USER_GUIDE.md](USER_GUIDE.md)** is the guide for callers: auth, the catalog, every
|
||||
endpoint with curl, the lifecycle, the error table, the CLI.
|
||||
|
||||
The API also documents itself — FastAPI generates OpenAPI from the same models and routes
|
||||
it serves, so the spec cannot drift the way a hand-written one does.
|
||||
|
||||
| What | Where |
|
||||
|---|---|
|
||||
| Swagger UI (try requests in the browser) | `https://svcforge.oci-oci.duckdns.org/docs` |
|
||||
| ReDoc (nicer to read) | `https://svcforge.oci-oci.duckdns.org/redoc` |
|
||||
| Raw spec, for generating clients | `https://svcforge.oci-oci.duckdns.org/openapi.json` |
|
||||
|
||||
Locally, `uv run uvicorn services.api.main:app --factory` then <http://127.0.0.1:8000/docs>.
|
||||
`tests/integration/test_api.py` pins the description, the tags and the bearer security
|
||||
scheme, so the docs fail CI if they rot.
|
||||
|
||||
## Where things live
|
||||
|
||||
| Module | Teaches | Read here |
|
||||
|
||||
+460
@@ -12,6 +12,298 @@ alias sfsql='psql "$SVCFORGE_PG_DSN_SESSION"'
|
||||
|
||||
---
|
||||
|
||||
## Setting up CI/CD from scratch
|
||||
|
||||
What it takes to get this repo building in Gitea Actions, including every trap that cost
|
||||
real time. Do it in this order; each step fails loudly if the one before it was skipped.
|
||||
|
||||
### 1. The repo and its secrets
|
||||
|
||||
```bash
|
||||
GITEA=https://gitea.oci-oci.duckdns.org
|
||||
USER=gitea_admin # the registry namespaces packages by OWNER, so this is IMAGE_NS too
|
||||
|
||||
curl -u "$USER:$PASS" -X POST "$GITEA/api/v1/user/repos" \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"name":"svcforge","default_branch":"master","private":false}'
|
||||
|
||||
# A PAT with exactly three scopes. Not admin.
|
||||
curl -u "$USER:$PASS" -X POST "$GITEA/api/v1/users/$USER/tokens" \
|
||||
-H 'content-type: application/json' \
|
||||
-d '{"name":"svcforge-ci","scopes":["write:package","write:repository","read:user"]}'
|
||||
```
|
||||
|
||||
Then set three repo Actions secrets (`Settings → Actions → Secrets`, or the API):
|
||||
|
||||
| Secret | Value | Why it exists |
|
||||
|---|---|---|
|
||||
| `REGISTRY_USER` | `gitea_admin` | |
|
||||
| `REGISTRY_TOKEN` | the PAT | **The auto-injected `GITEA_TOKEN` is rejected by the package registry with a 401.** This is the single most common reason a first pipeline fails at `docker push`. |
|
||||
| `CI_BOT_TOKEN` | the PAT | Used only by the `bump` job to push the digest commit. It is deliberately not a kubeconfig, so CI's maximum blast radius is a bad commit. |
|
||||
|
||||
### 2. The runner needs a cache server, and it fails SOFT without one
|
||||
|
||||
`cache: enabled: false` in the act_runner config is the default, and it does not fail the
|
||||
build. It prints:
|
||||
|
||||
```
|
||||
Warning: Failed to restore: getCacheEntry failed: Cache Service Url not found, unable to restore cache.
|
||||
```
|
||||
|
||||
…and carries on, re-downloading every wheel on every run, forever. **A cache that is off
|
||||
looks exactly like a cache that is always cold.** Enable it in
|
||||
`oci-k8s/k8s/roles/addons/tasks/main.yml` (Ansible owns this — never `kubectl edit` it):
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
enabled: true
|
||||
dir: /data/cache # the runner's PVC, so it survives a restart
|
||||
host: "" # auto-detect the pod IP; 127.0.0.1 would resolve to the JOB container
|
||||
port: 8088
|
||||
```
|
||||
|
||||
```bash
|
||||
cd oci-k8s/k8s && ansible-playbook 03_install_addons.yml --tags gitea
|
||||
```
|
||||
|
||||
Without this, `astral-sh/setup-uv`'s `enable-cache: true` and buildx's `--cache-to
|
||||
type=gha` are both no-ops.
|
||||
|
||||
### 3. Traps that are specific to Gitea, not GitHub
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---|---|---|
|
||||
| `Unable to resolve v0.2.2: reference not found` | A third-party action pinned by SHA still resolves **its own** dependencies by mutable tag. `trivy-action@v0.29.0` does `uses: setup-trivy@v0.2.2`, and that tag was removed upstream. | Run the tool directly from an image pinned by digest. Pinning the outer action bought nothing. |
|
||||
| `docker push` → 401 | Used `GITEA_TOKEN`. | Use a PAT with `write:package`. |
|
||||
| The pipeline retriggers itself forever | The `bump` job commits to the repo it is triggered by. | `[skip ci]` in the commit message (Gitea honours it), **and** a `git diff --quiet` guard so an unchanged digest commits nothing. |
|
||||
| Service container unreachable at `localhost` | Jobs run *inside* a container, so a service is reached by its **service name**, not localhost. | `postgres:5432`, not `localhost:5432`. |
|
||||
|
||||
### 4. Keeping trivy green
|
||||
|
||||
The image gate is a moving target: trivy's vulnerability DB updates daily, so an image that
|
||||
passed yesterday fails today without a single line of code changing. Two rules keep it sane.
|
||||
|
||||
**Bump the version, do not add an ignore.** The worker image went 39 findings (2 CRITICAL)
|
||||
→ 18 → 5 → 0 across three fixes, each a version bump or a removal:
|
||||
|
||||
| Change | Result |
|
||||
|---|---|
|
||||
| `alpine/helm` 3.16.2 → 3.21.3, `kubectl` 1.31.2 → 1.35.3 | 39 → 18, both CRITICALs cleared |
|
||||
| `kubectl` 1.35.3 → 1.36.2 (k8s 1.35.x vendors spdystream 0.5.0; the fix is 0.5.1) | 18 → 5 |
|
||||
| dropped kubectl entirely — `helm --create-namespace` replaced `kubectl apply` | **5 → 0** |
|
||||
|
||||
**The cheapest CVE is the binary you do not ship.** The last five findings lived in
|
||||
kubectl's vendored `golang.org/x/net` and Go stdlib, inside the newest kubectl that exists —
|
||||
no version cleared them. kubectl was in that image for exactly one call, and helm already
|
||||
does the same thing with a flag. Removing it removed the CVEs, a binary, and an adapter.
|
||||
|
||||
### 5. When every job fails at `Set up job` after ~14 minutes
|
||||
|
||||
Symptom: `Set up job` runs for 10–15 minutes and succeeds, then the first real step fails
|
||||
instantly at 0s, and every downstream job is skipped. It looks like the action broke.
|
||||
|
||||
Cause: the dind sidecar's image cache is empty, so each run re-pulls the ~1.6GB act job
|
||||
image (`ghcr.io/catthehacker/ubuntu:act-24.04`) before it can start. dind has no volume for
|
||||
`/var/lib/docker` — the cache lives in its container writable layer and is destroyed on
|
||||
every pod restart. A runner that is restart-looping therefore never keeps a cache, and each
|
||||
job pays the full pull.
|
||||
|
||||
Check it:
|
||||
```bash
|
||||
kubectl -n gitea exec gitea-actions-runner-0 -c dind -- docker images
|
||||
```
|
||||
Empty output is the diagnosis. Warm it once by hand:
|
||||
```bash
|
||||
kubectl -n gitea exec gitea-actions-runner-0 -c dind -- docker pull \
|
||||
ghcr.io/catthehacker/ubuntu:act-24.04@sha256:c710431fbad9eb3bcb102d04e5ff74fbd0ce6e383f78afebfb3770a1a817fdf9
|
||||
```
|
||||
|
||||
The durable fix is a persistent image store, which the runner now has: `/var/lib/docker`
|
||||
is a hostPath on node0 (see `oci-k8s/.../addons/tasks/main.yml`), so the act image survives
|
||||
a restart and is not re-pulled. The runner is pinned to **node0**, not node2 — node2 is a
|
||||
single-core control-plane node whose pod network was measured 21x slower under its own
|
||||
load, which starved every clone and pull. Its `/data` PVC is NFS ReadWriteMany, so a
|
||||
reschedule attaches immediately with no `Multi-Attach` wait. Entries 9 and 10 cover the
|
||||
caches and the node move in full.
|
||||
|
||||
### 6. Stopping a run, and reading a restarted runner correctly
|
||||
|
||||
Restarting the act_runner StatefulSet does not reliably orphan its in-flight jobs. Both
|
||||
outcomes have been observed on this cluster:
|
||||
|
||||
- run #14 was left with two jobs `in_progress` and nothing behind them, still stuck at
|
||||
15m45s, so `ZOMBIE_TASK_TIMEOUT` is not a rescue you can wait for
|
||||
- run #16 had its remaining jobs re-dispatched to the new pod and carried on normally
|
||||
|
||||
So `in_progress` after a restart is ambiguous on its own. Check whether the runner is
|
||||
actually working before concluding anything, or you will diagnose a healthy run as a zombie:
|
||||
|
||||
```bash
|
||||
kubectl -n gitea logs gitea-actions-runner-0 -c runner --tail=4
|
||||
```
|
||||
|
||||
Recent `NewParallelExecutor` lines mean it is executing, not stuck. At `capacity: 1` a later
|
||||
run sitting in `waiting` behind a live one is correct, not a block.
|
||||
|
||||
**Gitea 1.26 has no cancel endpoint at all** — the full swagger contains no path matching
|
||||
`cancel`. The red "Cancel workflow run" button in the web UI is a CSRF-protected web route,
|
||||
so an API token cannot drive it, and cancelling from a script is simply not available.
|
||||
|
||||
`DELETE .../actions/runs/{index}` exists and takes the **run index** (`14`), not the database
|
||||
id (`51`). It cleared the stuck #14, but returned 204 against the live #16 without stopping
|
||||
it and then 404 on retry. Treat it as a way to remove a finished run, not a cancel.
|
||||
|
||||
To stop a running job: click Cancel in the UI.
|
||||
|
||||
### 7. Gitea postgres: `Input/output error`, and when scale 0/1 is not enough
|
||||
|
||||
`gitea-postgresql-0` CrashLoopBackOff with:
|
||||
|
||||
mkdir: cannot create directory '/bitnami/postgresql/data': Input/output error
|
||||
|
||||
and the Gitea API returning 500, so every CI run dies at checkout with
|
||||
`Failed to connect to gitea-http:3000`. The "Initializing PostgreSQL database" line above
|
||||
that error is alarming and is not what it looks like: the data is fine, the mount is broken,
|
||||
so the container sees an empty directory.
|
||||
|
||||
The documented recovery — scale to 0, wait for `detached`, scale back to 1 — was **not
|
||||
enough** here. The volume came back `detached/faulted` and simply refused to attach, so the
|
||||
pod sat in ContainerCreating. `auto-salvage: true` does not help: salvage happens during
|
||||
attach, and a faulted volume never gets that far, so it cannot rescue itself.
|
||||
|
||||
What the volume was actually saying:
|
||||
|
||||
```bash
|
||||
V=$(kubectl -n gitea get pvc -o jsonpath='{.items[?(@.metadata.name=="data-gitea-postgresql-0")].spec.volumeName}')
|
||||
kubectl -n longhorn-system get volume $V -o jsonpath='{.status.state}/{.status.robustness}' # detached/faulted
|
||||
kubectl -n longhorn-system get replicas.longhorn.io -o json \
|
||||
| jq -r '.items[]|select(.spec.volumeName=="'$V'")|[.metadata.name,.spec.failedAt]|@tsv'
|
||||
```
|
||||
|
||||
The replica carries a `failedAt` timestamp, and that alone is what keeps the volume faulted.
|
||||
Clearing it is the salvage:
|
||||
|
||||
```bash
|
||||
kubectl -n longhorn-system patch replicas.longhorn.io <replica-name> --type merge \
|
||||
-p '{"spec":{"failedAt":"","lastFailedAt":""}}'
|
||||
```
|
||||
|
||||
The volume went `attached/healthy` and postgres reached 1/1 within 40 seconds, with the repo,
|
||||
its size and the whole CI run history intact.
|
||||
|
||||
**Check the backups before patching anything**, because this cluster runs Longhorn at one
|
||||
replica — there is no second copy to fall back on, only the nightly backup:
|
||||
|
||||
```bash
|
||||
kubectl -n longhorn-system get backups.longhorn.io -o json \
|
||||
| jq -r '.items[]|select(.status.volumeName=="'$V'")|[.status.backupCreatedAt,.status.state]|@tsv'
|
||||
```
|
||||
|
||||
Salvage reuses the replica exactly as it was when it failed, so Postgres may do crash
|
||||
recovery on start. If it cannot, restore the most recent Completed backup instead.
|
||||
|
||||
### 8. OPEN: dind is killed by its own liveness probe
|
||||
|
||||
Unresolved as of 2026-07-20. Recorded because it probably explains build failures that were
|
||||
diagnosed as something else.
|
||||
|
||||
The runner sits at `Init:1/2` and its dind sidecar accumulates restarts:
|
||||
|
||||
```
|
||||
Liveness probe failed: command timed out:
|
||||
"/usr/bin/test -S /var/run/docker.sock" timed out after 1s (x27 over 156m)
|
||||
Killing: Init container dind failed liveness probe
|
||||
```
|
||||
|
||||
The probe is hardcoded at `timeoutSeconds: 1`, `failureThreshold: 3`, `periodSeconds: 10`.
|
||||
`test -S` only asks whether a socket exists. When that cannot finish inside a second, the
|
||||
node is starved rather than dind being unhealthy, and kubelet kills a working daemon.
|
||||
|
||||
**Why this matters beyond the runner restarting.** Image builds failed with:
|
||||
|
||||
ERROR: failed to solve: DeadlineExceeded: no active session for <id>
|
||||
|
||||
which was attributed to CPU starvation alone and addressed by dropping the runner's
|
||||
`capacity` to 1. Starvation is real, but the mechanism is more likely that kubelet killed
|
||||
dind mid-build and the buildkit session died with it. Lowering capacity reduced the load
|
||||
that trips the probe, which is consistent with run #17 passing — it treated the cause of
|
||||
the trigger, not the trigger. Runs #18-#21 then failed anyway.
|
||||
|
||||
Treat this as a strong hypothesis, not a settled one. Confirming it means correlating the
|
||||
kill timestamps against the failed builds:
|
||||
|
||||
```bash
|
||||
kubectl -n gitea describe pod gitea-actions-runner-0 | grep -A10 Events:
|
||||
kubectl -n gitea get pod gitea-actions-runner-0 \
|
||||
-o jsonpath='{.status.initContainerStatuses[?(@.name=="dind")].lastState.terminated}'
|
||||
```
|
||||
|
||||
**The chart exposes no probe knobs** — `helm show values gitea-charts/actions` has no match
|
||||
for `probe`. So this cannot be fixed the way `capacity` was, and a `kubectl patch` is
|
||||
reverted by the next Ansible run. Same shape as the longhorn-csi-plugin probe problem.
|
||||
|
||||
The candidate fix is a Kyverno mutating policy authored in Ansible, relaxing the probe to
|
||||
roughly `timeoutSeconds: 5` and `failureThreshold: 6`. This cluster already mutates
|
||||
workloads that way — see `force-best-effort-cpu`, which rewrites every CPU request to 0 —
|
||||
so the precedent and the tooling are both in place.
|
||||
|
||||
### 9. The runner's three ephemeral caches, and the boot cascade they cause
|
||||
|
||||
Three separate caches on this runner were container-layer only, each found the same way —
|
||||
something was slow, and the cause was a cache that had never survived a restart:
|
||||
|
||||
| cache | path | fixed by |
|
||||
|---|---|---|
|
||||
| dind image store | `/var/lib/docker` | hostPath `/var/lib/gitea-dind` |
|
||||
| trivy vulnerability DB | `/root/.cache/trivy` in dind | named docker volume |
|
||||
| act's action clones | `/root/.cache/act` | hostPath `/var/lib/gitea-act-cache` |
|
||||
|
||||
act clones each action with **full history**, not shallow: 66.7MB/538 commits for
|
||||
`astral-sh/setup-uv`, 24.4MB/222 commits for `actions/checkout`, and this workflow uses
|
||||
five. After a restart that made `Set up job` an 11-minute step in which the job container
|
||||
sat idle running `sleep` while the runner cloned GitHub. If a job looks hung in setup, check
|
||||
the job container before blaming the network:
|
||||
|
||||
```bash
|
||||
C=$(kubectl -n gitea exec gitea-actions-runner-0 -c dind -- docker ps -q | head -1)
|
||||
kubectl -n gitea exec gitea-actions-runner-0 -c dind -- docker exec "$C" ps -eo pid,etime,comm
|
||||
```
|
||||
|
||||
Only `sleep` means the work is in the runner, not the job.
|
||||
|
||||
**The cascade this creates.** Giving dind a persistent image store made it boot slowly,
|
||||
because dockerd scans that store on startup — measured at 38s, 2m13s, and over 5 minutes
|
||||
depending on node load. Two independent timeouts then fire:
|
||||
|
||||
1. dind's startup probe. Widened to 5 minutes by the Kyverno policy in oci-k8s, and a cold
|
||||
boot has still exceeded it.
|
||||
2. **The runner container's own `Docker wait timeout of 5m0s`**, which is internal to the
|
||||
runner image and not configurable from the chart. When dind is late, the runner exits 1
|
||||
and restarts — and that restart kills whatever job was running, which surfaces as every
|
||||
step in the job failing at once with no error in the log, right after a green
|
||||
`Set up job`.
|
||||
|
||||
The pair self-heals: the second dind boot is fast because the store is warm, and the runner
|
||||
comes up behind it. The cost is roughly ten minutes of thrash after any runner restart, and
|
||||
one lost CI run. Restart the runner deliberately, not casually.
|
||||
|
||||
### 10. Verify the whole loop, not just the green checkmarks
|
||||
|
||||
```bash
|
||||
# the digest CI pushed
|
||||
docker buildx imagetools inspect gitea.oci-oci.duckdns.org/gitea_admin/svcforge-api:<sha> \
|
||||
--format '{{.Manifest.Digest}}'
|
||||
# the digest the chart deploys — these must be equal
|
||||
grep -A2 'api:' deploy/chart/values.yaml
|
||||
# what ArgoCD actually synced
|
||||
kubectl -n argocd get application svcforge -o jsonpath='{.status.sync.revision}'
|
||||
```
|
||||
|
||||
If those three disagree, the deploy is not what CI tested, and every other guarantee in
|
||||
this document is void.
|
||||
|
||||
---
|
||||
|
||||
## Measured numbers
|
||||
|
||||
From `scripts/load.py` + in-process workers on a `FakeProvisioner` (`delay=0.05s`), 200 tasks
|
||||
@@ -190,3 +482,171 @@ remembering.
|
||||
|
||||
**Escalate** if size is growing with `tasks` already pruned — that means `instances` is
|
||||
growing, i.e. tenants are real, i.e. the free tier is the wrong tier.
|
||||
|
||||
---
|
||||
|
||||
## Deploy stuck: ArgoCD says Synced at an old commit
|
||||
|
||||
**Symptom:** CI is green and the bump commit is on `master`, but the running pods are on the
|
||||
previous digest. `kubectl -n argocd get application svcforge` says **`Synced`** — at a
|
||||
revision several commits behind. Nothing looks broken, which is what makes this expensive.
|
||||
|
||||
**Diagnose.** Compare what ArgoCD thinks it synced against what `master` actually is:
|
||||
|
||||
```bash
|
||||
kubectl -n argocd get application svcforge \
|
||||
-o jsonpath='{.status.sync.revision}{" reconciledAt="}{.status.reconciledAt}{"\n"}'
|
||||
git -C ~/workspace/svcforge-reference log --oneline origin/master -1
|
||||
```
|
||||
|
||||
**`reconciledAt` alone does not tell you.** ArgoCD writes that field only when the computed
|
||||
status *changes*, so on a cluster where everything is Synced and nothing is deploying it can
|
||||
sit still while the controller is fine. It is evidence only when paired with a
|
||||
`sync.revision` that is *behind `master`* — which is exactly the case you are in if you are
|
||||
reading this section.
|
||||
|
||||
**One metric answers it, and only one.** The controller observes
|
||||
`argocd_app_reconcile_count` once per completed app reconciliation. Flat means it is doing
|
||||
nothing:
|
||||
|
||||
```bash
|
||||
kubectl -n monitoring exec prometheus-kube-prometheus-stack-prometheus-0 -c prometheus -- \
|
||||
wget -qO- --post-data='query=sum(increase(argocd_app_reconcile_count[10m]))' \
|
||||
http://localhost:9090/api/v1/query
|
||||
```
|
||||
|
||||
**Two metrics that look like they answer it and do not.** Both were tried on 2026-07-22
|
||||
against a controller that had reconciled nothing for 82 minutes, and both read healthy:
|
||||
|
||||
| metric | reading at the time | why it lies |
|
||||
|---|---|---|
|
||||
| `argocd_redis_request_total` | 45 reads / 15m, climbing | something in the process still touches the cache when nothing is reconciling — it measures "the pod is running", which `up` already covers |
|
||||
| `workqueue_unfinished_work_seconds{name="app_reconciliation_queue"}` | 0 | it only counts work already *in* the queue, and the queue is empty. Nothing is stuck; nothing is being enqueued |
|
||||
|
||||
Keep the second one anyway — it catches a genuinely stuck queue item, which is a different
|
||||
failure. Just never read a zero from it as "healthy".
|
||||
|
||||
**What is actually wrong: the periodic git poll does not run.** Read the controller's own
|
||||
metrics on an idle cluster:
|
||||
|
||||
```bash
|
||||
kubectl -n monitoring exec prometheus-kube-prometheus-stack-prometheus-0 -c prometheus -- \
|
||||
wget -qO- http://argocd-application-controller-metrics.argocd.svc:8082/metrics \
|
||||
| grep -E '^workqueue_(depth|adds_total|longest_running_processor_seconds)\{controller="app_reconciliation_queue"'
|
||||
```
|
||||
|
||||
Measured 2026-07-22, across two separate controller pods, on a fully healthy API server:
|
||||
|
||||
```
|
||||
03:37:03 controller starts, refreshes all 3 apps adds_total = 3
|
||||
03:38:31 adds_total = 3
|
||||
03:41:33 adds_total = 3
|
||||
03:44:34 adds_total = 3
|
||||
03:47:35 adds_total = 3 <- expiry is 2m0s, jitter 60s
|
||||
```
|
||||
|
||||
`workqueue_depth 0`, `longest_running_processor_seconds 0`, `adds_total` frozen. Nothing is
|
||||
*blocked* — nothing is being **enqueued**. The controller logs its own schedule at startup
|
||||
(`appResyncPeriod=2m0s, appResyncJitter=1m0s`) and `argocd-cm` carries the matching
|
||||
`timeout.reconciliation: 120s`, so the setting is read and then never acted on.
|
||||
|
||||
Refreshes still happen from two other paths, which is what makes this so easy to
|
||||
misread as working:
|
||||
|
||||
| path | fires when | observed |
|
||||
|---|---|---|
|
||||
| startup | controller (re)starts | 3 apps refreshed within ~2s of ready |
|
||||
| cluster events | a watched resource changes | 68 adds during one svcforge rollout, then flat the moment the cluster went quiet |
|
||||
| periodic poll | every 2m ± 60s | **never** |
|
||||
|
||||
**The consequence is the thing to take away: a commit that changes only the repo is never
|
||||
noticed.** Every "auto-sync" observed on 2026-07-22 happened within seconds of a controller
|
||||
restart, i.e. it was the startup refresh, not the poll. Do not read a successful deploy
|
||||
straight after a restart as evidence that polling works.
|
||||
|
||||
**A webhook now covers for it** (added 2026-07-22, `oci-k8s` `--tags argocd,gitea`). Gitea
|
||||
POSTs every push to `https://argocd.oci-oci.duckdns.org/api/webhook`, so a commit refreshes
|
||||
ArgoCD in under a second instead of waiting for a poll that never comes.
|
||||
|
||||
It is registered as Gitea's **`gogs`** type, which looks wrong and is not: ArgoCD's webhook
|
||||
handler dispatches on the `X-Gogs-Event` header and ships no Gitea parser. Gitea forked from
|
||||
Gogs and still emits that wire format on request. The shared secret lives in
|
||||
`argocd-secret` as `webhook.gogs.secret` and in Ansible as `argocd_webhook_secret`; both
|
||||
sides must match or every delivery fails signature validation *silently*, which looks
|
||||
identical to having no webhook at all.
|
||||
|
||||
Check a delivery when a push does not deploy:
|
||||
|
||||
```bash
|
||||
# Gitea's own record of the last attempt, including the response ArgoCD gave
|
||||
curl -s -u "$USER:$PASS" \
|
||||
https://gitea.oci-oci.duckdns.org/api/v1/repos/gitea_admin/svcforge/hooks | jq '.[].id'
|
||||
|
||||
# ArgoCD's side
|
||||
kubectl -n argocd logs deploy/argocd-server --tail=200 | grep -i webhook
|
||||
```
|
||||
|
||||
`Unknown webhook event` means the hook type is wrong (it must be `gogs`). A 400 on signature
|
||||
means the secrets have drifted — re-run `03_install_addons.yml --tags argocd,gitea`, which
|
||||
rewrites both ends from the same variable.
|
||||
|
||||
Manual nudge, still valid if the webhook is ever down:
|
||||
|
||||
```bash
|
||||
kubectl -n argocd annotate application svcforge argocd.argoproj.io/refresh=normal --overwrite
|
||||
```
|
||||
|
||||
Without Prometheus, fall back to the logs — this works and is what found the 2026-07-22
|
||||
wedge before the metrics were checked:
|
||||
|
||||
```bash
|
||||
# Healthy: a few hundred lines an hour. Stalled: exactly 6 — the 10-minute memory heartbeat.
|
||||
kubectl -n argocd logs statefulset/argocd-application-controller --tail=8000 \
|
||||
| grep -oE 'time="[0-9-]+T[0-9]{2}' | sort | uniq -c | tail
|
||||
```
|
||||
|
||||
A flat `Goroutines=NNN` across hours in those heartbeat lines means blocked goroutines, not
|
||||
an idle controller.
|
||||
|
||||
**Cause seen here (2026-07-21).** Not ArgoCD config — `timeout.reconciliation` was 120s the
|
||||
whole time. The controller's server-side dry-run applies go through the cluster's admission
|
||||
webhooks, and Kyverno's mutate webhook was `failurePolicy: Fail`. Kyverno restarts under
|
||||
this cluster's memory pressure, and each restart is a window where that webhook is
|
||||
unreachable, so the applies blocked and the controller wedged for **11 hours** — reconciling
|
||||
zero apps while still reporting `Synced`. Fixed in `oci-k8s` by setting `failurePolicy:
|
||||
Ignore` on both ClusterPolicies; see the comment there.
|
||||
|
||||
**Unstick it now:**
|
||||
|
||||
```bash
|
||||
# 1. Force a re-poll. If the revision advances, polling was the only problem.
|
||||
kubectl -n argocd annotate application svcforge argocd.argoproj.io/refresh=hard --overwrite
|
||||
|
||||
# 2. If it does not advance within ~60s, the controller is wedged. Restart it —
|
||||
# ArgoCD holds no state of its own; everything is in the cluster and in git.
|
||||
kubectl -n argocd rollout restart statefulset/argocd-application-controller
|
||||
kubectl -n argocd rollout status statefulset/argocd-application-controller --timeout=180s
|
||||
```
|
||||
|
||||
**Then check the actual chain, because `Synced` is not the same as `deployed`:**
|
||||
|
||||
```bash
|
||||
git show origin/master:deploy/chart/values.yaml | grep -A2 -E '^\s+(api|worker|reconciler):'
|
||||
kubectl -n svcforge get pods \
|
||||
-o jsonpath='{range .items[*]}{.spec.containers[0].image}{"\n"}{end}' | sort -u
|
||||
```
|
||||
|
||||
The digests must match. If the sync stalls with a Job stuck `Complete` but never deleted,
|
||||
it is holding `argocd.argoproj.io/hook-finalizer` — see below.
|
||||
|
||||
**Related: the migrate Job deadlock.** A PreSync hook Job that finished but keeps the
|
||||
finalizer blocks the sync forever:
|
||||
|
||||
```bash
|
||||
kubectl -n svcforge get job svcforge-migrate -o jsonpath='{.metadata.finalizers}{"\n"}'
|
||||
kubectl -n svcforge patch job svcforge-migrate --type=merge -p '{"metadata":{"finalizers":null}}'
|
||||
```
|
||||
|
||||
The Application goes `Synced` within seconds of the patch.
|
||||
|
||||
<!-- ci: exercising the runner CPU limit and the webhook end to end, 2026-07-22 -->
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
# svcforge — user guide
|
||||
|
||||
Ask for a managed service, get one. svcforge provisions Elasticsearch, Redis, Postgres and
|
||||
a couple of tiny test services into Kubernetes, one namespace per team, and keeps them
|
||||
matching what the database says they should be.
|
||||
|
||||
**Base URL:** `https://svcforge.oci-oci.duckdns.org`
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Swagger UI (send requests from the browser) | [`/docs`](https://svcforge.oci-oci.duckdns.org/docs) |
|
||||
| ReDoc (nicer to read) | [`/redoc`](https://svcforge.oci-oci.duckdns.org/redoc) |
|
||||
| Raw OpenAPI spec | [`/openapi.json`](https://svcforge.oci-oci.duckdns.org/openapi.json) |
|
||||
|
||||
The spec is generated from the same models and routes the server runs, so it cannot drift
|
||||
from the implementation. Generate a client from it rather than hand-rolling one.
|
||||
|
||||
---
|
||||
|
||||
## The three things that will surprise you
|
||||
|
||||
1. **Writes are asynchronous.** `POST` and `DELETE` return **202 Accepted**. Nothing is
|
||||
provisioned when you get the response — you have a row and a queued task. Poll
|
||||
`GET /v1/instances/{id}` and watch `state`.
|
||||
2. **Authorisation is a WHERE clause.** Another team's instance returns **404**, not 403.
|
||||
The API never confirms that an id you cannot access exists.
|
||||
3. **Every non-2xx body is `{"code", "message"}`** — including the 404s and 405s raised by
|
||||
the framework itself. Never branch on the body's shape.
|
||||
|
||||
---
|
||||
|
||||
## Getting a token
|
||||
|
||||
Every `/v1` route needs `Authorization: Bearer <jwt>`. The token is verified against the
|
||||
configured JWKS (RS256 only), and its **`team` claim** decides which instances you see.
|
||||
`aud`, `iss` and `exp` are all required and all checked.
|
||||
|
||||
> **The public deployment currently issues no tokens.** `SVCFORGE_JWKS_URL` points at
|
||||
> `https://auth.oci-oci.duckdns.org/realms/svcforge/...`, and no identity provider is
|
||||
> deployed there — the hostname resolves to the ingress, which answers with its default
|
||||
> self-signed certificate. The API logs `JWKS warm-up failed` at startup and every `/v1`
|
||||
> request returns `401`. Unauthenticated routes (`/healthz`, `/readyz`, `/metrics`,
|
||||
> `/docs`, `/openapi.json`) work normally. To make the live API usable, either deploy an
|
||||
> OIDC provider at that realm URL or repoint `auth.jwksUrl` / `auth.issuer` in
|
||||
> `deploy/chart/values.yaml` at one that exists.
|
||||
|
||||
Any OIDC provider works. With Keycloak, the client-credentials flow is:
|
||||
|
||||
```bash
|
||||
TOKEN=$(curl -s -X POST \
|
||||
https://auth.example.com/realms/svcforge/protocol/openid-connect/token \
|
||||
-d grant_type=client_credentials \
|
||||
-d client_id=svcforge-cli \
|
||||
-d client_secret="$CLIENT_SECRET" | jq -r .access_token)
|
||||
```
|
||||
|
||||
The provider must put a `team` claim in the token (a Keycloak protocol mapper, or the
|
||||
equivalent) and set `aud: svcforge`. A token without a non-empty string `team` is a 401.
|
||||
|
||||
### Running it locally instead
|
||||
|
||||
The fastest way to actually drive the API is to run it yourself with auth off:
|
||||
|
||||
```bash
|
||||
export SVCFORGE_PG_DSN="postgresql://svcforge:svcforge@127.0.0.1:5432/svcforge"
|
||||
export SVCFORGE_AUTH_DISABLED=true # refused unless SVCFORGE_ENVIRONMENT=local
|
||||
uv run uvicorn services.api.main:app --factory
|
||||
```
|
||||
|
||||
Every request is then team `platform` and no header is needed. `check_production()` refuses
|
||||
this flag whenever `SVCFORGE_ENVIRONMENT` is anything but `local`, so it cannot escape a
|
||||
laptop.
|
||||
|
||||
---
|
||||
|
||||
## The catalog
|
||||
|
||||
`service_type` and `size` must both exist in the catalog. An unknown `service_type` is a
|
||||
**404**; a real service type with an unknown size is a **422** that lists the valid sizes.
|
||||
|
||||
| `service_type` | `size` | Memory request | Notes |
|
||||
|---|---|---|---|
|
||||
| `elasticsearch` | `small`, `medium` | 1Gi / 4Gi per replica | 1 or 3 replicas |
|
||||
| `redis` | `small`, `medium` | 256Mi / 1Gi | 1 or 3 replicas |
|
||||
| `postgres` | `small`, `medium` | 512Mi / 2Gi | 1 or 2 replicas |
|
||||
| `podinfo` | `small`, `medium` | 16Mi / 32Mi | tiny, for exercising the platform |
|
||||
| `nginx` | `small`, `medium` | 32Mi / 64Mi | tiny, for exercising the platform |
|
||||
|
||||
Use `podinfo` or `nginx` to exercise the control plane: they are single small pods and fit
|
||||
on a node with no room for a real Elasticsearch. Both are pulled straight from an OCI
|
||||
registry. The three larger entries reference a `bitnamilegacy/` chart repo that the worker
|
||||
image does not currently configure, so they will fail at provision time until it is added.
|
||||
|
||||
---
|
||||
|
||||
## Endpoints
|
||||
|
||||
### `POST /v1/instances` → 202
|
||||
|
||||
```bash
|
||||
curl -X POST https://svcforge.oci-oci.duckdns.org/v1/instances \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"service_type": "podinfo", "size": "small", "ttl_days": 7}'
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "0f8b7d3e-1c2a-4f5b-9e6d-7a8b9c0d1e2f",
|
||||
"state": "requested",
|
||||
"service_type": "podinfo",
|
||||
"size": "small",
|
||||
"endpoint": null,
|
||||
"chart_version": "6.7.1",
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
The response carries a `Location` header pointing at the instance. `ttl_days` (1–30,
|
||||
optional) deletes the instance automatically; omit it for no expiry. Unknown body fields
|
||||
are rejected with a 422 rather than ignored.
|
||||
|
||||
### `GET /v1/instances/{id}` → 200
|
||||
|
||||
The polling endpoint. Repeat until `state` is `ready` or `failed`.
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
https://svcforge.oci-oci.duckdns.org/v1/instances/$ID
|
||||
```
|
||||
|
||||
### `GET /v1/instances` → 200
|
||||
|
||||
Your team's instances, newest first. `?limit=` accepts 1–200 and defaults to 50. No
|
||||
endpoint here returns "all rows".
|
||||
|
||||
### `DELETE /v1/instances/{id}` → 202
|
||||
|
||||
```bash
|
||||
curl -X DELETE -H "Authorization: Bearer $TOKEN" \
|
||||
https://svcforge.oci-oci.duckdns.org/v1/instances/$ID
|
||||
```
|
||||
|
||||
Moves the instance to `deleting` and queues the teardown; the helm uninstall has not
|
||||
happened when this returns. Deleting something already `deleting` or `deleted` is a **409**.
|
||||
|
||||
### Unauthenticated
|
||||
|
||||
`GET /healthz` (liveness, no I/O) · `GET /readyz` (readiness, checks Postgres only) ·
|
||||
`GET /metrics` (Prometheus exposition).
|
||||
|
||||
---
|
||||
|
||||
## The lifecycle
|
||||
|
||||
```
|
||||
requested ──> provisioning ──> ready ──> deleting ──> deleted
|
||||
│ │
|
||||
└──────────────┴──> failed
|
||||
```
|
||||
|
||||
| State | What it means |
|
||||
|---|---|
|
||||
| `requested` | The row exists and a provision task is queued. |
|
||||
| `provisioning` | A worker is running `helm upgrade --install`. |
|
||||
| `ready` | The release is up. **Only this state carries a usable `endpoint`.** |
|
||||
| `failed` | The provision exhausted its retries. `error` says why. |
|
||||
| `deleting` | Teardown queued or running. |
|
||||
| `deleted` | Terminal. |
|
||||
|
||||
`endpoint` is in-cluster DNS —
|
||||
`http://<release>.tenant-<team>.svc.cluster.local` — reachable from inside the cluster, not
|
||||
from your laptop.
|
||||
|
||||
A few things happen without you asking:
|
||||
|
||||
- **Drift repair.** If a `ready` instance's helm release disappears, the reconciler notices
|
||||
within ~60s and re-provisions it. You may see `ready → failed → provisioning → ready`.
|
||||
- **TTL.** An instance past `expires_at` is torn down automatically.
|
||||
- **Upgrades.** When the catalog pins a newer chart version, instances are upgraded inside
|
||||
their maintenance window, one at a time. Entries marked `security: true` skip the window.
|
||||
|
||||
A `failed` instance is not retried automatically — it needs a human.
|
||||
|
||||
---
|
||||
|
||||
## Errors
|
||||
|
||||
Every non-2xx body is the same shape. `code` is stable and meant for machines; `message` is
|
||||
for humans and must not be parsed.
|
||||
|
||||
```json
|
||||
{"code": "unknown_service_type", "message": "no such service_type: mongodb"}
|
||||
```
|
||||
|
||||
| Status | `code` | Cause |
|
||||
|---|---|---|
|
||||
| 401 | `unauthorized` | Missing, expired, malformed or unverifiable token. Never says which. |
|
||||
| 404 | `unknown_service_type` | Not in the catalog. |
|
||||
| 404 | `not_found` | No such instance — **or it belongs to another team**. |
|
||||
| 409 | `illegal_transition` | e.g. deleting something already deleted. |
|
||||
| 409 | `conflict` | The row changed between the read and the write. Retry. |
|
||||
| 422 | `unknown_size` | The message lists the sizes that exist. |
|
||||
| 422 | `validation_error` | Malformed body: bad type, extra field, `ttl_days` out of range. |
|
||||
| 429 | `rate_limited` | Over the per-team budget. Honour `Retry-After`. |
|
||||
| 503 | `not_ready` | `/readyz` only: Postgres is unreachable. |
|
||||
|
||||
**Rate limiting** is per team, 60 requests/minute by default, and it fails *open* — if the
|
||||
limiter's Redis is down you are unmetered rather than refused. A 429 always means a real,
|
||||
counted overage.
|
||||
|
||||
---
|
||||
|
||||
## The CLI
|
||||
|
||||
`services/cli/` is a thin API client. It holds a URL and a token and never touches the
|
||||
database.
|
||||
|
||||
```bash
|
||||
export SVCFORGE_API_URL=https://svcforge.oci-oci.duckdns.org
|
||||
export SVCFORGE_API_TOKEN="$TOKEN"
|
||||
|
||||
svcforge create podinfo --size small --ttl 7d --wait
|
||||
svcforge list --state ready
|
||||
svcforge status <instance-id>
|
||||
svcforge delete <instance-id> --yes
|
||||
```
|
||||
|
||||
`--wait` polls until the instance reaches `ready` or `failed`.
|
||||
|
||||
---
|
||||
|
||||
## Generating a client
|
||||
|
||||
```bash
|
||||
curl -s https://svcforge.oci-oci.duckdns.org/openapi.json > openapi.json
|
||||
openapi-generator-cli generate -i openapi.json -g python -o ./client
|
||||
```
|
||||
|
||||
The error models are declared on every route, so a generated client gets typed 401/404/409/
|
||||
422 bodies rather than guessing. `tests/integration/test_api.py` pins the description, the
|
||||
tags and the bearer security scheme, so these docs fail CI if they rot.
|
||||
|
||||
---
|
||||
|
||||
## Polling, end to end
|
||||
|
||||
```bash
|
||||
ID=$(curl -s -X POST https://svcforge.oci-oci.duckdns.org/v1/instances \
|
||||
-H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
|
||||
-d '{"service_type":"podinfo","size":"small","ttl_days":1}' | jq -r .id)
|
||||
|
||||
while :; do
|
||||
BODY=$(curl -s -H "Authorization: Bearer $TOKEN" \
|
||||
https://svcforge.oci-oci.duckdns.org/v1/instances/$ID)
|
||||
STATE=$(jq -r .state <<<"$BODY")
|
||||
echo "$STATE"
|
||||
case "$STATE" in
|
||||
ready) jq -r .endpoint <<<"$BODY"; break ;;
|
||||
failed) jq -r .error <<<"$BODY"; exit 1 ;;
|
||||
esac
|
||||
sleep 5
|
||||
done
|
||||
```
|
||||
|
||||
Poll every few seconds, not every few milliseconds. A provision is a helm install against a
|
||||
StatefulSet; `podinfo` takes seconds, Elasticsearch takes minutes.
|
||||
+105
-6
@@ -3,11 +3,33 @@
|
||||
#
|
||||
# security: true bypasses every tenant's maintenance window for this entry. Set it for a
|
||||
# CVE with a public exploit; leave it false and the bump waits for 03:00 Sunday.
|
||||
#
|
||||
# Every chart is addressed as `oci://`, which is not cosmetic: an OCI chart is pulled by
|
||||
# reference, with no `helm repo add` first. The three entries below previously named
|
||||
# `bitnamilegacy/<chart>`, a classic repo alias that nothing in the worker image
|
||||
# configures — so they could never resolve at provision time. OCI is the form that works
|
||||
# from a bare container.
|
||||
#
|
||||
# All of them go through mirror.gcr.io rather than registry-1.docker.io. Docker Hub
|
||||
# rate-limits anonymous pulls per source IP and every node here shares one NAT address, so
|
||||
# a busy afternoon becomes `toomanyrequests` on an unrelated deploy. mirror.gcr.io is a
|
||||
# pull-through cache with no such limit, verified digest-for-digest identical.
|
||||
#
|
||||
# The chart pull is only half of it. A bitnami chart defaults its own images to
|
||||
# `registry-1.docker.io/bitnami/<name>`, so the pods would still go to Docker Hub even
|
||||
# though the chart did not. `values:` on an entry is the fix: anything there is passed to
|
||||
# helm underneath the size's replicas and resources, so `global.imageRegistry` moves the
|
||||
# image pull too. `podinfo` needs none of this — its chart already points at ghcr.io.
|
||||
services:
|
||||
elasticsearch:
|
||||
chart: bitnamilegacy/elasticsearch
|
||||
chart_version: 21.3.15
|
||||
chart: oci://mirror.gcr.io/bitnamicharts/elasticsearch
|
||||
chart_version: 22.1.6
|
||||
security: false
|
||||
# Sends the chart's own image pulls through the mirror as well, so nothing in this
|
||||
# entry touches a rate-limited registry. Merged under the size below.
|
||||
values:
|
||||
global:
|
||||
imageRegistry: mirror.gcr.io
|
||||
sizes:
|
||||
small:
|
||||
replicas: 1
|
||||
@@ -29,9 +51,14 @@ services:
|
||||
memory: 8Gi
|
||||
|
||||
redis:
|
||||
chart: bitnamilegacy/redis
|
||||
chart_version: 20.6.2
|
||||
chart: oci://mirror.gcr.io/bitnamicharts/redis
|
||||
chart_version: 27.0.15
|
||||
security: false
|
||||
# Sends the chart's own image pulls through the mirror as well, so nothing in this
|
||||
# entry touches a rate-limited registry. Merged under the size below.
|
||||
values:
|
||||
global:
|
||||
imageRegistry: mirror.gcr.io
|
||||
sizes:
|
||||
small:
|
||||
replicas: 1
|
||||
@@ -53,9 +80,14 @@ services:
|
||||
memory: 2Gi
|
||||
|
||||
postgres:
|
||||
chart: bitnamilegacy/postgresql
|
||||
chart_version: 16.4.5
|
||||
chart: oci://mirror.gcr.io/bitnamicharts/postgresql
|
||||
chart_version: 18.8.0
|
||||
security: false
|
||||
# Sends the chart's own image pulls through the mirror as well, so nothing in this
|
||||
# entry touches a rate-limited registry. Merged under the size below.
|
||||
values:
|
||||
global:
|
||||
imageRegistry: mirror.gcr.io
|
||||
sizes:
|
||||
small:
|
||||
replicas: 1
|
||||
@@ -75,3 +107,70 @@ services:
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
|
||||
# --- Small services, for exercising the control plane on a cluster with no room ---------
|
||||
#
|
||||
# The three entries above are real products and size accordingly: one `elasticsearch`
|
||||
# small asks for 1Gi, and a medium asks for 4Gi across three replicas. On a test cluster
|
||||
# that is a request that never schedules, so provisioning them proves nothing about
|
||||
# svcforge and everything about the node.
|
||||
#
|
||||
# These two exist to exercise the actual loop — claim, helm install, CAS to ready, drift,
|
||||
# TTL, deprovision — in seconds and in tens of megabytes.
|
||||
|
||||
podinfo:
|
||||
# A single small Go binary with no dependencies, no PVC and a fast image pull. The e2e
|
||||
# test provisions exactly this for the same reason.
|
||||
chart: oci://ghcr.io/stefanprodan/charts/podinfo
|
||||
chart_version: 6.14.0
|
||||
security: false
|
||||
sizes:
|
||||
small:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
cpu: 50m
|
||||
memory: 32Mi
|
||||
medium:
|
||||
replicas: 2
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
|
||||
nginx:
|
||||
# A recognisable web server, still small. Useful when the thing being demonstrated needs
|
||||
# to look like a service someone would actually ask for.
|
||||
chart: oci://mirror.gcr.io/bitnamicharts/nginx
|
||||
chart_version: 25.0.14
|
||||
security: false
|
||||
# Sends the chart's own image pulls through the mirror as well, so nothing in this
|
||||
# entry touches a rate-limited registry. Merged under the size below.
|
||||
values:
|
||||
global:
|
||||
imageRegistry: mirror.gcr.io
|
||||
sizes:
|
||||
small:
|
||||
replicas: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
medium:
|
||||
replicas: 2
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
|
||||
+33
-1
@@ -42,7 +42,19 @@ spec:
|
||||
# non-zero exit fails the sync instead of rolling out pods onto an unmigrated schema.
|
||||
- ApplyOutOfSyncOnly=true
|
||||
retry:
|
||||
limit: 3
|
||||
# 0, not 3, and for the same reason migrate-job.yaml sets backoffLimit: 0.
|
||||
#
|
||||
# An ArgoCD retry re-runs the WHOLE sync including the PreSync phase, and the migrate
|
||||
# Job is a helm pre-install/pre-upgrade hook that ArgoCD maps onto PreSync. So
|
||||
# `limit: 3` quietly reinstated the retry-a-failed-DDL behaviour that backoffLimit: 0
|
||||
# exists to forbid — three attempts at the same failed migration, one readable error
|
||||
# turned into three, against a schema that may now be half-applied.
|
||||
#
|
||||
# There is no recovery path here that a retry helps with. Recovery from a failed
|
||||
# migration is a REVERT COMMIT: fix the SQL forward, push, and let ArgoCD sync the
|
||||
# new revision. Rolling back the app image does not roll back DDL that already
|
||||
# committed.
|
||||
limit: 0
|
||||
backoff:
|
||||
duration: 20s
|
||||
factor: 2
|
||||
@@ -55,3 +67,23 @@ spec:
|
||||
kind: Job
|
||||
jsonPointers:
|
||||
- /spec/template/metadata/labels
|
||||
# This cluster runs a Kyverno ClusterPolicy, `force-best-effort-cpu`, whose rule
|
||||
# set-cpu-request-to-zero rewrites every container's CPU request to "0" at admission.
|
||||
# It is deliberate and predates this app by well over a year: the nodes are
|
||||
# oversubscribed, and making pods BestEffort on CPU is how everything gets scheduled.
|
||||
#
|
||||
# The chart asks for 50m and the cluster writes 0, so without this the Deployments sit
|
||||
# permanently OutOfSync while being perfectly Healthy — the failure mode where a
|
||||
# dashboard is always yellow, everyone learns to ignore it, and it stops meaning
|
||||
# anything the day it goes yellow for a real reason.
|
||||
#
|
||||
# The chart deliberately keeps its real request rather than capitulating to 0. What the
|
||||
# chart asks for is the honest intent; what the cluster does with it is the cluster's
|
||||
# business, and a reader of the repo should see the former.
|
||||
#
|
||||
# jqPathExpressions, not jsonPointers: a pointer would have to name a container index,
|
||||
# and this has to hold for every container in every one of the three Deployments.
|
||||
- group: apps
|
||||
kind: Deployment
|
||||
jqPathExpressions:
|
||||
- .spec.template.spec.containers[].resources.requests.cpu
|
||||
|
||||
@@ -53,8 +53,18 @@ comments — the acceptance gate greps for it and does not know what a comment i
|
||||
{{- if not $img -}}
|
||||
{{- fail (printf "no image config for component %q" .component) -}}
|
||||
{{- end -}}
|
||||
{{- if not (hasPrefix "sha256:" ($img.digest | default "")) -}}
|
||||
{{- fail (printf "image.%s.digest must be a sha256 digest, not a tag — CI bumps it; got %q" .component ($img.digest | default "<empty>")) -}}
|
||||
{{- $digest := $img.digest | default "" -}}
|
||||
{{/*
|
||||
Full-shape match, not `hasPrefix "sha256:"`. A prefix check accepts the all-zeros
|
||||
placeholder in values.yaml, so a fresh clone rendered clean and the guard guarded nothing.
|
||||
Two conditions, both required: the digest must be sha256: plus exactly 64 lowercase hex
|
||||
characters, AND it must not be the placeholder literal.
|
||||
*/}}
|
||||
{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" $digest) -}}
|
||||
{{- fail (printf "image.%s.digest must be a sha256 digest (sha256: + 64 hex chars), not a tag — CI bumps it; got %q" .component ($digest | default "<empty>")) -}}
|
||||
{{- end -}}
|
||||
{{- if eq $digest "sha256:0000000000000000000000000000000000000000000000000000000000000000" -}}
|
||||
{{- fail (printf "image.%s.digest is still the all-zeros placeholder from values.yaml — this chart has never been bumped by CI and must not be deployed" .component) -}}
|
||||
{{- end -}}
|
||||
{{- printf "%s@%s" $img.repo $img.digest -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -13,6 +13,37 @@ after a failure — the one time you actually want it — and clears it on the n
|
||||
Deliberately no terminationGracePeriodSeconds: 60 here. Three Deployments carry it; a
|
||||
migration is not one of them.
|
||||
*/}}
|
||||
{{/*
|
||||
The hook needs its own ServiceAccount, and it has to be a hook itself.
|
||||
|
||||
It used to run as the api ServiceAccount, which is an ordinary chart resource. Hooks are
|
||||
created before the release's ordinary manifests, so on a first install that account does
|
||||
not exist yet and the Job never starts:
|
||||
|
||||
Error creating: pods "svcforge-migrate-" is forbidden: error looking up service
|
||||
account svcforge/svcforge-api: serviceaccount "svcforge-api" not found
|
||||
|
||||
This is not an ArgoCD quirk. `helm install` orders hooks the same way, so it failed
|
||||
identically on both paths. It survived review because the chart was only ever checked with
|
||||
`helm template` and `helm install --dry-run=server`, and neither creates a Job — the pod is
|
||||
what fails, so nothing short of a real install can catch it.
|
||||
|
||||
Weight -10 so it is created before the Job at -5. Deliberately bound to no Role: the
|
||||
migration talks to Postgres and needs nothing from the Kubernetes API.
|
||||
*/}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "svcforge.fullname" . }}-migrate
|
||||
labels:
|
||||
{{- include "svcforge.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: migrate
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install,pre-upgrade
|
||||
"helm.sh/hook-weight": "-10"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
@@ -36,7 +67,7 @@ spec:
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
|
||||
serviceAccountName: {{ include "svcforge.fullname" . }}-migrate
|
||||
{{- with .Values.image.pullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
@@ -59,6 +90,12 @@ spec:
|
||||
{{- include "svcforge.env" . | nindent 12 }}
|
||||
- name: OTEL_SERVICE_NAME
|
||||
value: svcforge-migrate
|
||||
# Where services/api/Dockerfile copies migrations/ to. The Dockerfile sets the
|
||||
# same value as an ENV; this states it in the manifest as well so the path is
|
||||
# visible to anyone reading the Job rather than only to whoever opens the
|
||||
# image. The two MUST agree — if one moves, move both.
|
||||
- name: SVCFORGE_MIGRATIONS_DIR
|
||||
value: /app/migrations
|
||||
resources:
|
||||
{{- toYaml .Values.migrate.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
|
||||
@@ -6,6 +6,14 @@ is banned: it drifts from the chart, survives a `helm uninstall`, and nothing ow
|
||||
The api is scraped through its Service. The worker and reconciler have no Service — they
|
||||
are scraped by pod, which is why their metrics port is named and their pods carry the
|
||||
component label.
|
||||
|
||||
A ServiceMonitor's selector matches the SERVICE OBJECT's own metadata labels, not the
|
||||
Service's pod selector. api-service.yaml labels itself with `svcforge.labels` plus
|
||||
`app.kubernetes.io/component: api` — it does NOT carry the `app: api` that
|
||||
`svcforge.selectorLabels` adds, because that label exists for the chaos experiments'
|
||||
`kubectl delete pod -l app=worker` and belongs on pods. So this must not reuse
|
||||
selectorLabels: doing so matched nothing and api metrics were never scraped. These three
|
||||
keys are exactly the ones the Service metadata carries, and are enough to be unambiguous.
|
||||
*/}}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
@@ -16,7 +24,9 @@ metadata:
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
|
||||
app.kubernetes.io/name: {{ include "svcforge.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: api
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
|
||||
@@ -25,9 +25,19 @@ spec:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
# The load-bearing one. On SIGTERM the loop stops claiming and finishes the task in
|
||||
# hand; 60s is the budget for that. Chaos experiment 2 asserts the pod exits 0 inside
|
||||
# it. Lower this and a rolling deploy starts orphaning tasks to lease expiry.
|
||||
terminationGracePeriodSeconds: 60
|
||||
# hand. Chaos experiment 2 asserts the pod exits 0 inside it. Lower this and a
|
||||
# rolling deploy starts orphaning tasks to lease expiry.
|
||||
#
|
||||
# 660s, and the number is derived, not chosen. The task being drained is a
|
||||
# `helm upgrade --install`, and this MUST exceed the longest that call can run before
|
||||
# it returns:
|
||||
# - settings.py helm_timeout_s = 300 (default)
|
||||
# - HelmProvisioner timeout_s = 600 (default, the ceiling)
|
||||
# 600s is the value to beat; 660 leaves a 60s margin for the pool to close and the
|
||||
# process to exit cleanly. At the old 60s the kubelet SIGKILLed the worker mid-helm —
|
||||
# precisely the orphaning this setting exists to prevent. If either timeout above is
|
||||
# raised, raise this past it first.
|
||||
terminationGracePeriodSeconds: 660
|
||||
securityContext:
|
||||
{{- include "svcforge.podSecurityContext" . | nindent 8 }}
|
||||
containers:
|
||||
|
||||
+43
-24
@@ -9,7 +9,6 @@
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
image:
|
||||
registry: gitea.oci-oci.duckdns.org
|
||||
pullPolicy: IfNotPresent
|
||||
@@ -19,14 +18,13 @@ image:
|
||||
# deploy, which is the intended failure mode. Never hand-edit these.
|
||||
api:
|
||||
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-api
|
||||
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
digest: sha256:51c9b6d09dc9f3861a18ed4116f03ce24717a34cb2f985fbcf2a33e96d96cae8
|
||||
worker:
|
||||
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-worker
|
||||
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
digest: sha256:5307e1e23a815dcc40b290c27b37b0847c12c654a84aa80643821789c95763d9
|
||||
reconciler:
|
||||
repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-reconciler
|
||||
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
|
||||
digest: sha256:bcf7c309c74bb59572c745f6a34e71cdbef39f3881aa779ee182bd67a63186be
|
||||
api:
|
||||
replicas: 2
|
||||
# One process per pod. Module 7 took the "scale with replicas" fix over
|
||||
@@ -47,7 +45,6 @@ api:
|
||||
tls:
|
||||
enabled: true
|
||||
secretName: svcforge-tls
|
||||
|
||||
worker:
|
||||
# Plain replicas. No HPA: the day `replicas: 2` stops keeping up, not before.
|
||||
replicas: 2
|
||||
@@ -55,7 +52,6 @@ worker:
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 192Mi}
|
||||
limits: {memory: 512Mi}
|
||||
|
||||
reconciler:
|
||||
# A singleton, and not by convention — the four checks are not safe to run twice
|
||||
# concurrently. replicas is deliberately not a value: there is nothing to tune.
|
||||
@@ -63,13 +59,11 @@ reconciler:
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 128Mi}
|
||||
limits: {memory: 256Mi}
|
||||
|
||||
# Postgres pool sizing. replicas × maxSize is spent against the Supabase pooler budget:
|
||||
# api(2 × 5) + worker(2 × 5) + reconciler(1 × 2) = 22 connections. Raise with care.
|
||||
pool:
|
||||
minSize: 1
|
||||
maxSize: 5
|
||||
|
||||
migrate:
|
||||
# backoffLimit: 0 — a failed migration must fail the release, not retry into a
|
||||
# half-applied schema. Migrations run here and only here; never on app startup.
|
||||
@@ -77,23 +71,40 @@ migrate:
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 128Mi}
|
||||
limits: {memory: 256Mi}
|
||||
|
||||
auth:
|
||||
jwksUrl: https://auth.oci-oci.duckdns.org/realms/svcforge/protocol/openid-connect/certs
|
||||
issuer: https://auth.oci-oci.duckdns.org/realms/svcforge
|
||||
audience: svcforge
|
||||
|
||||
otel:
|
||||
enabled: true
|
||||
endpoint: http://alloy.observability.svc.cluster.local:4317
|
||||
|
||||
log:
|
||||
level: info
|
||||
|
||||
# The DSNs are pulled from Vault by external-secrets into a Secret the pods envFrom.
|
||||
# No DSN is ever a chart value, a ConfigMap key, or a CI variable.
|
||||
# No DSN is ever a chart value, a ConfigMap key, or a CI variable. That invariant holds
|
||||
# either way here — what changes below is only who creates the Secret.
|
||||
#
|
||||
# DISABLED ON THIS CLUSTER, AND THIS IS A DEVIATION, NOT THE DESIGN.
|
||||
#
|
||||
# The block below describes a ClusterSecretStore named `vault` with HashiCorp-style
|
||||
# key/property refs. This cluster has `oci-vault` instead: OCI Vault via InstancePrincipal,
|
||||
# whose provider addresses a secret by NAME and takes a JSON property, so these remoteRefs
|
||||
# do not translate as written. There are also no ExternalSecrets anywhere on the cluster
|
||||
# yet, so nothing has ever exercised this path.
|
||||
#
|
||||
# With this false, `svcforge.secretName` still resolves through targetName, so the
|
||||
# deployments and the migrate hook read a Secret called `svcforge-secrets` that was created
|
||||
# out of band:
|
||||
#
|
||||
# kubectl -n svcforge create secret generic svcforge-secrets \
|
||||
# --from-env-file=~/.config/svcforge/secrets.env
|
||||
#
|
||||
# ArgoCD does not manage that Secret, so prune and selfHeal cannot touch it — which is also
|
||||
# why it is invisible in git, and the one part of this deployment you cannot read from the
|
||||
# repo. Restoring the intended design means adding oci_vault_secret resources to
|
||||
# oci-k8s/infra/vault.tf and repointing secretStoreRef at oci-vault.
|
||||
externalSecret:
|
||||
enabled: true
|
||||
enabled: false
|
||||
secretStoreRef:
|
||||
name: vault
|
||||
kind: ClusterSecretStore
|
||||
@@ -110,7 +121,6 @@ externalSecret:
|
||||
- secretKey: SVCFORGE_REDIS_DSN
|
||||
key: svcforge/redis
|
||||
property: dsn
|
||||
|
||||
rbac:
|
||||
# The worker helm-installs tenant releases into namespaces it creates. `namespaces` is a
|
||||
# cluster-scoped resource, so `create namespaces` cannot be granted by a namespaced Role
|
||||
@@ -118,17 +128,14 @@ rbac:
|
||||
# verbs, no `*`, no cluster-admin, and no rbac.authorization.k8s.io group at all, so the
|
||||
# worker cannot grant itself anything further.
|
||||
create: true
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
|
||||
# Chart-native only. A hand-authored ServiceMonitor/PrometheusRule CR is banned — the
|
||||
# chart owns these, gated by these flags.
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
interval: 30s
|
||||
|
||||
prometheusRule:
|
||||
enabled: true
|
||||
rules:
|
||||
@@ -148,27 +155,39 @@ prometheusRule:
|
||||
annotations:
|
||||
summary: svcforge p95 provision time is over 5 minutes
|
||||
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
|
||||
- alert: SvcforgeTaskFailed
|
||||
expr: sum(svcforge_tasks_failed_total) > 0
|
||||
# `increase(...[15m])`, not the raw counter. `sum(counter) > 0` on a monotonic counter
|
||||
# latches: one dead-lettered task at any point keeps this firing until the pod restarts,
|
||||
# and a restart silently clears it — so it can never distinguish "failing now" from
|
||||
# "failed last Tuesday". The dead-letter counter is also the right one: the attempts
|
||||
# counter increments on ordinary transient retries that later succeed.
|
||||
- alert: SvcforgeTaskDeadLettered
|
||||
expr: sum(increase(svcforge_tasks_dead_lettered_total[15m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: a svcforge task exhausted its retries
|
||||
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
|
||||
# Scoped to the reconciler job, and aggregated with max().
|
||||
#
|
||||
# RECONCILER_LAST_TICK is a module-level Gauge in obs.py, so EVERY service that imports
|
||||
# svcforge_core.obs registers and exports it — api and worker included, permanently at
|
||||
# 0. Unscoped, `time() - 0` is ~1.7e9, so this critical alert fires from the moment the
|
||||
# chart is installed, from pods that have no reconciler in them. Scope by job, then
|
||||
# max() so a rolling restart of the single reconciler does not flap it.
|
||||
- alert: SvcforgeReconcilerStale
|
||||
expr: time() - svcforge_reconciler_last_tick_timestamp_seconds > 300
|
||||
expr: time() - max(svcforge_reconciler_last_tick_timestamp_seconds{job=~".*reconciler.*"}) > 300
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: the svcforge reconciler has not ticked in 5 minutes
|
||||
runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#orphaned-release
|
||||
|
||||
# Not in the spec. Off by default: on a three-node k3s a PDB that cannot be satisfied
|
||||
# blocks drains, which is worse than the disruption it prevents.
|
||||
podDisruptionBudget:
|
||||
enabled: false
|
||||
minAvailable: 1
|
||||
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
name = "svcforge-core"
|
||||
version = "0.1.0"
|
||||
description = "svcforge shared core: domain, repo, adapters"
|
||||
requires-python = ">=3.12"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"pydantic>=2.9",
|
||||
"pydantic-settings>=2.6",
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
"""Time, as a dependency.
|
||||
|
||||
The centrepiece of the Day-2 module, and it is nine lines. `datetime.now()` called from
|
||||
inside domain logic is an untestable global read: a maintenance-window test that wants
|
||||
"03:00 next Sunday" would have to either sleep until Sunday or monkeypatch a stdlib symbol
|
||||
and hope nothing else in the process noticed. Passing a Clock makes the same test a
|
||||
`FakeClock(start=...)` and an `advance()`.
|
||||
`datetime.now()` inside domain logic is an untestable global read: a maintenance-window
|
||||
test wanting "03:00 next Sunday" must sleep until Sunday or monkeypatch a stdlib symbol.
|
||||
Passing a Clock makes it a `FakeClock(start=...)` and an `advance()`.
|
||||
|
||||
Aware UTC, always. A naive datetime is a bug that survives every test on a UTC CI box and
|
||||
detonates the first time it meets a tenant in Asia/Ho_Chi_Minh: `datetime.utcnow()` returns
|
||||
a naive value, and comparing it to a `timestamptz` from Postgres raises TypeError, or worse,
|
||||
silently compares wrong after somebody "fixes" it with a `.replace(tzinfo=...)`.
|
||||
Aware UTC, always. `datetime.utcnow()` returns a naive value that survives every test on a
|
||||
UTC CI box, then raises TypeError against a `timestamptz` from Postgres — or compares wrong
|
||||
after someone "fixes" it with `.replace(tzinfo=...)`.
|
||||
|
||||
The fake lives in `tests/fakes.py`, not here: shipping test doubles in the production
|
||||
package is how they end up imported by production code.
|
||||
The fake lives in `tests/fakes.py`: test doubles shipped in the production package end up
|
||||
imported by production code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
"""Driving helm from asyncio, with timeouts that actually kill helm.
|
||||
|
||||
The whole module exists for `_run`. Everything above it is argv construction.
|
||||
|
||||
Four things go wrong when you spawn a process from an event loop, and all four are
|
||||
handled here rather than in the caller:
|
||||
The module exists for `_run`; everything above it is argv construction. Four things go
|
||||
wrong when an event loop spawns a process, and all four are handled here:
|
||||
|
||||
1. `subprocess.run` blocks the loop. Use `create_subprocess_exec`.
|
||||
2. `stdout=PIPE` with `proc.wait()` and nobody draining deadlocks at ~64 KB of output —
|
||||
`helm --debug` clears that in one install. Use `communicate()`.
|
||||
3. `asyncio.wait_for` cancels the *coroutine*. The process does not know it was waited on:
|
||||
helm keeps running and keeps mutating the cluster. The timeout has to kill it.
|
||||
4. `proc.kill()` signals the direct child. `helm` forks; its children reparent to init and
|
||||
survive. Only `killpg` gets the whole tree, and only if the group exists — which needs
|
||||
`start_new_session=True` **at spawn time**, because setsid can only run in the window
|
||||
between fork and exec.
|
||||
2. `stdout=PIPE` with nobody draining deadlocks at ~64 KB — one `helm --debug` install.
|
||||
Use `communicate()`.
|
||||
3. `asyncio.wait_for` cancels the *coroutine*; helm keeps running and keeps mutating the
|
||||
cluster. The timeout has to kill it.
|
||||
4. `proc.kill()` signals the direct child, and helm's children reparent and survive. Only
|
||||
`killpg` gets the tree, and only with `start_new_session=True` at spawn time — setsid
|
||||
can only run between fork and exec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,32 +18,58 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
import yaml
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from svcforge_core.adapters.tempyaml import yaml_tempfile
|
||||
from svcforge_core.domain.models import CatalogEntry
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
|
||||
# How long the process group gets to honour SIGTERM before SIGKILL. Helm traps SIGTERM
|
||||
# and tries to leave the release in a coherent state; give it a moment to do so.
|
||||
# Grace for SIGTERM before SIGKILL. Helm traps SIGTERM and tries to leave the release
|
||||
# coherent; give it a moment.
|
||||
_TERM_GRACE_S = 5.0
|
||||
|
||||
# `_run` is the backstop, not the primary timeout: helm gets its own `--timeout` so that
|
||||
# `--atomic` can roll back cleanly. `_run` only fires when helm itself is wedged, so its
|
||||
# deadline sits this far past helm's.
|
||||
# `_run` is the backstop, not the primary timeout: helm gets its own `--timeout` so
|
||||
# `--atomic` can roll back cleanly. `_run` fires only when helm itself is wedged.
|
||||
_RUN_TIMEOUT_MARGIN_S = 30
|
||||
|
||||
_STDERR_TAIL_BYTES = 2048
|
||||
|
||||
# The label every svcforge release carries — written by install(), read by list_releases().
|
||||
# It is the only thing that tells svcforge's releases from the rest of the cluster's.
|
||||
# Changing one value without the other empties the reconciler's view, which reads as
|
||||
# "no drift" rather than as an error.
|
||||
MANAGED_BY_LABEL = "app.kubernetes.io/managed-by"
|
||||
MANAGED_BY_VALUE = "svcforge"
|
||||
|
||||
class HelmError(RuntimeError):
|
||||
"""Non-zero exit. str(self) is the stderr tail that lands in instances.error."""
|
||||
# Where the kubelet mounts the pod's ServiceAccount. Their presence is also how this module
|
||||
# decides it is in-cluster: in-cluster takes the fast release read, everything else shells
|
||||
# out to helm.
|
||||
_SA_DIR = Path("/var/run/secrets/kubernetes.io/serviceaccount")
|
||||
_SA_TOKEN = _SA_DIR / "token"
|
||||
_SA_CA = _SA_DIR / "ca.crt"
|
||||
|
||||
# helm's own label on every release secret it writes.
|
||||
_HELM_OWNER_LABEL = "owner=helm"
|
||||
|
||||
# The states `helm list` shows by default. In the selector so superseded revisions never
|
||||
# leave the API server — 96 release secrets here, 71 of them superseded.
|
||||
_LIVE_STATUSES = "status in (deployed,failed,pending-install,pending-upgrade,pending-rollback)"
|
||||
|
||||
_API_TIMEOUT_S = 15.0
|
||||
|
||||
|
||||
class HelmError(SvcforgeError, RuntimeError):
|
||||
"""Non-zero exit. str(self) is the stderr tail that lands in instances.error.
|
||||
|
||||
RuntimeError stays in the MRO so callers written against it keep catching; SvcforgeError
|
||||
comes first so `except SvcforgeError` separates a modelled failure from a stray bug.
|
||||
"""
|
||||
|
||||
|
||||
class ReleaseInfo(BaseModel):
|
||||
@@ -75,9 +98,9 @@ class Provisioner(Protocol):
|
||||
def _tail(raw: bytes, tail_bytes: int) -> str:
|
||||
"""The last `tail_bytes` of a stream, as text.
|
||||
|
||||
Truncation happens here, at the adapter boundary, and nowhere else. A helm failure can
|
||||
emit megabytes; `instances.error` is a text column read by humans. Slice the bytes, not
|
||||
the decoded string, then decode with `replace` — the cut can land mid-codepoint.
|
||||
Truncation happens here, at the adapter boundary, and nowhere else: a helm failure can
|
||||
emit megabytes and `instances.error` is read by humans. Slice the bytes and decode with
|
||||
`replace` — the cut can land mid-codepoint.
|
||||
"""
|
||||
return raw[-tail_bytes:].decode("utf-8", errors="replace").strip()
|
||||
|
||||
@@ -85,8 +108,8 @@ def _tail(raw: bytes, tail_bytes: int) -> str:
|
||||
def _signal_group(proc: asyncio.subprocess.Process, sig: int) -> None:
|
||||
"""Signal the process's whole group. No-op if it has already exited.
|
||||
|
||||
`os.getpgid` rather than `proc.pid`: with `start_new_session=True` they are equal, but
|
||||
that equality is an implementation detail, and asking the kernel costs nothing.
|
||||
`os.getpgid` rather than `proc.pid`: `start_new_session=True` makes them equal, but that
|
||||
equality is an implementation detail and asking the kernel costs nothing.
|
||||
"""
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
@@ -149,8 +172,8 @@ class HelmProvisioner:
|
||||
self, *, helm_bin: str = "helm", kubeconfig: Path | None = None, timeout_s: int = 600
|
||||
) -> None:
|
||||
"""kubeconfig=None means the ambient config: $KUBECONFIG, ~/.kube/config, or the
|
||||
in-cluster service account when svcforge runs as a pod. Keyword-only so that the
|
||||
three arguments can never be swapped by position at a call site.
|
||||
in-cluster service account. Keyword-only so the three can never be swapped by
|
||||
position at a call site.
|
||||
"""
|
||||
self._helm_bin = helm_bin
|
||||
self._kubeconfig = kubeconfig
|
||||
@@ -166,14 +189,26 @@ class HelmProvisioner:
|
||||
argv += ["--kubeconfig", str(self._kubeconfig)]
|
||||
return argv
|
||||
|
||||
async def _run_helm(self, argv: Sequence[str]) -> str:
|
||||
"""`_run`, with the timeout path translated to this adapter's declared error type.
|
||||
|
||||
`_run` raises a bare TimeoutError so the process-group test can assert on it, but
|
||||
every public method here is documented as raising HelmError. A wedged helm arriving
|
||||
as TimeoutError sails past a caller's `except HelmError` and fails the task as an
|
||||
unmodelled crash, so translate once, at the public boundary.
|
||||
"""
|
||||
try:
|
||||
return await _run(argv, timeout_s=self._run_timeout_s)
|
||||
except TimeoutError as exc:
|
||||
raise HelmError(f"{argv[0]} timed out after {self._run_timeout_s}s and was killed") from exc
|
||||
|
||||
async def install(self, release: str, ns: str, entry: CatalogEntry, values: dict[str, Any]) -> None:
|
||||
"""helm upgrade --install --wait --timeout. Idempotent by construction."""
|
||||
# `upgrade --install` is why this is idempotent: a retried task after a crash mid-provision
|
||||
# converges on the same release instead of erroring with "release already exists".
|
||||
# `--wait` is why `ready` in the DB means ready — it returns when the pods are up.
|
||||
# `--atomic` rolls back a failed upgrade; it doubles the worst case, which is what
|
||||
# `_RUN_TIMEOUT_MARGIN_S` and helm's own `--timeout` are sized around.
|
||||
with _values_file(values) as path:
|
||||
# `upgrade --install`: a task retried after a crash mid-provision converges on the
|
||||
# same release instead of erroring with "release already exists". `--wait` is why
|
||||
# `ready` in the DB means ready. `--atomic` rolls back a failed upgrade and doubles
|
||||
# the worst case, which is what the two timeouts are sized around.
|
||||
with yaml_tempfile(values, prefix="svcforge-values-", name="values.yaml") as path:
|
||||
argv = self._base_argv(
|
||||
"upgrade",
|
||||
"--install",
|
||||
@@ -181,6 +216,15 @@ class HelmProvisioner:
|
||||
entry.chart,
|
||||
"--namespace",
|
||||
ns,
|
||||
# `--namespace X` does not create X, and every tenant's first provision
|
||||
# targets one that does not exist yet. helm creates it here rather than a
|
||||
# `kubectl apply` step, which keeps kubectl out of the worker image — one
|
||||
# fewer binary and one fewer set of vendored Go CVEs. Idempotent.
|
||||
"--create-namespace",
|
||||
# Stamps MANAGED_BY_LABEL, which is what lets list_releases() ask for
|
||||
# svcforge's releases and nobody else's.
|
||||
"--labels",
|
||||
f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}",
|
||||
"--version",
|
||||
entry.chart_version,
|
||||
"--values",
|
||||
@@ -190,7 +234,7 @@ class HelmProvisioner:
|
||||
"--timeout",
|
||||
f"{self._timeout_s}s",
|
||||
)
|
||||
await _run(argv, timeout_s=self._run_timeout_s)
|
||||
await self._run_helm(argv)
|
||||
|
||||
async def uninstall(self, release: str, ns: str) -> None:
|
||||
"""helm uninstall --wait. `--ignore-not-found` makes the retry of a half-done delete a no-op."""
|
||||
@@ -204,12 +248,40 @@ class HelmProvisioner:
|
||||
"--timeout",
|
||||
f"{self._timeout_s}s",
|
||||
)
|
||||
await _run(argv, timeout_s=self._run_timeout_s)
|
||||
await self._run_helm(argv)
|
||||
|
||||
async def list_releases(self) -> list[ReleaseInfo]:
|
||||
"""Every release helm knows about, in every namespace. The reconciler's view of reality."""
|
||||
argv = self._base_argv("list", "--all-namespaces", "--output", "json")
|
||||
raw = await _run(argv, timeout_s=self._run_timeout_s)
|
||||
"""Every release svcforge provisioned, in every namespace. The reconciler's reality.
|
||||
|
||||
Scoped by label for correctness: the reconciler diffs this against the database in
|
||||
both directions, and `live - known` is reported as `drift.orphan_release` at ERROR.
|
||||
Unscoped, `live` is every release in the cluster, so argocd, longhorn, gitea and
|
||||
cert-manager are all reported as orphans on every sweep.
|
||||
|
||||
In-cluster this reads the release secrets off the API server instead of shelling
|
||||
out — 21ms and 31KB against helm's 4392ms, because helm applies `--selector` only
|
||||
after fetching and decompressing every release secret in the cluster. That cost was
|
||||
not theoretical: with the CPU request mutated to 0 by a cluster policy, helm's list
|
||||
took over 330s and timed out on every tick, and a check that never completes reports
|
||||
no drift. Out of cluster there is no ServiceAccount, so it falls back to helm and the
|
||||
e2e suite keeps working. See `_list_releases_via_api`.
|
||||
|
||||
Releases provisioned before the label existed do not match, so the first sweep sees
|
||||
them as missing and re-provisions. That is safe — provisioning is `upgrade --install`
|
||||
against a deterministic release name — and the re-provision applies the label.
|
||||
"""
|
||||
via_api = await self._list_releases_via_api()
|
||||
if via_api is not None:
|
||||
return via_api
|
||||
argv = self._base_argv(
|
||||
"list",
|
||||
"--all-namespaces",
|
||||
"--selector",
|
||||
f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}",
|
||||
"--output",
|
||||
"json",
|
||||
)
|
||||
raw = await self._run_helm(argv)
|
||||
try:
|
||||
parsed: Any = json.loads(raw or "[]")
|
||||
except json.JSONDecodeError as exc:
|
||||
@@ -218,28 +290,97 @@ class HelmProvisioner:
|
||||
raise HelmError(f"helm list returned {type(parsed).__name__}, expected a list")
|
||||
return [ReleaseInfo.model_validate(row) for row in parsed]
|
||||
|
||||
async def _list_releases_via_api(self) -> list[ReleaseInfo] | None:
|
||||
"""Release names and namespaces read straight off helm's release secrets.
|
||||
|
||||
class _ValuesFile:
|
||||
"""Context manager yielding a path to a values.yaml written from a dict.
|
||||
None when there is no in-cluster ServiceAccount, which is the caller's signal to
|
||||
fall back to helm.
|
||||
|
||||
A file, not `--set`: `--set` has its own escaping grammar (commas, dots, backslashes) and
|
||||
values carry tenant-shaped strings. Serialising YAML sidesteps the grammar entirely.
|
||||
"""
|
||||
helm gunzips every release payload to build its table. The only fields either caller
|
||||
reads are name and namespace, and both live in the secret's labels and metadata, so
|
||||
nothing has to be decompressed. Three details carry the correctness:
|
||||
|
||||
def __init__(self, values: dict[str, Any]) -> None:
|
||||
self._values = values
|
||||
self._dir: str | None = None
|
||||
* `PartialObjectMetadataList` in the Accept header asks for metadata only. Without
|
||||
it the response carries every release's gzipped manifest — megabytes fetched to be
|
||||
thrown away, which is the cost this method exists to avoid.
|
||||
* The status selector drops superseded revisions server-side (96 secrets here, 25
|
||||
live). The states kept are the ones `helm list` shows, so a failed release still
|
||||
counts as existing — it does, and calling it missing would re-provision on top.
|
||||
* helm writes one secret per revision, so a release can appear several times. The
|
||||
newest `version` label wins; without that, any caller counting releases over-counts.
|
||||
|
||||
def __enter__(self) -> Path:
|
||||
self._dir = tempfile.mkdtemp(prefix="svcforge-values-")
|
||||
path = Path(self._dir) / "values.yaml"
|
||||
path.write_text(yaml.safe_dump(self._values, default_flow_style=False), encoding="utf-8")
|
||||
return path
|
||||
`chart` comes back empty because the chart name lives only in the compressed payload.
|
||||
The field stays so the helm fallback, which does populate it, returns the same shape.
|
||||
"""
|
||||
try:
|
||||
token = _SA_TOKEN.read_text(encoding="utf-8").strip()
|
||||
except OSError:
|
||||
return None
|
||||
# The CA is checked here rather than left to httpx, which loads it eagerly at client
|
||||
# construction and raises OSError — not in the except below. A half-mounted
|
||||
# ServiceAccount would crash the tick as a bare bug instead of falling back. A
|
||||
# complete ServiceAccount is the in-cluster signal; a missing CA means "not
|
||||
# in-cluster" exactly as a missing token does.
|
||||
if not token or not os.access(_SA_CA, os.R_OK):
|
||||
return None
|
||||
host, port = (
|
||||
os.environ.get("KUBERNETES_SERVICE_HOST"),
|
||||
os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443"),
|
||||
)
|
||||
if not host:
|
||||
return None
|
||||
|
||||
def __exit__(self, *exc: object) -> None:
|
||||
if self._dir is not None:
|
||||
shutil.rmtree(self._dir, ignore_errors=True)
|
||||
selector = f"{_HELM_OWNER_LABEL},{MANAGED_BY_LABEL}={MANAGED_BY_VALUE},{_LIVE_STATUSES}"
|
||||
try:
|
||||
async with httpx.AsyncClient(verify=str(_SA_CA), timeout=_API_TIMEOUT_S) as client:
|
||||
# No `limit`, and that is load-bearing: the apiserver only returns a
|
||||
# `metadata.continue` token when the client sets one, so the single read
|
||||
# below is the complete set. Adding `limit` without looping on `continue`
|
||||
# would truncate silently, and the reconciler would read the missing
|
||||
# releases as orphans to delete or as vanished releases to re-provision.
|
||||
resp = await client.get(
|
||||
f"https://{host}:{port}/api/v1/secrets",
|
||||
params={"labelSelector": selector},
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
"accept": ("application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1"),
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
# `or []`, not `.get("items", [])`. Kubernetes serialises an empty list as
|
||||
# `"items": null`, so the key is present and the default never fires. This
|
||||
# shipped and failed on the first tick that matched no releases.
|
||||
items: Any = resp.json().get("items") or []
|
||||
except (httpx.HTTPError, json.JSONDecodeError) as exc:
|
||||
# Raise, do not fall back to helm. The fallback is for "there is no
|
||||
# ServiceAccount", a fact about the environment known before any request goes
|
||||
# out. Here the API server was reachable and something went wrong, and retrying
|
||||
# through helm would swap a visible error for the 330s timeout this method
|
||||
# exists to remove. The tick logs check.failed and tries again in 60s.
|
||||
raise HelmError(f"listing release secrets failed: {exc}") from exc
|
||||
|
||||
|
||||
def _values_file(values: dict[str, Any]) -> _ValuesFile:
|
||||
return _ValuesFile(values)
|
||||
newest: dict[tuple[str, str], tuple[int, ReleaseInfo]] = {}
|
||||
for item in items:
|
||||
meta = item.get("metadata") or {}
|
||||
labels = meta.get("labels") or {}
|
||||
name, namespace = labels.get("name"), meta.get("namespace")
|
||||
if not name or not namespace:
|
||||
continue # not a release secret this method understands; leave it alone
|
||||
try:
|
||||
revision = int(labels.get("version", 0))
|
||||
except ValueError:
|
||||
revision = 0
|
||||
key = (name, namespace)
|
||||
if key in newest and newest[key][0] >= revision:
|
||||
continue
|
||||
newest[key] = (
|
||||
revision,
|
||||
ReleaseInfo(
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
chart="",
|
||||
status=labels.get("status", ""),
|
||||
revision=max(revision, 0),
|
||||
),
|
||||
)
|
||||
return [info for _, info in newest.values()]
|
||||
|
||||
@@ -13,20 +13,22 @@ from __future__ import annotations
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from svcforge_core.adapters.helm import HelmError, _run
|
||||
from svcforge_core.adapters.helm import MANAGED_BY_LABEL, MANAGED_BY_VALUE, HelmError, _run
|
||||
from svcforge_core.adapters.tempyaml import yaml_tempfile
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
|
||||
_KUBECTL_TIMEOUT_S = 60
|
||||
|
||||
|
||||
class K8sError(RuntimeError):
|
||||
"""kubectl failed. str(self) is the stderr tail, already truncated by `_run`."""
|
||||
class K8sError(SvcforgeError, RuntimeError):
|
||||
"""kubectl failed. str(self) is the stderr tail, already truncated by `_run`.
|
||||
|
||||
RuntimeError stays in the MRO so existing `except RuntimeError` callers keep catching;
|
||||
SvcforgeError comes first so a modelled cluster failure is distinguishable from a bug.
|
||||
"""
|
||||
|
||||
|
||||
class SecretNotFound(K8sError):
|
||||
@@ -66,10 +68,10 @@ class KubectlClient:
|
||||
"kind": "Namespace",
|
||||
"metadata": {
|
||||
"name": ns,
|
||||
"labels": {"app.kubernetes.io/managed-by": "svcforge", **(labels or {})},
|
||||
"labels": {MANAGED_BY_LABEL: MANAGED_BY_VALUE, **(labels or {})},
|
||||
},
|
||||
}
|
||||
with _manifest_file(manifest) as path:
|
||||
with yaml_tempfile(manifest, prefix="svcforge-manifest-", name="manifest.yaml") as path:
|
||||
await self._kubectl("apply", "--filename", str(path))
|
||||
|
||||
async def read_secret(self, ns: str, name: str) -> dict[str, str]:
|
||||
@@ -112,25 +114,8 @@ class KubectlClient:
|
||||
return await _run(self._base_argv(*args), timeout_s=self._timeout_s)
|
||||
except HelmError as exc:
|
||||
raise K8sError(str(exc)) from exc
|
||||
|
||||
|
||||
class _ManifestFile:
|
||||
"""A temp file holding one YAML manifest, removed on exit."""
|
||||
|
||||
def __init__(self, manifest: dict[str, Any]) -> None:
|
||||
self._manifest = manifest
|
||||
self._dir: str | None = None
|
||||
|
||||
def __enter__(self) -> Path:
|
||||
self._dir = tempfile.mkdtemp(prefix="svcforge-manifest-")
|
||||
path = Path(self._dir) / "manifest.yaml"
|
||||
path.write_text(yaml.safe_dump(self._manifest, default_flow_style=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def __exit__(self, *exc: object) -> None:
|
||||
if self._dir is not None:
|
||||
shutil.rmtree(self._dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _manifest_file(manifest: dict[str, Any]) -> _ManifestFile:
|
||||
return _ManifestFile(manifest)
|
||||
except TimeoutError as exc:
|
||||
# `_run` re-raises the bare TimeoutError after killing the process group, so the
|
||||
# timeout path does not come through HelmError. Without this clause a wedged
|
||||
# kubectl surfaces as TimeoutError past a caller written to `except K8sError`.
|
||||
raise K8sError(f"kubectl {args[0] if args else ''} timed out after {self._timeout_s}s") from exc
|
||||
|
||||
@@ -1,25 +1,38 @@
|
||||
"""Telling someone a provision finished, or didn't.
|
||||
|
||||
Best-effort by construction: `send` never raises. A notifier that can fail a task is a
|
||||
notifier that lets a Slack outage roll back a successful provision. The instance is ready;
|
||||
the DB says so; failing the task would re-run helm for nothing. Delivery failures are
|
||||
logged and dropped on the floor, which is the correct amount of ceremony for a webhook.
|
||||
Best-effort by construction: `send` never raises. A notifier that can fail a task lets a
|
||||
Slack outage roll back a successful provision — the instance is ready and the DB says so,
|
||||
and failing the task would re-run helm for nothing. Delivery failures are logged and dropped.
|
||||
|
||||
Two implementations, so the Protocol earns its place: LogNotifier (the default, and what
|
||||
tests and local dev get) and WebhookNotifier (the one that leaves the process).
|
||||
Two implementations, so the Protocol earns its place: LogNotifier (the default) and
|
||||
WebhookNotifier (the one that leaves the process).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
from svcforge_core import obs
|
||||
|
||||
# obs imports nothing from adapters (it is settings + structlog + otel only), so this is a
|
||||
# plain top-level import and not a lazy one — the cycle it would otherwise create does not
|
||||
# exist. Keep it that way: obs must stay importable before any adapter is.
|
||||
|
||||
_DEFAULT_TIMEOUT_S = 5.0
|
||||
|
||||
# Keys a caller's `fields` must not be allowed to occupy. `event` is structlog's first
|
||||
# positional parameter, so passing it as a kwarg is a TypeError, not a shadowed value; the
|
||||
# rest are stdlib LogRecord attributes that the ProcessorFormatter bridge refuses to
|
||||
# overwrite. A tenant-supplied field named `event` must not be able to crash the notifier.
|
||||
_RESERVED_KEYS = frozenset({"event", "msg", "args", "name", "levelname", "exc_info", "stack_info"})
|
||||
|
||||
|
||||
def _safe_fields(fields: dict[str, str] | None) -> dict[str, str]:
|
||||
"""Fields with reserved names prefixed rather than dropped — the value still gets logged."""
|
||||
return {(f"field_{k}" if k in _RESERVED_KEYS else k): v for k, v in (fields or {}).items()}
|
||||
|
||||
|
||||
class Notifier(Protocol):
|
||||
"""Implemented by LogNotifier, WebhookNotifier, and FakeNotifier (tests/fakes.py)."""
|
||||
@@ -33,11 +46,16 @@ class LogNotifier:
|
||||
"""Writes the event to the log. The default: structured logs are already shipped somewhere."""
|
||||
|
||||
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
|
||||
# NOT extra={"message": ...}. `message` is a reserved LogRecord attribute, and
|
||||
# logging raises KeyError("Attempt to overwrite 'message' in LogRecord") at call
|
||||
# time — so the notifier would crash the caller it was meant to inform. Same trap
|
||||
# for `msg`, `args`, `name`, `levelname`, `exc_info`.
|
||||
_log.info("notify", extra={"event": event, "detail": message, "fields": fields or {}})
|
||||
# structlog kwargs, NOT logging's `extra=`: ProcessorFormatter builds the event dict
|
||||
# from `record.msg` alone, so `extra=` keys are dropped and this used to emit a bare
|
||||
# {"event": "notify"} with the payload gone.
|
||||
#
|
||||
# Fields are splatted rather than nested so each is its own queryable key in Loki.
|
||||
# `detail`, not `message` — a reserved LogRecord attribute the bridge raises on. And
|
||||
# `notify_event`, not `event` — structlog's first positional parameter is named
|
||||
# `event`, so passing it as a kwarg is a TypeError at the call.
|
||||
log = obs.get_logger(__name__)
|
||||
log.info("notify", notify_event=event, detail=message, **_safe_fields(fields))
|
||||
|
||||
|
||||
class WebhookNotifier:
|
||||
@@ -52,26 +70,32 @@ class WebhookNotifier:
|
||||
"""
|
||||
self._url = url
|
||||
self._timeout_s = timeout_s
|
||||
self._client = client
|
||||
self._owns_client = client is None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self._timeout_s)
|
||||
return self._client
|
||||
# Eager, not lazy. `AsyncClient()` does no I/O, so laziness bought nothing and cost a
|
||||
# race: two concurrent `send`s both see None, both construct a client, and the loser's
|
||||
# connection pool leaks because only one survives the assignment.
|
||||
self._client = client if client is not None else httpx.AsyncClient(timeout=self._timeout_s)
|
||||
|
||||
async def send(self, event: str, message: str, fields: dict[str, str] | None = None) -> None:
|
||||
payload = {"event": event, "message": message, "fields": fields or {}}
|
||||
try:
|
||||
client = await self._get_client()
|
||||
resp = await client.post(self._url, json=payload, timeout=self._timeout_s)
|
||||
resp = await self._client.post(self._url, json=payload, timeout=self._timeout_s)
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPError as exc:
|
||||
# Deliberately swallowed. See the module docstring: the work already succeeded.
|
||||
_log.warning("notify webhook failed", extra={"event": event, "error": str(exc)})
|
||||
except Exception as exc: # the bare `except Exception` IS the specification here
|
||||
# "send must not raise" is not satisfiable by catching httpx.HTTPError alone:
|
||||
# `httpx.InvalidURL` is not a subclass, and posting on an aclose()d client raises
|
||||
# RuntimeError — so a typo'd webhook URL would fail a task whose helm work
|
||||
# already succeeded. exc_info so the traceback survives the swallowing.
|
||||
obs.get_logger(__name__).warning(
|
||||
"notify webhook failed", notify_event=event, error=str(exc), exc_info=exc
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the client, if we made it."""
|
||||
if self._client is not None and self._owns_client:
|
||||
"""Close the client, if we made it. Call at process shutdown, next to the pool's close.
|
||||
|
||||
The reference is kept rather than cleared: a `send` racing shutdown then raises
|
||||
RuntimeError on a closed client and is swallowed like any other delivery failure,
|
||||
instead of resurrecting a pool nobody will close.
|
||||
"""
|
||||
if self._owns_client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
@@ -1,46 +1,34 @@
|
||||
"""Redis: derived state only. Never the truth, never the queue.
|
||||
|
||||
Everything in here is a shortcut past Postgres, and every one of them is optional. Postgres
|
||||
holds the instances, the tasks, the leases and the `release_name` UNIQUE constraint. Redis
|
||||
holds a counter, a claim marker and a copy — all of it rebuildable by doing nothing and
|
||||
waiting for a TTL.
|
||||
Everything here is an optional shortcut past Postgres, which holds the instances, the
|
||||
tasks, the leases and the `release_name` UNIQUE constraint. Redis holds a counter, a claim
|
||||
marker and a copy — all rebuildable by waiting for a TTL.
|
||||
|
||||
That framing is not philosophy, it decides the error handling, and the error handling is
|
||||
the module. Each class below catches `RedisError` and returns a *safe* answer rather than
|
||||
raising:
|
||||
That decides the error handling, and the error handling is the module. Each class catches
|
||||
`RedisError` and returns a safe answer rather than raising:
|
||||
|
||||
| Path | Redis is down | Why |
|
||||
|-------------|--------------------------|--------------------------------------------------|
|
||||
| Cache | miss -> read Postgres | It was an optimisation. Nobody notices. |
|
||||
| Rate limit | **allow** | An internal platform that refuses every request |
|
||||
| | | because the limiter is sick is worse than one |
|
||||
| | | that is briefly unmetered. |
|
||||
| Idempotency | fall through to the DB | `instances.release_name` is UNIQUE. That is the |
|
||||
| | | real guarantee; this is the fast path. |
|
||||
| Path | Redis is down | Why |
|
||||
|-------------|------------------------|--------------------------------------------------|
|
||||
| Cache | miss -> read Postgres | It was an optimisation. Nobody notices. |
|
||||
| Rate limit | **allow** | Briefly unmetered beats refusing every request. |
|
||||
| Idempotency | fall through to the DB | `instances.release_name` UNIQUE is the guarantee.|
|
||||
|
||||
Consequently nothing here raises out to a caller, and `/readyz` stays Postgres-only. A
|
||||
Redis outage must not make a single pod unready — that would convert "the cache is down"
|
||||
into "the platform is down", which is the exact inversion this module exists to prevent.
|
||||
Nothing here raises out to a caller and `/readyz` stays Postgres-only, so a Redis outage
|
||||
never makes a pod unready.
|
||||
|
||||
**The budget is a design constraint, not a footnote.** Upstash free tier:
|
||||
**The budget is a design constraint.** Upstash free tier is 500,000 commands/month =
|
||||
16,129/day = 0.19/second sustained. One worker polling every five seconds spends the entire
|
||||
budget producing nothing, so the rule is structural: Redis lives on the request path only,
|
||||
never in a poll or control loop. It is also why the limiter is a Lua script —
|
||||
`GET`/`INCR`/`EXPIRE` is three billed commands and a race, one `EVALSHA` is one and atomic.
|
||||
A pipeline batches round trips but still bills N.
|
||||
|
||||
500,000 commands / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
|
||||
|
||||
One worker polling Redis every five seconds spends 518,400/month: the entire budget,
|
||||
producing nothing. So the rule is structural — **Redis lives on the request path only**,
|
||||
where volume is bounded by the number of humans with an API token, and never inside a poll
|
||||
or control loop. That is also why the limiter is a Lua script: `GET`/`INCR`/`EXPIRE` is
|
||||
three billed commands and a race; one `EVALSHA` is one billed command and atomic. A
|
||||
pipeline would not help — it batches round trips but still bills N.
|
||||
|
||||
Every key gets a TTL. 256 MB with no expiry is a slow leak that ends by evicting the keys
|
||||
you cared about.
|
||||
Every key gets a TTL. 256 MB with no expiry is a leak that ends by evicting what mattered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
@@ -52,6 +40,7 @@ from pydantic import ValidationError
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from svcforge_core import obs
|
||||
from svcforge_core.adapters.clock import Clock, SystemClock
|
||||
from svcforge_core.domain.models import Instance
|
||||
|
||||
@@ -60,19 +49,18 @@ if TYPE_CHECKING:
|
||||
|
||||
from svcforge_core.settings import Settings
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
# structlog via obs, not stdlib logging: the stdlib bridge builds the event dict from the
|
||||
# record message alone and drops `extra=`. A bound logger takes fields as kwargs and keeps
|
||||
# them. Bound per instance in __init__, after obs.setup() runs, never at import time.
|
||||
|
||||
# --- The budget metric ------------------------------------------------------------------
|
||||
#
|
||||
# The counter that `scripts/redis_budget.py` projects month-end burn from. It counts
|
||||
# commands we *send*, incremented next to each call, because the number that matters is
|
||||
# the one Upstash bills — not the number of times a method was called. A cache miss calls
|
||||
# `get()` once and spends one command; a `put()` after it spends another.
|
||||
# What `scripts/redis_budget.py` projects month-end burn from. It counts commands *sent*,
|
||||
# incremented next to each call, because the billed number is what matters — a cache miss
|
||||
# spends one command on `get()` and another on the `put()` after it.
|
||||
#
|
||||
# The dangerous failure this makes visible: when Redis is down, every path degrades
|
||||
# silently and correctly, so nothing pages. Nothing fails until the month rolls over and
|
||||
# every Redis call starts erroring at once. A counter you can extrapolate from is the only
|
||||
# warning you get.
|
||||
# This is the only warning available: when Redis is down every path degrades silently and
|
||||
# correctly, so nothing pages until the month rolls over and every call starts erroring.
|
||||
|
||||
REDIS_COMMANDS = Counter(
|
||||
"svcforge_redis_commands_total",
|
||||
@@ -86,26 +74,24 @@ REDIS_ERRORS = Counter(
|
||||
["op"],
|
||||
)
|
||||
|
||||
# A hung Redis must not hang the request path. Without these, a TCP connection that is
|
||||
# open but unanswered blocks the handler until the client gives up — which turns "Redis is
|
||||
# slow" into "the API is down", the same inversion the fail-open policy prevents. Upstash
|
||||
# steady-state RTT is ~2.4 ms; two seconds is already pathological.
|
||||
# A hung Redis must not hang the request path: an open but unanswered TCP connection blocks
|
||||
# the handler until the client gives up, turning "Redis is slow" into "the API is down".
|
||||
# Upstash steady-state RTT is ~2.4 ms, so two seconds is already pathological.
|
||||
_SOCKET_TIMEOUT_S = 2.0
|
||||
_CONNECT_TIMEOUT_S = 2.0
|
||||
|
||||
# Errors that mean "Redis did not answer". Every public method below turns these into a
|
||||
# safe default. `OSError` because a DNS failure at connect time need not arrive wrapped,
|
||||
# `TimeoutError` because the socket timeouts above raise it.
|
||||
# "Redis did not answer" — every public method turns these into a safe default. `OSError`
|
||||
# because a DNS failure at connect time need not arrive wrapped, `TimeoutError` because the
|
||||
# socket timeouts above raise it.
|
||||
_REDIS_DOWN = (RedisError, OSError, asyncio.TimeoutError)
|
||||
|
||||
|
||||
def _as_text(value: bytes | str) -> str:
|
||||
"""`decode_responses=True` already did this; redis-py's annotations do not know it.
|
||||
|
||||
The client is configured for text, so the `bytes` branch is unreachable in this
|
||||
process. It stays because the type says it is reachable, and a `cast` here would hide
|
||||
the day someone builds a client without `decode_responses` and gets a `UUID(b'...')`
|
||||
TypeError from three frames away instead of a value that just works.
|
||||
The `bytes` branch is unreachable in this process. It stays rather than becoming a
|
||||
`cast` so that a client built without `decode_responses` gets a working value instead
|
||||
of a `UUID(b'...')` TypeError three frames away.
|
||||
"""
|
||||
return value.decode() if isinstance(value, bytes) else value
|
||||
|
||||
@@ -113,21 +99,14 @@ def _as_text(value: bytes | str) -> str:
|
||||
def make_redis(settings: Settings) -> Redis | None:
|
||||
"""One client per process, opened in lifespan next to the psycopg pool, closed on exit.
|
||||
|
||||
`None` when no DSN is configured, and that is a supported way to run: every consumer
|
||||
below is optional by construction, so "no Redis" and "Redis is down" take the same
|
||||
code path. The signature is `Redis | None` rather than `Redis` precisely so that
|
||||
"unconfigured" cannot be faked with a client pointed at nothing.
|
||||
`None` when no DSN is configured, which is a supported way to run: every consumer is
|
||||
optional, so "no Redis" and "Redis is down" take the same path. The return type is
|
||||
`Redis | None` so "unconfigured" cannot be faked with a client pointed at nothing.
|
||||
|
||||
Two settings are not negotiable:
|
||||
|
||||
`decode_responses=True` — the first bug everyone hits. Without it every read is
|
||||
`bytes` and the traceback is `AttributeError: 'bytes' object has no attribute
|
||||
'encode'`, several frames away from the cause.
|
||||
|
||||
`rediss://` (TLS) — Upstash rejects plaintext. The handshake is ~56 ms against a
|
||||
~2.4 ms steady-state RTT, which is the whole argument for one pooled client per
|
||||
process: a client per request pays the handshake every time and turns a cache into a
|
||||
latency regression.
|
||||
`decode_responses=True` is not optional — without it every read is `bytes` and the
|
||||
traceback is an `AttributeError` several frames from the cause. Neither is `rediss://`:
|
||||
Upstash rejects plaintext, and the ~56 ms handshake against a ~2.4 ms steady-state RTT
|
||||
is the whole argument for one pooled client per process.
|
||||
"""
|
||||
if settings.redis_dsn is None:
|
||||
return None
|
||||
@@ -141,15 +120,14 @@ def make_redis(settings: Settings) -> Redis | None:
|
||||
|
||||
# --- Rate limiting ----------------------------------------------------------------------
|
||||
|
||||
# One INCR; EXPIRE only when the counter is new. The `== 1` test is the entire trick: set
|
||||
# the TTL unconditionally and every request slides the window forward, so a caller at
|
||||
# steady load is never reset and the "window" is a sliding refusal that never lets up.
|
||||
# One INCR; EXPIRE only when the counter is new. The `== 1` test is the trick: set the TTL
|
||||
# unconditionally and every request slides the window forward, so a caller at steady load is
|
||||
# never reset and the window becomes a refusal that never lets up.
|
||||
#
|
||||
# KEYS and ARGV arrive as 1-based tables — Lua indexes from 1, and `ARGV[0]` is silently
|
||||
# nil rather than an error, which reads as "the limit is nil" and compares false forever.
|
||||
# KEYS and ARGV are 1-based — `ARGV[0]` is silently nil rather than an error, which reads as
|
||||
# "the limit is nil" and compares false forever.
|
||||
#
|
||||
# Everything derivable in Python is derived in Python: `reset_at` comes from the window
|
||||
# number the caller already computed, so there is no TTL round trip. One command, total.
|
||||
# `reset_at` is derived in Python from the window number, so there is no TTL round trip.
|
||||
_RATE_LIMIT_LUA = """
|
||||
local n = redis.call('INCR', KEYS[1])
|
||||
if n == 1 then
|
||||
@@ -176,6 +154,10 @@ class RateLimitResult:
|
||||
limit: int
|
||||
remaining: int
|
||||
reset_at: datetime
|
||||
# From the same injected clock as reset_at. The two must share a clock or retry_after_s
|
||||
# (their difference) is meaningless under a FakeClock and drifts by the request latency
|
||||
# in production.
|
||||
checked_at: datetime
|
||||
degraded: bool = False
|
||||
|
||||
@property
|
||||
@@ -185,7 +167,7 @@ class RateLimitResult:
|
||||
Rounded up and floored at one: `Retry-After: 0` invites an immediate retry into
|
||||
the same closed window, which is a busy loop with extra steps.
|
||||
"""
|
||||
delta = (self.reset_at - datetime.now(UTC)).total_seconds()
|
||||
delta = (self.reset_at - self.checked_at).total_seconds()
|
||||
return max(1, math.ceil(delta))
|
||||
|
||||
|
||||
@@ -200,16 +182,14 @@ class RateLimiterProto(Protocol):
|
||||
class RateLimiter:
|
||||
"""Fixed-window limiter. One EVALSHA per check. Fails OPEN.
|
||||
|
||||
Fixed window, not a token bucket or a sliding log, because the window boundary is the
|
||||
only thing a fixed window gets wrong and the cost of getting it wrong is that a caller
|
||||
can spend 2x the limit across a boundary. A sliding log is a sorted set, an
|
||||
`ZREMRANGEBYSCORE`, an `ZADD` and a `ZCARD` — four billed commands and unbounded key
|
||||
size — to fix a burst nobody is paying for. The limit is a courtesy, not a security
|
||||
control; the security control is the JWT.
|
||||
A fixed window's only error is the boundary, where a caller can spend 2x the limit. The
|
||||
sliding log that fixes it costs four billed commands and an unbounded key. The limit is
|
||||
a courtesy; the security control is the JWT.
|
||||
"""
|
||||
|
||||
def __init__(self, r: Redis, limit: int, window_s: int, *, clock: Clock | None = None) -> None:
|
||||
"""`clock` is injectable so the window boundary is testable without sleeping."""
|
||||
self._log = obs.get_logger(__name__)
|
||||
if limit < 1:
|
||||
raise ValueError("limit must be >= 1")
|
||||
if window_s < 1:
|
||||
@@ -217,10 +197,9 @@ class RateLimiter:
|
||||
self._limit = limit
|
||||
self._window_s = window_s
|
||||
self._clock = clock or SystemClock()
|
||||
# register_script() is local: it hashes the source and returns a callable. No round
|
||||
# trip here, and none wasted at import. The first call sends EVALSHA; redis-py
|
||||
# catches NOSCRIPT and replays it as EVAL, which is why a restarted Redis costs one
|
||||
# extra command once rather than an outage.
|
||||
# register_script() is local — it hashes the source and returns a callable, with no
|
||||
# round trip. The first call sends EVALSHA; redis-py catches NOSCRIPT and replays it
|
||||
# as EVAL, so a restarted Redis costs one extra command rather than an outage.
|
||||
self._script: AsyncScript = r.register_script(_RATE_LIMIT_LUA)
|
||||
|
||||
def _window(self) -> tuple[int, datetime]:
|
||||
@@ -234,9 +213,8 @@ class RateLimiter:
|
||||
async def check(self, team: str) -> RateLimitResult:
|
||||
"""Count one request against `team`. Never raises.
|
||||
|
||||
On any Redis error: allow, log loudly, count it. The metric is the point — a
|
||||
limiter that fails open silently is indistinguishable from no limiter at all, and
|
||||
you find out which one you shipped during the incident.
|
||||
On any Redis error: allow, log loudly, count it. The metric is the point — a limiter
|
||||
that fails open silently is indistinguishable from no limiter at all.
|
||||
"""
|
||||
window, reset_at = self._window()
|
||||
key = f"rl:{team}:{window}"
|
||||
@@ -245,16 +223,17 @@ class RateLimiter:
|
||||
allowed, remaining = await self._script(keys=[key], args=[self._limit, self._window_s])
|
||||
except _REDIS_DOWN:
|
||||
REDIS_ERRORS.labels(op="ratelimit").inc()
|
||||
_log.warning(
|
||||
self._log.warning(
|
||||
"rate limiter degraded: redis unavailable, failing OPEN",
|
||||
exc_info=True,
|
||||
extra={"team": team},
|
||||
team=team,
|
||||
)
|
||||
return RateLimitResult(
|
||||
allowed=True,
|
||||
limit=self._limit,
|
||||
remaining=self._limit,
|
||||
reset_at=reset_at,
|
||||
checked_at=self._clock.now(),
|
||||
degraded=True,
|
||||
)
|
||||
return RateLimitResult(
|
||||
@@ -262,6 +241,7 @@ class RateLimiter:
|
||||
limit=self._limit,
|
||||
remaining=int(remaining),
|
||||
reset_at=reset_at,
|
||||
checked_at=self._clock.now(),
|
||||
)
|
||||
|
||||
|
||||
@@ -279,22 +259,20 @@ class IdempotencyStoreProto(Protocol):
|
||||
class IdempotencyStore:
|
||||
"""`SET NX EX`. Maps an `Idempotency-Key` to the instance UUID it created.
|
||||
|
||||
Claimed BEFORE the DB transaction, so the marker exists before the row it names. The
|
||||
inversion matters: claim after the commit and a crash in between leaves a created
|
||||
instance with no marker, and the client's retry creates a second one.
|
||||
Claimed BEFORE the DB transaction: claim after the commit and a crash in between leaves
|
||||
a created instance with no marker, so the client's retry creates a second one. Claiming
|
||||
first has the opposite hole — a marker naming an instance that never committed, and the
|
||||
retry is told "already done" about nothing — and that is the better hole, because the
|
||||
client polls the id, gets a 404 and retries with a fresh key.
|
||||
|
||||
Claiming first has its own hole — a crash after the claim and before the commit leaves
|
||||
a marker pointing at an instance that never existed, and the retry is told "already
|
||||
done" about nothing. It is the better hole: the client polls the id, gets a 404, and
|
||||
retries with a fresh key. The alternative loses money to a duplicate Elasticsearch.
|
||||
|
||||
And neither hole is load-bearing, because `instances.release_name` is UNIQUE and
|
||||
deterministic from (team, service_type, id). **That constraint is the guarantee.** This
|
||||
class only saves the round trip to find out.
|
||||
Neither hole is load-bearing. `instances.release_name` is UNIQUE and deterministic from
|
||||
(team, service_type, id); that constraint is the guarantee and this only saves a round
|
||||
trip.
|
||||
"""
|
||||
|
||||
def __init__(self, r: Redis, ttl_s: int = 86400) -> None:
|
||||
"""A day is the window a client might reasonably retry in; then the key is garbage."""
|
||||
self._log = obs.get_logger(__name__)
|
||||
if ttl_s < 1:
|
||||
raise ValueError("ttl_s must be >= 1")
|
||||
self._r = r
|
||||
@@ -303,13 +281,11 @@ class IdempotencyStore:
|
||||
async def claim(self, key: str, instance_id: UUID) -> UUID | None:
|
||||
"""Try to bind `key` to `instance_id`. Never raises.
|
||||
|
||||
`None` from the happy path means "you won, go create it". `None` from a Redis
|
||||
failure means the same thing — the caller creates, and the UNIQUE constraint
|
||||
catches an actual duplicate. Degrading to "create it" is safe *only* because that
|
||||
constraint exists; without it this would have to fail closed.
|
||||
`None` means "you won, go create it", and a Redis failure returns the same thing —
|
||||
the caller creates and the UNIQUE constraint catches a real duplicate. Degrading to
|
||||
"create it" is safe only because that constraint exists.
|
||||
|
||||
One command when we win, which is the common case and the one the budget is sized
|
||||
for. Two when we lose: the loser pays a GET, and losers are rare by definition.
|
||||
One command when we win, two when we lose: the loser pays a GET, and losers are rare.
|
||||
"""
|
||||
redis_key = f"idem:{key}"
|
||||
try:
|
||||
@@ -321,20 +297,20 @@ class IdempotencyStore:
|
||||
existing = await self._r.get(redis_key)
|
||||
except _REDIS_DOWN:
|
||||
REDIS_ERRORS.labels(op="idempotency").inc()
|
||||
_log.warning(
|
||||
self._log.warning(
|
||||
"idempotency degraded: redis unavailable, falling through to the DB constraint",
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
if existing is None:
|
||||
# The key expired between the SET and the GET. Vanishingly rare, and the honest
|
||||
# answer is "no winner recorded" — let the caller create and let Postgres decide.
|
||||
# The key expired between the SET and the GET. The honest answer is "no winner
|
||||
# recorded" — let the caller create and let Postgres decide.
|
||||
return None
|
||||
try:
|
||||
return UUID(_as_text(existing))
|
||||
except ValueError:
|
||||
_log.warning("idempotency key holds a non-UUID value; ignoring it")
|
||||
self._log.warning("idempotency key holds a non-UUID value; ignoring it")
|
||||
return None
|
||||
|
||||
|
||||
@@ -360,18 +336,17 @@ class InstanceCacheProto(Protocol):
|
||||
class InstanceCache:
|
||||
"""Cache-aside for `GET /v1/instances/{id}`. TTL 30s.
|
||||
|
||||
Hit costs one command, miss costs two (the GET, then the SET after Postgres answers).
|
||||
That is ~1 per read at any useful hit rate, which is what keeps a read-heavy poller
|
||||
inside the budget.
|
||||
A hit costs one command and a miss two, so ~1 per read at any useful hit rate — which is
|
||||
what keeps a read-heavy poller inside the budget.
|
||||
|
||||
The TTL is short on purpose and is the actual correctness argument. `invalidate()` on
|
||||
every state transition is the fast path, not the guarantee: the worker can crash
|
||||
between the UPDATE and the DEL, and then the cache is wrong. Thirty seconds bounds how
|
||||
wrong. Trusting the invalidation instead — and raising the TTL to an hour — is how a
|
||||
deleted instance stays `ready` in the API for an hour.
|
||||
The short TTL is the correctness argument. `invalidate()` on every state transition is
|
||||
the fast path, not the guarantee: a worker can crash between the UPDATE and the DEL, and
|
||||
30 seconds bounds how wrong the cache gets. Trusting invalidation and raising the TTL to
|
||||
an hour is how a deleted instance stays `ready` in the API for an hour.
|
||||
"""
|
||||
|
||||
def __init__(self, r: Redis, ttl_s: int = 30) -> None:
|
||||
self._log = obs.get_logger(__name__)
|
||||
if ttl_s < 1:
|
||||
raise ValueError("ttl_s must be >= 1")
|
||||
self._r = r
|
||||
@@ -384,24 +359,24 @@ class InstanceCache:
|
||||
async def get(self, instance_id: UUID) -> Instance | None:
|
||||
"""One GET. A miss, a Redis outage and a corrupt entry are all the same answer.
|
||||
|
||||
Which is the point: the caller writes `cache.get() or repo.get()` and has no branch
|
||||
for "Redis is broken", because there is nothing different to do about it.
|
||||
The caller writes `cache.get() or repo.get()` and has no branch for "Redis is
|
||||
broken", because there is nothing different to do about it.
|
||||
"""
|
||||
try:
|
||||
REDIS_COMMANDS.labels(op="cache_get").inc()
|
||||
raw = await self._r.get(self._key(instance_id))
|
||||
except _REDIS_DOWN:
|
||||
REDIS_ERRORS.labels(op="cache_get").inc()
|
||||
_log.warning("cache read degraded: redis unavailable, falling through to Postgres")
|
||||
self._log.warning("cache read degraded: redis unavailable, falling through to Postgres")
|
||||
return None
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return Instance.model_validate_json(raw)
|
||||
except ValidationError:
|
||||
# A model change deployed over a warm cache. Treat it as a miss and let the TTL
|
||||
# take the old shape out. Not an error: the truth is in Postgres either way.
|
||||
_log.info("cache entry failed validation; treating as a miss")
|
||||
# A model change deployed over a warm cache. A miss, not an error — the TTL
|
||||
# takes the old shape out and the truth is in Postgres either way.
|
||||
self._log.info("cache entry failed validation; treating as a miss")
|
||||
return None
|
||||
|
||||
async def put(self, inst: Instance) -> None:
|
||||
@@ -411,17 +386,17 @@ class InstanceCache:
|
||||
await self._r.set(self._key(inst.id), inst.model_dump_json(), ex=self._ttl_s)
|
||||
except _REDIS_DOWN:
|
||||
REDIS_ERRORS.labels(op="cache_put").inc()
|
||||
_log.warning("cache write degraded: redis unavailable")
|
||||
self._log.warning("cache write degraded: redis unavailable")
|
||||
|
||||
async def invalidate(self, instance_id: UUID) -> None:
|
||||
"""One DEL. Called by the worker inside the code path that writes the state.
|
||||
|
||||
Inside that path, not after it and not from a subscriber: an invalidation that can
|
||||
be skipped by an early return is an invalidation that will be.
|
||||
Inside that path, not after it and not from a subscriber: an invalidation an early
|
||||
return can skip is an invalidation that will be skipped.
|
||||
"""
|
||||
try:
|
||||
REDIS_COMMANDS.labels(op="cache_del").inc()
|
||||
await self._r.delete(self._key(instance_id))
|
||||
except _REDIS_DOWN:
|
||||
REDIS_ERRORS.labels(op="cache_del").inc()
|
||||
_log.warning("cache invalidate degraded: redis unavailable; entry expires within the TTL")
|
||||
self._log.warning("cache invalidate degraded: redis unavailable; entry expires within the TTL")
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""One temp YAML file, written from a dict and removed on exit.
|
||||
|
||||
helm and kubectl both take their input as a file rather than on the command line: `--set`
|
||||
and inline manifests each have their own escaping grammar, and tenant-shaped values would
|
||||
have to be escaped into it. Serialising YAML to a file sidesteps the grammar entirely. Both
|
||||
adapters needed the same throwaway-file dance, so it lives here once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@contextmanager
|
||||
def yaml_tempfile(payload: dict[str, Any], *, prefix: str, name: str) -> Iterator[Path]:
|
||||
"""Yield a path to `name` inside a fresh temp dir, holding `payload` as YAML.
|
||||
|
||||
The whole dir is removed on exit, `ignore_errors` so a cleanup race never masks the real
|
||||
error from the block.
|
||||
"""
|
||||
tmpdir = tempfile.mkdtemp(prefix=prefix)
|
||||
try:
|
||||
path = Path(tmpdir) / name
|
||||
path.write_text(yaml.safe_dump(payload, default_flow_style=False), encoding="utf-8")
|
||||
yield path
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
@@ -10,9 +10,10 @@ import yaml
|
||||
from pydantic import ValidationError
|
||||
|
||||
from svcforge_core.domain.models import CatalogEntry
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
|
||||
|
||||
class CatalogError(Exception):
|
||||
class CatalogError(SvcforgeError):
|
||||
"""A catalog file could not be parsed or validated.
|
||||
|
||||
`key` names the offending service type, or None when the failure is file-level.
|
||||
|
||||
@@ -55,6 +55,16 @@ class CatalogEntry(BaseModel):
|
||||
# Bypass tenant maintenance windows for this entry's upgrades. Defaults False: a
|
||||
# normal version bump waits for 03:00 Sunday; a CVE with a public exploit does not.
|
||||
security: bool = False
|
||||
# Chart values that apply to every size of this entry, merged UNDER the size's own
|
||||
# replicas and resources. This is where a chart's own knobs go — `global.imageRegistry`
|
||||
# to keep image pulls off a rate-limited registry, a storageClass, a disabled subchart.
|
||||
# Without it the only expressible values are replicas and resources, and anything else
|
||||
# a chart needs is a code change, which is the line between a platform and a script.
|
||||
#
|
||||
# Operator-supplied, never tenant-supplied: the catalog is a file only the platform team
|
||||
# edits. A tenant reaching this dict would be handing arbitrary helm values — image
|
||||
# references, securityContext, hostPath mounts — straight to the cluster.
|
||||
values: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class Instance(BaseModel):
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
from enum import StrEnum
|
||||
from typing import Final
|
||||
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
|
||||
|
||||
class InstanceState(StrEnum):
|
||||
"""Lifecycle of a provisioned service instance."""
|
||||
@@ -15,7 +17,7 @@ class InstanceState(StrEnum):
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class IllegalTransition(Exception):
|
||||
class IllegalTransition(SvcforgeError):
|
||||
"""Raised by transition() when cur -> nxt is not in LEGAL."""
|
||||
|
||||
|
||||
|
||||
@@ -19,11 +19,13 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
|
||||
CRON_FIELDS = 5
|
||||
SEPARATOR = "|"
|
||||
|
||||
|
||||
class BadWindow(ValueError):
|
||||
class BadWindow(SvcforgeError, ValueError):
|
||||
"""A maintenance window spec is not a 5-field cron plus a known IANA zone."""
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""The one base class every svcforge-raised exception shares.
|
||||
|
||||
Without it, "the cluster failed" has to be caught as `except Exception`, which also swallows
|
||||
the `AttributeError` from a typo three frames down. One is retried and the other is a bug
|
||||
that must dead-letter loudly; a single root makes that expressible in one clause.
|
||||
|
||||
Subclasses keep their stdlib base too (`HelmError(SvcforgeError, RuntimeError)`), so code
|
||||
written against `except RuntimeError` keeps working. `SvcforgeError` comes first in the MRO.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class SvcforgeError(Exception):
|
||||
"""Root of every exception svcforge raises on purpose.
|
||||
|
||||
Catch this to mean "an operation svcforge models failed". Anything not deriving from it
|
||||
that escapes a handler is, by definition, a programming error rather than a modelled one.
|
||||
"""
|
||||
@@ -13,6 +13,7 @@ Run: python -m svcforge_core.migrate
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -20,7 +21,30 @@ import psycopg
|
||||
|
||||
from svcforge_core.settings import load_settings
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parents[3] / "migrations"
|
||||
|
||||
def _migrations_dir() -> Path:
|
||||
"""Where the .sql files live, in a way that survives being installed as a wheel.
|
||||
|
||||
`Path(__file__).parents[3] / "migrations"` works only for the editable dev install,
|
||||
where the package really does sit three levels under the repo root. The images install
|
||||
a built wheel into site-packages (deliberately — an editable install in an image ships
|
||||
a path, not a package), so the same expression resolves to
|
||||
`/app/.venv/lib/migrations`, which does not exist. The migration Job then finds zero
|
||||
files and exits 1, and the Helm pre-upgrade hook fails on every single sync.
|
||||
|
||||
That failure is invisible in dev and in CI, because both run editable. It only appears
|
||||
the first time you deploy — which is the worst time to discover it.
|
||||
|
||||
So: the image sets SVCFORGE_MIGRATIONS_DIR and copies the directory in; the repo
|
||||
checkout falls back to the path relative to this file.
|
||||
"""
|
||||
env = os.getenv("SVCFORGE_MIGRATIONS_DIR")
|
||||
if env:
|
||||
return Path(env)
|
||||
return Path(__file__).resolve().parents[3] / "migrations"
|
||||
|
||||
|
||||
MIGRATIONS_DIR = _migrations_dir()
|
||||
|
||||
# Advisory lock: two Jobs racing (a retried hook, a hand-run) must not both apply DDL.
|
||||
# Session-scoped, so this needs the session pooler (5432), not pgbouncer (6543).
|
||||
|
||||
@@ -1,33 +1,27 @@
|
||||
"""Logs, traces, metrics. One setup() call, made once, before anything else.
|
||||
|
||||
Three libraries, one module, because the three are one decision. A log line without the
|
||||
trace id it belongs to is a log line you cannot join to anything; a span without the
|
||||
`instance_id` the request is about is a span you cannot search for. They are wired here
|
||||
together so that no service can configure two of the three and ship.
|
||||
Three libraries in one module because they are one decision: a log line without its trace
|
||||
id joins to nothing, and a span without the `instance_id` cannot be searched for. Wiring
|
||||
them together here stops a service configuring two of the three and shipping.
|
||||
|
||||
The three things that make this module worth reading:
|
||||
Three things worth knowing:
|
||||
|
||||
1. **Context does not cross a queue.** A trace is a chain of parent/child span contexts
|
||||
passed in-process or over a wire header. `POST /v1/instances` inserts a row and
|
||||
returns; the worker picks that row up ninety seconds later in a different pod. Nothing
|
||||
carries the context across — unless we carry it ourselves. So: `inject_traceparent()`
|
||||
at enqueue, a `traceparent` column, `context_from_traceparent()` at claim. Two
|
||||
disconnected traces in Tempo is the symptom of skipping this.
|
||||
1. **Context does not cross a queue.** `POST /v1/instances` inserts a row and returns; the
|
||||
worker picks it up ninety seconds later in another pod, with no ambient context. So:
|
||||
`inject_traceparent()` at enqueue, a `traceparent` column, `context_from_traceparent()`
|
||||
at claim. Two disconnected traces in Tempo is the symptom of skipping this.
|
||||
|
||||
2. **Histogram buckets are a domain decision.** prometheus_client's defaults top out at
|
||||
10 seconds because they were chosen for HTTP handlers. A provision is `helm --wait` on
|
||||
a StatefulSet: minutes. With the defaults every observation lands in `+Inf`,
|
||||
`histogram_quantile` interpolates inside a bucket that spans 10s→infinity, and the p95
|
||||
it prints is a number with no relationship to reality. The buckets below are sized for
|
||||
what is being measured.
|
||||
2. **Histogram buckets are a domain decision.** prometheus_client's defaults were chosen
|
||||
for HTTP handlers and top out at 10s; a provision is `helm --wait` on a StatefulSet, so
|
||||
every observation lands in `+Inf` and the p95 is interpolated inside a bucket spanning
|
||||
10s→infinity. The buckets below are sized for what is measured.
|
||||
|
||||
3. **One process per pod.** prometheus_client keeps its registry in process memory. Run
|
||||
`uvicorn --workers 4` and Prometheus scrapes whichever of the four children the socket
|
||||
happens to hand it, so counters appear to jump backwards. There are two fixes:
|
||||
`PROMETHEUS_MULTIPROC_DIR` + `MultiProcessCollector` (a shared mmap directory, a
|
||||
gauge-mode decision at every call site, and dead files to garbage-collect after every
|
||||
crash), or one process per pod and scale with replicas. This repo takes the second.
|
||||
`PROMETHEUS_MULTIPROC_DIR` is deliberately not set, and nothing here reads it.
|
||||
3. **One process per pod.** prometheus_client keeps its registry in process memory, so
|
||||
`uvicorn --workers 4` has Prometheus scraping whichever child the socket hands it and
|
||||
counters appear to jump backwards. The alternative fix — `PROMETHEUS_MULTIPROC_DIR` and
|
||||
`MultiProcessCollector` — costs a shared mmap directory, a gauge-mode decision at every
|
||||
call site, and dead files to collect after every crash. This repo scales with replicas
|
||||
instead; nothing here reads that variable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -52,9 +46,8 @@ if TYPE_CHECKING:
|
||||
# --- Metrics ----------------------------------------------------------------------------
|
||||
#
|
||||
# Module level, created exactly once at import. A second registration of the same name
|
||||
# against the default registry raises ValueError, which is a feature: it turns "two modules
|
||||
# each defined their own copy of this counter" into an ImportError at startup instead of a
|
||||
# metric that silently reports half the truth.
|
||||
# raises ValueError, which turns "two modules each defined their own copy" into a startup
|
||||
# failure instead of a metric that silently reports half the truth.
|
||||
|
||||
TASKS_CLAIMED = Counter(
|
||||
"svcforge_tasks_claimed_total",
|
||||
@@ -62,18 +55,25 @@ TASKS_CLAIMED = Counter(
|
||||
["kind"],
|
||||
)
|
||||
|
||||
TASKS_FAILED = Counter(
|
||||
"svcforge_tasks_failed_total",
|
||||
"Tasks that exhausted their attempts and went to 'failed'.",
|
||||
TASK_ATTEMPTS_FAILED = Counter(
|
||||
"svcforge_task_attempts_failed_total",
|
||||
"Task ATTEMPTS that raised. Not the same as tasks that dead-lettered: one task that "
|
||||
"succeeds on its third try increments this twice. Alert on increase(), not on the raw "
|
||||
"total, or ordinary transient retries page you.",
|
||||
["kind"],
|
||||
)
|
||||
|
||||
TASKS_DEAD_LETTERED = Counter(
|
||||
"svcforge_tasks_dead_lettered_total",
|
||||
"Tasks that exhausted max_attempts and went to 'failed'. These need a human.",
|
||||
["kind"],
|
||||
)
|
||||
|
||||
PROVISION_TIME = Histogram(
|
||||
"svcforge_provision_duration_seconds",
|
||||
"Wall time of a provision task, claim to terminal report.",
|
||||
# NOT the defaults. See the module docstring: the defaults end at 10s and a provision
|
||||
# takes minutes. The top finite bucket is 1800 because helm's own --timeout is 600 and
|
||||
# a provision past thirty minutes is not slow, it is broken and belongs in +Inf.
|
||||
# Not the defaults — see the module docstring. The top finite bucket is 1800 because
|
||||
# helm's own --timeout is 600, so a provision past thirty minutes belongs in +Inf.
|
||||
buckets=(10, 30, 60, 120, 300, 600, 1800, float("inf")),
|
||||
)
|
||||
|
||||
@@ -97,6 +97,9 @@ RECONCILER_LAST_TICK = Gauge(
|
||||
|
||||
_TRACER_NAME = "svcforge"
|
||||
|
||||
# Set by setup(); re-bound by bind_task_context after it clears the context.
|
||||
_service_name: str = "svcforge"
|
||||
|
||||
# setup() is idempotent because it is called from three entrypoints and from tests, and
|
||||
# because configuring structlog twice silently discards the first configuration while
|
||||
# adding a second stdout handler to the root logger — every line then prints twice.
|
||||
@@ -112,9 +115,9 @@ def _add_trace_ids(
|
||||
) -> structlog.typing.EventDict:
|
||||
"""Stamp the active trace/span id onto the line, if there is one.
|
||||
|
||||
This is the join key. Without it, "find the logs for this trace" is a full-text search
|
||||
over a time window and a guess; with it, it is one query. Hex-formatted to the widths
|
||||
the W3C spec uses, so the value pasted from Tempo matches the value in Loki.
|
||||
The join key: without it, "find the logs for this trace" is a full-text search over a
|
||||
time window and a guess. Hex-formatted to the W3C widths, so a value pasted from Tempo
|
||||
matches the value in Loki.
|
||||
"""
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context()
|
||||
@@ -127,10 +130,9 @@ def _add_trace_ids(
|
||||
def setup(service_name: str, settings: Settings) -> None:
|
||||
"""Configure structlog, the tracer provider, and the metric registry. Idempotent.
|
||||
|
||||
Called once from each service's entrypoint, before anything else — "before anything
|
||||
else" because any logger bound before this runs keeps the default configuration
|
||||
(`cache_logger_on_first_use`), and a module-level `log = structlog.get_logger()` in an
|
||||
import that lands first will print unstructured text forever.
|
||||
Called once from each service's entrypoint, before anything else: a logger bound before
|
||||
this runs keeps the default configuration (`cache_logger_on_first_use`), so a
|
||||
module-level `log = structlog.get_logger()` prints unstructured text forever.
|
||||
"""
|
||||
global _configured # process-wide config is process-wide state
|
||||
if _configured:
|
||||
@@ -144,9 +146,9 @@ def setup(service_name: str, settings: Settings) -> None:
|
||||
def _setup_logging(service_name: str, settings: Settings) -> None:
|
||||
"""structlog + stdlib logging, both rendering JSON to stdout through one handler.
|
||||
|
||||
The stdlib half is not optional. `psycopg`, `httpx`, `uvicorn` and the OTEL SDK all log
|
||||
through `logging`; without the ProcessorFormatter bridge below, their lines arrive as
|
||||
bare text on the same stdout and every one of them is a parse failure in the collector.
|
||||
The stdlib half is not optional: `psycopg`, `httpx`, `uvicorn` and the OTEL SDK all log
|
||||
through `logging`, and without the ProcessorFormatter bridge their lines arrive as bare
|
||||
text on the same stdout — a parse failure each in the collector.
|
||||
"""
|
||||
level = getattr(logging, settings.log_level.upper(), logging.INFO)
|
||||
|
||||
@@ -189,21 +191,24 @@ def _setup_logging(service_name: str, settings: Settings) -> None:
|
||||
|
||||
root = logging.getLogger()
|
||||
# Replace rather than append: basicConfig may already have run, and two handlers means
|
||||
# two copies of every line. stdout only — a container writes logs to stdout and the
|
||||
# collector tails them from there. A log file inside a pod is deleted with the pod.
|
||||
# two copies of every line. stdout only — a log file inside a pod dies with the pod.
|
||||
root.handlers = [handler]
|
||||
root.setLevel(level)
|
||||
|
||||
# Remembered so bind_task_context can restore it after clearing. Without it every line
|
||||
# emitted inside a task loses `service`, which is what tells worker output from
|
||||
# reconciler output.
|
||||
global _service_name
|
||||
_service_name = service_name
|
||||
structlog.contextvars.bind_contextvars(service=service_name)
|
||||
|
||||
|
||||
def _setup_tracing(service_name: str, settings: Settings) -> None:
|
||||
"""Set the global tracer provider, exporting over OTLP when an endpoint is configured.
|
||||
|
||||
Skipped entirely when something already set a provider: the API runs under
|
||||
`opentelemetry-instrument`, whose auto-instrumentation installs one before our
|
||||
`main()` is reached. Overwriting it drops the FastAPI and psycopg instrumentation's
|
||||
spans on the floor, and the SDK only logs a warning about it.
|
||||
Skipped when something already set one: the API runs under `opentelemetry-instrument`,
|
||||
which installs a provider before `main()` is reached. Overwriting it drops the FastAPI
|
||||
and psycopg spans on the floor, and the SDK only logs a warning.
|
||||
"""
|
||||
if isinstance(trace.get_tracer_provider(), TracerProvider):
|
||||
return
|
||||
@@ -214,7 +219,7 @@ def _setup_tracing(service_name: str, settings: Settings) -> None:
|
||||
exporter = _otlp_exporter(settings.otel_endpoint)
|
||||
if exporter is not None:
|
||||
# Batch, not Simple: SimpleSpanProcessor exports inline on span end, so every
|
||||
# helm span would block on a network round trip to the collector.
|
||||
# helm span would block on a round trip to the collector.
|
||||
provider.add_span_processor(BatchSpanProcessor(exporter))
|
||||
|
||||
trace.set_tracer_provider(provider)
|
||||
@@ -223,9 +228,9 @@ def _setup_tracing(service_name: str, settings: Settings) -> None:
|
||||
def _otlp_exporter(endpoint: str) -> Any | None: # noqa: ANN401 - one of two exporter classes
|
||||
"""The OTLP exporter, if the optional exporter package is installed.
|
||||
|
||||
Optional on purpose. In the cluster the API runs under `opentelemetry-instrument`,
|
||||
which brings its own exporter and configures it from `OTEL_EXPORTER_OTLP_*`. Making it
|
||||
a hard dependency of the shared library would mean every unit test imports gRPC.
|
||||
Optional on purpose: in the cluster the API runs under `opentelemetry-instrument`, which
|
||||
brings its own exporter configured from `OTEL_EXPORTER_OTLP_*`. As a hard dependency of
|
||||
the shared library it would make every unit test import gRPC.
|
||||
"""
|
||||
try:
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||
@@ -247,8 +252,8 @@ def get_logger(name: str) -> structlog.stdlib.BoundLogger:
|
||||
def tracer() -> trace.Tracer:
|
||||
"""The svcforge tracer. Manual spans wrap helm calls, and nothing else.
|
||||
|
||||
Everything else is auto-instrumented (FastAPI, psycopg). A hand-rolled span around a
|
||||
function that the SDK already wraps is a duplicated span and a maintenance cost.
|
||||
FastAPI and psycopg are auto-instrumented, and a hand-rolled span around something the
|
||||
SDK already wraps is a duplicate to maintain.
|
||||
"""
|
||||
return trace.get_tracer(_TRACER_NAME)
|
||||
|
||||
@@ -269,15 +274,16 @@ def start_metrics_server(port: int) -> None:
|
||||
def bind_task_context(instance_id: UUID, task_id: int, team: str) -> None:
|
||||
"""Bind the three keys every log line in a task must carry. Called at claim time.
|
||||
|
||||
`clear_contextvars()` first, and this is the whole reason the function exists rather
|
||||
than three `bind_contextvars` calls at the call site. A worker coroutine reuses its
|
||||
context across loop iterations; without the clear, task 41's `instance_id` is still
|
||||
bound when task 42 starts logging, and the log for the incident you are debugging
|
||||
names the wrong tenant. Contextvars are per-task in asyncio, which makes this safe
|
||||
under the concurrency semaphore: two handlers running at once do not see each other's.
|
||||
`clear_contextvars()` first, which is why this is a function rather than three
|
||||
`bind_contextvars` calls at the call site: a worker coroutine reuses its context across
|
||||
iterations, so without the clear, task 41's `instance_id` is still bound when task 42
|
||||
logs and the incident names the wrong tenant. Contextvars are per-task in asyncio, so
|
||||
two handlers under the concurrency semaphore do not see each other's.
|
||||
"""
|
||||
structlog.contextvars.clear_contextvars()
|
||||
structlog.contextvars.bind_contextvars(
|
||||
# Re-bound because the indiscriminate clear above took it; it is not per-task.
|
||||
service=_service_name,
|
||||
instance_id=str(instance_id),
|
||||
task_id=task_id,
|
||||
team=team,
|
||||
@@ -288,8 +294,7 @@ def inject_traceparent() -> str | None:
|
||||
"""Serialise the active span context to a W3C traceparent, for the tasks row.
|
||||
|
||||
None when there is no recording span — a task enqueued by the reconciler's own tick has
|
||||
no inbound request to be part of. Nullable column, nullable return: an untraced task is
|
||||
normal, not an error.
|
||||
no inbound request to belong to. Nullable column, nullable return.
|
||||
"""
|
||||
carrier: dict[str, str] = {}
|
||||
_propagator.inject(carrier)
|
||||
@@ -299,9 +304,9 @@ def inject_traceparent() -> str | None:
|
||||
def context_from_traceparent(traceparent: str | None) -> Context:
|
||||
"""Inverse of inject_traceparent. Used at claim to parent the worker span to the API's.
|
||||
|
||||
An empty Context for None or for a malformed value — `extract` does not raise on a
|
||||
traceparent that fails to parse, it returns the carrier's context unchanged, and the
|
||||
resulting span starts a new trace. A bad header must never fail a provision.
|
||||
An empty Context for None or for a malformed value: `extract` does not raise on an
|
||||
unparseable traceparent, it returns the carrier's context unchanged and the span starts
|
||||
a new trace. A bad header must never fail a provision.
|
||||
"""
|
||||
if not traceparent:
|
||||
return Context()
|
||||
|
||||
@@ -12,11 +12,15 @@ from psycopg import AsyncConnection
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
# The pool hands out dict-row connections because of `row_factory=dict_row` below. Say so
|
||||
# in the type system too, or every `row["attempts"]` in this codebase is a mypy error
|
||||
# against a bare `AsyncConnectionPool`, which resolves to tuple rows. The runtime was
|
||||
# always right; without these aliases the annotations quietly disagree with it, and the
|
||||
# fix people reach for is `# type: ignore`, which throws away the checking entirely.
|
||||
# The cap on error text written to `instances.error` and `tasks.last_error`. A helm failure
|
||||
# can emit megabytes and these columns are read by humans. Defined once so the two call
|
||||
# paths that feed the same columns agree.
|
||||
ERROR_MAX_CHARS = 2000
|
||||
|
||||
# The pool hands out dict-row connections because of `row_factory=dict_row` below, and the
|
||||
# type system has to say so: against a bare `AsyncConnectionPool`, which resolves to tuple
|
||||
# rows, every `row["attempts"]` is a mypy error. The reach-for fix is `# type: ignore`,
|
||||
# which throws away the checking entirely.
|
||||
type DictRow = dict[str, Any]
|
||||
type DictConnection = AsyncConnection[DictRow]
|
||||
type DictPool = AsyncConnectionPool[DictConnection]
|
||||
@@ -25,28 +29,25 @@ type DictPool = AsyncConnectionPool[DictConnection]
|
||||
def make_pool(dsn: str, min_size: int = 1, max_size: int = 5) -> DictPool:
|
||||
"""Construct the pool. Does NOT open it — the caller owns open/close.
|
||||
|
||||
`open=False` is deliberate: the constructor does zero I/O, so building a pool at
|
||||
import time and never opening it fails later as a PoolTimeout at first use, far
|
||||
from the cause. The caller (a FastAPI lifespan, a worker main) opens and closes it.
|
||||
`open=False` because the constructor does zero I/O: a pool built at import time and
|
||||
never opened fails later as a PoolTimeout at first use, far from the cause. The caller
|
||||
(a FastAPI lifespan, a worker main) opens and closes it.
|
||||
|
||||
kwargs are per-connection:
|
||||
Two per-connection kwargs matter:
|
||||
|
||||
* `prepare_threshold=None` — REQUIRED through pgbouncer in transaction mode.
|
||||
psycopg3 auto-prepares a statement after it sees it 5 times. pgbouncer may hand
|
||||
the next execution to a different backend, which has never heard of that prepared
|
||||
statement. Symptom: everything works for exactly five calls, then
|
||||
`prepared statement "_pg3_0" does not exist` — intermittent, only under
|
||||
concurrency, never in a unit test.
|
||||
|
||||
* `row_factory=dict_row` — rows arrive as dicts, so `Instance.model_validate(row)`
|
||||
works directly instead of unpacking tuples by position.
|
||||
* `prepare_threshold=None` — REQUIRED through pgbouncer in transaction mode. psycopg3
|
||||
auto-prepares a statement after five executions, and pgbouncer may hand the sixth to a
|
||||
backend that has never heard of it. Symptom: `prepared statement "_pg3_0" does not
|
||||
exist`, intermittent, only under concurrency, never in a unit test.
|
||||
* `row_factory=dict_row` — rows arrive as dicts, so `Instance.model_validate(row)` works
|
||||
without unpacking tuples by position.
|
||||
|
||||
Also gone on 6543: LISTEN/NOTIFY, session-level SET, cross-statement advisory locks.
|
||||
`SELECT ... FOR UPDATE SKIP LOCKED` inside one transaction is unaffected — which is
|
||||
exactly why the queue is built on it. Use the session pooler (5432) for migrations.
|
||||
`SELECT ... FOR UPDATE SKIP LOCKED` inside one transaction is unaffected, which is why
|
||||
the queue is built on it. Migrations use the session pooler (5432).
|
||||
|
||||
max_size is a database-capacity decision, not a throughput knob: the free tier has a
|
||||
small connection budget, and replicas multiply this number.
|
||||
small connection budget and replicas multiply this number.
|
||||
"""
|
||||
return AsyncConnectionPool(
|
||||
conninfo=dsn,
|
||||
|
||||
@@ -20,7 +20,7 @@ from svcforge_core.domain.models import Instance
|
||||
from svcforge_core.domain.states import InstanceState
|
||||
from svcforge_core.repo.db import DictPool
|
||||
|
||||
_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
|
||||
INSTANCE_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
|
||||
endpoint, error, expires_at, created_at, updated_at"""
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ class InstanceRepo:
|
||||
release_name, chart_version, endpoint, error, expires_at)
|
||||
values (%(id)s, %(team)s, %(service_type)s, %(size)s, %(state)s, %(namespace)s,
|
||||
%(release_name)s, %(chart_version)s, %(endpoint)s, %(error)s, %(expires_at)s)
|
||||
returning {_COLUMNS}""", # noqa: S608 - _COLUMNS is a module constant, not input
|
||||
returning {INSTANCE_COLUMNS}""", # noqa: S608 - INSTANCE_COLUMNS is a module constant, not input
|
||||
{
|
||||
"id": inst.id,
|
||||
"team": inst.team,
|
||||
@@ -80,7 +80,7 @@ class InstanceRepo:
|
||||
"""Fetch one instance owned by `team`. None if it does not exist OR is not theirs."""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
f"select {_COLUMNS} from instances where id = %s and team = %s", # noqa: S608
|
||||
f"select {INSTANCE_COLUMNS} from instances where id = %s and team = %s", # noqa: S608
|
||||
(id, team),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
@@ -90,7 +90,7 @@ class InstanceRepo:
|
||||
"""The team's instances, newest first."""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
f"""select {_COLUMNS} from instances
|
||||
f"""select {INSTANCE_COLUMNS} from instances
|
||||
where team = %s order by created_at desc limit %s""", # noqa: S608
|
||||
(team, limit),
|
||||
)
|
||||
@@ -129,14 +129,6 @@ class InstanceRepo:
|
||||
)
|
||||
return cur.rowcount == 1
|
||||
|
||||
async def set_error(self, id: UUID, error: str) -> None:
|
||||
"""Record a terminal failure message. Used when a task exhausts its attempts."""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"update instances set error = %s, state = %s, updated_at = now() where id = %s",
|
||||
(error[-2000:], InstanceState.FAILED.value, id),
|
||||
)
|
||||
|
||||
async def list_upgradable(
|
||||
self,
|
||||
service_type: str,
|
||||
@@ -153,8 +145,8 @@ class InstanceRepo:
|
||||
rollouts table, no state machine, no pause/resume CLI. You clear it with SQL.
|
||||
|
||||
`order by team = %(own_team)s desc` — your own instances upgrade first, so you are
|
||||
the tenant who finds out the chart is broken. Eating your own dog food is a
|
||||
`order by`, not a policy document.
|
||||
the tenant who finds out the chart is broken. Eating your own dog food is enforced
|
||||
by an `order by` rather than a policy document.
|
||||
|
||||
`limit %(max_in_flight)s` — a config value, not a scheduler. It stays at 1 until 1
|
||||
is too slow, and 1 is what makes the halt meaningful: the fleet stops after the
|
||||
@@ -165,7 +157,7 @@ class InstanceRepo:
|
||||
# and would otherwise type these rows as tuples.
|
||||
async with self._pool.connection() as conn, conn.cursor(row_factory=dict_row) as cur:
|
||||
await cur.execute(
|
||||
f"""select {_COLUMNS}, maintenance_window from instances
|
||||
f"""select {INSTANCE_COLUMNS}, maintenance_window from instances
|
||||
where state = 'ready'
|
||||
and service_type = %(service_type)s
|
||||
and chart_version <> %(catalog_version)s
|
||||
@@ -174,7 +166,7 @@ class InstanceRepo:
|
||||
where cv.service_type = %(service_type)s
|
||||
and cv.rollout_state = 'halted')
|
||||
order by team = %(own_team)s desc, created_at
|
||||
limit %(max_in_flight)s""", # noqa: S608 - _COLUMNS is a module constant, not input
|
||||
limit %(max_in_flight)s""", # noqa: S608 - INSTANCE_COLUMNS is a module constant, not input
|
||||
{
|
||||
"service_type": service_type,
|
||||
"catalog_version": catalog_version,
|
||||
|
||||
@@ -1,17 +1,14 @@
|
||||
"""The reconciler's SQL.
|
||||
|
||||
Why this file exists rather than the queries living in `services/reconciler/main.py`: the
|
||||
layer rule says transport knows nothing about SQL, and the reconciler is transport — a CLI
|
||||
entrypoint. It gets its own repo module rather than growing `InstanceRepo` and `TaskRepo`
|
||||
because everything here is a *sweep*: it reads rows nobody asked about and it writes an
|
||||
instance state and a task row in the same transaction. `InstanceRepo.update_state` owns its
|
||||
own connection by design, so the reconciler cannot get atomicity from it without reaching
|
||||
around the repo — which is the thing the layer rule exists to prevent.
|
||||
Its own module rather than queries in `services/reconciler/main.py` (transport knows no
|
||||
SQL) and rather than more methods on `InstanceRepo`/`TaskRepo`, because everything here is
|
||||
a *sweep*: it reads rows nobody asked about and writes an instance state and a task row in
|
||||
one transaction. `InstanceRepo.update_state` owns its own connection by design, so the
|
||||
reconciler cannot get that atomicity without reaching around the repo.
|
||||
|
||||
The recurring shape below is: lock the row, re-check the condition under the lock, act.
|
||||
The re-check is not paranoia about concurrency — the reconciler is a singleton. It is what
|
||||
makes the sweep idempotent against *itself*: a tick that crashes after the insert and
|
||||
before the commit must leave nothing behind, and the next tick must not double-enqueue.
|
||||
The recurring shape is: lock the row, re-check the condition under the lock, act. The
|
||||
re-check makes the sweep idempotent against *itself* — the reconciler is a singleton, but a
|
||||
tick that crashes before its commit must leave nothing behind for the next one to double.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -25,15 +22,12 @@ from psycopg import AsyncCursor
|
||||
from svcforge_core.domain.models import Instance, TaskKind, TaskState
|
||||
from svcforge_core.domain.states import InstanceState, transition
|
||||
from svcforge_core.obs import inject_traceparent
|
||||
from svcforge_core.repo.db import DictPool
|
||||
from svcforge_core.repo.db import ERROR_MAX_CHARS, DictPool
|
||||
from svcforge_core.repo.instances import INSTANCE_COLUMNS
|
||||
|
||||
_COLUMNS = """id, team, service_type, size, state, namespace, release_name, chart_version,
|
||||
endpoint, error, expires_at, created_at, updated_at"""
|
||||
|
||||
# A task nobody will ever run again. The idempotency guard on every enqueue below asks
|
||||
# "is one already outstanding?", and 'done'/'failed' are not outstanding: a failed
|
||||
# deprovision that exhausted its attempts must be re-enqueueable by the next sweep, or a
|
||||
# transient cluster outage would permanently strand the instance.
|
||||
# What "already outstanding" means to the idempotency guard on every enqueue below.
|
||||
# 'done'/'failed' are not outstanding: a deprovision that exhausted its attempts must be
|
||||
# re-enqueueable, or a transient cluster outage strands the instance permanently.
|
||||
_UNFINISHED = (TaskState.QUEUED.value, TaskState.RUNNING.value)
|
||||
|
||||
|
||||
@@ -48,9 +42,8 @@ class ReconcileRepo:
|
||||
async def queue_depth(self) -> int:
|
||||
"""Tasks waiting to be claimed.
|
||||
|
||||
Counts every `queued` row, not just the runnable ones (`run_after <= now()`). The
|
||||
alert on this gauge is `deriv(...) > 0` — "the backlog is growing" — and a backlog
|
||||
of tasks parked on backoff is exactly the backlog you want to see growing.
|
||||
Every `queued` row, not just the runnable ones. The alert is `deriv(...) > 0` — "the
|
||||
backlog is growing" — and tasks parked on backoff are part of that backlog.
|
||||
"""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("select count(*) as n from tasks where state = %s", (TaskState.QUEUED.value,))
|
||||
@@ -70,7 +63,7 @@ class ReconcileRepo:
|
||||
"""Every instance the DB believes is running. The drift check's expectation."""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
f"select {_COLUMNS} from instances where state = %s", # noqa: S608 - module constant
|
||||
f"select {INSTANCE_COLUMNS} from instances where state = %s", # noqa: S608 - module constant
|
||||
(InstanceState.READY.value,),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
@@ -79,9 +72,9 @@ class ReconcileRepo:
|
||||
async def known_releases(self) -> set[tuple[str, str]]:
|
||||
"""(release_name, namespace) for every instance row, in any state.
|
||||
|
||||
Any state, deliberately. An instance that is still `requested` has no release yet,
|
||||
but a worker may be installing it *right now* — treating it as unknown would
|
||||
report a healthy in-flight provision as an orphan on every tick.
|
||||
Any state, deliberately. A `requested` instance has no release yet, but a worker may
|
||||
be installing it right now, and treating it as unknown reports a healthy in-flight
|
||||
provision as an orphan.
|
||||
"""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("select release_name, namespace from instances")
|
||||
@@ -93,20 +86,16 @@ class ReconcileRepo:
|
||||
|
||||
None if the row moved, or if a provision is already outstanding.
|
||||
|
||||
The two-hop state change is the interesting part, and it is not busywork:
|
||||
The state change takes two hops. `LEGAL` has no `ready -> provisioning` edge — the
|
||||
tenant-visible lifecycle leaves `ready` only through `deleting` or `failed`, and
|
||||
drift is a failure — so it goes `ready -> failed -> provisioning`, both edges legal
|
||||
and asserted below rather than assumed. It has to land in `provisioning`, not
|
||||
`failed`: `handle_provision` finishes with a `provisioning -> ready` CAS, and given a
|
||||
`failed` row helm runs, the CAS matches nothing, and the instance sits in `failed`
|
||||
forever with a healthy release behind it.
|
||||
|
||||
* `LEGAL` has no `ready -> provisioning` edge. The tenant-visible lifecycle only
|
||||
leaves `ready` through `deleting` or `failed`, and drift is a failure — the
|
||||
service the tenant is paying for is gone. So: `ready -> failed -> provisioning`,
|
||||
both edges legal, asserted below by the domain function rather than assumed.
|
||||
* The row must land in `provisioning`, not `failed`, before the worker sees the
|
||||
task. `handle_provision` CASes `requested -> provisioning` best-effort and then
|
||||
CASes `provisioning -> ready` for real; hand it a `failed` row and helm runs, the
|
||||
final CAS matches nothing, and the instance sits in `failed` forever with a
|
||||
healthy release behind it.
|
||||
|
||||
Both hops and the insert are one transaction, so the row is never observable in the
|
||||
intermediate `failed` state and a crash mid-sweep leaves nothing half-done.
|
||||
Both hops and the insert are one transaction, so the intermediate `failed` is never
|
||||
observable and a crash mid-sweep leaves nothing half-done.
|
||||
"""
|
||||
async with self._pool.connection() as conn:
|
||||
async with conn.transaction(), conn.cursor() as cur:
|
||||
@@ -119,14 +108,14 @@ class ReconcileRepo:
|
||||
if await _has_unfinished(cur, instance_id, TaskKind.PROVISION):
|
||||
return None
|
||||
|
||||
# Assert the path through the state machine instead of trusting the SQL.
|
||||
# If someone edits LEGAL, this raises here rather than corrupting rows.
|
||||
# Assert the path through the state machine instead of trusting the SQL: an
|
||||
# edit to LEGAL raises here rather than corrupting rows.
|
||||
failed = transition(InstanceState.READY, InstanceState.FAILED)
|
||||
provisioning = transition(failed, InstanceState.PROVISIONING)
|
||||
|
||||
await cur.execute(
|
||||
"update instances set state = %s, error = %s, updated_at = now() where id = %s",
|
||||
(provisioning.value, reason[-2000:], instance_id),
|
||||
(provisioning.value, reason[-ERROR_MAX_CHARS:], instance_id),
|
||||
)
|
||||
return await _insert_task(cur, instance_id, TaskKind.PROVISION)
|
||||
|
||||
@@ -137,21 +126,19 @@ class ReconcileRepo:
|
||||
|
||||
Two populations, one query:
|
||||
|
||||
* `ready` and past `expires_at` — the TTL sweep proper. The whole reason a
|
||||
throwaway Elasticsearch does not become a permanent line on the cloud bill.
|
||||
* `deleting` with nothing to do the deleting — the API CASes to `deleting` and then
|
||||
* `ready` past `expires_at` — the TTL sweep, which is what stops a throwaway
|
||||
Elasticsearch becoming a permanent line on the cloud bill.
|
||||
* `deleting` with nothing doing the deleting — the API CASes to `deleting` and
|
||||
enqueues in a second statement, and a crash between the two leaves exactly this.
|
||||
That ordering is chosen *because* this sweep exists; the other order would leave
|
||||
a deprovision task pointing at a `ready` instance, and a worker would tear down a
|
||||
live service nobody asked to delete.
|
||||
That order is chosen because this sweep exists; the reverse would leave a
|
||||
deprovision task on a `ready` instance and tear down a live service.
|
||||
|
||||
Note the parentheses around the OR. Without them, `and not exists (...)` binds to
|
||||
the second branch alone and the query re-enqueues a deprovision for every deleting
|
||||
instance on every tick, forever.
|
||||
Note the parentheses around the OR: without them `and not exists (...)` binds to the
|
||||
second branch alone and every deleting instance is re-enqueued on every tick.
|
||||
"""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
f"""select {_COLUMNS} from instances i
|
||||
f"""select {INSTANCE_COLUMNS} from instances i
|
||||
where ((i.state = %(ready)s and i.expires_at < now()) or i.state = %(deleting)s)
|
||||
and not exists (
|
||||
select 1 from tasks t
|
||||
@@ -172,10 +159,10 @@ class ReconcileRepo:
|
||||
async def enqueue_deprovision(self, instance_id: UUID) -> int | None:
|
||||
"""CAS to `deleting` if needed, and enqueue the task. One transaction. None if moot.
|
||||
|
||||
The instance must be in `deleting` before the worker claims the task, for the same
|
||||
reason as `enqueue_reprovision`: `handle_deprovision` finishes with a
|
||||
`deleting -> deleted` CAS, and a `ready` row would make helm uninstall the release
|
||||
and the DB keep advertising an endpoint that no longer resolves.
|
||||
The instance must reach `deleting` before the worker claims the task, for the same
|
||||
reason as `enqueue_reprovision`: `handle_deprovision` ends with a `deleting ->
|
||||
deleted` CAS, and on a `ready` row helm uninstalls the release while the DB keeps
|
||||
advertising an endpoint that no longer resolves.
|
||||
"""
|
||||
async with self._pool.connection() as conn:
|
||||
async with conn.transaction(), conn.cursor() as cur:
|
||||
@@ -210,15 +197,13 @@ class ReconcileRepo:
|
||||
"""Enqueue an upgrade unless one is already outstanding. None if it is.
|
||||
|
||||
The guard is what keeps the fleet at `max_in_flight`. The work list is a query over
|
||||
`chart_version`, and that column is only written *after* helm reports success — so
|
||||
an instance stays on the work list for the entire duration of its own upgrade, and
|
||||
for the hours it spends parked waiting for its 03:00 window. Without this check the
|
||||
sweep enqueues one more upgrade for the same instance every 60 seconds, and
|
||||
`max_in_flight=1` becomes sixty tasks an hour against one release.
|
||||
`chart_version`, which is written only after helm reports success, so an instance
|
||||
stays on the list for the whole duration of its own upgrade and for the hours it
|
||||
spends parked waiting for its 03:00 window. Without the check, `max_in_flight=1`
|
||||
becomes sixty tasks an hour against one release.
|
||||
|
||||
`verify` counts as outstanding too: an upgrade whose verify has not reported is an
|
||||
upgrade still in progress, and re-enqueueing it would race the probe that decides
|
||||
whether the whole rollout halts.
|
||||
`verify` counts as outstanding too: re-enqueueing an upgrade whose verify has not
|
||||
reported would race the probe that decides whether the rollout halts.
|
||||
"""
|
||||
async with self._pool.connection() as conn:
|
||||
async with conn.transaction(), conn.cursor() as cur:
|
||||
@@ -234,9 +219,8 @@ async def _has_unfinished(
|
||||
) -> bool:
|
||||
"""Is a task of any of these kinds queued or running for this instance?
|
||||
|
||||
Takes the caller's cursor on purpose: the answer is only true for as long as the
|
||||
transaction that asked, and checking on a separate connection would be a check against
|
||||
a different snapshot than the insert that follows it.
|
||||
Takes the caller's cursor: the answer holds only for the asking transaction, and a
|
||||
separate connection would check a different snapshot than the insert that follows.
|
||||
"""
|
||||
await cur.execute(
|
||||
"""select 1 from tasks
|
||||
@@ -260,10 +244,9 @@ async def _insert_task(
|
||||
"""Insert one task in the caller's transaction, carrying the current trace context.
|
||||
|
||||
`traceparent` is written here rather than left to `TaskRepo.enqueue` because these rows
|
||||
are inserted inside a transaction the reconciler owns. Nothing propagates a trace
|
||||
through a table on its own — see `obs.inject_traceparent`. It is null when the sweep is
|
||||
not itself inside a span, which is fine and expected: a nullable column for an untraced
|
||||
task.
|
||||
go in inside a transaction the reconciler owns, and nothing propagates a trace through a
|
||||
table on its own (see `obs.inject_traceparent`). Null when the sweep is not itself inside
|
||||
a span, which is normal.
|
||||
"""
|
||||
await cur.execute(
|
||||
"""insert into tasks (instance_id, kind, run_after, traceparent)
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
"""The queue.
|
||||
"""The queue: a Postgres table, not Redis.
|
||||
|
||||
The queue is a Postgres table, not Redis. The reason is one sentence: a task and the
|
||||
instance state it describes must commit atomically. Split them across two stores and you
|
||||
own a distributed commit problem that has no winning move — the process can die between
|
||||
the two writes, and whichever you write first is the one that lies.
|
||||
|
||||
Everything else here follows from that.
|
||||
A task and the instance state it describes must commit atomically. Split across two stores
|
||||
that is a distributed commit problem with no winning move — the process can die between the
|
||||
two writes, and whichever went first is the one that lies. Everything here follows from that.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,36 +16,33 @@ from psycopg import AsyncConnection
|
||||
from svcforge_core.domain.backoff import next_attempt_at
|
||||
from svcforge_core.domain.models import Task, TaskKind
|
||||
from svcforge_core.domain.states import LEGAL, InstanceState
|
||||
from svcforge_core.obs import inject_traceparent
|
||||
from svcforge_core.repo.db import DictPool
|
||||
from svcforge_core.obs import TASKS_DEAD_LETTERED, inject_traceparent
|
||||
from svcforge_core.repo.db import ERROR_MAX_CHARS, DictPool
|
||||
|
||||
# Which states may legally become `failed`, derived from the domain's own table rather
|
||||
# than restated here. Without this guard the UPDATE below would happily move a `deleted`
|
||||
# instance to `failed` — a transition domain.transition() explicitly forbids, performed
|
||||
# by raw SQL that never asks it. The state machine has to be the same one everywhere, or
|
||||
# it is decoration.
|
||||
# Which states may legally become `failed`, derived from the domain's table rather than
|
||||
# restated here. Without this guard the UPDATE below would move a `deleted` instance to
|
||||
# `failed` — a transition domain.transition() forbids, performed by SQL that never asks it.
|
||||
_CAN_FAIL: Final[tuple[str, ...]] = tuple(
|
||||
state.value for state, allowed in LEGAL.items() if InstanceState.FAILED in allowed
|
||||
)
|
||||
|
||||
# The claim query. Do not "simplify" this into two statements.
|
||||
#
|
||||
# Postgres has no `UPDATE ... LIMIT`, so the row is chosen by a subquery. That subquery
|
||||
# takes a row lock (`for update`) and steps over rows other workers already hold
|
||||
# (`skip locked`) instead of blocking behind them — which is what makes N workers scale
|
||||
# instead of queueing single-file behind the oldest task.
|
||||
# Postgres has no `UPDATE ... LIMIT`, so a subquery picks the row. It takes a row lock
|
||||
# (`for update`) and steps over rows other workers hold (`skip locked`) instead of blocking
|
||||
# behind them, which is what lets N workers scale instead of queueing behind the oldest
|
||||
# task. Select-then-update as two statements leaves a gap where a second worker reads the
|
||||
# same id and both provision — small enough to miss in testing and hit in production.
|
||||
#
|
||||
# The whole thing is ONE statement on purpose. Select-then-update as two statements
|
||||
# leaves a gap in which a second worker reads the same id, and both provision. The gap
|
||||
# is small, which means you will not hit it in testing and will hit it in production.
|
||||
# The `with claimed as (...)` wrapper changes nothing about the locking: the UPDATE and its
|
||||
# subquery are still one statement, and a data-modifying CTE runs exactly once. The outer
|
||||
# SELECT only joins `instances.team` onto the claimed row so the worker can bind `team` to
|
||||
# its log context without a second round trip.
|
||||
#
|
||||
# The `with claimed as (...)` wrapper is the ONLY addition to the canonical form, and it
|
||||
# changes nothing about the locking: the UPDATE and its `for update skip locked` subquery
|
||||
# are still one statement, executed once. The outer SELECT only joins `instances.team`
|
||||
# onto the row that was already claimed, so the worker can bind `team` to its log context
|
||||
# before it has loaded anything. A data-modifying CTE runs exactly once and cannot claim
|
||||
# twice. The alternative — a second SELECT for the team — would be a second round trip per
|
||||
# task to fetch a column the database already had in hand.
|
||||
# LEFT join, not inner. The CTE's UPDATE has already taken effect when the outer select
|
||||
# runs, so an inner join matching nothing returns no row — `claim()` would report "queue
|
||||
# empty" for a task it just marked `running`, stranding it until the lease expires and
|
||||
# burning an attempt. `Task.team` is already `str | None`.
|
||||
_CLAIM_SQL = """
|
||||
with claimed as (
|
||||
update tasks set state='running', attempts=attempts+1, locked_by=%(worker)s, locked_at=now()
|
||||
@@ -62,7 +56,7 @@ with claimed as (
|
||||
returning *
|
||||
)
|
||||
select claimed.*, instances.team
|
||||
from claimed join instances on instances.id = claimed.instance_id;
|
||||
from claimed left join instances on instances.id = claimed.instance_id;
|
||||
"""
|
||||
|
||||
|
||||
@@ -81,16 +75,14 @@ class TaskRepo:
|
||||
) -> Task:
|
||||
"""Insert a task inside the CALLER's transaction.
|
||||
|
||||
Takes `conn` so the API can insert the instance and enqueue its provision task in
|
||||
one transaction. Rolling back must lose both, or you get an orphan task pointing
|
||||
at an instance that was never committed.
|
||||
Takes `conn` so the API can insert the instance and enqueue its provision task
|
||||
together. A rollback must lose both, or an orphan task points at an instance that
|
||||
was never committed.
|
||||
|
||||
The `traceparent` is captured here, at enqueue time, because this is the last
|
||||
moment the caller's span context still exists. Trace context does NOT survive a
|
||||
queue on its own: the worker picks the row up in a different process, minutes
|
||||
later, with no ambient context. Writing the W3C traceparent onto the row is the
|
||||
thread that lets the worker re-parent its span to the POST that caused it — the
|
||||
difference between one trace spanning API → queue → helm and two unrelated ones.
|
||||
`traceparent` is captured here because this is the last moment the caller's span
|
||||
context exists. Trace context does not survive a queue on its own — the worker picks
|
||||
the row up in another process minutes later — so writing the W3C traceparent onto
|
||||
the row is what lets it re-parent its span to the POST that caused it.
|
||||
"""
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
@@ -111,10 +103,8 @@ class TaskRepo:
|
||||
) -> int:
|
||||
"""Enqueue in its own transaction, returning the new task id.
|
||||
|
||||
For callers with nothing to commit alongside it — the reconciler, tests. The
|
||||
module specs disagree about enqueue's shape (Module 2 passes a conn, Module 4
|
||||
does not); rather than making `conn` optional and quietly hiding the transaction
|
||||
question, both callers get an honest method name.
|
||||
For callers with nothing to commit alongside it — the reconciler, tests. A separate
|
||||
method rather than an optional `conn`, which would hide the transaction question.
|
||||
"""
|
||||
async with self._pool.connection() as conn:
|
||||
task = await self.enqueue(conn, instance_id, kind, run_after)
|
||||
@@ -135,43 +125,58 @@ class TaskRepo:
|
||||
async def complete(
|
||||
self,
|
||||
task_id: int,
|
||||
worker_id: str,
|
||||
conn: AsyncConnection[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
"""Mark done. Pass `conn` to commit alongside the caller's instance update.
|
||||
) -> bool:
|
||||
"""Mark done, but only if this worker still owns the task. False if it does not.
|
||||
|
||||
Completing the task and recording what it accomplished belong in one transaction:
|
||||
commit them separately and a crash in between leaves a task marked done whose
|
||||
work never landed, or work that landed and will be redone.
|
||||
Pass `conn` to commit alongside the caller's instance update: completing the task
|
||||
and recording what it accomplished belong in one transaction, or a crash between
|
||||
them leaves a task marked done whose work never landed.
|
||||
|
||||
`and state='running' and locked_by=%s` is a lost-update guard with a real trigger. A
|
||||
worker that hangs past `lease_seconds` has its task requeued and re-claimed; when it
|
||||
returns, an unconditional UPDATE marks the task `done` while the new owner is still
|
||||
running it. The loser gets False and treats it as "someone else owns this", not an
|
||||
error.
|
||||
"""
|
||||
sql = "update tasks set state='done', locked_by=null where id = %s"
|
||||
sql = "update tasks set state='done', locked_by=null where id=%s and state='running' and locked_by=%s"
|
||||
if conn is not None:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(sql, (task_id,))
|
||||
return
|
||||
await cur.execute(sql, (task_id, worker_id))
|
||||
return cur.rowcount == 1
|
||||
async with self._pool.connection() as own, own.cursor() as cur:
|
||||
await cur.execute(sql, (task_id,))
|
||||
await cur.execute(sql, (task_id, worker_id))
|
||||
return cur.rowcount == 1
|
||||
|
||||
async def fail(self, task_id: int, err: str, max_attempts: int = 5) -> None:
|
||||
"""Retry with backoff, or give up.
|
||||
async def fail(self, task_id: int, err: str, worker_id: str, max_attempts: int = 5) -> bool:
|
||||
"""Retry with backoff, or give up. False if this worker no longer owns the task.
|
||||
|
||||
Under max_attempts: back to 'queued' with run_after pushed out by exponential
|
||||
backoff with full jitter. Jitter matters — a cluster-wide outage fails every task
|
||||
at once, and without it every worker retries in the same instant, forever.
|
||||
Under max_attempts: back to 'queued', run_after pushed out by exponential backoff
|
||||
with full jitter. Jitter matters — a cluster-wide outage fails every task at once,
|
||||
and without it every worker retries in the same instant, forever.
|
||||
|
||||
At max_attempts: 'failed', and the error is copied onto the instance so the tenant
|
||||
can see it. A dead-letter state, not an infinite retry: a task that cannot succeed
|
||||
must stop and become someone's problem.
|
||||
At max_attempts: 'failed', a dead-letter state rather than an infinite retry, and
|
||||
for a provision the error is copied onto the instance so the tenant can see it.
|
||||
|
||||
The ownership check is the same lost-lease guard as `complete`, and matters more
|
||||
here: a stale worker reporting failure would push a task the new owner is running
|
||||
back to `queued`, letting a third worker claim it.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
async with self._pool.connection() as conn:
|
||||
async with conn.transaction(), conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"select attempts, instance_id from tasks where id = %s for update",
|
||||
(task_id,),
|
||||
"""select attempts, instance_id, kind from tasks
|
||||
where id=%s and state='running' and locked_by=%s
|
||||
for update""",
|
||||
(task_id, worker_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return
|
||||
# Task gone, or the lease was stolen. Either way it is not ours to
|
||||
# report on, and writing here would corrupt the new owner's run.
|
||||
return False
|
||||
attempts = int(row["attempts"])
|
||||
instance_id = row["instance_id"]
|
||||
|
||||
@@ -181,32 +186,47 @@ class TaskRepo:
|
||||
set state='queued', locked_by=null, locked_at=null,
|
||||
last_error=%s, run_after=%s
|
||||
where id = %s""",
|
||||
(err[-2000:], next_attempt_at(attempts - 1, now=now), task_id),
|
||||
(err[-ERROR_MAX_CHARS:], next_attempt_at(attempts - 1, now=now), task_id),
|
||||
)
|
||||
return
|
||||
return True
|
||||
|
||||
await cur.execute(
|
||||
"""update tasks
|
||||
set state='failed', locked_by=null, locked_at=null, last_error=%s
|
||||
where id = %s""",
|
||||
(err[-2000:], task_id),
|
||||
)
|
||||
# `state = any(%s)` keeps this honest: a deprovision that exhausts its
|
||||
# retries against an already-deleted instance records nothing rather than
|
||||
# resurrecting it into `failed`.
|
||||
await cur.execute(
|
||||
"""update instances set error=%s, state=%s, updated_at=now()
|
||||
where id=%s and state = any(%s)""",
|
||||
(err[-2000:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
|
||||
(err[-ERROR_MAX_CHARS:], task_id),
|
||||
)
|
||||
# Dead-lettering the task is right for every kind; moving the INSTANCE to
|
||||
# `failed` is right only for provision, where nothing but a human recovers
|
||||
# it. For the other three the instance is still healthy and something else
|
||||
# owns recovery:
|
||||
# deprovision — stays `deleting`, which is what lets due_for_deprovision
|
||||
# re-enqueue it. `failed` drops it out of that query and
|
||||
# leaks the release forever.
|
||||
# upgrade — helm --atomic rolled back, so it is `ready` on the previous
|
||||
# version. check_version_drift retries next window; `failed`
|
||||
# would drop it off the upgrade work-list.
|
||||
# verify — handle_verify already halted the rollout; drift
|
||||
# re-provisions if the release vanished.
|
||||
# The dead-letter metric and its alert cover all four, so leaving the
|
||||
# instance alone loses no visibility.
|
||||
if row["kind"] == TaskKind.PROVISION.value:
|
||||
await cur.execute(
|
||||
"""update instances set error=%s, state=%s, updated_at=now()
|
||||
where id=%s and state = any(%s)""",
|
||||
(err[-ERROR_MAX_CHARS:], InstanceState.FAILED.value, instance_id, list(_CAN_FAIL)),
|
||||
)
|
||||
# Counted here, not in the worker: this is the only place that knows the
|
||||
# difference between "attempt 2 of 5 failed" and "this task is done trying".
|
||||
TASKS_DEAD_LETTERED.labels(kind=str(row["kind"])).inc()
|
||||
return True
|
||||
|
||||
async def reset_expired_leases(self, lease_seconds: int) -> int:
|
||||
"""Return tasks whose worker died back to the queue. Called by the reconciler.
|
||||
|
||||
No distributed lock survives a power cut. A worker that is SIGKILLed leaves
|
||||
`state='running'` and `locked_by` set with nobody running it, and that row would
|
||||
sit there forever. The lease is the only thing that recovers it: not a lock, a
|
||||
timeout. This is why `locked_at` exists.
|
||||
No distributed lock survives a power cut. A SIGKILLed worker leaves `state='running'`
|
||||
and `locked_by` set with nobody running it, and that row sits there forever. The
|
||||
lease is the only thing that recovers it, which is why `locked_at` exists.
|
||||
"""
|
||||
async with self._pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Shared asyncio scaffolding for the long-lived services.
|
||||
|
||||
The worker and the reconciler are both a loop that runs until SIGTERM. They wake immediately
|
||||
on shutdown rather than sleeping through it, and they install the same loop-safe signal
|
||||
handlers. Both lived in each service before; keeping one copy means the shutdown behaviour
|
||||
cannot drift between them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import signal
|
||||
|
||||
|
||||
async def sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
"""Sleep for `seconds`, but return the instant `stop` is set.
|
||||
|
||||
`await asyncio.sleep(seconds)` would make every SIGTERM cost up to `seconds` of
|
||||
Kubernetes waiting on terminationGracePeriod for nothing.
|
||||
"""
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(stop.wait(), timeout=seconds)
|
||||
|
||||
|
||||
def install_stop_signals(stop: asyncio.Event) -> None:
|
||||
"""Set `stop` on SIGTERM and SIGINT, loop-safely.
|
||||
|
||||
add_signal_handler, not signal.signal: the latter runs at an arbitrary bytecode boundary
|
||||
on whatever thread the C-level handler lands on, and the loop does not notice until its
|
||||
next timer fires — up to a full sleep interval away.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
@@ -23,6 +23,11 @@ class Settings(BaseSettings):
|
||||
frozen=True,
|
||||
)
|
||||
|
||||
# Anything other than "local" makes check_production() enforce. The chart sets it;
|
||||
# a laptop does not. Defaulting to "local" means a forgotten env var costs you a
|
||||
# refused dev shortcut, never an unauthenticated production API.
|
||||
environment: str = "local"
|
||||
|
||||
# --- Postgres -----------------------------------------------------------------
|
||||
# Transaction pooler (6543 on Supabase). Everything the services do at runtime.
|
||||
pg_dsn: PostgresDsn
|
||||
@@ -35,6 +40,9 @@ class Settings(BaseSettings):
|
||||
|
||||
# --- Redis (derived state only; never the source of truth) ---------------------
|
||||
redis_dsn: RedisDsn | None = None
|
||||
# Per-team request budget. Generous on purpose: this exists to stop one team's runaway
|
||||
# script from starving the others, not to meter usage.
|
||||
rate_limit_per_minute: int = Field(default=60, ge=1)
|
||||
|
||||
# --- API ----------------------------------------------------------------------
|
||||
jwks_url: str | None = None
|
||||
@@ -59,12 +67,8 @@ class Settings(BaseSettings):
|
||||
kubectl_bin: str = "kubectl"
|
||||
helm_timeout_s: float = Field(default=300.0, gt=0)
|
||||
|
||||
# --- CLI ----------------------------------------------------------------------
|
||||
# The CLI is an API client and nothing more. It gets a URL and a token; it does not
|
||||
# get a DSN, because the moment a human can reach the database directly, someone will
|
||||
# "just fix one row" and the state machine stops being true.
|
||||
api_url: str = "http://localhost:8000"
|
||||
api_token: str | None = None
|
||||
# The CLI's own settings live in `services/cli/main.py`, not here: this model requires
|
||||
# SVCFORGE_PG_DSN, and the CLI is an API client that must never hold one.
|
||||
|
||||
# --- Observability ------------------------------------------------------------
|
||||
log_level: str = "info"
|
||||
@@ -75,15 +79,35 @@ class Settings(BaseSettings):
|
||||
# just for /metrics. 9000 matches the chart's PodMonitor; change both or neither.
|
||||
metrics_port: int = Field(default=9000, ge=1, le=65535)
|
||||
|
||||
@property
|
||||
def runtime_dsn(self) -> str:
|
||||
"""The transaction-pooler DSN the services open their pool against, as a string.
|
||||
|
||||
A property so the three entrypoints do not each pick between `str(pg_dsn)` and
|
||||
`pg_dsn.unicode_string()`, which had drifted across the services.
|
||||
"""
|
||||
return str(self.pg_dsn)
|
||||
|
||||
@property
|
||||
def migration_dsn(self) -> str:
|
||||
"""Migrations need a session-mode connection; fall back to the runtime DSN locally."""
|
||||
return str(self.pg_dsn_session or self.pg_dsn)
|
||||
|
||||
def check_production(self) -> None:
|
||||
"""Refuse the dev escape hatches when they would matter."""
|
||||
"""Refuse the dev escape hatches outside local development. Call at startup.
|
||||
|
||||
A no-op unless `SVCFORGE_ENVIRONMENT` says otherwise, which is what makes it safe to
|
||||
call unconditionally — and unconditionally is the point. A version invoked only from
|
||||
a branch that already knew it was production never ran at all, and
|
||||
`SVCFORGE_AUTH_DISABLED=true` in prod would silently serve every unauthenticated
|
||||
request as team `platform`.
|
||||
"""
|
||||
if self.environment == "local":
|
||||
return
|
||||
if self.auth_disabled:
|
||||
raise ValueError("SVCFORGE_AUTH_DISABLED=true is refused outside local development")
|
||||
raise ValueError(
|
||||
f"SVCFORGE_AUTH_DISABLED=true is refused when SVCFORGE_ENVIRONMENT={self.environment!r}"
|
||||
)
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
-- 005_task_indexes.sql — index the lookups the control loops actually do.
|
||||
--
|
||||
-- Postgres does NOT auto-index the referencing side of a foreign key. `tasks.instance_id`
|
||||
-- had no index, and three hot paths look rows up by it:
|
||||
--
|
||||
-- * ReconcileRepo._has_unfinished — the idempotency guard on every enqueue, so once per
|
||||
-- candidate per 60s tick.
|
||||
-- * due_for_deprovision's `not exists` correlated subquery — once per instance per tick.
|
||||
-- * the ON DELETE CASCADE itself, on every instance delete.
|
||||
--
|
||||
-- `tasks` is append-only in practice (done/failed rows are never pruned by the app; the
|
||||
-- runbook prunes them by hand), so a sequential scan there gets slower every day the
|
||||
-- system runs. This is the index that stops a 60-second control loop degrading into a
|
||||
-- full table scan of all history.
|
||||
--
|
||||
-- Composite on (instance_id, kind, state) because _has_unfinished filters all three.
|
||||
create index tasks_by_instance on tasks (instance_id, kind, state);
|
||||
|
||||
-- `instances.state` is filtered by ready_instances, instance_counts, due_for_deprovision
|
||||
-- and list_upgradable — every reconciler check. Partial would not help: the sweeps look
|
||||
-- for different states, so the whole column earns its index.
|
||||
create index instances_by_state on instances (state);
|
||||
+14
-3
@@ -2,7 +2,7 @@
|
||||
name = "svcforge"
|
||||
version = "0.1.0"
|
||||
description = "X-as-a-Service control plane — reference implementation"
|
||||
requires-python = ">=3.12"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"svcforge-core",
|
||||
"fastapi>=0.115",
|
||||
@@ -54,7 +54,18 @@ svcforge-core = { path = "libs/svcforge_core", editable = true }
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 110
|
||||
target-version = "py312"
|
||||
# py313, while the project RUNS on 3.14. `target-version` controls the syntax the
|
||||
# formatter is allowed to EMIT, and at py314 it rewrites `except (A, B):` into PEP 758's
|
||||
# unparenthesized `except A, B:`.
|
||||
#
|
||||
# That rewrite is rejected here for two reasons. It is visually identical to Python 2's
|
||||
# `except E, name:`, which binds a variable rather than catching two types — a reader who
|
||||
# learned Python 2, or an LLM trained on it, reads the opposite of what it does. And it
|
||||
# makes the source a hard SyntaxError on 3.13 and earlier for no semantic gain, in a repo
|
||||
# whose job is to be read.
|
||||
#
|
||||
# Raise this only for syntax that earns its incompatibility.
|
||||
target-version = "py313"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B", "ANN", "S", "C4", "RUF"]
|
||||
@@ -91,7 +102,7 @@ exclude_dirs = [".venv", "tests"]
|
||||
|
||||
[tool.mypy]
|
||||
strict = true
|
||||
python_version = "3.12"
|
||||
python_version = "3.14"
|
||||
warn_unreachable = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
|
||||
+36
-7
@@ -22,21 +22,50 @@ set -euo pipefail
|
||||
SERVICES=(api worker reconciler)
|
||||
CHART_VALUES="deploy/chart/values.yaml"
|
||||
|
||||
# yq, pinned by digest. Not python+pyyaml: a yaml round-trip strips every comment in
|
||||
# values.yaml, and those comments are the only thing explaining why the digests are there.
|
||||
# yq edits in place and leaves the rest of the file alone.
|
||||
YQ_IMAGE="mikefarah/yq:4.44.6@sha256:b1d117c609ba990436ad1649299e2f6c378f62cb562caf30b6f2fb6144713422"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
cleanup() {
|
||||
rm -rf "${WORKDIR}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
yq() {
|
||||
docker run --rm -v "${PWD}:/work" -w /work -u "$(id -u):$(id -g)" "${YQ_IMAGE}" "$@"
|
||||
# yq as a checksum-pinned binary, NOT `docker run -v "$PWD:/work"`.
|
||||
#
|
||||
# The container form cannot see the checkout on the Gitea runner: the -v source path is
|
||||
# resolved by the dind sidecar's daemon, not by the job container, so yq reported
|
||||
# "stat deploy/chart/values.yaml: no such file or directory" for a file that plainly
|
||||
# exists. The same boundary silently broke helm and gitleaks.
|
||||
#
|
||||
# Not python+pyyaml: a yaml round-trip strips every comment in values.yaml, and those
|
||||
# comments are the only thing explaining why the digests are there. yq edits in place.
|
||||
YQ_VERSION="4.53.3"
|
||||
YQ_SHA256="578648e463a11c1b6db6010cbf41eafed6bee79466fcffa1bb446672cf7945ea"
|
||||
|
||||
ensure_yq() {
|
||||
if command -v yq >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
local arch
|
||||
case "$(uname -m)" in
|
||||
aarch64|arm64) arch=arm64 ;;
|
||||
x86_64) arch=amd64 ;;
|
||||
*) echo "!! unsupported arch $(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
curl -fsSL -o "${WORKDIR}/yq" \
|
||||
"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${arch}"
|
||||
# Only the arm64 digest is pinned; this runner is Ampere. On another arch the download
|
||||
# is still version-pinned, and the mismatch is reported rather than silently accepted.
|
||||
if [ "${arch}" = "arm64" ]; then
|
||||
echo "${YQ_SHA256} ${WORKDIR}/yq" | sha256sum -c -
|
||||
else
|
||||
echo "warning: no pinned checksum for ${arch}; version-pinned only" >&2
|
||||
fi
|
||||
chmod +x "${WORKDIR}/yq"
|
||||
PATH="${WORKDIR}:${PATH}"
|
||||
export PATH
|
||||
}
|
||||
|
||||
ensure_yq
|
||||
|
||||
echo "==> resolving digests for tag ${IMAGE_TAG}"
|
||||
for svc in "${SERVICES[@]}"; do
|
||||
image="${REGISTRY}/${IMAGE_NS}/svcforge-${svc}"
|
||||
|
||||
+56
-29
@@ -1,9 +1,9 @@
|
||||
"""Throwaway load generator. Enqueue N instances, watch the queue drain, print three numbers.
|
||||
|
||||
The point is not a benchmark. It is to find the ceiling on purpose, in a place where finding
|
||||
it is free, so that the number in RUNBOOK.md comes from an observation instead of a guess.
|
||||
The point is to find the ceiling on purpose, in a place where finding it is free, so that
|
||||
the number in RUNBOOK.md comes from an observation instead of a guess.
|
||||
|
||||
The ceiling you are looking for is arithmetic, not mysterious:
|
||||
The ceiling you are looking for is arithmetic:
|
||||
|
||||
total connections = (api_replicas + worker_replicas) x pool_max_size
|
||||
|
||||
@@ -23,20 +23,28 @@ import argparse
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from svcforge_core.domain.catalog import load_catalog
|
||||
from svcforge_core.settings import load_settings
|
||||
|
||||
_SERVICE_TYPE = "elasticsearch"
|
||||
|
||||
async def _seed_direct(dsn: str, count: int) -> float:
|
||||
|
||||
async def _seed_direct(dsn: str, count: int, chart_version: str) -> float:
|
||||
"""Insert `count` instances + provision tasks. Returns seconds taken.
|
||||
|
||||
--direct exists to separate two questions that a single POST run conflates: "how fast can
|
||||
the API accept work" and "how fast can workers drain it". Measure them apart or you will
|
||||
tune the wrong one.
|
||||
|
||||
`chart_version` comes from the catalog rather than a literal. Hardcoding it meant the
|
||||
seeded rows carried a version the catalog could not resolve, so every task failed fast
|
||||
and the drain measurement — the whole point of the script — timed the failure path.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
|
||||
@@ -46,9 +54,9 @@ async def _seed_direct(dsn: str, count: int) -> float:
|
||||
await cur.execute(
|
||||
"""insert into instances (id, team, service_type, size, state, namespace,
|
||||
release_name, chart_version)
|
||||
values (%s, 'loadtest', 'elasticsearch', 'small', 'requested',
|
||||
'tenant-loadtest', %s, '21.3.19')""",
|
||||
(iid, f"loadtest-elasticsearch-{str(iid)[:8]}"),
|
||||
values (%s, 'loadtest', %s, 'small', 'requested',
|
||||
'tenant-loadtest', %s, %s)""",
|
||||
(iid, _SERVICE_TYPE, f"loadtest-{_SERVICE_TYPE}-{str(iid)[:8]}", chart_version),
|
||||
)
|
||||
await cur.execute(
|
||||
"insert into tasks (instance_id, kind) values (%s, 'provision')",
|
||||
@@ -57,38 +65,49 @@ async def _seed_direct(dsn: str, count: int) -> float:
|
||||
return time.monotonic() - started
|
||||
|
||||
|
||||
async def _depth(dsn: str) -> dict[str, int]:
|
||||
async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row) as conn:
|
||||
cur = await conn.execute("select state, count(*) as n from tasks group by 1")
|
||||
return {str(r["state"]): int(r["n"]) for r in await cur.fetchall()}
|
||||
async def _depth(conn: psycopg.AsyncConnection[dict[str, Any]]) -> dict[str, int]:
|
||||
"""Task counts by state, on a connection the caller owns."""
|
||||
cur = await conn.execute("select state, count(*) as n from tasks group by 1")
|
||||
return {str(r["state"]): int(r["n"]) for r in await cur.fetchall()}
|
||||
|
||||
|
||||
async def _watch(dsn: str, timeout_s: float) -> None:
|
||||
"""Print queue depth once a second until it drains. The slope is the number you want."""
|
||||
"""Print queue depth once a second until it drains. The slope is the number you want.
|
||||
|
||||
One connection for the whole loop, held open. Reconnecting every second added a
|
||||
connection to the pooler on every tick of a script whose entire purpose is finding the
|
||||
connection ceiling — the measurement was perturbing the thing being measured.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
peak = 0
|
||||
print(f"{'t(s)':>6} {'queued':>7} {'running':>8} {'done':>6} {'failed':>7} slope/s")
|
||||
prev_done, prev_t = 0, started
|
||||
|
||||
while time.monotonic() - started < timeout_s:
|
||||
d = await _depth(dsn)
|
||||
queued, running = d.get("queued", 0), d.get("running", 0)
|
||||
done, failed = d.get("done", 0), d.get("failed", 0)
|
||||
peak = max(peak, queued + running)
|
||||
# autocommit: a held connection without it sits idle-in-transaction between polls, which
|
||||
# pins a snapshot on the pooler and is exactly the pathology this script hunts for.
|
||||
async with await psycopg.AsyncConnection.connect(dsn, row_factory=dict_row, autocommit=True) as conn:
|
||||
while time.monotonic() - started < timeout_s:
|
||||
d = await _depth(conn)
|
||||
queued, running = d.get("queued", 0), d.get("running", 0)
|
||||
done, failed = d.get("done", 0), d.get("failed", 0)
|
||||
peak = max(peak, queued + running)
|
||||
|
||||
now = time.monotonic()
|
||||
slope = (done - prev_done) / max(now - prev_t, 1e-9)
|
||||
prev_done, prev_t = done, now
|
||||
now = time.monotonic()
|
||||
slope = (done - prev_done) / max(now - prev_t, 1e-9)
|
||||
prev_done, prev_t = done, now
|
||||
|
||||
print(f"{now - started:6.1f} {queued:7d} {running:8d} {done:6d} {failed:7d} {slope:7.1f}")
|
||||
print(f"{now - started:6.1f} {queued:7d} {running:8d} {done:6d} {failed:7d} {slope:7.1f}")
|
||||
|
||||
if queued == 0 and running == 0:
|
||||
elapsed = now - started
|
||||
print(f"\ndrained in {elapsed:.1f}s peak depth {peak} throughput {done / elapsed:.1f} task/s")
|
||||
if failed:
|
||||
print(f"WARNING: {failed} tasks failed — the number above is not a clean drain")
|
||||
return
|
||||
await asyncio.sleep(1.0)
|
||||
if queued == 0 and running == 0:
|
||||
elapsed = now - started
|
||||
print(
|
||||
f"\ndrained in {elapsed:.1f}s peak depth {peak} "
|
||||
f"throughput {done / elapsed:.1f} task/s"
|
||||
)
|
||||
if failed:
|
||||
print(f"WARNING: {failed} tasks failed — the number above is not a clean drain")
|
||||
return
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
print(f"\nstill draining after {timeout_s}s — that IS the result. Record it.")
|
||||
|
||||
@@ -117,8 +136,16 @@ async def _amain() -> None:
|
||||
"with k6 (one dependency, not two — do not add locust for this)."
|
||||
)
|
||||
|
||||
# The version the workers will actually resolve. Read it rather than restate it: a seeded
|
||||
# row whose chart_version disagrees with the catalog drains through the failure path.
|
||||
catalog = load_catalog(settings.catalog_path)
|
||||
entry = catalog.get(_SERVICE_TYPE)
|
||||
if entry is None:
|
||||
raise SystemExit(f"{settings.catalog_path} has no '{_SERVICE_TYPE}' entry to load-test with")
|
||||
|
||||
print(f"seeding {args.count} instances at {datetime.now(UTC).isoformat()} ...")
|
||||
took = await _seed_direct(dsn, args.count)
|
||||
print(f" service_type={_SERVICE_TYPE} chart_version={entry.chart_version} (from catalog)")
|
||||
took = await _seed_direct(dsn, args.count, entry.chart_version)
|
||||
print(f"enqueued {args.count} in {took:.2f}s ({args.count / took:.0f}/s)\n")
|
||||
|
||||
if args.watch:
|
||||
|
||||
+61
-13
@@ -3,8 +3,17 @@
|
||||
|
||||
$ python3 scripts/redis_budget.py
|
||||
$ python3 scripts/redis_budget.py --url http://localhost:8000/metrics --budget 500000
|
||||
$ python3 scripts/redis_budget.py --url http://api:8000/metrics \
|
||||
--url http://worker:9100/metrics \
|
||||
--url http://reconciler:9100/metrics
|
||||
|
||||
Upstash's free tier is 500,000 commands/month, which sounds enormous and is not:
|
||||
The budget is per Upstash database; the counters are per process. api, worker and
|
||||
reconciler each keep their own registry (see obs.py — one process per pod, no multiproc
|
||||
directory), so scraping one endpoint measures one third of the burn. Repeat `--url` to sum
|
||||
them; a run that covers fewer than three sources says so in its output rather than printing
|
||||
a reassuring number derived from one process.
|
||||
|
||||
Upstash's free tier is 500,000 commands/month, which is far smaller than it sounds:
|
||||
|
||||
500,000 / month = 16,129 / day = 11 / minute = 0.19 / second, sustained
|
||||
|
||||
@@ -14,7 +23,7 @@ Redis is only ever on the request path here, and why this script exists: the rul
|
||||
to state and invisible to violate. A `cache.get()` added inside the reconciler's
|
||||
per-instance loop is one line in review and 2,160,000 commands/month in production.
|
||||
|
||||
**Why a projection and not an alarm on the counter.** Exhausting the budget is a slow,
|
||||
**This projects the burn rather than alarming on the counter.** Exhausting the budget is a slow,
|
||||
silent failure with a cliff at the end: nothing degrades, nothing pages, every call
|
||||
succeeds, and then the month rolls over and every Redis call starts erroring at once. By
|
||||
then the fix is a bill or an outage. A burn rate extrapolated from the counter is visible
|
||||
@@ -113,7 +122,23 @@ def collect(text: str) -> tuple[dict[str, float], float]:
|
||||
return per_op, started_at
|
||||
|
||||
|
||||
def report(per_op: dict[str, float], started_at: float, budget: int, now: float) -> int:
|
||||
def merge(scrapes: list[tuple[dict[str, float], float]]) -> tuple[dict[str, float], float]:
|
||||
"""Fold several processes' scrapes into one budget view.
|
||||
|
||||
The budget is per-Upstash-database, but the counters are per-process: api, worker and
|
||||
reconciler each hold their own prometheus_client registry, so scraping one of them
|
||||
projects a third of the truth. Command totals sum across processes; the window is the
|
||||
EARLIEST start time, because a counter that has been running longest bounds how far back
|
||||
the summed total can be attributed — using the latest would inflate the rate.
|
||||
"""
|
||||
per_op: dict[str, float] = {}
|
||||
for scraped, _ in scrapes:
|
||||
for op, value in scraped.items():
|
||||
per_op[op] = per_op.get(op, 0.0) + value
|
||||
return per_op, min(started for _, started in scrapes)
|
||||
|
||||
|
||||
def report(per_op: dict[str, float], started_at: float, budget: int, now: float, sources: int = 1) -> int:
|
||||
"""Print the projection. Returns the process exit code."""
|
||||
elapsed_s = max(1.0, now - started_at)
|
||||
total = sum(per_op.values())
|
||||
@@ -128,6 +153,17 @@ def report(per_op: dict[str, float], started_at: float, budget: int, now: float)
|
||||
print(f"rate {rate:.4f} /s (budget allows {budget / _MONTH_S:.4f} /s sustained)")
|
||||
print(f"projected {projected:,.0f} / month")
|
||||
print(f"budget {budget:,} / month")
|
||||
print(f"sources {sources} process(es) scraped")
|
||||
if sources < 3:
|
||||
# Be honest about what was measured. api/worker/reconciler each keep their own
|
||||
# in-process registry (obs.py: one process per pod, no multiproc dir), so a
|
||||
# single-endpoint run undercounts the shared Upstash budget by however many
|
||||
# processes were left out. An optimistic verdict here is worse than no verdict.
|
||||
print(
|
||||
" UNDERCOUNT: svcforge runs api + worker + reconciler, each with "
|
||||
"its own\n registry. Pass --url once per process for the real "
|
||||
"total; the numbers\n above cover only what was scraped."
|
||||
)
|
||||
|
||||
if total < 100:
|
||||
# Extrapolating a month from a handful of commands is astrology. Say so rather than
|
||||
@@ -147,20 +183,32 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
parser.add_argument("--url", default="http://localhost:8000/metrics", help="Prometheus endpoint")
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
action="append",
|
||||
dest="urls",
|
||||
metavar="URL",
|
||||
help="Prometheus endpoint; repeat once per process (api, worker, reconciler)",
|
||||
)
|
||||
parser.add_argument("--budget", type=int, default=_FREE_TIER_BUDGET, help="commands per month")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
per_op, started_at = collect(scrape(args.url))
|
||||
except BudgetError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except OSError as exc:
|
||||
print(f"error: cannot scrape {args.url}: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
# action="append" cannot carry a default (argparse appends to it), so apply it here.
|
||||
urls: list[str] = args.urls or ["http://localhost:8000/metrics"]
|
||||
|
||||
return report(per_op, started_at, args.budget, time.time())
|
||||
scrapes: list[tuple[dict[str, float], float]] = []
|
||||
for url in urls:
|
||||
try:
|
||||
scrapes.append(collect(scrape(url)))
|
||||
except BudgetError as exc:
|
||||
print(f"error: {url}: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except OSError as exc:
|
||||
print(f"error: cannot scrape {url}: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
per_op, started_at = merge(scrapes)
|
||||
return report(per_op, started_at, args.budget, time.time(), sources=len(scrapes))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+11
-4
@@ -7,9 +7,9 @@
|
||||
# Two syncs, not one: deps change rarely and our own code changes every commit, so the
|
||||
# expensive layer (third-party wheels) must land before the cheap one (our source).
|
||||
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 AS builder
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.29@sha256:eb2843a1e56fd9e30c7276ce1a52cba86e64c7b385f5e3279a0e08e02dd058fc /uv /usr/local/bin/uv
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
|
||||
WORKDIR /app
|
||||
@@ -28,6 +28,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
COPY libs/ libs/
|
||||
COPY services/api/ services/api/
|
||||
COPY catalog.yaml ./
|
||||
# The migrate Job runs from THIS image (migrate-job.yaml pins the api digest), so the SQL
|
||||
# has to be in it. svcforge_core.migrate's fallback resolves MIGRATIONS_DIR relative to
|
||||
# its own __file__, which lands under site-packages here — a directory that does not and
|
||||
# should not contain SQL — so main() returned 1 and the pre-install/pre-upgrade hook
|
||||
# failed every sync. SVCFORGE_MIGRATIONS_DIR below points it at this copy instead.
|
||||
COPY migrations/ migrations/
|
||||
# --no-editable is what turns svcforge-core into a real wheel in site-packages.
|
||||
# pyproject.toml declares it `editable = true` for local dev; an editable install in an
|
||||
# image points at /app/libs, which is a source tree that need not survive the final stage.
|
||||
@@ -38,7 +44,7 @@ p = svcforge_core.__file__; \
|
||||
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
|
||||
|
||||
# --- runtime --------------------------------------------------------------------------
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
|
||||
|
||||
ARG BUILD_SHA=unknown
|
||||
LABEL org.opencontainers.image.title="svcforge-api" \
|
||||
@@ -50,6 +56,7 @@ WORKDIR /app
|
||||
COPY --from=builder --chown=10001:10001 /app /app
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
SVCFORGE_MIGRATIONS_DIR=/app/migrations
|
||||
USER 10001
|
||||
ENTRYPOINT ["python", "-m", "services.api"]
|
||||
|
||||
+45
-27
@@ -1,8 +1,8 @@
|
||||
"""Dependency injection: how a handler gets a pool, a repo, a catalog, and a team.
|
||||
|
||||
Everything expensive — the pool, the JWKS client, the parsed catalog — is built once in
|
||||
`lifespan` and parked on `app.state`. These functions only hand it out. A `Depends` that
|
||||
does I/O per request is a `Depends` that does that I/O on every request forever.
|
||||
`lifespan` and parked on `app.state`; these functions only hand it out. A `Depends` that
|
||||
does I/O per request does that I/O on every request forever.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,16 +15,16 @@ from fastapi import Depends, HTTPException, Request, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from jwt import PyJWKClient
|
||||
|
||||
from svcforge_core.adapters.redis import RateLimiterProto
|
||||
from svcforge_core.domain.models import CatalogEntry
|
||||
from svcforge_core.repo.db import DictPool
|
||||
from svcforge_core.repo.instances import InstanceRepo
|
||||
from svcforge_core.repo.tasks import TaskRepo
|
||||
from svcforge_core.settings import Settings
|
||||
|
||||
# The algorithm allow-list is the whole point of naming algorithms explicitly.
|
||||
# `jwt.decode(..., algorithms=...)` without it accepts whatever the *token* claims in its
|
||||
# own header — including `none`, and including HS256 verified with the RSA public key as
|
||||
# an HMAC secret. Both are forgery. The list is not configuration.
|
||||
# The algorithm allow-list is not configuration. Without it, `jwt.decode` accepts whatever
|
||||
# the *token* claims in its own header — including `none`, and including HS256 verified
|
||||
# with the RSA public key as an HMAC secret. Both are forgery.
|
||||
ALLOWED_ALGORITHMS = ["RS256"]
|
||||
|
||||
# What `auth_disabled` returns. Settings.check_production() refuses that flag in prod.
|
||||
@@ -41,9 +41,9 @@ _bearer = HTTPBearer(auto_error=False)
|
||||
def _unauthorized() -> HTTPException:
|
||||
"""One shape for every auth failure.
|
||||
|
||||
Expired, wrong issuer, wrong audience, bad signature, malformed, no header: all the
|
||||
same 401 with the same body. Telling a caller *which* one turns the endpoint into an
|
||||
oracle they can tune a forgery against.
|
||||
Expired, wrong issuer, wrong audience, bad signature, malformed, no header: the same 401
|
||||
with the same body. Naming which one turns the endpoint into an oracle a forger can tune
|
||||
against.
|
||||
"""
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
@@ -67,8 +67,8 @@ async def get_pool(request: Request) -> DictPool:
|
||||
def get_catalog(request: Request) -> dict[str, CatalogEntry]:
|
||||
"""The catalog, parsed once at startup.
|
||||
|
||||
Read from disk per request and a mid-flight edit to catalog.yaml changes the answer
|
||||
between two requests of the same deploy. Load it at startup; a change is a restart.
|
||||
Read per request, a mid-flight edit to catalog.yaml would change the answer between two
|
||||
requests of the same deploy. A catalog change is a restart.
|
||||
"""
|
||||
catalog: dict[str, CatalogEntry] = request.app.state.catalog
|
||||
return catalog
|
||||
@@ -101,16 +101,14 @@ async def get_current_team(
|
||||
|
||||
jwks_client: PyJWKClient | None = getattr(request.app.state, "jwks_client", None)
|
||||
if jwks_client is None:
|
||||
# Auth is on but there is no key source. Fail closed. Answering 500 here would be
|
||||
# honest about the cause and would also let a misconfigured deploy be told apart
|
||||
# from a bad token; 401 is the same answer a forger gets.
|
||||
# Auth is on but there is no key source. Fail closed. A 500 would be honest about
|
||||
# the cause and would also let a forger tell a misconfigured deploy from a bad token.
|
||||
raise _unauthorized()
|
||||
|
||||
try:
|
||||
# PyJWKClient keeps its own TTL cache, so this is a dict lookup on the hot path.
|
||||
# It is only blocking on a cache MISS (key rotation) — hence to_thread, which
|
||||
# costs a thread hop we take a handful of times a day rather than an event loop
|
||||
# stalled on someone else's HTTP call once per rotation.
|
||||
# PyJWKClient keeps a TTL cache, so this is a dict lookup on the hot path and only
|
||||
# blocks on a miss (key rotation) — hence to_thread, a thread hop a few times a day
|
||||
# rather than an event loop stalled on someone else's HTTP call.
|
||||
signing_key = await _signing_key(jwks_client, creds.credentials)
|
||||
claims: dict[str, Any] = jwt.decode(
|
||||
creds.credentials,
|
||||
@@ -138,21 +136,41 @@ async def get_current_team(
|
||||
async def _signing_key(client: PyJWKClient, token: str) -> jwt.PyJWK:
|
||||
"""Fetch the signing key off the event loop.
|
||||
|
||||
PyJWKClient.get_signing_key_from_jwt() does a synchronous urlopen on a cache miss.
|
||||
Called directly from `async def`, that blocks the loop — every other in-flight request
|
||||
on this worker stops until the identity provider answers, and if it hangs, so does the
|
||||
pod, and /readyz keeps saying it is fine.
|
||||
`get_signing_key_from_jwt()` does a synchronous urlopen on a cache miss. Called directly
|
||||
from `async def` it blocks the loop: every other in-flight request stops until the IdP
|
||||
answers, and if the IdP hangs so does the pod, with /readyz still saying it is fine.
|
||||
"""
|
||||
return await asyncio.to_thread(client.get_signing_key_from_jwt, token)
|
||||
|
||||
|
||||
async def rate_limit(team: Annotated[str, Depends(get_current_team)]) -> None:
|
||||
"""Per-team rate limiting. Seam only — Module 10 fills this in (Redis, Lua, token bucket).
|
||||
def get_rate_limiter(request: Request) -> RateLimiterProto | None:
|
||||
"""The limiter lifespan built, or None when no Redis is configured."""
|
||||
limiter: RateLimiterProto | None = getattr(request.app.state, "rate_limiter", None)
|
||||
return limiter
|
||||
|
||||
It exists now, wired into the routes, so that turning it on is an edit to one function
|
||||
body rather than a change to every handler signature.
|
||||
|
||||
async def rate_limit(
|
||||
request: Request,
|
||||
team: Annotated[str, Depends(get_current_team)],
|
||||
) -> None:
|
||||
"""Per-team rate limiting. One Redis command per check, and it fails OPEN.
|
||||
|
||||
Redis holds derived state, so losing it must degrade the platform rather than stop it: a
|
||||
limiter that fails closed turns a cache outage into a total outage, a worse incident
|
||||
than the burst it was guarding against. `RateLimiter.check` swallows its own errors and
|
||||
returns `allowed=True`, so the 429 below only comes from a real, counted overage.
|
||||
"""
|
||||
return None
|
||||
limiter = get_rate_limiter(request)
|
||||
if limiter is None:
|
||||
return
|
||||
|
||||
result = await limiter.check(team)
|
||||
if not result.allowed:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail={"code": "rate_limited", "message": "too many requests"},
|
||||
headers={"Retry-After": str(result.retry_after_s)},
|
||||
)
|
||||
|
||||
|
||||
async def idempotency_key(request: Request) -> str | None:
|
||||
|
||||
+132
-29
@@ -1,63 +1,74 @@
|
||||
"""The app factory and its lifespan.
|
||||
|
||||
`create_app(settings)` is a factory, not a module-level `app = FastAPI()`, for one reason:
|
||||
a test needs an app pointed at a throwaway Postgres, and an import-time app reads the real
|
||||
`create_app(settings)` is a factory rather than a module-level `app = FastAPI()` because a
|
||||
test needs an app pointed at a throwaway Postgres, and an import-time app reads the real
|
||||
environment at import time — before any fixture can say otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from jwt import PyJWKClient
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from services.api.models import ErrorBody
|
||||
from services.api.routes import health, instances
|
||||
from svcforge_core import obs
|
||||
from svcforge_core.adapters.redis import RateLimiter, make_redis
|
||||
from svcforge_core.domain.catalog import load_catalog
|
||||
from svcforge_core.repo.db import make_pool
|
||||
from svcforge_core.settings import Settings, load_settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log = obs.get_logger("svcforge.api")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Open the pool, yield, close the pool.
|
||||
|
||||
A lifespan context, not the deprecated startup/shutdown event decorators: those cannot
|
||||
express "this resource lives for exactly as long as the app", and give you no place to
|
||||
put the teardown next to the setup. Closing the pool matters — an unclosed pool means
|
||||
connections linger server-side after SIGTERM, and on a pooled Postgres with a small
|
||||
connection budget a few rolling deploys exhaust it.
|
||||
A lifespan context, not the deprecated startup/shutdown decorators: those cannot express
|
||||
"this resource lives exactly as long as the app" and leave no place to put teardown next
|
||||
to setup. Closing matters — an unclosed pool leaves connections open server-side after
|
||||
SIGTERM, and on a pooled Postgres with a small budget a few rolling deploys exhaust it.
|
||||
|
||||
(The old decorator's name is spelled nowhere in this package on purpose: CI greps for
|
||||
the literal string, and a comment quoting it fails the gate just as loudly as a call.)
|
||||
(The old decorator's name is spelled nowhere here on purpose: CI greps for the literal
|
||||
string, so a comment quoting it fails the gate as loudly as a call would.)
|
||||
"""
|
||||
settings: Settings = app.state.settings
|
||||
|
||||
app.state.catalog = load_catalog(settings.catalog_path)
|
||||
|
||||
pool = make_pool(str(settings.pg_dsn), settings.pool_min_size, settings.pool_max_size)
|
||||
# Redis is optional by construction: `make_redis` returns None when no DSN is set and
|
||||
# every consumer treats None as "skip", so a deployment without Redis loses rate
|
||||
# limiting and keeps everything else. Built once here, not per request.
|
||||
redis = make_redis(settings)
|
||||
app.state.redis = redis
|
||||
app.state.rate_limiter = (
|
||||
RateLimiter(redis, limit=settings.rate_limit_per_minute, window_s=60) if redis is not None else None
|
||||
)
|
||||
|
||||
pool = make_pool(settings.runtime_dsn, settings.pool_min_size, settings.pool_max_size)
|
||||
# wait=True fails NOW, loudly, if the DSN is wrong — instead of at the first request,
|
||||
# as a PoolTimeout, in front of a user.
|
||||
await pool.open(wait=True)
|
||||
app.state.pool = pool
|
||||
|
||||
# The pool is open from here on, so everything below is inside the try: an exception
|
||||
# in JWKS setup must still close it, or a crash-looping pod leaks a connection per
|
||||
# restart until the database refuses new ones.
|
||||
# The pool is open from here, so everything below is inside the try: an exception in
|
||||
# JWKS setup must still close it, or a crash-looping pod leaks a connection per restart
|
||||
# until the database refuses new ones.
|
||||
try:
|
||||
if settings.jwks_url and not settings.auth_disabled:
|
||||
client = PyJWKClient(settings.jwks_url, cache_keys=True, lifespan=300)
|
||||
app.state.jwks_client = client
|
||||
# Warm the cache off the loop so the first authenticated request does not pay
|
||||
# a blocking urlopen. Best-effort: a slow identity provider must not stop the
|
||||
# pod from starting — a cache miss later just costs one to_thread hop.
|
||||
# Warm the cache off the loop so the first authenticated request does not pay a
|
||||
# blocking urlopen. Best-effort: a slow IdP must not stop the pod from starting,
|
||||
# and a miss later costs one to_thread hop.
|
||||
try:
|
||||
await asyncio.to_thread(client.get_signing_keys)
|
||||
except Exception: # deliberate catch-all: startup must not hinge on the IdP being up
|
||||
@@ -68,19 +79,81 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
yield
|
||||
finally:
|
||||
await pool.close()
|
||||
if redis is not None:
|
||||
await redis.aclose()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- API docs
|
||||
|
||||
# What the generated schema cannot express. Kept next to create_app because /docs is what
|
||||
# someone integrating reads, and they do not have this repo. USER_GUIDE.md is the longer form.
|
||||
API_DESCRIPTION = """
|
||||
Provision managed service instances into Kubernetes. The catalog offers Elasticsearch,
|
||||
Redis and Postgres, plus two deliberately tiny entries — `podinfo` and `nginx` — for
|
||||
exercising the platform where there is no room for the real thing.
|
||||
|
||||
## Authentication
|
||||
|
||||
Every `/v1` route needs a bearer JWT: `Authorization: Bearer <token>`. The token is
|
||||
verified against the configured JWKS (RS256), and its `team` claim decides which instances
|
||||
you can see. **Authorisation is a WHERE clause** — asking for another team's instance
|
||||
returns `404`, not `403`, so the API never confirms that an id you cannot access exists.
|
||||
|
||||
## Writes are asynchronous
|
||||
|
||||
`POST` and `DELETE` return **202 Accepted**, not 201/204. They enqueue work and return
|
||||
immediately; nothing is provisioned yet when you get the response. Poll
|
||||
`GET /v1/instances/{id}` and watch `state`.
|
||||
|
||||
## Instance lifecycle
|
||||
|
||||
requested -> provisioning -> ready
|
||||
|
|
||||
v
|
||||
deleting -> deleted
|
||||
|
||||
`failed` is reachable from `requested` and `provisioning` when a provision exhausts its
|
||||
retries. A `ready` instance whose release vanished is re-provisioned automatically by the
|
||||
reconciler, so `ready` is the only state that carries a usable `endpoint`.
|
||||
|
||||
## Errors
|
||||
|
||||
Every non-2xx body is the same shape — `{"code": ..., "message": ...}` — including the
|
||||
404s and 405s raised by the framework itself. `code` is stable and meant for machines;
|
||||
`message` is for humans.
|
||||
"""
|
||||
|
||||
OPENAPI_TAGS = [
|
||||
{
|
||||
"name": "instances",
|
||||
"description": "Create, inspect and delete service instances. All writes are 202 + poll.",
|
||||
},
|
||||
{
|
||||
"name": "ops",
|
||||
"description": (
|
||||
"Liveness, readiness and Prometheus metrics. Unauthenticated, and not part of "
|
||||
"the tenant API surface."
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
async def _http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
"""Render HTTPException bodies as ErrorBody, so every error has one shape.
|
||||
|
||||
Handlers raise `detail={"code": ..., "message": ...}`; FastAPI's default would nest
|
||||
that under `{"detail": {...}}`. Plain-string details (raised by FastAPI itself, e.g.
|
||||
a 405) are wrapped so clients never have to branch on the body's type.
|
||||
Handlers raise `detail={"code": ..., "message": ...}`, which FastAPI's default would
|
||||
nest under `{"detail": {...}}`. Plain-string details (a framework 405, say) are wrapped
|
||||
so clients never branch on the body's type.
|
||||
|
||||
Registered on starlette's HTTPException, not fastapi's. The FastAPI class is a subclass
|
||||
and Starlette matches handlers by walking `type(exc).__mro__`, so a handler keyed on the
|
||||
subclass never fires for a framework-raised 404 or 405. Keying on the parent catches
|
||||
both, and the branch below renders each into ErrorBody.
|
||||
"""
|
||||
assert isinstance(exc, HTTPException) # noqa: S101 - registered only for HTTPException
|
||||
# Widened to object deliberately. Starlette types `detail` as str, but FastAPI passes
|
||||
# through whatever a handler raised — our handlers raise dicts. Narrowing off the
|
||||
# declared type would make mypy call the dict branch unreachable and delete it.
|
||||
# Widened to object deliberately: Starlette types `detail` as str, but FastAPI passes
|
||||
# through whatever a handler raised, and ours raise dicts. Narrowing off the declared
|
||||
# type would let mypy call the dict branch unreachable and delete it.
|
||||
detail: object = exc.detail
|
||||
if isinstance(detail, dict) and "code" in detail and "message" in detail:
|
||||
body = ErrorBody(code=str(detail["code"]), message=str(detail["message"]))
|
||||
@@ -89,14 +162,44 @@ async def _http_exception_handler(request: Request, exc: Exception) -> JSONRespo
|
||||
return JSONResponse(status_code=exc.status_code, content=body.model_dump(), headers=exc.headers)
|
||||
|
||||
|
||||
async def _validation_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
"""Render request-validation failures as ErrorBody too.
|
||||
|
||||
A forbidden extra field, a bad type or an out-of-range ttl_days raises
|
||||
RequestValidationError, which the handler above never sees. Without this, FastAPI's
|
||||
default `{"detail": [...]}` is a second 422 shape alongside the handlers' ErrorBody.
|
||||
"""
|
||||
assert isinstance(exc, RequestValidationError) # noqa: S101 - registered only for this
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=ErrorBody(code="validation_error", message=str(exc.errors())).model_dump(),
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
"""App factory: lifespan, routers, exception handler, /metrics."""
|
||||
settings = settings or load_settings()
|
||||
|
||||
# FIRST, before any router is built and any logger is bound. Without it the API is the
|
||||
# one service of three that never configures structlog, and its lines go out through
|
||||
# logging.lastResort as bare text on stderr — no service, no trace_id, no JSON envelope.
|
||||
# `settings.log_json` was silently inert here for the same reason.
|
||||
obs.setup("svcforge-api", settings)
|
||||
|
||||
# Refuse the dev escape hatches when SVCFORGE_ENVIRONMENT says this is not a laptop.
|
||||
# Unconditional and early: a check that runs only from a branch someone remembered to
|
||||
# write is a check that does not run.
|
||||
settings.check_production()
|
||||
|
||||
# The description is the API's documentation, rendered as markdown at /docs. It is the
|
||||
# only place a caller without this repo learns the two things the schema cannot say:
|
||||
# every write is asynchronous, and the lifecycle is a state machine they have to poll.
|
||||
app = FastAPI(
|
||||
title="svcforge",
|
||||
version="0.1.0",
|
||||
summary="X-as-a-Service control plane",
|
||||
description=API_DESCRIPTION,
|
||||
openapi_tags=OPENAPI_TAGS,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.state.settings = settings
|
||||
@@ -107,6 +210,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.include_router(instances.router)
|
||||
|
||||
app.add_exception_handler(HTTPException, _http_exception_handler)
|
||||
app.add_exception_handler(RequestValidationError, _validation_exception_handler)
|
||||
return app
|
||||
|
||||
|
||||
@@ -115,7 +219,6 @@ def app() -> FastAPI:
|
||||
return create_app()
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("services.api.main:app", factory=True, host="0.0.0.0", port=8000) # noqa: S104
|
||||
# No `if __name__ == "__main__"` here on purpose. `services/api/__main__.py` is the single
|
||||
# entrypoint and the image's ENTRYPOINT uses it. A second one in this module drifted from
|
||||
# it — different log_level, different access_log — so the same app started two ways.
|
||||
|
||||
+60
-20
@@ -1,9 +1,8 @@
|
||||
"""Wire types.
|
||||
"""Wire types, deliberately not the domain models.
|
||||
|
||||
These are deliberately NOT the domain models. `Instance` carries `team`, `namespace` and
|
||||
`release_name` — placement details a tenant has no business seeing and no business
|
||||
setting. The response model is the allow-list that keeps them off the wire, which is why
|
||||
it is written out by hand instead of derived from `Instance`.
|
||||
`Instance` carries `team`, `namespace` and `release_name` — placement details a tenant has
|
||||
no business seeing or setting. The response model is the allow-list that keeps them off the
|
||||
wire, which is why it is written by hand instead of derived from `Instance`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,34 +18,75 @@ class CreateInstanceRequest(BaseModel):
|
||||
"""What a tenant may ask for.
|
||||
|
||||
`service_type` and `size` are plain strings, not enums: the catalog is data loaded at
|
||||
runtime, so baking its keys into a type would mean a redeploy to add a service type,
|
||||
and a 422 (schema) where the spec wants a 404 (unknown resource). They are validated
|
||||
against the catalog in the handler.
|
||||
runtime, so baking its keys into a type would mean a redeploy to add a service type and
|
||||
a 422 where the spec wants a 404. The handler validates them against the catalog.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
json_schema_extra={"examples": [{"service_type": "redis", "size": "small", "ttl_days": 7}]},
|
||||
)
|
||||
|
||||
service_type: str = Field(min_length=1)
|
||||
size: str
|
||||
ttl_days: int | None = Field(default=None, ge=1, le=30)
|
||||
service_type: str = Field(
|
||||
min_length=1,
|
||||
description=(
|
||||
"A service type in the catalog, e.g. `elasticsearch`, `redis`, `postgres`, "
|
||||
"`podinfo`, `nginx`. "
|
||||
"Unknown values return 404."
|
||||
),
|
||||
)
|
||||
size: str = Field(
|
||||
description=(
|
||||
"A size the catalog defines for that service type, e.g. `small`. Unknown values return 422."
|
||||
),
|
||||
)
|
||||
ttl_days: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
le=30,
|
||||
description="Delete the instance automatically after this many days. Omit for no expiry.",
|
||||
)
|
||||
|
||||
|
||||
class InstanceResponse(BaseModel):
|
||||
"""What a tenant gets back. A subset of Instance, on purpose."""
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
model_config = ConfigDict(
|
||||
from_attributes=True,
|
||||
json_schema_extra={
|
||||
"examples": [
|
||||
{
|
||||
"id": "0f8b7d3e-1c2a-4f5b-9e6d-7a8b9c0d1e2f",
|
||||
"state": "ready",
|
||||
"service_type": "redis",
|
||||
"size": "small",
|
||||
"endpoint": "http://acme-redis-0f8b7d3e.tenant-acme.svc.cluster.local",
|
||||
"chart_version": "20.6.2",
|
||||
"error": None,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
id: UUID
|
||||
state: InstanceState
|
||||
id: UUID = Field(description="Poll `GET /v1/instances/{id}` with this to watch the state change.")
|
||||
state: InstanceState = Field(description="Lifecycle state. Only `ready` carries a usable endpoint.")
|
||||
service_type: str
|
||||
size: str
|
||||
endpoint: str | None
|
||||
chart_version: str
|
||||
error: str | None
|
||||
endpoint: str | None = Field(description="In-cluster DNS name. Null until the instance is `ready`.")
|
||||
chart_version: str = Field(
|
||||
description="The chart version actually deployed, written only after helm succeeds."
|
||||
)
|
||||
error: str | None = Field(description="Why the last attempt failed. Null unless `state` is `failed`.")
|
||||
|
||||
|
||||
class ErrorBody(BaseModel):
|
||||
"""Every non-2xx body. `code` is for machines, `message` is for humans."""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
model_config = ConfigDict(
|
||||
json_schema_extra={
|
||||
"examples": [{"code": "unknown_service_type", "message": "no such service_type: mongodb"}]
|
||||
}
|
||||
)
|
||||
|
||||
code: str = Field(description="Stable machine-readable identifier for the failure.")
|
||||
message: str = Field(description="Human-readable detail. Do not parse this.")
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
"""Liveness, readiness, metrics.
|
||||
|
||||
The distinction between the first two is not pedantry, it is the difference between a
|
||||
30-second blip and a fleet-wide outage:
|
||||
The distinction between the first two is the difference between a 30-second blip and a
|
||||
fleet-wide outage:
|
||||
|
||||
* `/healthz` (liveness) answers "is this process wedged?" A failure here gets the
|
||||
container KILLED. It must therefore touch NOTHING external. Wire it to the DB and a
|
||||
20-second Postgres failover restarts every pod at once; they come back, find the DB
|
||||
still down, and CrashLoopBackOff with exponential restart delays — so the fleet is now
|
||||
down for minutes after the database recovered.
|
||||
* `/healthz` (liveness) answers "is this process wedged?" A failure here KILLS the
|
||||
container, so it must touch nothing external. Wired to the DB, a 20-second Postgres
|
||||
failover restarts every pod at once; they come back, find the DB still down, and
|
||||
CrashLoopBackOff with exponential delays — the fleet stays down for minutes after the
|
||||
database recovered.
|
||||
* `/readyz` (readiness) answers "should this pod get traffic?" A failure here only removes
|
||||
it from the Service endpoints. It is allowed to check dependencies, and it recovers by
|
||||
itself the moment the check passes.
|
||||
@@ -26,10 +26,10 @@ from services.api.models import ErrorBody
|
||||
|
||||
router = APIRouter(tags=["ops"])
|
||||
|
||||
# No PROMETHEUS_MULTIPROC_DIR here, deliberately: it exists for prefork servers where each
|
||||
# worker process holds a slice of the counters. One uvicorn process per container means
|
||||
# the default in-process registry is already correct, and multiproc mode would add a
|
||||
# shared temp dir, a cleanup obligation, and a class of stale-file bugs for nothing.
|
||||
# No PROMETHEUS_MULTIPROC_DIR, deliberately: it exists for prefork servers where each
|
||||
# process holds a slice of the counters. One uvicorn process per container makes the
|
||||
# in-process registry correct, and multiproc mode would add a shared temp dir, a cleanup
|
||||
# obligation, and a class of stale-file bugs for nothing.
|
||||
|
||||
|
||||
@router.get("/healthz", status_code=status.HTTP_200_OK)
|
||||
@@ -45,10 +45,9 @@ async def healthz() -> dict[str, str]:
|
||||
async def readyz(pool: PoolDep) -> dict[str, str]:
|
||||
"""Readiness. Postgres only.
|
||||
|
||||
Postgres-only is the rule, and Redis is the temptation. Redis holds derived state —
|
||||
rate-limit buckets, caches — and everything degrades gracefully without it. Put it in
|
||||
this check and an Upstash hiccup marks every pod unready, Kubernetes empties the
|
||||
Service, and a cache outage becomes a total API outage.
|
||||
Redis is the temptation and stays out: it holds derived state that degrades gracefully,
|
||||
so checking it here would let an Upstash hiccup mark every pod unready, empty the
|
||||
Service, and turn a cache outage into a total API outage.
|
||||
"""
|
||||
try:
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
@@ -68,14 +67,13 @@ async def readyz(pool: PoolDep) -> dict[str, str]:
|
||||
async def metrics(request: Request) -> Response:
|
||||
"""The Prometheus scrape endpoint.
|
||||
|
||||
A route rather than `app.mount("/metrics", make_asgi_app())`, for two reasons. A
|
||||
Starlette `Mount` compiles to `^/metrics(?P<path>/.*)$`, which does not match a bare
|
||||
`/metrics` — the exact URL every scrape config uses — and a `Mount` is invisible to
|
||||
OpenAPI, while the deliverable asks for `/metrics` in `openapi.json`.
|
||||
A route rather than `app.mount("/metrics", make_asgi_app())`: a Starlette `Mount`
|
||||
compiles to `^/metrics(?P<path>/.*)$`, which does not match the bare `/metrics` every
|
||||
scrape config uses, and a Mount is invisible to OpenAPI.
|
||||
|
||||
The encoding is still prometheus_client's: `choose_encoder` reads the Accept header and
|
||||
picks the exposition format (Prometheus text vs OpenMetrics) with its matching content
|
||||
type. Hand-rolling either is how you end up serving text/plain that a scraper rejects.
|
||||
The encoding stays prometheus_client's — `choose_encoder` reads Accept and picks the
|
||||
exposition format with its matching content type. Hand-rolling it serves text/plain a
|
||||
scraper rejects.
|
||||
"""
|
||||
encoder, content_type = choose_encoder(request.headers.get("Accept", ""))
|
||||
return Response(content=encoder(REGISTRY), media_type=content_type)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
"""The tenant-facing API.
|
||||
|
||||
Two rules run through every handler here:
|
||||
Two rules run through every handler:
|
||||
|
||||
* **AuthZ is the WHERE clause.** No handler ever compares `inst.team` to the caller's
|
||||
team, because the repo never returns another team's row to compare. A wrong-team id is
|
||||
a 404. 403 would confirm the id exists, which is the leak.
|
||||
* **The instance and its task commit together.** A committed instance with no task is an
|
||||
instance that never provisions and that nothing will ever retry.
|
||||
* **AuthZ is the WHERE clause.** No handler compares `inst.team` to the caller's team,
|
||||
because the repo never returns another team's row to compare. A wrong-team id is a 404;
|
||||
403 would confirm the id exists.
|
||||
* **The instance and its task commit together.** A committed instance with no task never
|
||||
provisions and nothing retries it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -30,9 +30,9 @@ from services.api.models import CreateInstanceRequest, ErrorBody, InstanceRespon
|
||||
from svcforge_core.domain.models import CatalogEntry, Instance, TaskKind
|
||||
from svcforge_core.domain.states import IllegalTransition, InstanceState, transition
|
||||
|
||||
# Declared on the router so every error shape lands in openapi.json under ErrorBody.
|
||||
# The exception handler already renders this at runtime; without declaring it, generated
|
||||
# clients see the contract for 2xx only and invent their own guess for the rest.
|
||||
# Declared on the router so every error shape lands in openapi.json under ErrorBody. The
|
||||
# exception handler already renders these at runtime; undeclared, a generated client sees
|
||||
# the contract for 2xx only and guesses the rest.
|
||||
ERROR_RESPONSES: dict[int | str, dict[str, Any]] = {
|
||||
401: {"model": ErrorBody, "description": "Missing or invalid credentials"},
|
||||
404: {"model": ErrorBody, "description": "No such instance, or not this team's"},
|
||||
@@ -46,14 +46,12 @@ router = APIRouter(prefix="/v1/instances", tags=["instances"], responses=ERROR_R
|
||||
def release_name_for(team: str, service_type: str, instance_id: UUID) -> str:
|
||||
"""The helm release name. Deterministic, and `unique` in the schema.
|
||||
|
||||
This is the idempotency anchor. A worker that dies after `helm install` but before it
|
||||
marks the task done will retry, compute the same name, and `helm upgrade --install`
|
||||
onto the same release instead of creating a second one. Derive it from anything that
|
||||
is not already durable — a timestamp, a random suffix, the retry count — and a retry
|
||||
provisions a duplicate.
|
||||
The idempotency anchor: a worker that dies after `helm install` but before marking the
|
||||
task done retries, computes the same name, and upgrades the same release instead of
|
||||
creating a second one. Derive it from anything not already durable — a timestamp, a
|
||||
random suffix, the retry count — and a retry provisions a duplicate.
|
||||
|
||||
Truncated to the uuid's first 8 chars to stay inside the 53-char limit helm imposes
|
||||
on release names (Kubernetes label values, minus room for chart-generated suffixes).
|
||||
Truncated to the uuid's first 8 chars to stay inside helm's 53-char release-name limit.
|
||||
"""
|
||||
return f"{team}-{service_type}-{str(instance_id)[:8]}"
|
||||
|
||||
@@ -66,9 +64,8 @@ def namespace_for(team: str) -> str:
|
||||
def _resolve(catalog: dict[str, CatalogEntry], service_type: str, size: str) -> CatalogEntry:
|
||||
"""Look up service_type + size, or raise the right 4xx.
|
||||
|
||||
The two failures are different HTTP problems and the spec asks for different codes:
|
||||
an unknown service_type is a resource that does not exist (404); an unknown size for a
|
||||
real service_type is a body the server understood and cannot process (422).
|
||||
Two different HTTP problems: an unknown service_type is a resource that does not exist
|
||||
(404), an unknown size for a real one is a body understood and unprocessable (422).
|
||||
"""
|
||||
entry = catalog.get(service_type)
|
||||
if entry is None:
|
||||
@@ -107,9 +104,9 @@ async def create_instance(
|
||||
) -> Instance:
|
||||
"""Accept a provisioning request. 202, never 201.
|
||||
|
||||
Nothing is provisioned when this returns. The row exists and a task is queued; a
|
||||
worker will do the work seconds or minutes from now. 201 Created would be a lie about
|
||||
a resource that does not exist yet, and clients would stop polling.
|
||||
Nothing is provisioned when this returns: the row exists and a task is queued, and a
|
||||
worker does the work seconds or minutes later. 201 Created would be a lie about a
|
||||
resource that does not exist yet, and clients would stop polling.
|
||||
"""
|
||||
entry = _resolve(catalog, body.service_type, body.size)
|
||||
|
||||
@@ -123,9 +120,9 @@ async def create_instance(
|
||||
state=InstanceState.REQUESTED,
|
||||
namespace=namespace_for(team),
|
||||
release_name=release_name_for(team, body.service_type, instance_id),
|
||||
# Pinned from the catalog AT CREATION TIME, not read from the catalog later.
|
||||
# This column records what is actually deployed; bumping catalog.yaml must show up
|
||||
# as drift the reconciler can see, not silently rewrite history.
|
||||
# Pinned at creation time, not read from the catalog later. The column records what
|
||||
# is deployed, so bumping catalog.yaml shows up as drift the reconciler can see
|
||||
# rather than silently rewriting history.
|
||||
chart_version=entry.chart_version,
|
||||
expires_at=now + timedelta(days=body.ttl_days) if body.ttl_days is not None else None,
|
||||
created_at=now,
|
||||
@@ -181,12 +178,11 @@ async def delete_instance(
|
||||
) -> Instance:
|
||||
"""state -> deleting, enqueue deprovision. 202: the helm uninstall has not happened yet.
|
||||
|
||||
Ordering note. `InstanceRepo.update_state` owns its own connection, so the CAS and the
|
||||
enqueue cannot share one transaction without reaching around the repo. Given two
|
||||
statements, the order is chosen for its failure mode: CAS first, enqueue second. A
|
||||
crash in between leaves an instance in `deleting` with no task, which the reconciler's
|
||||
sweep re-enqueues. The other order leaves a deprovision task pointing at a `ready`
|
||||
instance, and a worker would tear down a live service nobody asked to delete.
|
||||
`InstanceRepo.update_state` owns its own connection, so the CAS and the enqueue cannot
|
||||
share a transaction without reaching around the repo. Given two statements, the order is
|
||||
chosen for its failure mode: a crash between CAS and enqueue leaves an instance in
|
||||
`deleting` with no task, which the reconciler's sweep re-enqueues. The reverse would
|
||||
leave a deprovision task on a `ready` instance and tear down a live service.
|
||||
"""
|
||||
inst = await instances.get(instance_id, team)
|
||||
if inst is None:
|
||||
|
||||
+25
-6
@@ -1,9 +1,9 @@
|
||||
"""svcforge — the control plane client.
|
||||
|
||||
This talks to the API over HTTP and never touches the database. That restraint is the
|
||||
whole design: if the CLI could write to Postgres, every invariant the API enforces
|
||||
(the state machine, the one-transaction create, AuthZ in the WHERE clause) would have a
|
||||
back door, and the first 3am incident would go through it.
|
||||
Talks to the API over HTTP and never touches the database. If the CLI could write to
|
||||
Postgres, every invariant the API enforces — the state machine, the one-transaction create,
|
||||
AuthZ in the WHERE clause — would have a back door, and the first 3am incident would go
|
||||
through it. `ClientSettings` below is what keeps that true in practice.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,9 +16,9 @@ from typing import Annotated, Any
|
||||
|
||||
import httpx
|
||||
import typer
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from svcforge_core.domain.states import InstanceState
|
||||
from svcforge_core.settings import load_settings
|
||||
|
||||
app = typer.Typer(help="svcforge control plane client", no_args_is_help=True)
|
||||
|
||||
@@ -31,6 +31,9 @@ class ServiceType(StrEnum):
|
||||
ELASTICSEARCH = "elasticsearch"
|
||||
REDIS = "redis"
|
||||
POSTGRES = "postgres"
|
||||
# Small enough to provision on a cluster with no spare memory; see catalog.yaml.
|
||||
PODINFO = "podinfo"
|
||||
NGINX = "nginx"
|
||||
|
||||
|
||||
class Size(StrEnum):
|
||||
@@ -38,8 +41,24 @@ class Size(StrEnum):
|
||||
MEDIUM = "medium"
|
||||
|
||||
|
||||
class ClientSettings(BaseSettings):
|
||||
"""The two values the CLI needs, and nothing else.
|
||||
|
||||
Its own model rather than `svcforge_core.settings.Settings`, which requires
|
||||
`SVCFORGE_PG_DSN`: loading that here would refuse to run the CLI without a database URL
|
||||
it then never opens, on a laptop that has no reason to hold one.
|
||||
"""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="SVCFORGE_", env_file=".env", env_file_encoding="utf-8", extra="ignore", frozen=True
|
||||
)
|
||||
|
||||
api_url: str = "http://localhost:8000"
|
||||
api_token: str | None = None
|
||||
|
||||
|
||||
def _client() -> httpx.Client:
|
||||
settings = load_settings()
|
||||
settings = ClientSettings()
|
||||
headers = {"authorization": f"Bearer {settings.api_token}"} if settings.api_token else {}
|
||||
return httpx.Client(base_url=settings.api_url, headers=headers, timeout=10.0)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# syntax=docker/dockerfile:1.10
|
||||
# syntax=mirror.gcr.io/docker/dockerfile:1.10@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
|
||||
#
|
||||
# svcforge reconciler. Build from the REPO ROOT:
|
||||
# docker buildx build -f services/reconciler/Dockerfile -t svcforge/reconciler:dev .
|
||||
@@ -7,9 +7,9 @@
|
||||
# writes: the four checks enqueue tasks, they do not provision. Orphans are logged, never
|
||||
# deleted.
|
||||
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 AS builder
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.29@sha256:eb2843a1e56fd9e30c7276ce1a52cba86e64c7b385f5e3279a0e08e02dd058fc /uv /usr/local/bin/uv
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
|
||||
WORKDIR /app
|
||||
@@ -30,7 +30,7 @@ p = svcforge_core.__file__; \
|
||||
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
|
||||
|
||||
# --- runtime --------------------------------------------------------------------------
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
|
||||
|
||||
ARG BUILD_SHA=unknown
|
||||
LABEL org.opencontainers.image.title="svcforge-reconciler" \
|
||||
@@ -41,7 +41,10 @@ RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
|
||||
WORKDIR /app
|
||||
COPY --from=builder --chown=10001:10001 /app /app
|
||||
|
||||
COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
|
||||
# helm 3.21.3, not 3.16.2 — see services/worker/Dockerfile. 3.16.2 is a Go 1.22.9 build
|
||||
# carrying CRITICAL CVE-2025-68121 (crypto/tls) and CVE-2026-33186 (grpc) and HIGH
|
||||
# CVE-2026-35469 (spdystream). Kept on 3.x on purpose: helm 4 is a breaking change.
|
||||
COPY --from=mirror.gcr.io/alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm
|
||||
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
|
||||
+91
-122
@@ -1,36 +1,33 @@
|
||||
"""The control loop.
|
||||
|
||||
Every other service in svcforge is edge-triggered: a tenant POSTs, a row appears, a worker
|
||||
claims it. Edge-triggered systems are correct exactly as long as nothing is ever missed —
|
||||
and things are missed. A worker is SIGKILLed holding a lease. An operator runs
|
||||
`helm uninstall` by hand. A pod dies between the CAS and the enqueue. Nobody sends an event
|
||||
for any of that, because the thing that would have sent it is the thing that died.
|
||||
Every other service here is edge-triggered: a tenant POSTs, a row appears, a worker claims
|
||||
it. That is correct only as long as nothing is missed, and things are missed — a worker
|
||||
SIGKILLed holding a lease, an operator running `helm uninstall` by hand, a pod dying
|
||||
between the CAS and the enqueue. Nothing sends an event for any of it, because the thing
|
||||
that would have sent it is the thing that died.
|
||||
|
||||
So: level-triggered. Every 60 seconds, compare the world to the database and enqueue what
|
||||
is missing. The four checks below do not know or care what went wrong, or whether anything
|
||||
did; they are the same code on the happy path and after an outage. That property is the
|
||||
entire reason this service exists, and it is why each check is written as a *query for
|
||||
work*, never as a reaction to an event.
|
||||
is missing. The four checks below do not know what went wrong, or whether anything did;
|
||||
they are the same code on the happy path and after an outage. That is why each is written
|
||||
as a query for work rather than a reaction to an event.
|
||||
|
||||
Three rules hold the design together:
|
||||
|
||||
* **Singleton.** `replicas: 1`, `strategy: Recreate` in the chart. Two reconcilers
|
||||
double-enqueue drift and race on TTL. There is no leader election here on purpose: the
|
||||
correct lease for that lives in Postgres next to the data, not in a Redis lock, and
|
||||
until there is a second replica to elect between, an election is a subsystem that can
|
||||
only fail. One pod, and the `SvcforgeReconcilerStale` alert is what notices it is gone.
|
||||
* **Each check is independent.** One failing check must not skip the other three. A helm
|
||||
binary that cannot reach the API server must not stop TTLs from expiring.
|
||||
* **Enqueue, never act.** The reconciler diagnoses; workers treat. It writes task rows and
|
||||
instance states, and never calls `helm install`. The one exception is reading — the drift
|
||||
check runs `helm list`, because seeing reality is the job.
|
||||
* **Singleton.** `replicas: 1`, `strategy: Recreate`. Two reconcilers double-enqueue drift
|
||||
and race on TTL. No leader election on purpose — the right lease for that lives in
|
||||
Postgres next to the data, and until there is a second replica to elect between, an
|
||||
election is a subsystem that can only fail. The `SvcforgeReconcilerStale` alert notices
|
||||
when the one pod is gone.
|
||||
* **Each check is independent.** A helm binary that cannot reach the API server must not
|
||||
stop TTLs from expiring.
|
||||
* **Enqueue, never act.** The reconciler diagnoses and workers treat: it writes task rows
|
||||
and instance states and never calls `helm install`. Reading is the exception, since
|
||||
seeing reality is the job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import signal
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -55,21 +52,19 @@ from svcforge_core.repo.db import DictPool, make_pool
|
||||
from svcforge_core.repo.instances import InstanceRepo
|
||||
from svcforge_core.repo.reconcile import ReconcileRepo
|
||||
from svcforge_core.repo.tasks import TaskRepo
|
||||
from svcforge_core.runtime import install_stop_signals, sleep_or_stop
|
||||
from svcforge_core.settings import Settings, load_settings
|
||||
|
||||
log = get_logger("svcforge.reconciler")
|
||||
|
||||
# The chart's PodMonitor scrapes the port named `metrics` on 9000. Keep them in step.
|
||||
DEFAULT_METRICS_PORT = 9000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReconcilerDeps:
|
||||
"""Everything a check is allowed to touch. Built once in main(), passed down.
|
||||
|
||||
Same shape as `WorkerDeps` for the same reason: the checks take `deps` instead of
|
||||
reaching for globals, so the integration tests below run every check against a real
|
||||
Postgres and a `FakeProvisioner` without a cluster anywhere in sight.
|
||||
Same shape as `WorkerDeps` and for the same reason: checks take `deps` instead of
|
||||
reaching for globals, so the integration tests run every check against a real Postgres
|
||||
and a `FakeProvisioner` with no cluster in sight.
|
||||
"""
|
||||
|
||||
pool: DictPool
|
||||
@@ -81,8 +76,8 @@ class ReconcilerDeps:
|
||||
clock: Clock
|
||||
catalog: dict[str, CatalogEntry]
|
||||
settings: Settings
|
||||
# Whose instances go first in the day-2 work list. Eating your own dog food is an
|
||||
# `order by`, not a policy document — see InstanceRepo.list_upgradable.
|
||||
# Whose instances go first in the day-2 work list. Eating your own dog food is enforced
|
||||
# by an `order by` rather than a policy document — see InstanceRepo.list_upgradable.
|
||||
own_team: str
|
||||
# A config value, not a scheduler. Leave it at 1 until 1 is too slow.
|
||||
max_in_flight: int
|
||||
@@ -92,23 +87,19 @@ class ReconcilerDeps:
|
||||
|
||||
|
||||
async def check_drift(deps: ReconcilerDeps) -> None:
|
||||
"""`helm list -A -o json` versus what the database believes.
|
||||
"""The live helm releases versus what the database believes.
|
||||
|
||||
This is the only check that looks outside Postgres, and the only one that can catch the
|
||||
failure nothing else can: someone ran `helm uninstall` by hand, or a node was drained
|
||||
and the release never came back. The DB still says `ready` and still hands the tenant an
|
||||
endpoint that resolves to nothing.
|
||||
The only check that looks outside Postgres, and the only one that catches someone
|
||||
running `helm uninstall` by hand or a drained node whose release never came back — where
|
||||
the DB still says `ready` and still hands the tenant an endpoint resolving to nothing.
|
||||
|
||||
Two directions, two very different answers:
|
||||
Two directions, two different answers:
|
||||
|
||||
* **Release gone, DB says `ready`** -> re-enqueue provision. Safe, because provisioning
|
||||
is `helm upgrade --install` against a deterministic release name: converging on
|
||||
desired state, not a blind re-install.
|
||||
* **Release exists, DB knows nothing** -> log at error with release and namespace, and
|
||||
stop. **Never delete in v1.** The reconciler's view of "the DB knows nothing" is one
|
||||
query against one database; the release might belong to another team, another tool,
|
||||
or a migration half-finished. Deleting on that evidence is how an automated system
|
||||
takes down production faster than any human could. A human reads the log and decides.
|
||||
* **Release gone, DB says `ready`** -> re-enqueue provision. Safe because provisioning
|
||||
is `helm upgrade --install` against a deterministic release name.
|
||||
* **Release exists, DB knows nothing** -> log at error and stop. **Never delete in v1.**
|
||||
"The DB knows nothing" is one query against one database, and the release might belong
|
||||
to another team, another tool, or a half-finished migration. A human decides.
|
||||
"""
|
||||
with tracer().start_as_current_span("helm.list"):
|
||||
releases = await deps.provisioner.list_releases()
|
||||
@@ -136,31 +127,30 @@ async def check_drift(deps: ReconcilerDeps) -> None:
|
||||
{"instance_id": str(inst.id), "team": inst.team},
|
||||
)
|
||||
except Exception:
|
||||
# The task is already committed; the notification is a courtesy. A webhook
|
||||
# timing out must not abandon the rest of the sweep — the instances after this
|
||||
# one in the loop have the same problem and nobody else is coming to find them.
|
||||
# The task is committed; the notification is a courtesy. A webhook timing out
|
||||
# must not abandon the rest of the sweep — the instances after this one have the
|
||||
# same problem and nobody else is coming to find them.
|
||||
log.exception("notify.failed", instance_id=str(inst.id))
|
||||
|
||||
known = await deps.reconcile.known_releases()
|
||||
for name, namespace in sorted(live - known):
|
||||
# error, not warning: this is a resource nobody is billing for and nobody owns.
|
||||
# It will sit here every 60s until a human deletes it or adopts it. That is the
|
||||
# intended pressure.
|
||||
# error, not warning: a resource nobody owns and nobody is billing for. It repeats
|
||||
# every 60s until a human deletes or adopts it, which is the intended pressure.
|
||||
log.error("drift.orphan_release", release=name, namespace=namespace, action="none (v1 never deletes)")
|
||||
|
||||
|
||||
async def check_lease_expiry(deps: ReconcilerDeps) -> None:
|
||||
"""Tasks whose worker died -> back to `queued`.
|
||||
|
||||
A lease, not a lock. No lock survives a power cut: a worker SIGKILLed mid-provision
|
||||
leaves `state='running'` with `locked_by` set and nobody running it, and no amount of
|
||||
cleanup code in the worker helps, because the worker is the part that died. `locked_at`
|
||||
plus a timeout is the only thing that recovers the row, which is why `locked_at` exists.
|
||||
A lease, not a lock: no lock survives a power cut. A worker SIGKILLed mid-provision
|
||||
leaves `state='running'` with `locked_by` set and nobody running it, and cleanup code in
|
||||
the worker cannot help because the worker is what died. `locked_at` plus a timeout is
|
||||
the only thing that recovers the row.
|
||||
|
||||
The 5-minute default is not arbitrary: it must exceed the longest a healthy task can
|
||||
hold a lease, or the reconciler hands a still-running provision to a second worker.
|
||||
Handlers are idempotent, so that is survivable rather than fatal — but survivable is
|
||||
not free, and `lease_seconds` sits above helm's `--timeout` for that reason.
|
||||
The 5-minute default must exceed the longest a healthy task can hold a lease, or a
|
||||
still-running provision is handed to a second worker. Handlers are idempotent so that is
|
||||
survivable, but it costs a duplicated helm run — hence `lease_seconds` > helm's
|
||||
`--timeout`.
|
||||
"""
|
||||
freed = await deps.tasks.reset_expired_leases(deps.settings.lease_seconds)
|
||||
if freed:
|
||||
@@ -170,14 +160,13 @@ async def check_lease_expiry(deps: ReconcilerDeps) -> None:
|
||||
async def check_ttl(deps: ReconcilerDeps) -> None:
|
||||
"""Expired instances -> `deleting`, plus a deprovision task.
|
||||
|
||||
The line item that stops a demo cluster from becoming a permanent cloud bill. Also the
|
||||
sweep the API's DELETE route depends on: it CASes to `deleting` and enqueues in two
|
||||
statements, and a crash in between lands here on the next tick.
|
||||
What stops a demo cluster becoming a permanent cloud bill, and the sweep the API's
|
||||
DELETE route depends on: DELETE CASes and enqueues in two statements, and a crash
|
||||
between them lands here on the next tick.
|
||||
|
||||
Idempotent by construction — the work list excludes anything that already has a queued
|
||||
or running deprovision, and the CAS and the insert share one transaction. Without that
|
||||
guard, a deprovision that takes longer than 60 seconds gets a second task on the next
|
||||
tick, and a third on the tick after.
|
||||
Idempotent by construction — the work list excludes anything with a queued or running
|
||||
deprovision, and the CAS and insert share one transaction. Without that, a deprovision
|
||||
taking longer than 60 seconds collects a new task every tick.
|
||||
"""
|
||||
for inst in await deps.reconcile.due_for_deprovision():
|
||||
task_id = await deps.reconcile.enqueue_deprovision(inst.id)
|
||||
@@ -196,20 +185,18 @@ async def check_ttl(deps: ReconcilerDeps) -> None:
|
||||
async def check_version_drift(deps: ReconcilerDeps) -> None:
|
||||
"""The day-2 rollout: the work-list query, one service type at a time.
|
||||
|
||||
Everything that makes this safe is somewhere else, which is the point:
|
||||
Everything that makes it safe is somewhere else, which is the point:
|
||||
|
||||
* `list_upgradable` limits to `max_in_flight` and returns nothing while
|
||||
`rollout_state='halted'`, so a bad chart stops after one tenant.
|
||||
* `schedule_upgrade_at` turns the tenant's maintenance window into a `run_after`; the
|
||||
queue does the waiting, in `where run_after <= now()`. There is no scheduler here and
|
||||
there must not be one — a task parked in Postgres until 03:00 Sunday survives a
|
||||
reconciler restart, and an in-memory timer does not.
|
||||
* `security: true` in the catalog bypasses the window. A CVE with a public exploit does
|
||||
not wait until Sunday.
|
||||
* `schedule_upgrade_at` turns the maintenance window into a `run_after` and the queue
|
||||
does the waiting, in `where run_after <= now()`. No scheduler here, and there must not
|
||||
be one: a task parked in Postgres until 03:00 Sunday survives a restart, a timer does
|
||||
not.
|
||||
* `security: true` in the catalog bypasses the window.
|
||||
|
||||
A bad window spec is this instance's problem, not the fleet's: log it and move to the
|
||||
next one. Failing the whole check would let one tenant's typo freeze everyone's
|
||||
security rollout.
|
||||
A bad window spec is one instance's problem: log it and move on. Failing the check would
|
||||
let one tenant's typo freeze everyone's security rollout.
|
||||
"""
|
||||
now = deps.clock.now()
|
||||
|
||||
@@ -264,23 +251,20 @@ CHECKS: dict[str, Callable[[ReconcilerDeps], Awaitable[None]]] = {
|
||||
async def tick(deps: ReconcilerDeps) -> None:
|
||||
"""One pass: all four checks, then the gauges, then the heartbeat.
|
||||
|
||||
Checks first, gauges second: `svcforge_queue_depth` is read straight after the checks
|
||||
that add to the queue, so the value scraped is the value the tick left behind rather
|
||||
than one from before its own work.
|
||||
Checks first, gauges second, so `svcforge_queue_depth` reports what this tick left
|
||||
behind rather than what preceded its own work.
|
||||
|
||||
The heartbeat is set unconditionally, and that is deliberate. It answers "is the loop
|
||||
running", not "is everything fine" — the checks have their own alerts. Gating it on
|
||||
success would make `SvcforgeReconcilerStale` fire for a helm blip and mean two things
|
||||
at once, and an alert that means two things gets muted.
|
||||
The heartbeat is set unconditionally. It answers "is the loop running", not "is
|
||||
everything fine" — the checks have their own alerts. Gating it on success would make
|
||||
`SvcforgeReconcilerStale` fire for a helm blip and mean two things at once, and an alert
|
||||
that means two things gets muted.
|
||||
|
||||
The whole tick runs inside one span, which is a considered exception to "manual spans go
|
||||
around helm calls only". That rule exists so the API does not hand-roll spans that
|
||||
`opentelemetry-instrument` already creates for it. Nothing auto-instruments the
|
||||
reconciler: without a span here it emits no traces at all, and — because
|
||||
`inject_traceparent` serialises the *active* context — every task it enqueues would be
|
||||
written with a null `traceparent` and be unjoinable to the tick that decided to create
|
||||
it. One span per tick is what makes "why was this instance re-provisioned at 03:00?" a
|
||||
question the traces can answer.
|
||||
The whole tick runs in one span, a considered exception to "manual spans wrap helm calls
|
||||
only". That rule keeps the API from hand-rolling spans `opentelemetry-instrument`
|
||||
already makes; nothing auto-instruments the reconciler, so without this it emits no
|
||||
traces at all and — since `inject_traceparent` serialises the *active* context — every
|
||||
task it enqueues would carry a null `traceparent` and be unjoinable to the tick that
|
||||
created it.
|
||||
"""
|
||||
with tracer().start_as_current_span("reconciler.tick"):
|
||||
await _run_checks(deps)
|
||||
@@ -295,12 +279,11 @@ async def _run_checks(deps: ReconcilerDeps) -> None:
|
||||
try:
|
||||
await check(deps)
|
||||
except Exception: # the tick is the error boundary
|
||||
# The swallow is the design. These four checks share nothing but a database
|
||||
# handle, and the value of a level-triggered loop is that it keeps running: an
|
||||
# unreachable cluster must not stop TTLs from expiring, and one tenant's broken
|
||||
# The swallow is the design. The four checks share nothing but a database
|
||||
# handle, and a level-triggered loop is only worth having if it keeps running:
|
||||
# an unreachable cluster must not stop TTLs expiring, and one tenant's broken
|
||||
# window spec must not stop drift detection. This means "this check achieved
|
||||
# nothing for 60 seconds", which the log says out loud. It never means "the
|
||||
# reconciler stops".
|
||||
# nothing for 60 seconds", never "the reconciler stops".
|
||||
log.exception("check.failed", check=name)
|
||||
|
||||
try:
|
||||
@@ -312,23 +295,17 @@ async def _run_checks(deps: ReconcilerDeps) -> None:
|
||||
log.exception("gauges.failed")
|
||||
|
||||
|
||||
async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
"""Sleep, but wake immediately on SIGTERM. A 60s nap must not cost 60s of shutdown."""
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(stop.wait(), timeout=seconds)
|
||||
|
||||
|
||||
async def run_reconciler(deps: ReconcilerDeps, stop: asyncio.Event) -> None:
|
||||
"""Tick, sleep, repeat, until told to stop.
|
||||
|
||||
Tick first, then sleep: a pod that has just been restarted should reconcile now, not in
|
||||
sixty seconds. Fixed interval rather than a fixed period — a tick that overruns simply
|
||||
delays the next one, instead of stacking a second tick on top of the first, which for a
|
||||
singleton would be exactly the concurrent reconciler `replicas: 1` exists to prevent.
|
||||
Tick first, then sleep: a just-restarted pod should reconcile now, not in sixty seconds.
|
||||
Fixed interval rather than fixed period, so a tick that overruns delays the next one
|
||||
instead of stacking a second on top — which for a singleton is exactly the concurrent
|
||||
reconciler `replicas: 1` exists to prevent.
|
||||
"""
|
||||
while not stop.is_set():
|
||||
await tick(deps)
|
||||
await _sleep_or_stop(stop, deps.settings.reconcile_interval_s)
|
||||
await sleep_or_stop(stop, deps.settings.reconcile_interval_s)
|
||||
|
||||
|
||||
def build_deps(
|
||||
@@ -353,32 +330,27 @@ def build_deps(
|
||||
)
|
||||
|
||||
|
||||
async def _amain(once: bool, metrics_port: int, own_team: str, max_in_flight: int) -> None:
|
||||
async def _amain(once: bool, own_team: str, max_in_flight: int) -> None:
|
||||
settings = load_settings()
|
||||
setup("svcforge-reconciler", settings)
|
||||
|
||||
pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
|
||||
pool = make_pool(settings.runtime_dsn, settings.pool_min_size, settings.pool_max_size)
|
||||
await pool.open(wait=True)
|
||||
deps = build_deps(pool, settings, own_team, max_in_flight)
|
||||
|
||||
try:
|
||||
if once:
|
||||
# One pass and exit: the acceptance path, and how you drive a reconcile by hand
|
||||
# from a shell. No metrics server — nothing would ever scrape it.
|
||||
# One pass and exit: the acceptance path, and how to drive a reconcile by hand.
|
||||
# No metrics server — nothing would ever scrape it.
|
||||
await tick(deps)
|
||||
return
|
||||
|
||||
start_metrics_server(metrics_port)
|
||||
# settings.metrics_port, like the worker. SVCFORGE_METRICS_PORT overrides it through
|
||||
# pydantic rather than a second CLI option, so the port has one definition.
|
||||
start_metrics_server(settings.metrics_port)
|
||||
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
# add_signal_handler, NOT signal.signal. signal.signal fires the handler at an
|
||||
# arbitrary bytecode boundary on the main thread and the loop does not notice
|
||||
# until its next timer — which here is up to a full 60s tick away. This one is
|
||||
# scheduled as an ordinary loop callback, so the `stop.wait()` above returns
|
||||
# immediately.
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
install_stop_signals(stop)
|
||||
|
||||
await run_reconciler(deps, stop)
|
||||
finally:
|
||||
@@ -391,9 +363,6 @@ app = typer.Typer(add_completion=False, help="svcforge reconciler: the control l
|
||||
@app.command()
|
||||
def main(
|
||||
once: bool = typer.Option(False, "--once", help="Run one tick and exit."),
|
||||
metrics_port: int = typer.Option(
|
||||
DEFAULT_METRICS_PORT, envvar="SVCFORGE_METRICS_PORT", help="Port for /metrics."
|
||||
),
|
||||
own_team: str = typer.Option(
|
||||
"platform", envvar="SVCFORGE_OWN_TEAM", help="Team whose instances upgrade first."
|
||||
),
|
||||
@@ -403,7 +372,7 @@ def main(
|
||||
) -> None:
|
||||
"""Run the reconciler."""
|
||||
# One asyncio.run, at the top, never nested. Everything below it is already async.
|
||||
asyncio.run(_amain(once, metrics_port, own_team, max_in_flight))
|
||||
asyncio.run(_amain(once, own_team, max_in_flight))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# syntax=docker/dockerfile:1.10
|
||||
# syntax=mirror.gcr.io/docker/dockerfile:1.10@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
|
||||
#
|
||||
# svcforge worker. Build from the REPO ROOT:
|
||||
# docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
|
||||
#
|
||||
# The only service that shells out to helm/kubectl, so the only one carrying those two
|
||||
# The only service that shells out to helm, so the only one carrying that binary
|
||||
# binaries. They are copied from pinned images rather than curl'd, so the version is a
|
||||
# reviewable line in a Dockerfile instead of a network call at build time.
|
||||
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de AS builder
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 AS builder
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5.11@sha256:0ac957607303916420297a4c9c213bb33fbd3c888f9cd7f4f7273596ebf42b85 /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.29@sha256:eb2843a1e56fd9e30c7276ce1a52cba86e64c7b385f5e3279a0e08e02dd058fc /uv /usr/local/bin/uv
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
|
||||
WORKDIR /app
|
||||
@@ -30,7 +30,7 @@ p = svcforge_core.__file__; \
|
||||
sys.exit(0) if "site-packages" in p else sys.exit("not a wheel install: " + p)'
|
||||
|
||||
# --- runtime --------------------------------------------------------------------------
|
||||
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
|
||||
FROM mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
|
||||
|
||||
ARG BUILD_SHA=unknown
|
||||
LABEL org.opencontainers.image.title="svcforge-worker" \
|
||||
@@ -41,8 +41,11 @@ RUN useradd -u 10001 -m -s /usr/sbin/nologin svcforge
|
||||
WORKDIR /app
|
||||
COPY --from=builder --chown=10001:10001 /app /app
|
||||
|
||||
COPY --from=alpine/helm:3.16.2@sha256:a19a2968fd672336d39771f6c899781424d725229148656dbc2a1e305003cdec /usr/bin/helm /usr/local/bin/helm
|
||||
COPY --from=bitnamilegacy/kubectl:1.31.2@sha256:0eab9ec8f5e0f75271277467ebb7513b36dea0122bc68615d18c47fece4fd82c /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/kubectl
|
||||
# helm 3.21.3, not 3.16.2. 3.16.2 was built with Go 1.22.9 and carries CRITICAL
|
||||
# CVE-2025-68121 (crypto/tls) and CVE-2026-33186 (grpc), plus HIGH CVE-2026-35469
|
||||
# (spdystream, fixed in 0.5.1) — trivy fails the build on them and is right to.
|
||||
# Deliberately 3.x: helm 4 is a breaking change and is not a CVE fix.
|
||||
COPY --from=mirror.gcr.io/alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm
|
||||
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
|
||||
+75
-29
@@ -1,13 +1,12 @@
|
||||
"""Task handlers.
|
||||
|
||||
Every handler here obeys one rule: running it twice must equal running it once.
|
||||
Every handler obeys one rule: running it twice must equal running it once.
|
||||
|
||||
That is not a nicety. A worker can be SIGKILLed after helm has installed the release but
|
||||
before the DB row says so; the lease expires; another worker claims the same task and runs
|
||||
this function again. If the handler is not idempotent, the tenant gets two Elasticsearches
|
||||
and you get a bill. Idempotency is what makes the crash safe, and it is bought in two
|
||||
places: a deterministic `release_name`, and adapters that state desired state
|
||||
(`helm upgrade --install`) instead of issuing imperative commands.
|
||||
A worker can be SIGKILLed after helm installed the release but before the DB row says so;
|
||||
the lease expires, another worker claims the same task, and this function runs again. A
|
||||
handler that is not idempotent gives the tenant two Elasticsearches and you a bill.
|
||||
Idempotency is bought in two places: a deterministic `release_name`, and adapters that
|
||||
state desired state (`helm upgrade --install`) instead of issuing imperative commands.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,18 +17,21 @@ from typing import Any
|
||||
from services.worker.deps import WorkerDeps
|
||||
from svcforge_core.domain.models import CatalogEntry, Instance, Task, TaskKind
|
||||
from svcforge_core.domain.states import InstanceState
|
||||
from svcforge_core.errors import SvcforgeError
|
||||
from svcforge_core.obs import get_logger
|
||||
from svcforge_core.repo.instances import INSTANCE_COLUMNS
|
||||
|
||||
log = get_logger("svcforge.worker")
|
||||
|
||||
|
||||
class HandlerError(RuntimeError):
|
||||
class HandlerError(SvcforgeError, RuntimeError):
|
||||
"""A task failed in a way worth retrying. The message lands in tasks.last_error."""
|
||||
|
||||
|
||||
async def _load_instance(task: Task, deps: WorkerDeps) -> Instance:
|
||||
async with deps.pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""select id, team, service_type, size, state, namespace, release_name,
|
||||
chart_version, endpoint, error, expires_at, created_at, updated_at
|
||||
from instances where id = %s""",
|
||||
f"select {INSTANCE_COLUMNS} from instances where id = %s", # noqa: S608 - module constant
|
||||
(task.instance_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
@@ -38,12 +40,35 @@ async def _load_instance(task: Task, deps: WorkerDeps) -> Instance:
|
||||
return Instance.model_validate(row)
|
||||
|
||||
|
||||
def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`override` wins, except where both sides hold a dict — then merge those too.
|
||||
|
||||
Shallow `base | override` would be wrong the moment two layers touch different keys of
|
||||
the same nested map: `{"global": {"imageRegistry": ...}}` overridden by
|
||||
`{"global": {"storageClass": ...}}` silently drops the registry, and the pod pulls from
|
||||
somewhere nobody chose.
|
||||
"""
|
||||
out = dict(base)
|
||||
for key, value in override.items():
|
||||
current = out.get(key)
|
||||
if isinstance(current, dict) and isinstance(value, dict):
|
||||
out[key] = _deep_merge(current, value)
|
||||
else:
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def _values_for(inst: Instance, entry: CatalogEntry) -> dict[str, Any]:
|
||||
"""Turn a catalog size into helm values."""
|
||||
"""Catalog values, with the requested size's replicas and resources on top.
|
||||
|
||||
Size last, deliberately. An entry that sets `replicaCount` in its own `values:` would
|
||||
otherwise beat the size the tenant actually asked for, and every size would deploy the
|
||||
same shape.
|
||||
"""
|
||||
size = entry.sizes.get(inst.size)
|
||||
if size is None:
|
||||
raise HandlerError(f"size {inst.size!r} not in catalog for {inst.service_type!r}")
|
||||
return {"replicaCount": size.replicas, "resources": size.resources}
|
||||
return _deep_merge(entry.values, {"replicaCount": size.replicas, "resources": size.resources})
|
||||
|
||||
|
||||
async def handle_provision(task: Task, deps: WorkerDeps) -> None:
|
||||
@@ -75,11 +100,18 @@ async def handle_provision(task: Task, deps: WorkerDeps) -> None:
|
||||
inst.id, InstanceState.PROVISIONING, InstanceState.READY, endpoint=endpoint
|
||||
)
|
||||
if ok:
|
||||
await deps.notifier.send(
|
||||
"instance.ready",
|
||||
f"instance {inst.id} is ready at {endpoint}",
|
||||
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
|
||||
)
|
||||
try:
|
||||
await deps.notifier.send(
|
||||
"instance.ready",
|
||||
f"instance {inst.id} is ready at {endpoint}",
|
||||
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
|
||||
)
|
||||
except Exception:
|
||||
# The provision succeeded and the row is READY; the notification is a courtesy.
|
||||
# Propagating a webhook timeout would fail the task, and the retry would hit the
|
||||
# READY early-return and drop the notification anyway — so a flaky notifier
|
||||
# would turn every provision into a "failed" task.
|
||||
log.exception("notify.failed", instance_id=str(inst.id))
|
||||
|
||||
|
||||
async def handle_deprovision(task: Task, deps: WorkerDeps) -> None:
|
||||
@@ -92,7 +124,14 @@ async def handle_deprovision(task: Task, deps: WorkerDeps) -> None:
|
||||
# `helm uninstall` of an already-gone release is not an error to us: the adapter
|
||||
# swallows not-found, because the desired state — no release — is already true.
|
||||
await deps.provisioner.uninstall(release=inst.release_name, ns=inst.namespace)
|
||||
await deps.instances.update_state(inst.id, InstanceState.DELETING, InstanceState.DELETED)
|
||||
|
||||
# Raise rather than ignore the CAS result. Swallowing it leaves the release gone, the
|
||||
# row on `state=ready` with a dangling endpoint, the task marked done — and 60 seconds
|
||||
# later the drift check re-provisions the thing the tenant asked to delete.
|
||||
if not await deps.instances.update_state(inst.id, InstanceState.DELETING, InstanceState.DELETED):
|
||||
raise HandlerError(
|
||||
f"instance {inst.id} was {inst.state.value}, expected {InstanceState.DELETING.value}"
|
||||
)
|
||||
|
||||
|
||||
async def handle_upgrade(task: Task, deps: WorkerDeps) -> None:
|
||||
@@ -127,10 +166,9 @@ async def handle_upgrade(task: Task, deps: WorkerDeps) -> None:
|
||||
async def handle_verify(task: Task, deps: WorkerDeps) -> None:
|
||||
"""Post-upgrade health probe. On failure, halt the whole rollout for this service type.
|
||||
|
||||
One column decides whether the fleet keeps rolling. The work-list query returns nothing
|
||||
while `rollout_state='halted'`, so a bad chart stops after the first tenant instead of
|
||||
after all of them. You clear it with SQL, deliberately: an automatic un-halt would just
|
||||
resume breaking things.
|
||||
The work-list query returns nothing while `rollout_state='halted'`, so a bad chart stops
|
||||
after the first tenant instead of all of them. Clearing it is a deliberate SQL statement:
|
||||
an automatic un-halt would resume breaking things.
|
||||
"""
|
||||
inst = await _load_instance(task, deps)
|
||||
releases = {r.name for r in await deps.provisioner.list_releases()}
|
||||
@@ -138,18 +176,26 @@ async def handle_verify(task: Task, deps: WorkerDeps) -> None:
|
||||
if inst.release_name in releases:
|
||||
return
|
||||
|
||||
# `returning` plus a `where` on the update half says whether THIS call halted the
|
||||
# rollout. The halt is idempotent; the page is not. Without the distinction, a verify
|
||||
# that burns its full retry budget sends five identical notifications for one incident.
|
||||
async with deps.pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute(
|
||||
"""insert into catalog_versions (service_type, rollout_state)
|
||||
values (%s, 'halted')
|
||||
on conflict (service_type) do update set rollout_state = 'halted'""",
|
||||
on conflict (service_type) do update set rollout_state = 'halted'
|
||||
where catalog_versions.rollout_state <> 'halted'
|
||||
returning service_type""",
|
||||
(inst.service_type,),
|
||||
)
|
||||
await deps.notifier.send(
|
||||
"rollout.halted",
|
||||
f"rollout halted for {inst.service_type}: {inst.release_name} failed verify",
|
||||
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
|
||||
)
|
||||
newly_halted = await cur.fetchone() is not None
|
||||
|
||||
if newly_halted:
|
||||
await deps.notifier.send(
|
||||
"rollout.halted",
|
||||
f"rollout halted for {inst.service_type}: {inst.release_name} failed verify",
|
||||
{"instance_id": str(inst.id), "team": inst.team, "service_type": inst.service_type},
|
||||
)
|
||||
raise HandlerError(f"verify failed for {inst.release_name}; rollout halted")
|
||||
|
||||
|
||||
|
||||
+59
-44
@@ -1,18 +1,16 @@
|
||||
"""The claim loop.
|
||||
|
||||
Poll every 5 seconds. Claim while a semaphore slot is free. Run the handler. Report.
|
||||
That is the whole design, and the restraint is the point: LISTEN/NOTIFY would shave the
|
||||
latency, is fire-and-forget so it can never replace the poll anyway, is strictly extra
|
||||
code, and does not exist on pgbouncer's transaction pooler. The poll is not a placeholder
|
||||
for something better.
|
||||
Poll every 5 seconds. Claim while a semaphore slot is free. Run the handler. Report. The
|
||||
poll is not a placeholder for something better: LISTEN/NOTIFY would shave latency, but it
|
||||
is fire-and-forget so it can never replace the poll, and it does not exist on pgbouncer's
|
||||
transaction pooler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import signal
|
||||
import time
|
||||
from collections.abc import Awaitable
|
||||
|
||||
from opentelemetry import trace
|
||||
|
||||
@@ -23,44 +21,56 @@ from svcforge_core.adapters.clock import SystemClock
|
||||
from svcforge_core.adapters.helm import HelmProvisioner
|
||||
from svcforge_core.adapters.notify import LogNotifier
|
||||
from svcforge_core.domain.catalog import load_catalog
|
||||
from svcforge_core.domain.models import Task
|
||||
from svcforge_core.domain.models import Task, TaskKind
|
||||
from svcforge_core.repo.db import make_pool
|
||||
from svcforge_core.repo.instances import InstanceRepo
|
||||
from svcforge_core.repo.tasks import TaskRepo
|
||||
from svcforge_core.runtime import install_stop_signals, sleep_or_stop
|
||||
from svcforge_core.settings import Settings, load_settings
|
||||
|
||||
log = obs.get_logger("svcforge.worker")
|
||||
|
||||
|
||||
async def _sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
"""Sleep, but wake immediately on shutdown.
|
||||
async def _report(coro: Awaitable[bool], task_id: int, what: str) -> None:
|
||||
"""Run a terminal report, and never let its failure escape.
|
||||
|
||||
`await asyncio.sleep(5)` would make every SIGTERM cost up to five seconds of
|
||||
Kubernetes waiting on terminationGracePeriod for no reason.
|
||||
`_run_one` runs inside a TaskGroup, which cancels every sibling the moment one child
|
||||
raises — so a DB blip during `tasks.fail()` would abort every other in-flight provision
|
||||
on this pod. The task itself is safe either way: it stays `running` and the lease sweep
|
||||
returns it to the queue. Losing the report costs one lease interval; losing the siblings
|
||||
costs their work.
|
||||
"""
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(stop.wait(), timeout=seconds)
|
||||
try:
|
||||
if not await coro:
|
||||
# The lease was stolen while we were working: another worker owns this task now
|
||||
# and is mid-run. Reporting is theirs, not ours.
|
||||
log.warning("lease lost before report; another worker owns this task", task_id=task_id)
|
||||
except Exception:
|
||||
log.exception("could not report task %s (%s); lease will expire", task_id, what)
|
||||
|
||||
|
||||
async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None:
|
||||
"""Run one task to a terminal report. Never lets an exception escape the TaskGroup."""
|
||||
worker_id = deps.settings.worker_id
|
||||
try:
|
||||
# Every log line from here carries instance_id/task_id/team. Bound once, at claim,
|
||||
# rather than passed down: the alternative is threading three arguments through
|
||||
# every function that might log, and the first one anyone forgets is the one you
|
||||
# need at 3am.
|
||||
# Every log line from here carries instance_id/task_id/team. Bound once at claim
|
||||
# rather than threaded through every function that might log.
|
||||
obs.bind_task_context(task.instance_id, task.id, team=task.team or "unknown")
|
||||
log.info("task claimed", kind=task.kind.value, attempt=task.attempts)
|
||||
obs.TASKS_CLAIMED.labels(kind=task.kind.value).inc()
|
||||
|
||||
handler = HANDLERS.get(task.kind)
|
||||
if handler is None:
|
||||
await deps.tasks.fail(task.id, f"no handler for {task.kind}", max_attempts=1)
|
||||
await _report(
|
||||
deps.tasks.fail(task.id, f"no handler for {task.kind}", worker_id, max_attempts=1),
|
||||
task.id,
|
||||
"no-handler",
|
||||
)
|
||||
return
|
||||
|
||||
# Re-parent to the span that enqueued this task. Without the stored traceparent
|
||||
# the worker's span starts a brand-new trace, and the POST that caused the work
|
||||
# is in a different trace to the helm call that did it.
|
||||
# Re-parent to the span that enqueued this task. Without the stored traceparent the
|
||||
# worker's span starts a new trace, putting the POST that caused the work in a
|
||||
# different trace from the helm call that did it.
|
||||
ctx = obs.context_from_traceparent(task.traceparent)
|
||||
started = time.monotonic()
|
||||
with obs.tracer().start_as_current_span(
|
||||
@@ -77,17 +87,29 @@ async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None
|
||||
# CancelledError inherits from BaseException, so `except Exception` below
|
||||
# would never see it. Catch it only to release the claim, then ALWAYS
|
||||
# re-raise: swallowing it breaks cancellation for everyone above us.
|
||||
await deps.tasks.fail(task.id, "cancelled", max_attempts=deps.settings.max_attempts)
|
||||
await _report(
|
||||
deps.tasks.fail(task.id, "cancelled", worker_id, deps.settings.max_attempts),
|
||||
task.id,
|
||||
"cancelled",
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.exception("task failed", kind=task.kind.value, error=str(exc))
|
||||
span.record_exception(exc)
|
||||
span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc)))
|
||||
obs.TASKS_FAILED.labels(kind=task.kind.value).inc()
|
||||
await deps.tasks.fail(task.id, str(exc), max_attempts=deps.settings.max_attempts)
|
||||
obs.TASK_ATTEMPTS_FAILED.labels(kind=task.kind.value).inc()
|
||||
await _report(
|
||||
deps.tasks.fail(task.id, str(exc), worker_id, deps.settings.max_attempts),
|
||||
task.id,
|
||||
"fail",
|
||||
)
|
||||
else:
|
||||
obs.PROVISION_TIME.observe(time.monotonic() - started)
|
||||
await deps.tasks.complete(task.id)
|
||||
# Only provisions go in the provision histogram. Its buckets run 10s..1800s
|
||||
# for helm installs, so a sub-second `verify` in the same series drags the
|
||||
# p95 down and quietly stops SvcforgeProvisionSlow from ever firing.
|
||||
if task.kind is TaskKind.PROVISION:
|
||||
obs.PROVISION_TIME.observe(time.monotonic() - started)
|
||||
await _report(deps.tasks.complete(task.id, worker_id), task.id, "complete")
|
||||
finally:
|
||||
sem.release()
|
||||
|
||||
@@ -95,10 +117,9 @@ async def _run_one(deps: WorkerDeps, task: Task, sem: asyncio.Semaphore) -> None
|
||||
async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
|
||||
"""Claim and run until told to stop, then drain what is in flight.
|
||||
|
||||
Draining is what makes a rolling deploy invisible. Exiting the `async with` block
|
||||
awaits every in-flight handler, so a pod that is being replaced finishes the provision
|
||||
it already started instead of abandoning it half-done for the lease to clean up
|
||||
five minutes later.
|
||||
Draining is what makes a rolling deploy invisible: exiting the `async with` awaits every
|
||||
in-flight handler, so a pod being replaced finishes the provision it started instead of
|
||||
abandoning it for the lease to clean up five minutes later.
|
||||
"""
|
||||
sem = asyncio.Semaphore(deps.settings.worker_concurrency)
|
||||
worker_id = deps.settings.worker_id
|
||||
@@ -116,12 +137,12 @@ async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
|
||||
# A DB blip must not kill the worker; back off and try again.
|
||||
log.exception("claim failed")
|
||||
sem.release()
|
||||
await _sleep_or_stop(stop, deps.settings.poll_interval_s)
|
||||
await sleep_or_stop(stop, deps.settings.poll_interval_s)
|
||||
continue
|
||||
|
||||
if task is None:
|
||||
sem.release()
|
||||
await _sleep_or_stop(stop, deps.settings.poll_interval_s)
|
||||
await sleep_or_stop(stop, deps.settings.poll_interval_s)
|
||||
continue
|
||||
|
||||
tg.create_task(_run_one(deps, task, sem))
|
||||
@@ -131,13 +152,13 @@ async def run_worker(deps: WorkerDeps, stop: asyncio.Event) -> None:
|
||||
async def _amain() -> None:
|
||||
settings: Settings = load_settings()
|
||||
|
||||
# Before anything else: nothing logged above this line is structured, and the metrics
|
||||
# the SvcforgeTaskFailed / SvcforgeProvisionSlow alerts query do not exist until the
|
||||
# registry is up.
|
||||
# Before anything else: nothing above this line logs structured, and the metrics the
|
||||
# SvcforgeTaskFailed / SvcforgeProvisionSlow alerts query do not exist until it runs.
|
||||
obs.setup("svcforge-worker", settings)
|
||||
settings.check_production()
|
||||
obs.start_metrics_server(settings.metrics_port)
|
||||
|
||||
pool = make_pool(settings.pg_dsn.unicode_string(), settings.pool_min_size, settings.pool_max_size)
|
||||
pool = make_pool(settings.runtime_dsn, settings.pool_min_size, settings.pool_max_size)
|
||||
await pool.open(wait=True)
|
||||
|
||||
deps = WorkerDeps(
|
||||
@@ -152,13 +173,7 @@ async def _amain() -> None:
|
||||
)
|
||||
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
# add_signal_handler, NOT signal.signal. signal.signal runs the handler at an
|
||||
# arbitrary bytecode boundary on whatever thread the C-level handler lands on,
|
||||
# and the event loop will not notice until its next timer fires. This one is
|
||||
# loop-safe: the callback runs as a normal loop callback.
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
install_stop_signals(stop)
|
||||
|
||||
try:
|
||||
await run_worker(deps, stop)
|
||||
|
||||
+7
-1
@@ -145,7 +145,12 @@ class FakeRateLimiter:
|
||||
# Fails OPEN, exactly like the real one. A limiter that refused here would make
|
||||
# "Redis is down" indistinguishable from "you are over quota".
|
||||
return RateLimitResult(
|
||||
allowed=True, limit=self.limit, remaining=self.limit, reset_at=reset_at, degraded=True
|
||||
allowed=True,
|
||||
limit=self.limit,
|
||||
remaining=self.limit,
|
||||
reset_at=reset_at,
|
||||
checked_at=self.clock.now(),
|
||||
degraded=True,
|
||||
)
|
||||
key = f"rl:{team}:{window}"
|
||||
n = self.counts.get(key, 0) + 1
|
||||
@@ -155,6 +160,7 @@ class FakeRateLimiter:
|
||||
limit=self.limit,
|
||||
remaining=max(0, self.limit - n),
|
||||
reset_at=reset_at,
|
||||
checked_at=self.clock.now(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ Two ways to get a database, in priority order:
|
||||
1. `SVCFORGE_TEST_DSN` in the environment — an already-running Postgres. This is the
|
||||
path on a host that has no Docker daemon (for example one whose containerd belongs
|
||||
to a Kubernetes kubelet, where installing Docker would evict the runtime).
|
||||
2. testcontainers, which starts `postgres:16-alpine` and throws it away after. This is
|
||||
2. testcontainers, which starts `postgres:18-alpine` and throws it away after. This is
|
||||
the CI path.
|
||||
|
||||
Same tests either way.
|
||||
@@ -88,7 +88,7 @@ def pg_dsn() -> Iterator[str]:
|
||||
pytest.skip("set SVCFORGE_TEST_DSN or install testcontainers")
|
||||
|
||||
# A container is already private to this process; no need to clone inside it.
|
||||
with PostgresContainer("postgres:16-alpine", driver=None) as pg:
|
||||
with PostgresContainer("postgres:18-alpine", driver=None) as pg:
|
||||
dsn = pg.get_connection_url()
|
||||
_apply_migrations(dsn)
|
||||
yield dsn
|
||||
@@ -98,7 +98,7 @@ def pg_dsn() -> Iterator[str]:
|
||||
async def pool(pg_dsn: str) -> AsyncIterator[DictPool]:
|
||||
"""A clean database and an open pool, per test.
|
||||
|
||||
max_size=60 is not a performance choice. test_skip_locked_claims_each_task_exactly_once
|
||||
max_size=60 is a correctness requirement. test_skip_locked_claims_each_task_exactly_once
|
||||
races 50 concurrent claims; a pool smaller than that serialises them at the pool
|
||||
instead of at the database, and the test passes while proving nothing.
|
||||
"""
|
||||
|
||||
@@ -28,6 +28,7 @@ from fastapi import FastAPI
|
||||
|
||||
from services.api.main import create_app
|
||||
from services.api.routes.instances import release_name_for
|
||||
from svcforge_core.domain.catalog import load_catalog
|
||||
from svcforge_core.repo.db import DictPool
|
||||
from svcforge_core.settings import Settings
|
||||
|
||||
@@ -182,8 +183,10 @@ async def test_post_returns_202_and_location(client: httpx.AsyncClient, token: s
|
||||
assert resp.headers["location"] == f"/v1/instances/{body['id']}"
|
||||
assert body["state"] == "requested"
|
||||
assert body["service_type"] == "elasticsearch"
|
||||
# Pinned from catalog.yaml at creation time, not echoed from the request.
|
||||
assert body["chart_version"] == "21.3.15"
|
||||
# Pinned from catalog.yaml at creation time, not echoed from the request. Read from the
|
||||
# catalog rather than hardcoded: the literal made a routine version bump fail here, on a
|
||||
# test whose subject is *where the value comes from*, not what it is.
|
||||
assert body["chart_version"] == load_catalog(CATALOG)["elasticsearch"].chart_version
|
||||
assert body["endpoint"] is None
|
||||
# The response model is an allow-list: placement details stay off the wire.
|
||||
assert "team" not in body and "namespace" not in body and "release_name" not in body
|
||||
@@ -298,6 +301,46 @@ async def test_ttl_out_of_range_is_422(client: httpx.AsyncClient, token: str) ->
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
def _is_error_body(payload: object) -> bool:
|
||||
"""The uniform error shape: a dict with `code` and `message`, and no default `detail`."""
|
||||
return isinstance(payload, dict) and "code" in payload and "message" in payload
|
||||
|
||||
|
||||
async def test_framework_404_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
|
||||
"""A 404 raised by the router, not a handler, must still be ErrorBody.
|
||||
|
||||
Starlette raises its own HTTPException for an unknown route. The exception handler is
|
||||
registered on that parent class precisely so this body is ErrorBody and not FastAPI's
|
||||
default `{"detail": "Not Found"}` — one shape for every error.
|
||||
"""
|
||||
resp = await client.get("/v1/no-such-route")
|
||||
assert resp.status_code == 404
|
||||
assert _is_error_body(resp.json()), resp.text
|
||||
|
||||
|
||||
async def test_framework_405_uses_the_error_body_shape(client: httpx.AsyncClient) -> None:
|
||||
"""A wrong-method 405 comes from the router too, and must be ErrorBody."""
|
||||
resp = await client.delete("/v1/instances") # collection route has no DELETE
|
||||
assert resp.status_code == 405
|
||||
assert _is_error_body(resp.json()), resp.text
|
||||
|
||||
|
||||
async def test_body_validation_422_uses_the_error_body_shape(client: httpx.AsyncClient, token: str) -> None:
|
||||
"""A RequestValidationError 422 must match the handler-raised 422 shape.
|
||||
|
||||
A forbidden extra field trips pydantic's `extra="forbid"` and raises
|
||||
RequestValidationError, which the dedicated handler renders as ErrorBody rather than the
|
||||
default `{"detail": [...]}`.
|
||||
"""
|
||||
resp = await client.post(
|
||||
"/v1/instances",
|
||||
headers=auth(token),
|
||||
json={"service_type": "redis", "size": "small", "surprise": "field"},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
assert _is_error_body(resp.json()), resp.text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- authn
|
||||
|
||||
|
||||
@@ -545,11 +588,25 @@ async def test_202_is_on_the_decorator_not_the_response_object(client: httpx.Asy
|
||||
assert "202" in schema["paths"]["/v1/instances/{instance_id}"]["delete"]["responses"]
|
||||
|
||||
|
||||
def test_auth_disabled_short_circuits(settings: Settings) -> None:
|
||||
"""The dev escape hatch exists, and Settings.check_production() refuses it in prod."""
|
||||
dev = settings.model_copy(update={"auth_disabled": True})
|
||||
with pytest.raises(ValueError, match="refused outside local development"):
|
||||
dev.check_production()
|
||||
def test_auth_disabled_is_allowed_locally_and_refused_everywhere_else(settings: Settings) -> None:
|
||||
"""The dev escape hatch exists, and check_production() refuses it outside `local`.
|
||||
|
||||
Both halves matter. The permissive half is why every entrypoint can call this
|
||||
unconditionally at startup; the refusing half is the actual safety property. An
|
||||
earlier version raised unconditionally, which meant it could only be called from a
|
||||
branch that already knew it was production — so nobody ever wrote that branch and the
|
||||
check never ran at all.
|
||||
"""
|
||||
dev_locally = settings.model_copy(update={"auth_disabled": True, "environment": "local"})
|
||||
dev_locally.check_production() # must not raise
|
||||
|
||||
for env in ("prod", "staging"):
|
||||
dev_in_prod = settings.model_copy(update={"auth_disabled": True, "environment": env})
|
||||
with pytest.raises(ValueError, match="refused"):
|
||||
dev_in_prod.check_production()
|
||||
|
||||
# And a correctly-configured production app starts fine.
|
||||
settings.model_copy(update={"auth_disabled": False, "environment": "prod"}).check_production()
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@@ -580,3 +637,40 @@ async def test_auth_disabled_accepts_an_unauthenticated_request(settings: Settin
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as c:
|
||||
resp = await c.get("/v1/instances")
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- openapi
|
||||
|
||||
|
||||
async def test_openapi_documents_the_api_for_a_caller_without_this_repo(
|
||||
client: httpx.AsyncClient,
|
||||
) -> None:
|
||||
"""/openapi.json is the contract other teams integrate against, so it gets a test.
|
||||
|
||||
Pins the parts a generated schema does not give you for free and that silently rot: the
|
||||
prose description, the tag docs, and the bearer scheme that makes the Authorize button
|
||||
in /docs work. Without the security scheme a caller cannot try a single authenticated
|
||||
route from the UI.
|
||||
"""
|
||||
resp = await client.get("/openapi.json")
|
||||
assert resp.status_code == 200
|
||||
spec = resp.json()
|
||||
|
||||
assert spec["info"]["title"] == "svcforge"
|
||||
# The description carries the two things the schema cannot express: writes are async,
|
||||
# and authorisation is a WHERE clause that 404s rather than 403s.
|
||||
description = spec["info"]["description"]
|
||||
assert "202" in description and "404" in description
|
||||
|
||||
assert {t["name"] for t in spec["tags"]} == {"instances", "ops"}
|
||||
assert "HTTPBearer" in spec["components"]["securitySchemes"]
|
||||
assert "/v1/instances" in spec["paths"]
|
||||
# An example payload, so a caller can see a valid body rather than infer one.
|
||||
assert spec["components"]["schemas"]["CreateInstanceRequest"]["examples"]
|
||||
|
||||
|
||||
async def test_swagger_and_redoc_are_served(client: httpx.AsyncClient) -> None:
|
||||
"""The human-facing docs. Both are on by default; a `docs_url=None` would drop them."""
|
||||
for path in ("/docs", "/redoc"):
|
||||
resp = await client.get(path)
|
||||
assert resp.status_code == 200, path
|
||||
|
||||
@@ -1,29 +1,59 @@
|
||||
"""The one test that proves the timeout is real.
|
||||
|
||||
`bash -c "sleep 300 & sleep 300"` is a miniature helm: a process that forks a child and
|
||||
waits on another. Kill the direct child only and the backgrounded `sleep` reparents to init
|
||||
and keeps running — which, when the process is helm, means a timed-out task retries while
|
||||
the original helm is still mutating the same release.
|
||||
`bash -c "sleep N & sleep N"` is a miniature helm: a process that forks a child and waits on
|
||||
another. Kill the direct child only and the backgrounded `sleep` reparents to init and keeps
|
||||
running — which, when the process is helm, means a timed-out task retries while the original
|
||||
helm is still mutating the same release.
|
||||
|
||||
This test needs a real process tree, so it lives in integration/. It needs no database:
|
||||
the `pool` fixture in conftest is not autouse.
|
||||
This test needs a real process tree, so it lives in integration/. It needs no database: the
|
||||
`pool` fixture in conftest is not autouse.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from svcforge_core.adapters.helm import _run
|
||||
|
||||
# A duration nothing else on the machine will be sleeping for, derived from the pid so two
|
||||
# concurrent runs cannot collide either.
|
||||
#
|
||||
# The obvious version of this test hardcodes `sleep 300` and then asserts
|
||||
# `pgrep -f "sleep 300"` is empty. That assertion is machine-global: it matches ANY process
|
||||
# whose command line contains the string, including a leftover from an earlier run of this
|
||||
# same test, an unrelated `sleep 300` somewhere on the box, and — the subtle one — the shell
|
||||
# that is running pgrep, whose own command line contains the pattern it is searching for.
|
||||
# The result is a test that fails for reasons that have nothing to do with the code, which
|
||||
# is as useless as one that cannot fail at all. Observed: three spurious matches on a
|
||||
# developer box, and a red run blamed on a Python upgrade that was innocent.
|
||||
_SLEEP_S = 30000 + (os.getpid() % 1000)
|
||||
_PATTERN = f"sleep {_SLEEP_S}"
|
||||
|
||||
|
||||
def _survivors() -> list[str]:
|
||||
"""PIDs still matching this run's unique sleep. Empty means the group really died."""
|
||||
out = subprocess.run( # noqa: S603
|
||||
["pgrep", "-f", _PATTERN], # noqa: S607 - resolved via PATH, fixed argv
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return [line for line in out.stdout.split() if line.strip()]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_kills_the_whole_process_group() -> None:
|
||||
argv = ["bash", "-c", "sleep 300 & sleep 300"] # child forks a grandchild
|
||||
# Guard the guard: if the pattern already matches something, the assertion below would
|
||||
# be meaningless. Fail loudly rather than report a false negative.
|
||||
assert not _survivors(), f"{_PATTERN!r} matched before the test started; pick another marker"
|
||||
|
||||
argv = ["bash", "-c", f"{_PATTERN} & {_PATTERN}"] # child forks a grandchild
|
||||
with pytest.raises(TimeoutError):
|
||||
await _run(argv, timeout_s=1)
|
||||
await asyncio.sleep(0.5)
|
||||
out = subprocess.run(["pgrep", "-f", "sleep 300"], capture_output=True, text=True) # noqa: S607
|
||||
assert out.stdout.strip() == "", "grandchild survived: you killed the child, not the group"
|
||||
|
||||
assert not _survivors(), "grandchild survived: you killed the child, not the group"
|
||||
|
||||
@@ -267,8 +267,8 @@ async def test_lease_expiry_returns_a_dead_workers_task_to_the_queue(pool: DictP
|
||||
async def test_lease_expiry_leaves_a_live_worker_alone(pool: DictPool) -> None:
|
||||
"""A task claimed a second ago is not a dead worker. Reclaiming it would double-provision.
|
||||
|
||||
Handlers are idempotent, so a wrongly-freed lease is survivable — but survivable is not
|
||||
free, and this is why lease_seconds sits above helm's own --timeout.
|
||||
Handlers are idempotent, so a wrongly-freed lease is survivable, though it still costs a
|
||||
duplicated helm run — which is why lease_seconds sits above helm's own --timeout.
|
||||
"""
|
||||
instance_id, _, _ = await _seed(pool)
|
||||
tasks = TaskRepo(pool)
|
||||
@@ -419,8 +419,9 @@ async def test_version_drift_parks_the_upgrade_until_the_maintenance_window(
|
||||
) -> None:
|
||||
"""The queue does the waiting, in `where run_after <= now()`.
|
||||
|
||||
Not a scheduler and not an in-memory timer: a task parked in Postgres until 03:00 Sunday
|
||||
survives a reconciler restart. That is the whole reason `run_after` exists.
|
||||
The waiting is done by the queue rather than by a scheduler or an in-memory timer: a
|
||||
task parked in Postgres until 03:00 Sunday survives a reconciler restart. That is the
|
||||
whole reason `run_after` exists.
|
||||
"""
|
||||
instance_id, _, _ = await _seed(pool, maintenance_window=SUNDAY_0300_HCM)
|
||||
|
||||
@@ -520,8 +521,19 @@ def tracing() -> InMemorySpanExporter:
|
||||
|
||||
Global because OTEL's is: `trace.set_tracer_provider` takes once per process, and
|
||||
`obs.tracer()` resolves it at call time. Session-scoped so the second call never
|
||||
happens.
|
||||
happens from HERE.
|
||||
|
||||
The internals reset is load-bearing rather than cosmetic. `set_tracer_provider` is
|
||||
one-shot: a second call logs "Overriding of current TracerProvider is not allowed" at
|
||||
WARNING and is otherwise ignored. Any earlier test that builds a FastAPI app calls
|
||||
`obs.setup()` and burns that one shot, after which this fixture silently installs
|
||||
nothing, `get_finished_spans()` returns `[]`, and the failure reads as "the reconciler
|
||||
stopped writing traceparents" rather than "another test got there first". Clearing the
|
||||
module globals is the only way to take the shot back.
|
||||
"""
|
||||
trace._TRACER_PROVIDER = None
|
||||
trace._TRACER_PROVIDER_SET_ONCE._done = False
|
||||
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
|
||||
@@ -206,7 +206,7 @@ async def test_real_lua_refuses_the_eleventh(upstash: Redis) -> None:
|
||||
async def test_a_check_costs_exactly_one_redis_command(upstash: Redis) -> None:
|
||||
"""One EVALSHA. Not GET+INCR+EXPIRE, which is three billed commands and a race.
|
||||
|
||||
At 500K/month the difference is not academic: three commands per request caps the
|
||||
At 500K/month the difference is material: three commands per request caps the
|
||||
platform at 166K requests/month instead of 500K, for a limiter that is also wrong.
|
||||
|
||||
The first call is excluded from the count on purpose. redis-py sends EVALSHA, Upstash
|
||||
@@ -434,7 +434,7 @@ async def test_the_budget_metric_is_exposed_and_labelled_by_op() -> None:
|
||||
"""`curl -s localhost:8000/metrics | grep svcforge_redis_commands_total`."""
|
||||
limiter = FakeRateLimiter(limit=10, window_s=60, clock=FakeClock(start=_T0))
|
||||
await limiter.check("acme") # the fake does not touch the real counter
|
||||
RateLimitResult(allowed=True, limit=10, remaining=9, reset_at=_T0)
|
||||
RateLimitResult(allowed=True, limit=10, remaining=9, reset_at=_T0, checked_at=_T0)
|
||||
|
||||
text = generate_latest(REGISTRY).decode()
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ async def test_complete_marks_done_and_releases_lock(pool: DictPool) -> None:
|
||||
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
claimed = await repo.claim("w1")
|
||||
assert claimed is not None
|
||||
await repo.complete(claimed.id)
|
||||
await repo.complete(claimed.id, "w1")
|
||||
row = await _task_row(pool, tid)
|
||||
assert row["state"] == "done"
|
||||
assert row["locked_by"] is None
|
||||
@@ -49,7 +49,7 @@ async def test_fail_under_max_attempts_requeues_with_future_run_after(pool: Dict
|
||||
assert claimed is not None
|
||||
assert claimed.attempts == 1 # attempts increments at CLAIM time, not on failure
|
||||
|
||||
await repo.fail(tid, "helm exploded", max_attempts=5)
|
||||
await repo.fail(tid, "helm exploded", "w1", max_attempts=5)
|
||||
row = await _task_row(pool, tid)
|
||||
assert row["state"] == "queued"
|
||||
assert row["last_error"] == "helm exploded"
|
||||
@@ -66,10 +66,12 @@ async def test_fail_at_max_attempts_dead_letters_and_marks_instance(pool: DictPo
|
||||
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
|
||||
tid = await tasks.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
|
||||
claimed = await tasks.claim("w1")
|
||||
assert claimed is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
|
||||
|
||||
await tasks.fail(tid, "chart not found", max_attempts=5)
|
||||
assert await tasks.fail(tid, "chart not found", "w1", max_attempts=5) is True
|
||||
|
||||
row = await _task_row(pool, tid)
|
||||
assert row["state"] == "failed"
|
||||
@@ -90,10 +92,12 @@ async def test_fail_does_not_resurrect_a_deleted_instance(pool: DictPool) -> Non
|
||||
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
|
||||
tid = await tasks.enqueue_standalone(iid, TaskKind.DEPROVISION)
|
||||
|
||||
claimed = await tasks.claim("w1")
|
||||
assert claimed is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
|
||||
|
||||
await tasks.fail(tid, "helm uninstall kept failing", max_attempts=5)
|
||||
assert await tasks.fail(tid, "helm uninstall kept failing", "w1", max_attempts=5) is True
|
||||
|
||||
# The task still dead-letters — that part is unconditional.
|
||||
row = await _task_row(pool, tid)
|
||||
@@ -106,12 +110,92 @@ async def test_fail_does_not_resurrect_a_deleted_instance(pool: DictPool) -> Non
|
||||
assert inst.error is None
|
||||
|
||||
|
||||
async def test_fail_of_deprovision_leaves_the_instance_deleting_to_be_retried(
|
||||
pool: DictPool,
|
||||
) -> None:
|
||||
"""A dead-lettered deprovision must not strand the instance in `failed`.
|
||||
|
||||
The instance is `deleting`, which is one of the states that CAN legally become `failed`,
|
||||
so the naive blanket UPDATE would move it there. due_for_deprovision only re-selects
|
||||
`ready`(expired) and `deleting`, so `failed` would take the instance out of the recovery
|
||||
sweep and leak the helm release forever. reconcile.py documents that a deprovision which
|
||||
exhausts its retries stays re-enqueueable; this pins that guarantee.
|
||||
"""
|
||||
iid = await make_instance(pool, state=InstanceState.DELETING)
|
||||
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
|
||||
tid = await tasks.enqueue_standalone(iid, TaskKind.DEPROVISION)
|
||||
|
||||
claimed = await tasks.claim("w1")
|
||||
assert claimed is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
|
||||
|
||||
assert await tasks.fail(tid, "cluster unreachable", "w1", max_attempts=5) is True
|
||||
|
||||
assert (await _task_row(pool, tid))["state"] == "failed"
|
||||
inst = await instances.get(iid, team="platform")
|
||||
assert inst is not None
|
||||
assert inst.state is InstanceState.DELETING, "a stranded deprovision leaks the release"
|
||||
assert inst.error is None
|
||||
|
||||
|
||||
async def test_fail_of_upgrade_leaves_a_working_instance_ready(pool: DictPool) -> None:
|
||||
"""A dead-lettered upgrade must not mark a healthy instance `failed`.
|
||||
|
||||
helm --atomic rolls the release back, so after a failed upgrade the instance is still
|
||||
`ready` and serving the previous version. Marking it `failed` mislabels a working
|
||||
service and drops it off the upgrade work-list. check_version_drift retries on the next
|
||||
window; the dead-letter metric is the operator signal.
|
||||
"""
|
||||
iid = await make_instance(pool, state=InstanceState.READY)
|
||||
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
|
||||
tid = await tasks.enqueue_standalone(iid, TaskKind.UPGRADE)
|
||||
|
||||
claimed = await tasks.claim("w1")
|
||||
assert claimed is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
|
||||
|
||||
assert await tasks.fail(tid, "upgrade to 1.4.0 kept timing out", "w1", max_attempts=5) is True
|
||||
|
||||
assert (await _task_row(pool, tid))["state"] == "failed"
|
||||
inst = await instances.get(iid, team="platform")
|
||||
assert inst is not None
|
||||
assert inst.state is InstanceState.READY, "a failed upgrade mislabelled a healthy instance"
|
||||
assert inst.error is None
|
||||
|
||||
|
||||
async def test_fail_of_verify_leaves_the_instance_ready(pool: DictPool) -> None:
|
||||
"""A dead-lettered verify must not mark the instance `failed`.
|
||||
|
||||
handle_verify halts the rollout for the service type; the instance itself is `ready`,
|
||||
and drift re-provisions it if its release vanished. `failed` would take it out of both
|
||||
recovery paths.
|
||||
"""
|
||||
iid = await make_instance(pool, state=InstanceState.READY)
|
||||
tasks, instances = TaskRepo(pool), InstanceRepo(pool)
|
||||
tid = await tasks.enqueue_standalone(iid, TaskKind.VERIFY)
|
||||
|
||||
claimed = await tasks.claim("w1")
|
||||
assert claimed is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set attempts = 5 where id = %s", (tid,))
|
||||
|
||||
assert await tasks.fail(tid, "release vanished after upgrade", "w1", max_attempts=5) is True
|
||||
|
||||
assert (await _task_row(pool, tid))["state"] == "failed"
|
||||
inst = await instances.get(iid, team="platform")
|
||||
assert inst is not None
|
||||
assert inst.state is InstanceState.READY
|
||||
assert inst.error is None
|
||||
|
||||
|
||||
async def test_fail_truncates_error_to_2kb(pool: DictPool) -> None:
|
||||
iid = await make_instance(pool)
|
||||
repo = TaskRepo(pool)
|
||||
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
await repo.claim("w1")
|
||||
await repo.fail(tid, "x" * 9000, max_attempts=5)
|
||||
await repo.fail(tid, "x" * 9000, "w1", max_attempts=5)
|
||||
row = await _task_row(pool, tid)
|
||||
assert isinstance(row["last_error"], str)
|
||||
assert len(row["last_error"]) == 2000
|
||||
@@ -153,3 +237,56 @@ async def test_fresh_lease_is_not_reset(pool: DictPool) -> None:
|
||||
await repo.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
await repo.claim("w1")
|
||||
assert await repo.reset_expired_leases(lease_seconds=300) == 0
|
||||
|
||||
|
||||
async def test_stale_worker_cannot_complete_a_task_another_worker_now_owns(pool: DictPool) -> None:
|
||||
"""The lost-lease race, as a regression test.
|
||||
|
||||
Worker A hangs past its lease. The reconciler requeues the task. Worker B claims it and
|
||||
starts working. Worker A finally returns and reports success. Without the ownership
|
||||
check in `complete`, A marks the task done while B is still running it: B's helm
|
||||
install is unaccounted for, and if B then fails, the task is resurrected and a THIRD
|
||||
worker provisions the same instance. That is the double-provision the claim query's
|
||||
whole design exists to prevent, arriving through the back door.
|
||||
"""
|
||||
iid = await make_instance(pool)
|
||||
repo = TaskRepo(pool)
|
||||
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
|
||||
a = await repo.claim("worker-A")
|
||||
assert a is not None
|
||||
|
||||
# A's lease expires and the reconciler hands the task back to the queue.
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set locked_at = now() - interval '10 minutes' where id=%s", (tid,))
|
||||
assert await repo.reset_expired_leases(lease_seconds=300) == 1
|
||||
|
||||
b = await repo.claim("worker-B")
|
||||
assert b is not None, "worker-B should have been able to claim the requeued task"
|
||||
|
||||
# A reports. It must lose.
|
||||
assert await repo.complete(tid, "worker-A") is False, "stale worker completed a task it no longer owns"
|
||||
|
||||
row = await _task_row(pool, tid)
|
||||
assert row["state"] == "running", "the task must still belong to worker-B"
|
||||
assert row["locked_by"] == "worker-B"
|
||||
|
||||
|
||||
async def test_stale_worker_cannot_fail_a_task_another_worker_now_owns(pool: DictPool) -> None:
|
||||
"""The mirror case: a stale failure report must not requeue someone else's task."""
|
||||
iid = await make_instance(pool)
|
||||
repo = TaskRepo(pool)
|
||||
tid = await repo.enqueue_standalone(iid, TaskKind.PROVISION)
|
||||
|
||||
assert await repo.claim("worker-A") is not None
|
||||
async with pool.connection() as conn, conn.cursor() as cur:
|
||||
await cur.execute("update tasks set locked_at = now() - interval '10 minutes' where id=%s", (tid,))
|
||||
await repo.reset_expired_leases(lease_seconds=300)
|
||||
assert await repo.claim("worker-B") is not None
|
||||
|
||||
assert await repo.fail(tid, "stale report", "worker-A", max_attempts=5) is False
|
||||
|
||||
row = await _task_row(pool, tid)
|
||||
assert row["state"] == "running"
|
||||
assert row["locked_by"] == "worker-B"
|
||||
assert row["last_error"] is None, "a stale worker wrote its error onto another worker's task"
|
||||
|
||||
@@ -200,4 +200,12 @@ async def test_concurrency_cap_is_respected(pool: DictPool, concurrency: int) ->
|
||||
stop.set()
|
||||
await asyncio.wait_for(worker, timeout=10)
|
||||
|
||||
# BOTH bounds. `<= concurrency` alone passes on a worker whose semaphore is broken to 1,
|
||||
# or that lost concurrency entirely — it only proves the cap is not exceeded, never that
|
||||
# concurrency exists. With 6 tasks seeded and a 0.2s handler, a working worker reaches
|
||||
# its cap exactly.
|
||||
assert prov.max_concurrent <= concurrency, f"ran {prov.max_concurrent} at once, cap was {concurrency}"
|
||||
assert prov.max_concurrent == concurrency, (
|
||||
f"only reached {prov.max_concurrent} concurrent with a cap of {concurrency}: "
|
||||
"the worker is not actually running tasks in parallel"
|
||||
)
|
||||
|
||||
@@ -150,6 +150,60 @@ def test_bare_mapping_without_services_key_is_accepted(tmp_path: Path) -> None:
|
||||
def test_repo_catalog_yaml_is_valid() -> None:
|
||||
catalog = load_catalog(Path(__file__).parents[2] / "catalog.yaml")
|
||||
|
||||
assert set(catalog) == {"elasticsearch", "redis", "postgres"}
|
||||
assert set(catalog) == {"elasticsearch", "redis", "postgres", "podinfo", "nginx"}
|
||||
for entry in catalog.values():
|
||||
assert set(entry.sizes) == {"small", "medium"}
|
||||
|
||||
|
||||
def test_entry_values_default_to_empty(tmp_path: Path) -> None:
|
||||
"""`values:` is optional — an entry that needs no chart knobs says nothing."""
|
||||
body = textwrap.dedent("""
|
||||
services:
|
||||
podinfo:
|
||||
chart: oci://ghcr.io/stefanprodan/charts/podinfo
|
||||
chart_version: "6.14.0"
|
||||
sizes:
|
||||
small: {replicas: 1, resources: {}}
|
||||
""")
|
||||
assert load_catalog(_write(tmp_path, body))["podinfo"].values == {}
|
||||
|
||||
|
||||
def test_entry_values_are_parsed(tmp_path: Path) -> None:
|
||||
"""Nested values survive the load, which is what `global.imageRegistry` needs."""
|
||||
body = textwrap.dedent("""
|
||||
services:
|
||||
redis:
|
||||
chart: oci://mirror.gcr.io/bitnamicharts/redis
|
||||
chart_version: "27.0.15"
|
||||
values:
|
||||
global:
|
||||
imageRegistry: mirror.gcr.io
|
||||
sizes:
|
||||
small: {replicas: 1, resources: {}}
|
||||
""")
|
||||
assert load_catalog(_write(tmp_path, body))["redis"].values == {
|
||||
"global": {"imageRegistry": "mirror.gcr.io"}
|
||||
}
|
||||
|
||||
|
||||
def test_no_catalog_entry_pulls_from_docker_hub() -> None:
|
||||
"""Every chart, and every image registry an entry pins, avoids Docker Hub.
|
||||
|
||||
Docker Hub rate-limits anonymous pulls per source IP and the whole cluster shares one
|
||||
NAT address, so a Docker Hub reference here is a provision that fails under load for a
|
||||
reason no log in this repo will explain.
|
||||
"""
|
||||
catalog = load_catalog(Path(__file__).parents[2] / "catalog.yaml")
|
||||
banned = ("docker.io", "registry-1.docker.io", "index.docker.io")
|
||||
|
||||
for name, entry in catalog.items():
|
||||
assert not entry.chart.startswith(banned), f"{name}: chart on Docker Hub"
|
||||
assert "docker.io" not in entry.chart, f"{name}: chart on Docker Hub"
|
||||
|
||||
registry = entry.values.get("global", {}).get("imageRegistry")
|
||||
# A bitnami chart defaults its images to Docker Hub, so any entry pointing at one
|
||||
# has to redirect them. podinfo's chart already names ghcr.io and needs nothing.
|
||||
if "bitnamicharts" in entry.chart:
|
||||
assert registry == "mirror.gcr.io", f"{name}: bitnami chart without a registry override"
|
||||
if registry is not None:
|
||||
assert "docker.io" not in registry, f"{name}: imageRegistry on Docker Hub"
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
"""The helm argv that the reconciler's correctness depends on.
|
||||
|
||||
install() writes a label; list_releases() reads it back. Neither is checked by anything at
|
||||
runtime — if the two ever disagree, `helm list --selector` matches nothing, the reconciler
|
||||
sees an empty cluster, and every ready instance looks like it lost its release. That failure
|
||||
is silent and reads as "no drift", so it gets a test rather than a comment.
|
||||
|
||||
These assert argv, not behaviour against a real cluster: tests/e2e covers that. The point
|
||||
here is that the two sides of the label agree, and that neither drops out under a refactor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from svcforge_core.adapters import helm
|
||||
from svcforge_core.adapters.helm import (
|
||||
MANAGED_BY_LABEL,
|
||||
MANAGED_BY_VALUE,
|
||||
HelmError,
|
||||
HelmProvisioner,
|
||||
)
|
||||
from svcforge_core.domain.models import CatalogEntry, SizeSpec
|
||||
|
||||
ENTRY = CatalogEntry(
|
||||
service_type="redis",
|
||||
chart="oci://example/redis",
|
||||
chart_version="1.2.3",
|
||||
sizes={"small": SizeSpec(replicas=1, resources={})},
|
||||
)
|
||||
|
||||
|
||||
def _capture(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
|
||||
"""Record every argv HelmProvisioner would exec, and run none of them."""
|
||||
seen: list[list[str]] = []
|
||||
|
||||
async def fake_run_helm(self: HelmProvisioner, argv: list[str]) -> str:
|
||||
seen.append(list(argv))
|
||||
return "[]"
|
||||
|
||||
monkeypatch.setattr(HelmProvisioner, "_run_helm", fake_run_helm, raising=True)
|
||||
return seen
|
||||
|
||||
|
||||
def _pair(argv: list[str], flag: str) -> str | None:
|
||||
"""The value following `flag`, or None. Positional, because helm takes `--flag value`."""
|
||||
return argv[argv.index(flag) + 1] if flag in argv else None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_install_labels_the_release_as_svcforge_managed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
seen = _capture(monkeypatch)
|
||||
await HelmProvisioner(kubeconfig=Path("/dev/null")).install(
|
||||
"acme-redis", "tenant-acme", ENTRY, {"replicas": 1}
|
||||
)
|
||||
|
||||
assert len(seen) == 1
|
||||
assert _pair(seen[0], "--labels") == f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_releases_asks_only_for_svcforge_releases(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
|
||||
|
||||
assert len(seen) == 1
|
||||
argv = seen[0]
|
||||
assert _pair(argv, "--selector") == f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}"
|
||||
# Still every namespace. Tenants get their own, so scoping to one would hide releases
|
||||
# rather than the cluster's; the label is what narrows this, not the namespace.
|
||||
assert "--all-namespaces" in argv
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_label_written_is_the_label_read(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The regression that motivated this file.
|
||||
|
||||
Asserted against each other rather than against a literal on both sides, so a rename
|
||||
that updates only one of install/list_releases fails here instead of in production as
|
||||
an empty drift check.
|
||||
"""
|
||||
seen = _capture(monkeypatch)
|
||||
prov = HelmProvisioner(kubeconfig=Path("/dev/null"))
|
||||
|
||||
await prov.install("acme-redis", "tenant-acme", ENTRY, {})
|
||||
await prov.list_releases()
|
||||
|
||||
assert _pair(seen[0], "--labels") == _pair(seen[1], "--selector")
|
||||
|
||||
|
||||
def _meta(name: str, ns: str, version: str, status: str = "deployed") -> dict[str, object]:
|
||||
"""One PartialObjectMetadata item as the API server returns it for a release secret."""
|
||||
return {
|
||||
"metadata": {
|
||||
"name": f"sh.helm.release.v1.{name}.v{version}",
|
||||
"namespace": ns,
|
||||
"labels": {"name": name, "owner": "helm", "status": status, "version": version},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, items: list[dict[str, object]]) -> None:
|
||||
self._items = items
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, object]:
|
||||
return {"items": self._items}
|
||||
|
||||
|
||||
def _fake_api(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, items: list[dict[str, object]]
|
||||
) -> dict[str, Any]:
|
||||
"""Stand in for the in-cluster ServiceAccount and the API call it authenticates.
|
||||
|
||||
The token is a real file at a repointed constant rather than a patched method: the
|
||||
module decides it is in-cluster by whether that file reads, so faking the decision at
|
||||
the filesystem keeps the test honest about what it is exercising.
|
||||
"""
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
token = tmp_path / "token"
|
||||
token.write_text("tok", encoding="utf-8")
|
||||
ca = tmp_path / "ca.crt"
|
||||
ca.write_text("ca", encoding="utf-8") # a complete SA has both; the readability check needs it
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", token)
|
||||
monkeypatch.setattr(helm, "_SA_CA", ca)
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_PORT_HTTPS", "443")
|
||||
|
||||
class _Client:
|
||||
def __init__(self, **kw: object) -> None:
|
||||
captured["client_kwargs"] = kw
|
||||
|
||||
async def __aenter__(self) -> _Client:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def get(self, url: str, **kw: object) -> _FakeResponse:
|
||||
captured["url"] = url
|
||||
captured.update(kw)
|
||||
return _FakeResponse(items)
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _Client)
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_collapses_a_release_to_its_newest_revision(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""helm writes one secret per revision. Ten revisions is one release, not ten."""
|
||||
_fake_api(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
[_meta("acme-redis", "tenant-acme", v) for v in ("1", "2", "10", "9")],
|
||||
)
|
||||
|
||||
out = await HelmProvisioner().list_releases()
|
||||
|
||||
assert [(r.name, r.namespace) for r in out] == [("acme-redis", "tenant-acme")]
|
||||
# 10, not 9: string ordering would pick "9" and quietly report a stale revision.
|
||||
assert out[0].revision == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_keeps_same_named_releases_in_different_namespaces(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Two tenants may both call their instance `redis`. Namespace is part of the identity."""
|
||||
_fake_api(monkeypatch, tmp_path, [_meta("redis", "tenant-a", "1"), _meta("redis", "tenant-b", "1")])
|
||||
|
||||
out = await HelmProvisioner().list_releases()
|
||||
|
||||
assert {(r.name, r.namespace) for r in out} == {("redis", "tenant-a"), ("redis", "tenant-b")}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_asks_for_metadata_only_and_scopes_the_selector(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The Accept header is the difference between metadata and megabytes of gzipped payload."""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
|
||||
await HelmProvisioner().list_releases()
|
||||
|
||||
assert "PartialObjectMetadataList" in cap["headers"]["accept"]
|
||||
selector = cap["params"]["labelSelector"]
|
||||
assert "owner=helm" in selector
|
||||
assert f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE}" in selector
|
||||
assert "superseded" not in selector # dropped server-side by asking only for live states
|
||||
assert cap["url"].endswith("/api/v1/secrets")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_ignores_secrets_that_are_not_releases(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A malformed or unrelated secret must not become a phantom release."""
|
||||
_fake_api(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
[{"metadata": {"namespace": "x", "labels": {}}}, _meta("real", "tenant-a", "1")],
|
||||
)
|
||||
|
||||
out = await HelmProvisioner().list_releases()
|
||||
|
||||
assert [r.name for r in out] == ["real"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_service_account_falls_back_to_helm(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""Out of cluster there is nothing to authenticate with, so e2e and laptops keep working."""
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "absent")
|
||||
|
||||
await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
|
||||
|
||||
assert seen and seen[0][:2] == ["helm", "list"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_handles_kubernetes_serialising_empty_as_null(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""`"items": null`, not `[]`, is what an empty list looks like on the wire.
|
||||
|
||||
The key is present, so `.get("items", [])` returns None and the default never fires.
|
||||
This shipped and failed in production on the first tick that matched no releases:
|
||||
TypeError: 'NoneType' object is not iterable. The earlier tests all passed a real
|
||||
empty list, which is the one shape that cannot catch it.
|
||||
"""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
cap["force_null_items"] = True
|
||||
|
||||
class _NullResponse:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, object]:
|
||||
return {"apiVersion": "meta.k8s.io/v1", "kind": "PartialObjectMetadataList", "items": None}
|
||||
|
||||
class _NullClient:
|
||||
def __init__(self, **kw: object) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self) -> _NullClient:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def get(self, url: str, **kw: object) -> _NullResponse:
|
||||
return _NullResponse()
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _NullClient)
|
||||
|
||||
assert await HelmProvisioner().list_releases() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_passes_tls_verify_and_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The API client must verify against the SA CA and carry the read timeout.
|
||||
|
||||
A refactor that dropped `verify` to the default (or None) is a TLS regression the happy
|
||||
path would not reveal, so it is pinned here off the captured client kwargs.
|
||||
"""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
|
||||
await HelmProvisioner().list_releases()
|
||||
|
||||
assert cap["client_kwargs"]["verify"] == str(helm._SA_CA)
|
||||
assert cap["client_kwargs"]["timeout"] == helm._API_TIMEOUT_S
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_read_sends_no_limit_param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""No `limit`, so the apiserver returns the full set and the single read is complete.
|
||||
|
||||
Pins the pagination invariant: adding `limit` without consuming `metadata.continue`
|
||||
would silently truncate the release list.
|
||||
"""
|
||||
cap = _fake_api(monkeypatch, tmp_path, [])
|
||||
|
||||
await HelmProvisioner().list_releases()
|
||||
|
||||
assert "limit" not in cap["params"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_error_raises_helmerror_and_does_not_fall_back(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A reachable-but-erroring apiserver raises HelmError; it must not shell out to helm.
|
||||
|
||||
Falling back would swap a visible error for the 330s helm timeout this method exists to
|
||||
remove. The fallback is only for a ServiceAccount that is not present at all.
|
||||
"""
|
||||
(tmp_path / "ca.crt").write_text("ca", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
|
||||
(tmp_path / "token").write_text("tok", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "ca.crt")
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
class _ErrClient:
|
||||
def __init__(self, **kw: object) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self) -> _ErrClient:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def get(self, url: str, **kw: object) -> object:
|
||||
raise httpx.ConnectError("connection reset by peer")
|
||||
|
||||
monkeypatch.setattr(httpx, "AsyncClient", _ErrClient)
|
||||
|
||||
with pytest.raises(HelmError):
|
||||
await HelmProvisioner().list_releases()
|
||||
assert seen == [], "an API error must not fall back to `helm list`"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_ca_falls_back_to_helm(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""A token without a readable CA is a half-mounted SA: fall back rather than crash.
|
||||
|
||||
httpx loads the CA when the client is built, raising OSError that the API except clause
|
||||
does not catch, so the readability check has to happen before the request. A missing CA
|
||||
means "not in-cluster", the same as a missing token.
|
||||
"""
|
||||
monkeypatch.setattr(helm, "_SA_TOKEN", tmp_path / "token")
|
||||
(tmp_path / "token").write_text("tok", encoding="utf-8")
|
||||
monkeypatch.setattr(helm, "_SA_CA", tmp_path / "absent-ca.crt") # never created
|
||||
monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1")
|
||||
seen = _capture(monkeypatch)
|
||||
|
||||
await HelmProvisioner(kubeconfig=Path("/dev/null")).list_releases()
|
||||
|
||||
assert seen and seen[0][:2] == ["helm", "list"]
|
||||
+31
-16
@@ -13,11 +13,13 @@ import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
import structlog
|
||||
import yaml
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from prometheus_client import REGISTRY
|
||||
@@ -89,8 +91,8 @@ def test_provision_histogram_has_a_bucket_for_a_thirty_minute_provision() -> Non
|
||||
|
||||
With the library defaults the top finite bucket is 10s, every provision lands in +Inf,
|
||||
and histogram_quantile interpolates across a bucket spanning 10s to infinity. The p95
|
||||
it returns is not slow or fast, it is meaningless — and it is meaningless silently,
|
||||
which is why this is asserted rather than eyeballed on a dashboard.
|
||||
it returns is meaningless, and it is meaningless silently, which is why this is
|
||||
asserted rather than eyeballed on a dashboard.
|
||||
"""
|
||||
bounds = obs.PROVISION_TIME._upper_bounds
|
||||
assert 1800.0 in bounds
|
||||
@@ -105,21 +107,34 @@ def test_provision_histogram_exports_the_1800_bucket() -> None:
|
||||
assert bucket is not None
|
||||
|
||||
|
||||
def test_metric_names_are_the_ones_the_alerts_query() -> None:
|
||||
"""The four alerts are PromQL strings in values.yaml; nothing type-checks them.
|
||||
def test_every_metric_the_alerts_query_actually_exists() -> None:
|
||||
"""Parse the alerts out of values.yaml and check each metric is really registered.
|
||||
|
||||
A rename here is a silent alert that never fires again. This test is the link between
|
||||
the chart's rules and the code.
|
||||
Reading the chart is the whole point. An earlier version of this test asserted against
|
||||
a hardcoded list, which meant it could only catch a rename on the *code* side — rename
|
||||
a metric in values.yaml and it stayed green while the alert silently never fired again.
|
||||
A test that duplicates the thing it is meant to cross-check is not a cross-check.
|
||||
"""
|
||||
for name in (
|
||||
"svcforge_tasks_claimed_total",
|
||||
"svcforge_tasks_failed_total",
|
||||
"svcforge_provision_duration_seconds",
|
||||
"svcforge_queue_depth",
|
||||
"svcforge_instances",
|
||||
"svcforge_reconciler_last_tick_timestamp_seconds",
|
||||
):
|
||||
assert REGISTRY._names_to_collectors.get(name) is not None, f"{name} is queried by an alert"
|
||||
values = yaml.safe_load(
|
||||
(Path(__file__).resolve().parents[2] / "deploy" / "chart" / "values.yaml").read_text()
|
||||
)
|
||||
rules = (values.get("prometheusRule") or {}).get("rules") or []
|
||||
assert rules, "no alert rules found under prometheusRule.rules — this test would silently pass"
|
||||
|
||||
# Prometheus appends _bucket/_sum/_count to a histogram's series; the collector is
|
||||
# registered under the base name.
|
||||
suffixes = ("_bucket", "_sum", "_count")
|
||||
registered = set(REGISTRY._names_to_collectors)
|
||||
|
||||
checked = 0
|
||||
for rule in rules:
|
||||
for token in re.findall(r"\bsvcforge_[a-z0-9_]+\b", str(rule.get("expr", ""))):
|
||||
base = next((token[: -len(s)] for s in suffixes if token.endswith(s)), token)
|
||||
assert base in registered, (
|
||||
f"alert {rule.get('alert')!r} queries {token!r}, but no such metric is registered"
|
||||
)
|
||||
checked += 1
|
||||
assert checked >= len(rules), "expected at least one metric reference per alert"
|
||||
|
||||
|
||||
# --- Trace context across the queue -------------------------------------------------------
|
||||
@@ -232,7 +247,7 @@ def test_json_renderer_is_last_and_output_is_one_object_per_line(logs: io.String
|
||||
|
||||
|
||||
def test_setup_is_idempotent(logs: io.StringIO) -> None:
|
||||
"""Called twice, one handler, one line. Not a nicety: two handlers is two of every line.
|
||||
"""Called twice, one handler, one line. Two handlers would be two of every line.
|
||||
|
||||
Each service calls setup() from its entrypoint, and an entrypoint that imports another
|
||||
entrypoint (the CLI shelling into the reconciler) calls it twice.
|
||||
|
||||
@@ -79,8 +79,8 @@ def take_cache(c: InstanceCacheProto) -> None: ...
|
||||
def test_redis_implementations_conform() -> None:
|
||||
"""A client at a closed port is still a Redis. Nothing here connects — no commands billed.
|
||||
|
||||
That is not a trick to keep the test fast; it is the module's thesis restated as a
|
||||
fixture. Every one of these classes has to be constructible and callable with Redis
|
||||
That is the module's thesis restated as a fixture, rather than a trick to keep the
|
||||
test fast. Every one of these classes has to be constructible and callable with Redis
|
||||
unreachable, because that is the state they are designed for.
|
||||
"""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""What reaches `helm --values`: the catalog entry's values, with the size on top."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from services.worker.handlers import HandlerError, _deep_merge, _values_for
|
||||
from svcforge_core.domain.models import CatalogEntry, Instance, SizeSpec
|
||||
from svcforge_core.domain.states import InstanceState
|
||||
|
||||
RESOURCES: dict[str, Any] = {"requests": {"cpu": "10m", "memory": "16Mi"}}
|
||||
|
||||
|
||||
def _entry(values: dict[str, Any] | None = None, replicas: int = 1) -> CatalogEntry:
|
||||
return CatalogEntry(
|
||||
service_type="redis",
|
||||
chart="oci://mirror.gcr.io/bitnamicharts/redis",
|
||||
chart_version="27.0.15",
|
||||
sizes={"small": SizeSpec(replicas=replicas, resources=RESOURCES)},
|
||||
values=values or {},
|
||||
)
|
||||
|
||||
|
||||
def _instance(size: str = "small") -> Instance:
|
||||
now = datetime.now(UTC)
|
||||
return Instance(
|
||||
id=uuid4(),
|
||||
team="acme",
|
||||
service_type="redis",
|
||||
size=size,
|
||||
state=InstanceState.REQUESTED,
|
||||
namespace="tenant-acme",
|
||||
release_name="acme-redis-0f8b7d3e",
|
||||
chart_version="27.0.15",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- the merge
|
||||
|
||||
|
||||
def test_deep_merge_keeps_both_sides_of_a_shared_nested_key() -> None:
|
||||
"""The reason this is not `base | override`.
|
||||
|
||||
A shallow merge replaces the whole `global` map and silently drops imageRegistry, so
|
||||
the pod pulls from a registry nobody chose.
|
||||
"""
|
||||
merged = _deep_merge(
|
||||
{"global": {"imageRegistry": "mirror.gcr.io"}},
|
||||
{"global": {"storageClass": "longhorn"}},
|
||||
)
|
||||
assert merged == {"global": {"imageRegistry": "mirror.gcr.io", "storageClass": "longhorn"}}
|
||||
|
||||
|
||||
def test_deep_merge_override_wins_on_a_scalar() -> None:
|
||||
assert _deep_merge({"a": 1}, {"a": 2}) == {"a": 2}
|
||||
|
||||
|
||||
def test_deep_merge_does_not_mutate_its_inputs() -> None:
|
||||
"""The catalog is loaded once at startup and shared by every provision.
|
||||
|
||||
Mutating `entry.values` here would leak one instance's size into the next one's values,
|
||||
and the second tenant would get the first tenant's replica count.
|
||||
"""
|
||||
base = {"global": {"imageRegistry": "mirror.gcr.io"}}
|
||||
_deep_merge(base, {"global": {"storageClass": "longhorn"}, "replicaCount": 3})
|
||||
assert base == {"global": {"imageRegistry": "mirror.gcr.io"}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- what helm gets
|
||||
|
||||
|
||||
def test_entry_values_reach_helm() -> None:
|
||||
values = _values_for(_instance(), _entry({"global": {"imageRegistry": "mirror.gcr.io"}}))
|
||||
|
||||
assert values["global"] == {"imageRegistry": "mirror.gcr.io"}
|
||||
assert values["replicaCount"] == 1
|
||||
assert values["resources"] == RESOURCES
|
||||
|
||||
|
||||
def test_size_beats_entry_values() -> None:
|
||||
"""An entry that sets replicaCount must not override the size the tenant asked for.
|
||||
|
||||
Without this ordering every size deploys the same shape, and `medium` is a lie.
|
||||
"""
|
||||
entry = _entry({"replicaCount": 99}, replicas=3)
|
||||
|
||||
assert _values_for(_instance(), entry)["replicaCount"] == 3
|
||||
|
||||
|
||||
def test_entry_without_values_is_unchanged() -> None:
|
||||
assert _values_for(_instance(), _entry()) == {"replicaCount": 1, "resources": RESOURCES}
|
||||
|
||||
|
||||
def test_unknown_size_raises() -> None:
|
||||
with pytest.raises(HandlerError, match="not in catalog"):
|
||||
_values_for(_instance(size="enormous"), _entry())
|
||||
@@ -1,6 +1,6 @@
|
||||
version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12"
|
||||
requires-python = ">=3.14"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.15'",
|
||||
"python_full_version < '3.15'",
|
||||
@@ -30,7 +30,6 @@ version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
@@ -105,32 +104,6 @@ dependencies = [
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/cf/2b684132056f438567b61e19d690dd31cd0921ace051e0a458be6074369e/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0", size = 208801, upload-time = "2026-07-06T21:33:13.617Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/08/f2e7d62c460faae0926f2d6e423694aa409ced3bc1fe2927a0a6e5f05416/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224", size = 221808, upload-time = "2026-07-06T21:33:15.466Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/37/04f54b8e63a02f3d908332c9effbf8c366167c6f733ed8a3d4f79b7e2a1e/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c", size = 225241, upload-time = "2026-07-06T21:33:16.869Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/d6/c72eecca433cd3e681c65ed313ab4835d9d4a379704d0f628a6a05f51c2e/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a", size = 223588, upload-time = "2026-07-06T21:33:18.239Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/4b/e706f67279140f92939da3475ad610df18bfd52d50f14953a8e5fede71d5/cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2", size = 175248, upload-time = "2026-07-06T21:33:19.799Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/47/59eb7975cb0e4ef0afa764ea945b29a5bb4537a9f771cb7d6c8a5dd74c95/cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512", size = 185717, upload-time = "2026-07-06T21:33:21.47Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" },
|
||||
@@ -196,32 +169,6 @@ version = "3.4.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" },
|
||||
@@ -278,36 +225,6 @@ version = "7.15.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/76/d0/55fe630f4cf94e3fcba868240fad8c8cdd1f764e2a932f8926347e6ec4cd/coverage-7.15.2.tar.gz", hash = "sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d", size = 927741, upload-time = "2026-07-15T18:56:19.558Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/50/eb5bf42e531611a9f8d272556b1ed4de503f84a91413584094487cf69f8f/coverage-7.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9", size = 221587, upload-time = "2026-07-15T18:54:18.439Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/d1/da99af464c335d4e023a6efcd7ec30f63b88a43c93745154ab74ffb31cea/coverage-7.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73", size = 221943, upload-time = "2026-07-15T18:54:20.062Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/8a/13c42723d61ca447eafa18732e8141dd6a63f2732e1c7e1502c182dd88d7/coverage-7.15.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d", size = 253450, upload-time = "2026-07-15T18:54:21.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/29/99021303f98fbdcb63504b4d07bea4cc025b9b2dd907c4f07c85d50a0dab/coverage-7.15.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b", size = 256187, upload-time = "2026-07-15T18:54:23.4Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/a8/fd503715ed6ca9c5d742923aa5209257340b367a867b2ced0c7d4ba8a0b9/coverage-7.15.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296", size = 257301, upload-time = "2026-07-15T18:54:25.183Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/40/3f4b8fb409810036ebc2857d36adc0498c6e957b5df0290c5036b2e143f1/coverage-7.15.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6", size = 259562, upload-time = "2026-07-15T18:54:27.204Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/8a/9bdffbef47db77cce3d6b02a28f7e919b19f0106c4b080c2c2246040f885/coverage-7.15.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098", size = 253841, upload-time = "2026-07-15T18:54:29.134Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/1e/9031efde019d31a06646261fce6dfc5c3c74e951e27a71e5c9a424563178/coverage-7.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a", size = 255221, upload-time = "2026-07-15T18:54:31.142Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/db/787acde872389fc84a9ef9d8cd1ccc658e391ab4cb5b28092a714426a394/coverage-7.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b", size = 253366, upload-time = "2026-07-15T18:54:32.886Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/9b/6f57bc4b93c842eef1695f8cdaf2318e35e7ba54f5ba80d84be213ab7858/coverage-7.15.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2", size = 257434, upload-time = "2026-07-15T18:54:34.7Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/26/b3186a21b2acc83e451118978905c81c7072c3333707804db09a78c096a2/coverage-7.15.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440", size = 252935, upload-time = "2026-07-15T18:54:36.548Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/c2/c9f3376b2e717ea69ed7a6e9a5fcab968fb0b290db6cf4bd9a1fc7541b75/coverage-7.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e", size = 254807, upload-time = "2026-07-15T18:54:38.296Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/e1/dfc15401f4a8aaeb486e1ba3e9e3c40522a6e38bd0ecf0b3f29cb8082957/coverage-7.15.2-cp312-cp312-win32.whl", hash = "sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd", size = 223641, upload-time = "2026-07-15T18:54:40.103Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/40/81b6d809d320cd366ec5bdf8176575e897dcb8efe7fb4b489ef9e93e4d13/coverage-7.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40", size = 224172, upload-time = "2026-07-15T18:54:41.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/28/9f14ec438149f7de557f45518f09b4a7917b795cc37083aa7db482693f8c/coverage-7.15.2-cp312-cp312-win_arm64.whl", hash = "sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3", size = 223556, upload-time = "2026-07-15T18:54:43.674Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/d5/f8c838e6b7282976f7c918884b792df7a0c42c5bba5d99c60ad2d221d56d/coverage-7.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8", size = 221606, upload-time = "2026-07-15T18:54:45.448Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/37/97c926376364f66298cc44893b89cdf17b8bc406376497c4061ae4b8a8ff/coverage-7.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1", size = 221982, upload-time = "2026-07-15T18:54:47.341Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/30/a36050a6e83c2135ee0776f452ca3948224befc6d7f26acecc082d0c106a/coverage-7.15.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578", size = 252972, upload-time = "2026-07-15T18:54:49.2Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/d3/06b5f1daf95f0f15ab05bd75f26ba5f3c8b33d0bb72f3aaa3cf41d1bad3a/coverage-7.15.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1", size = 255569, upload-time = "2026-07-15T18:54:51.098Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/1c/9afb3f8de2b8d36960391c48559a2e3ff96594b58099f115921549ea8d0d/coverage-7.15.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6", size = 256806, upload-time = "2026-07-15T18:54:53.145Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/d8/b989f96061a5e32d82fddd1b1b9ff48a7c8f8ae7606f0e80fd9de54b1e33/coverage-7.15.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7", size = 258936, upload-time = "2026-07-15T18:54:55.015Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/fa/f99771f5110457c7b511c1935ca49ddf288218eaa84322e028b9334146ae/coverage-7.15.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d", size = 253178, upload-time = "2026-07-15T18:54:57.527Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/96/c098a6044d119c751ceede7be91035fa8310170ec24a6523aff72f0a5793/coverage-7.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026", size = 254934, upload-time = "2026-07-15T18:54:59.41Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/a2/1457b3a7a50c8d77500103b97a046db863e2f59a1cf6d2f814595f349885/coverage-7.15.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa", size = 252898, upload-time = "2026-07-15T18:55:01.338Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/0e/76958874c471ecfcdde0d2b2747bb2c61bdbf34a40636f4ce9db9923e643/coverage-7.15.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d", size = 257056, upload-time = "2026-07-15T18:55:03.243Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/7c/3d7c4e3bf58baa40327dc7edc2272b17cf02299366d52763db1b0ca1556a/coverage-7.15.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b", size = 252718, upload-time = "2026-07-15T18:55:05.029Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/b8/1cecffed9ce14fb25be9ba42d37b6bb61485c9a3ddd43cd3dde36b6087d8/coverage-7.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188", size = 254490, upload-time = "2026-07-15T18:55:06.889Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/2c/42984561bc7f4c045dca67516a0c50ee5ef8d84352dbeb5559dc86c4823e/coverage-7.15.2-cp313-cp313-win32.whl", hash = "sha256:434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050", size = 223647, upload-time = "2026-07-15T18:55:08.941Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/9f/39c7c9245efc583beddf89a87683574e663ed93637f3afb6cd7b88405676/coverage-7.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c", size = 224190, upload-time = "2026-07-15T18:55:10.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/de/3a2883cf8a213659280ef4b403059e17a9acaeb7fc7fd4105e1226ff2e6d/coverage-7.15.2-cp313-cp313-win_arm64.whl", hash = "sha256:3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b", size = 223583, upload-time = "2026-07-15T18:55:12.678Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/5f/aed265fd7a3551a394f36dfe41868aee709b7f95db4052205b4ad1563ac3/coverage-7.15.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a", size = 221650, upload-time = "2026-07-15T18:55:14.527Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/2c/222ba12a545189017120f8eddfc1a0bd4616b47d5d4a8d99421edb2fe4c6/coverage-7.15.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2", size = 221988, upload-time = "2026-07-15T18:55:16.674Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/38/304b5877ab46e6c290b4292cfcf3fe28245f0e5597cad7f6acc91fc7e0a4/coverage-7.15.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138", size = 253029, upload-time = "2026-07-15T18:55:18.856Z" },
|
||||
@@ -444,11 +361,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "filelock"
|
||||
version = "3.30.2"
|
||||
version = "3.31.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/02/f7/2165ef325da22d854b8f81ca4799395f2eb6afa55cdb52c7710f028b5336/filelock-3.30.2.tar.gz", hash = "sha256:1ea7c857465c897a4a6e64c1aace28ff6b83f5bc66c1c06ea148efa65bc2ec5d", size = 176823, upload-time = "2026-07-16T19:50:42.724Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4a/9f/994e80905542b748eb5b9f36d71458f0aea51a7be0fcb52ad959787dc1b7/filelock-3.31.0.tar.gz", hash = "sha256:c188cbc4307c18894c5424fa73f97ea7fa127ddf62192487546da3a214d0a381", size = 180931, upload-time = "2026-07-18T05:53:29.262Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/02/df/05118016cad66cd0d7c9417b2d4fc245be35decc4c36810f3c8dbf729d88/filelock-3.30.2-py3-none-any.whl", hash = "sha256:a64b58f75048ec39589983e97f5117163f822261dcb6ba843e098f05aac9663f", size = 94092, upload-time = "2026-07-16T19:50:41.189Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/4a/e213905d3b8ad3d35d14fc056b36134a274e7f6a1050e94428b5be10a94c/filelock-3.31.0-py3-none-any.whl", hash = "sha256:739b73e580fe88bb78d830aeddbc492519ece3d97ac8368de13a2032c61010c1", size = 96080, upload-time = "2026-07-18T05:53:27.732Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -479,20 +396,6 @@ version = "0.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/e5/8cfcabc5546e8022f168be28bcdaa128a240a0befdd03b59d558b4f18bd6/httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8", size = 205148, upload-time = "2026-05-25T22:17:16.333Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/0e/0fb14848c19a686c8062ff9067c1a48793e3224b47bc5b201535b6036fce/httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c", size = 111368, upload-time = "2026-05-25T22:17:17.586Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/66/b96623b27e51a68199ef4efdda0613cced9233fe3062ac74e50749c5ad37/httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1", size = 90117, upload-time = "2026-05-25T22:17:23.074Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" },
|
||||
@@ -526,51 +429,37 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "hypothesis"
|
||||
version = "6.156.6"
|
||||
version = "6.157.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "sortedcontainers" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/20/83/8dbe89bdb8c6f25a7a52e7898af6d82fe35dfef08e5c702f6e33231ce6c6/hypothesis-6.156.6.tar.gz", hash = "sha256:96de02faefa3ce079873541da96f42595583bb001e8e4219294ed7d4501cc4cc", size = 476304, upload-time = "2026-07-10T20:56:49.96Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bf/ea/3d93589be7784d396a5a4d6aca6eb26783dfe9f40813a1f82ef71d7f6df7/hypothesis-6.157.0.tar.gz", hash = "sha256:5e4cd0af9261b06fc79e8aabe7d840384b3c24eaceae7e7e25ee3800a6d6ac58", size = 477477, upload-time = "2026-07-19T08:02:21.964Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/dc/0c2a851f06c91d5ac9ef0f3b9615efc1ed650411d2eee23b6334f491c85e/hypothesis-6.156.6-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:caf6a93d011c10972da111c38ceb34ced20feaa8581e2b350c0655b022e27875", size = 747998, upload-time = "2026-07-10T20:56:16.311Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/f8/59203ca978ab51595d12d6bc7e7a63300d7373431ab42ca3f1742e45db68/hypothesis-6.156.6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:07f2bc9df1aeba80e12029c1618e2ee54abc440068c305d7075ffd6b85251843", size = 743073, upload-time = "2026-07-10T20:55:36.825Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/d8/86a0023740434098d1b187a62bd5f99b198f098fb43e7fc58342283a8270/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baca17f4803ad4aa151732326f3990baf54c3127df44aa872ac5bdf8a98a9a6", size = 1070169, upload-time = "2026-07-10T20:55:49.47Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/82/673453915fd0c67673f35a4876ba88f48c621335f293f3537d77b27d4286/hypothesis-6.156.6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8083806645f84243aade727f4978185caaa0b7190af4318673999ee15fdbf424", size = 1121760, upload-time = "2026-07-10T20:55:53.502Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/c3/3a5557f52912f2fecc6ed59642dcf80dd8e89d0d9664502b68e23d66bf3d/hypothesis-6.156.6-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a922eedcd8618f9c2e17b79fa7b3f3f0b2df34e201958611cc3f0f46cca33c10", size = 1111440, upload-time = "2026-07-10T20:55:43.054Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/a6/ae636d4ca7f996a1ccb4b3d5997d949f1718fba52b01559b3ab53b237b3f/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5291bd33c4704d274d7c214d5c200e77f372a06644f5cbbe96dcbe53cb2fbf10", size = 1244944, upload-time = "2026-07-10T20:55:56.109Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/79/c425d22d734be0268ca60d120c6296299e4220a1783cb1a4cc76232807bb/hypothesis-6.156.6-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55f3ec50161b4a95bae63bff2b5166e45935b493013d3be30ede279bf6192318", size = 1288808, upload-time = "2026-07-10T20:56:06.249Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/3a/cc9f479d22cbdd36ddfc55a978378eddadd183b09339ebdb81be33bb18e7/hypothesis-6.156.6-cp310-abi3-win32.whl", hash = "sha256:e96570ca5cdd9a5f2ff9e80a6fb2fd5420ebf33b833d7de5b09b6ebb26a3eb6c", size = 634868, upload-time = "2026-07-10T20:55:37.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/89/2008d287289841a936456cb13443ca89d88da6e4527d611d482e9544164d/hypothesis-6.156.6-cp310-abi3-win_amd64.whl", hash = "sha256:32710718c22fe8c5571464e898bb87d282837b02617d6ad68130abf7cb4843cb", size = 640382, upload-time = "2026-07-10T20:55:30.634Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/45/9f009005b9c796f4a40424484ac7e70847bc088456fd940a937f96bb4b6d/hypothesis-6.156.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a2a728b514fceb81e3f0464508911d5220fd74dadc3270f859427a686b60c4cf", size = 748844, upload-time = "2026-07-10T20:56:38.036Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/2f/4d852bb8a9c73a68b18eca9b5b085285282122166e158f4d2a477639bfee/hypothesis-6.156.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7489b9a8f9df8227edd6c7cd8b9ccfab2483bab24da6a474c175973ca2294f58", size = 741936, upload-time = "2026-07-10T20:55:27.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/89/b9968070ae042f9bf3149bb6ba6399d5f28f452e0fb7f638cafc69ff0b9a/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42760873d6db1069d6edbaa355a61b9078a9950259efcfc72fc695741d7db7cd", size = 1069749, upload-time = "2026-07-10T20:56:43.017Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/a9/753806f5292b40aeab1d269e408e3a7e85be3c0d88828fb78ab4a34d6626/hypothesis-6.156.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d", size = 1120983, upload-time = "2026-07-10T20:56:25.424Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/88/8386d064d680be27e936eba94f1448bc93ef6fa05473ee5034139f1c4284/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:08796b674c0b31a5dd4119b2173823390055921588d13eb77324e861b00fd7f8", size = 1243911, upload-time = "2026-07-10T20:55:54.799Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/8c/7524c1e5279e7728eb47c99f2357cbc5f08ae92e9bce49bf50118b53f9c9/hypothesis-6.156.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4ca8cc26ea2d31d22cf7710e92951cfaa921f0f8aa1b6db33a5176335f583a4f", size = 1287806, upload-time = "2026-07-10T20:56:02.176Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/b3/c347ad913e1c5f2988956fe17826c0400b4ce470b973e6c248e97b6a0acf/hypothesis-6.156.6-cp312-cp312-win_amd64.whl", hash = "sha256:c3363d3fb8015594636689572510bb6090602d8e8e838a5693c2d52d3b5b09d8", size = 637679, upload-time = "2026-07-10T20:55:39.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/5d/9583fe153573523dac27226c89e041a86ad4aeeae08c868160cbb93d39d2/hypothesis-6.156.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:59a8def90d9a5a9b67e1ac529e903a2363ceb6cf873c209da6b4284c5daab671", size = 749264, upload-time = "2026-07-10T20:56:46.118Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/35/e4113d06769b544f0fb77ffea9195b598b4c56a298905c21fd47c4eed388/hypothesis-6.156.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c574c3224563d730848bc5d1ef1683c4f83993400c0167899fe328f4bfcd4725", size = 742095, upload-time = "2026-07-10T20:56:41.412Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/5c/a47666ede10384e8978722cade7ab96a42df71d2ab577317092d0fed7c8a/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01bb8270c46b3ef53b0c2d23ff613ea506d609d06f936d823ea57c58b66b05f7", size = 1069917, upload-time = "2026-07-10T20:56:04.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/79/93/75f6057dadd9dc0134f37c08d5d14d04d3cd7374debbcb0cc4569c6712f1/hypothesis-6.156.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ea6559c13606e13b645927f2e0906e52b5ac5d99b40d3abaaeb2e8c7ceeb75", size = 1121204, upload-time = "2026-07-10T20:55:52.008Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/87/308efef08bc60d1e673d035e8ca8e9663f4b6b3ba519c3cdebf6583c2b76/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2d47054d0230f0dd9b6868fc030126c7a6c25527144272ff376cc4e9c39f7540", size = 1244168, upload-time = "2026-07-10T20:55:40.288Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/66/de8fff5bd9a40a4056dafbe7f904887ef12632282bbbac90f1977c30dd3b/hypothesis-6.156.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:050c8c0815f88d47dd0875a92698d20d61639b7b721ee043a6d687c7f14ff7d8", size = 1288127, upload-time = "2026-07-10T20:56:00.541Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/8d/794fb26e1fd3ff004978f8f18b7aa7e1c2270ba72e1f977b987a812064f8/hypothesis-6.156.6-cp313-cp313-win_amd64.whl", hash = "sha256:f0d73edab7b8a0051b3634f2d04d62b7e7282f8f274963b11188ee4957d672ef", size = 637954, upload-time = "2026-07-10T20:56:33.35Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/be/5b4b27984cb43c60e95f570b069660335dad34cb67f7d226017c5d35d31e/hypothesis-6.156.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:34a70a7b8226e34d658072d8fb81d03f97f0a75ceb536329a321b94ce2232fd6", size = 749312, upload-time = "2026-07-10T20:55:46.902Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/11/709cceffc28666c9d4cb75ffc6df5ce30db8c7dd5cc2c8b38a2fd837427f/hypothesis-6.156.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f1969646beead7d8cf6a2537d2765af89d73056e2cb218e7fae92b83802250a3", size = 742332, upload-time = "2026-07-10T20:56:30.254Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/52/cfc79b13d8dd3cd6de6b9df921c557efe8528a9c90a3a7cd93b37188d57e/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbc2ec7b7d905e6b6ec1635f6340bfa52aaab718101c59f052bc012a6b486cd8", size = 1070109, upload-time = "2026-07-10T20:55:48.244Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/ac/1da4def1f006b5ad01187ff96379e24c37439d659ec10c3e944c03436c0f/hypothesis-6.156.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9367ae25dfa6dc1af37904785e43c4f8fe1c4118cafdc2f06514154fbdd90992", size = 1121528, upload-time = "2026-07-10T20:56:39.665Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/47/744e4f5e3d635dea20dbedf3fa486e2a6fa5210e0a52a0d5c4da56babd84/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:455f09107ec07c78f2a83cb8fc19e23879c9d51cdc831de6f9cb6ec4059cb9af", size = 1244690, upload-time = "2026-07-10T20:56:31.854Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/8a/42252fcd5e521d140dac532f29c2a13ca8f22908cb545ffdd64b5e225680/hypothesis-6.156.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c76634c45a3ceee4c4fdfed39aebd08b8b822ec8b0c556877ef82846fd777730", size = 1288519, upload-time = "2026-07-10T20:56:03.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/e7/176df9e47cf583d2b8d234b78c0aac3a47075ad5d147e60b2c21a1338bb1/hypothesis-6.156.6-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:eb7e9f8343bc6b948937e6ec12e6879ed25a17b53ceccbd2b84adadd3d511698", size = 586452, upload-time = "2026-07-10T20:56:22.285Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/75/2c8a0411bbe76429f3ae738ef9a00107201bf6146d9534350014ce369d98/hypothesis-6.156.6-cp314-cp314-win_amd64.whl", hash = "sha256:f9631cd604ae6032c3edf99160dc1b9e33873f2e52762246b24f07fb758652ae", size = 637774, upload-time = "2026-07-10T20:56:34.97Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/22/8115005e9aa72c8d63d90e9db5e0b8425fd8950fbc5d6e332805d4d32c9e/hypothesis-6.156.6-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:1f81163d36d3763b09ffaef7c3a71e88174ca3e6816201fca9d1d159f448fdb5", size = 747428, upload-time = "2026-07-10T20:56:44.611Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/c2/66bfe9337f4a4b1f7754ee6d01d950280152a81d0d797e6c1d9eb0909750/hypothesis-6.156.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:556b905767e36147918634a64356aa05d8c956576f00aee01eb351678f193908", size = 740889, upload-time = "2026-07-10T20:55:57.656Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/3b/69f45af2d4f0950b7d1af3cdbdd800b88a6c2370331481eda79d6171fbe3/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3f2604b28d16d696aaaf4954d20f907b27e54034df98e64746a20c74c319f03", size = 1069270, upload-time = "2026-07-10T20:56:12.024Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/43/6b2549885da08f5e50ba34fb8e0d0a60b2f190ffd516fac220f8db5b5869/hypothesis-6.156.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffe012ad66dbe7b8e8ddef6f6992ab1b36719ea64430c2bf1ff7135521052a15", size = 1120409, upload-time = "2026-07-10T20:55:34.551Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/97/745c778c3eb29befa2367b1ded8437eecfbbe6932359d0f831275bc32170/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5bfa3c7b758f7278081c6bfec5f89b43c4eb075c0c9eb095323f7a9eb019b513", size = 1243111, upload-time = "2026-07-10T20:56:17.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/d7/c5ec6a442dc9b822f47064bda4b6d3e739dccdd1c5bf44c9a57fb6136830/hypothesis-6.156.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d0db1f4573800c618773622f03cb6533bb3377430ef938c9476ba10c39d22591", size = 1287262, upload-time = "2026-07-10T20:56:23.749Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/0c/c134d61710e14b68b010215dcf6bd57d2ec05cd169dff8bfab8fefc2d410/hypothesis-6.156.6-cp314-cp314t-win_amd64.whl", hash = "sha256:38cd0c4a7b9f809f1e23a4d15adfa9c5d99869b9afc327350a5e563350b78e48", size = 637862, upload-time = "2026-07-10T20:56:13.347Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/f6/867b57c8eb2005cf801bb03b0158f0524cccf57868a271eb9c0132082bf7/hypothesis-6.157.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c958d78cfe93e00fc410d8dadfafac379bff3633348e87a4c5a5af47a0ef0873", size = 749334, upload-time = "2026-07-19T08:01:44.006Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/a8/c0bae5c4332cd9183f133d4234fba49befd7285946fbfba2096c50fa91fe/hypothesis-6.157.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ffb3948acffacd2fe9cd56b5b259f48002ed97d8addc2087db55d2e76ab8183", size = 743963, upload-time = "2026-07-19T08:02:20.403Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/f4/c19460524a08fc5a78e90a2fc7b472e1d3c23fbeb71b6a6302cc0025f49f/hypothesis-6.157.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764ea1cb63e8d75b0b3889da30640466425b5a2fcaec803952b3fb13e56515a", size = 1071472, upload-time = "2026-07-19T08:01:48.132Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/c2/64ed0b609f9d90865373fb067f11a8444977b2b4547b42ff7f500758bdcf/hypothesis-6.157.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116734981c3c59e06e6ed1446b054fa92db999cbdbc354c3335c593ce917ebb9", size = 1122954, upload-time = "2026-07-19T08:01:54.528Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/5a/a6cf428f2b0e4f67f0e7e48f8a72eb4aad4780a736d01887c69bc565dbaa/hypothesis-6.157.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b93a357d1ff0dae972e8accbfe15cd91c1eeb8fa766a2a1c361b56bd0414f80", size = 1112453, upload-time = "2026-07-19T08:01:27.225Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/bd/0d9661dce553d5977c6e9f9ed46c06ab89fc64d471892362ce87cc02eddb/hypothesis-6.157.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e8778a10749cfd3cd236224dc78b354b14970a07cd5630345269ffbf8830db9a", size = 1246171, upload-time = "2026-07-19T08:01:07.635Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/81/f50fd9e502c11780a250df8cf6f9f3b0cef47792226f5138f6e55647967b/hypothesis-6.157.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b718c800bfd1fbec5a11ac1e56c28a5115a31af499ebb6acfc6e13922a6dd0d4", size = 1289928, upload-time = "2026-07-19T08:01:58.976Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/02/6c8fbe77ea67b31b777402ee28ba783e831a5724d098eb167905f6b882c1/hypothesis-6.157.0-cp310-abi3-win32.whl", hash = "sha256:17e9264974ffdeaa95e48caa36db418d49bb1f0fa164be616adea64f5547d900", size = 636444, upload-time = "2026-07-19T08:01:12.732Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/06/8e3262f176cea35bffcdfbd6b2a2d80b1719f6d16d1516f3ba197dfbc054/hypothesis-6.157.0-cp310-abi3-win_amd64.whl", hash = "sha256:c8ac434ead091077b1fc9ca69ed7162cd644e42bf75e59a11ae35258e8922225", size = 642232, upload-time = "2026-07-19T08:01:42.615Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/a3/170b589ee0e07c1dbf7dbfff68ec55138fb67585d924967654c47aaf0586/hypothesis-6.157.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c3084b75511d50a912eb597ab5d1d19a12734a38155c93b061a6beead84a55cd", size = 750646, upload-time = "2026-07-19T08:01:38.359Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/05/63a3261d9609807a382206d15bd4ae8022abb33abc0a90b7d3fe1a2d232e/hypothesis-6.157.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:46519de255107a12213470aebea23fb457acec1157ca9aafc2ce4278d073d61e", size = 743180, upload-time = "2026-07-19T08:01:49.814Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/0e/c8a350907bf330b2fb0ab3029ad26ccd914fe2e5cc1e3cf527a88a562e00/hypothesis-6.157.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:702c39588097cdaacbbd9b3c7c26abc2321a91f023163af364698d79d57adcd2", size = 1071546, upload-time = "2026-07-19T08:01:10.034Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/2c/eaafbf8e0cf473a68d22ee0962acaab932663c1ff4075e15e9d24bdf2eae/hypothesis-6.157.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f1f48231ba294c0eff490403ff1580c5ba8534679ee5462171f7c23a7096592", size = 1122801, upload-time = "2026-07-19T08:01:35.619Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/60/e0faf53a9d0b89eb3a8fe541305f41125be4283c5647c7943b27c6c705a2/hypothesis-6.157.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcc1cf1d961e9fa4043e5938ebee8b81de559c5253253ef42a523bb73f9a6be4", size = 1246079, upload-time = "2026-07-19T08:02:07.38Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/6e/a322bc611d3bb725cc06cb97a829dffe7be7a0499667b2b153ddb8f7152b/hypothesis-6.157.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:48a75902f2a5b6fbf8b3e22e5d1c9c099ecb1accb1b32860dbc07ef8b66d5cc5", size = 1289525, upload-time = "2026-07-19T08:02:02.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/c4/4a11576b60ec964e92c4968c944270cccd171619a599ca410ea01d19b3aa/hypothesis-6.157.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:a48197661d22f0a6d6fad9ae43fc630901c996e7914e8a22eb0fa3c041295b0c", size = 587912, upload-time = "2026-07-19T08:02:12.195Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fe/94/8a39d37b7523fc838e634f300140b98973d8033ad1c65ff98c6305d0c5bd/hypothesis-6.157.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8d7c38f06cd43f68eca7532fc55171b265c3b55cf29317e7d330fbdfdb19ab", size = 639589, upload-time = "2026-07-19T08:01:45.423Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/99/dc5fd3e30467b4f3c326b5f94b59d955f769a89f71aadbcfcde22170e942/hypothesis-6.157.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:a32ae18769c095331eedd1921e999075660631d532370497116dc5dc1975024a", size = 748530, upload-time = "2026-07-19T08:01:34.253Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/9e/a88f7a5f968930ef4348b17b19c37194a4d8246c3f9385f0a991f19bc30b/hypothesis-6.157.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d6947a4978496aeafc94815b16643a3aa753e12da090ccee65bf2f72c29b0b96", size = 741820, upload-time = "2026-07-19T08:01:21.885Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/30/9d9dcb36512f9a724c95a56694272dbf82d7b51ec56722e993d92f88f986/hypothesis-6.157.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c32d9da2d7f52d6cd04cfaaab6e32793c99525f07c5cb44a04c3895df73506b", size = 1070720, upload-time = "2026-07-19T08:01:39.651Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/87/9fc11ce9545315e489ef6694226b88bc606561223f987fd57ae66efbd0e8/hypothesis-6.157.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9404b73097ef8deb2f0a72d7a8c38c15c86f4aa3d2fcad89540e17c33eb6259", size = 1121766, upload-time = "2026-07-19T08:01:00.31Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/f2/847a0fe262deab8077f17dd87448a3fc14f30c353b61831eb482acf64672/hypothesis-6.157.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b97ebad2018f39287b5c8fdfd66d4372c005bc914fe2db6c283facbac7ddb81", size = 1244522, upload-time = "2026-07-19T08:02:10.713Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/ba/fe9e0db0c0dafb186ded1caf0d5dc3ac9f4cc98de20bad1e677ab52b8103/hypothesis-6.157.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cb4670b6f9a2c863025d9c4a8e7307bf083bce391e37dcb8857d9ec1a422ae5e", size = 1288305, upload-time = "2026-07-19T08:01:52.896Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/c1/1198203c07514252da690baecc97ff3bea9cea8029495c2a345c563114d9/hypothesis-6.157.0-cp314-cp314t-win_amd64.whl", hash = "sha256:42861f32c1ca6fb999528a5feb553a967fc6b18c39411a224002cdf0d6e91364", size = 639666, upload-time = "2026-07-19T08:01:26.051Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -606,33 +495,6 @@ version = "0.13.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/dc/2f/3908645ddddab7120b46295e541ead308109fa48dbec7d67d7a778870d60/librt-0.13.0.tar.gz", hash = "sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781", size = 211402, upload-time = "2026-07-08T12:26:29.834Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/f4/b2933ddae222dac338476abb872641169a5cfed2c2bb5444a5b07b32b0c3/librt-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0", size = 150990, upload-time = "2026-07-08T12:25:02.42Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/ef/db98f744ca50e6efc9c95c70ee49b77aefac31f6a3fc7c83754a42d6a74f/librt-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5", size = 155238, upload-time = "2026-07-08T12:25:03.681Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/e7/a197e7bc72baf2c61ce7fdc6906a5054dc05bd8da0819aa894e4857bf87e/librt-0.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9", size = 503073, upload-time = "2026-07-08T12:25:05.049Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/e7/7887712e27da7c1ab80fcabb1de6eb24243964f6557cae530d4b70706dbd/librt-0.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b", size = 496528, upload-time = "2026-07-08T12:25:06.26Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/f0/f2283385bb6b950b26a1410f4ce51ec27231e0b3a4b925c46366d218b198/librt-0.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03", size = 531786, upload-time = "2026-07-08T12:25:07.658Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/11/69ac3b54766ffba5fd7e5acebfb048d66dbe1f9f2d14516c2b3edc59cf87/librt-0.13.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e", size = 524393, upload-time = "2026-07-08T12:25:09.121Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/5f/d72f95fd444a926a3c14b4e24979474116988dd57a45be242077c45d3c22/librt-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd", size = 543026, upload-time = "2026-07-08T12:25:10.459Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/08/dcd9993ad192737a004ba263d549f8ea605b326b952e7d6205c7d4170b76/librt-0.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348", size = 546829, upload-time = "2026-07-08T12:25:11.716Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/d5/6d9bb2f54e4109a956b7128836529653eb9d740f784bc47ed10a02c1000e/librt-0.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7", size = 535700, upload-time = "2026-07-08T12:25:13.144Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/f2/10946922503858a359492fa27f13e86228bde702116a740ac7b3cd185f24/librt-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82", size = 573566, upload-time = "2026-07-08T12:25:14.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/a8/94f00e3c99479a18088af3685ea016c42f3c7d5d1964d8dbb40c08d7f1aa/librt-0.13.0-cp312-cp312-win32.whl", hash = "sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3", size = 106099, upload-time = "2026-07-08T12:25:16.159Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/7b/2da9c74c1ed25a89cc4e1c8e007ea2eb4a0f1fafa3e70d757fe3242c5c5c/librt-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa", size = 126934, upload-time = "2026-07-08T12:25:17.275Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/3b/18e7b63255297a2bdc9c25c8d6d4ca8eca9f63aceb1252c0f7427ac7099e/librt-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3", size = 151027, upload-time = "2026-07-08T12:25:19.638Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/68/e2248452c00d1a03b45fee1752cdc8f790a476efd2402b75181da88a9e61/librt-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c", size = 155152, upload-time = "2026-07-08T12:25:20.851Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/16/52b1c99bf19057a062aac39c900cbb81499f6f75d6c537c14463d247ba78/librt-0.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c", size = 502499, upload-time = "2026-07-08T12:25:22.055Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/54/b811151805c795f55e0dedee6ec687b75f9982a8105d240ea3910737a77b/librt-0.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b", size = 496108, upload-time = "2026-07-08T12:25:23.296Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/f8/094d6b2bd93f3fdaa54db54cc788c4a365333bddad65ab02e04da0b1d004/librt-0.13.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9", size = 531576, upload-time = "2026-07-08T12:25:24.648Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/40/541733d5755824f968f7ec39d78ffbd75d145964157ae5e69a09ec6d7326/librt-0.13.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db", size = 524390, upload-time = "2026-07-08T12:25:25.898Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/b5/255673cfdbf5ba663339d36cd863c897289ab4337577e19f9405ce059f36/librt-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6", size = 543053, upload-time = "2026-07-08T12:25:27.436Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/11/ab5005e9c9850710f21e354201bf090646349d3fabf5f951eaf70235729e/librt-0.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7", size = 546387, upload-time = "2026-07-08T12:25:28.65Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/04/a5d7ce1d1df1afd15ca283dcdf7530ac073e12d69ae8c40879dda96f7868/librt-0.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1", size = 535970, upload-time = "2026-07-08T12:25:30.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/76/927e267a6daa290174ac281b23c9804c8829b042ade9c6f24a065f540958/librt-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a", size = 573582, upload-time = "2026-07-08T12:25:31.507Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/24/b6c5213efe39c19f9e13605644d0cf063b4ddaa33ac2e45b088e23a70e2e/librt-0.13.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628", size = 82189, upload-time = "2026-07-08T12:25:32.675Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/00/d29736be177a906ac0b84a5b04b4fbfa22c776dc2f366de4172b0f968c08/librt-0.13.0-cp313-cp313-win32.whl", hash = "sha256:79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927", size = 106193, upload-time = "2026-07-08T12:25:33.692Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/ac/aff6fb45393cb8912f39dfb156ef6b2d1cadb207ff465fc8f66141054be8/librt-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650", size = 126962, upload-time = "2026-07-08T12:25:34.769Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/3a/d68cb2b334d53fd30fac81d3a489ce4ba0d9506f4df43fcf676b68352b19/librt-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566", size = 112127, upload-time = "2026-07-08T12:25:35.981Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/66/f49ae0d592bd45b6941e9a8bafcb6a87cddcd501ee7874707e767f01b585/librt-0.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71", size = 149818, upload-time = "2026-07-08T12:25:37.203Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/50/51c76d74014d04fb95b6506d286808984b78a2f7a41039094e6b2194ac48/librt-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6", size = 154071, upload-time = "2026-07-08T12:25:39.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/fe/f19b0f5f82d5a1f2da736586bc840abd00ce07d6388136ae80b7333883fc/librt-0.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f", size = 494168, upload-time = "2026-07-08T12:25:40.641Z" },
|
||||
@@ -696,20 +558,6 @@ dependencies = [
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/12/af/4e516a05d3ca2eb9283e9ec45b2c02225c1514dd6da49fd3c9eaa6639370/mypy-2.3.0.tar.gz", hash = "sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e", size = 3988104, upload-time = "2026-07-13T11:34:53.387Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/94/0e7e592619e2133596a47cdd642534b0456545c218430bd3b9d8fefdd1b1/mypy-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5", size = 15026523, upload-time = "2026-07-13T11:34:49.206Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/d2/1e1731df090a857df2807177a4626863e5ac0f0256513c35780efe53986f/mypy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c", size = 14032189, upload-time = "2026-07-13T11:33:57.168Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/95/cab921f4a806e171f34113e6181dd23c55358ccf6a80741269ef594a410e/mypy-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491", size = 14198696, upload-time = "2026-07-13T11:32:12.767Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/80/e6d008bb19fe446e3662d85e0e2717bf9f2d611a2164fb29d6e067dbf46c/mypy-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7", size = 15286904, upload-time = "2026-07-13T11:34:27.594Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/83/94397c9293608a364aa03e8084fb34ede4ae976a260384b9b52929308135/mypy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3", size = 15528342, upload-time = "2026-07-13T11:34:07.819Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/96/d8b37d819adec6cfccfb1fd3afc1735d94717ddeafb45536db9c6943e09b/mypy-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c", size = 11218346, upload-time = "2026-07-13T11:28:27.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2b/cd/cd9f725b19b19e5b530a154cf9bcf9e94279c5d55b3c34fb42b3aa48ea1b/mypy-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595", size = 10204525, upload-time = "2026-07-13T11:31:02.552Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/ae/f7d056eb0294586a572d0d0d89580ec633c064db520f11d37d5a2fb833bd/mypy-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97", size = 14947298, upload-time = "2026-07-13T11:27:47.734Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/d5/db3e7af01e7844d21662c6ddc1f7825ec7cb4053f0391ac02faf3638396f/mypy-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac", size = 13950768, upload-time = "2026-07-13T11:27:57.726Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/fb/43c031f0190513d1ec248ed037eceb742ddd2a4d74bbf406658a28173837/mypy-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6", size = 14151586, upload-time = "2026-07-13T11:29:18.615Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/c3/f8b2ffc60883084da91be51af58e88a7ffd4ff9795acb7d902ff88d31eb1/mypy-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b", size = 15227411, upload-time = "2026-07-13T11:30:29.904Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/2e/16b917fc7adcf03f1aadddfc93aab804ffb234b1ab09c0ffd6d92a5d34a2/mypy-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2", size = 15478790, upload-time = "2026-07-13T11:33:14.686Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/88/aaa65a93c73d0cdae7e42f8adb302bf6885bb281302084f99d0290a35347/mypy-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757", size = 11234919, upload-time = "2026-07-13T11:33:39.28Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/19/b40de63f1a80e63bc2d40f0679a6a8dbd34e95176c8122119bdf406aa552/mypy-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1", size = 10201510, upload-time = "2026-07-13T11:31:52.619Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/58/fa0ae047da911f540284009b4f44b96fe09d83c076d7c103e9d645f46303/mypy-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db", size = 14941909, upload-time = "2026-07-13T11:32:34.332Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/14/2ba1d61452d7c2a7fe12741e8d374e52b183476b07aa7f9e2a0d02b0720a/mypy-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762", size = 13967581, upload-time = "2026-07-13T11:30:00.587Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/5a/483fb9e5ffbbb1a28dccc7b0a13d141b17ac769b6c9f488c0a0c63698962/mypy-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef", size = 14168807, upload-time = "2026-07-13T11:28:48.6Z" },
|
||||
@@ -890,11 +738,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "platformdirs"
|
||||
version = "4.10.0"
|
||||
version = "4.10.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/52/cd/4f25b2f95b23f5d2c9c1fe43e49841bff5800562149b2666afc09309aa8f/platformdirs-4.10.1.tar.gz", hash = "sha256:ceab4084426fe6319ce18e86deada8ab1b7487c7aee7040c55e277c9ae793695", size = 31678, upload-time = "2026-07-18T03:53:43.808Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/73/6fd0bb9ce84138c3857f12e9de63bc901852975a092d545f18087a204aa2/platformdirs-4.10.1-py3-none-any.whl", hash = "sha256:0e4eff26be2d75293977f7cddc153fd9b8eaa7fb0c7b64ffe4076cb443117443", size = 22906, upload-time = "2026-07-18T03:53:42.576Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -936,7 +784,6 @@ name = "psycopg"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tzdata", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" }
|
||||
@@ -957,28 +804,6 @@ name = "psycopg-binary"
|
||||
version = "3.3.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" },
|
||||
@@ -1037,36 +862,6 @@ dependencies = [
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" },
|
||||
@@ -1097,10 +892,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1162,7 +953,6 @@ version = "1.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pytest" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" }
|
||||
wheels = [
|
||||
@@ -1222,12 +1012,6 @@ name = "pywin32"
|
||||
version = "312"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" },
|
||||
@@ -1242,26 +1026,6 @@ version = "6.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
|
||||
@@ -1377,7 +1141,6 @@ version = "1.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" }
|
||||
wheels = [
|
||||
@@ -1597,18 +1360,6 @@ version = "0.22.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" },
|
||||
@@ -1647,45 +1398,6 @@ dependencies = [
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/54/a9c7ea9a82a4ac65e7004c0a03920b5cdd2f9c3b678757d9cd425aa51d53/watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4", size = 400205, upload-time = "2026-05-18T04:32:05.153Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/5d/c9ab3534374a4a67450696905d6ef16a04405448b8dc52bd752ae50423d4/watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281", size = 392508, upload-time = "2026-05-18T04:30:54.849Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" },
|
||||
@@ -1726,79 +1438,45 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "websockets"
|
||||
version = "16.1"
|
||||
version = "16.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8c/02/b9a097e1e16fee4e2fd1ec8c39f6a9c5d6257bae8fa12640caf869f54436/websockets-16.1.tar.gz", hash = "sha256:299468cbe42e2b9981134c7c51d99387d8a7bf562b00183b3eec53f882846dad", size = 182530, upload-time = "2026-07-10T06:32:57.734Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/21/f7/bc3a25c5ec26ce62ce487690becc2f3710bbc7b33338f005ad390db0b986/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57", size = 182204, upload-time = "2026-07-17T22:51:05.858Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/52/748c014f07f4e0e170c8932de7e647a1511d5ab3049cd978797136aee577/websockets-16.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b6aa3f7ad345cf3862c21f4fbf2ef5e14d911348476c2845e137c091fe3a3f0b", size = 179798, upload-time = "2026-07-10T06:31:09.664Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/5e/2a2e64d977d084e49d37c187c26c056daaff41965be7300cd5dbde6f8b07/websockets-16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b43fcfb521ac2f34ba80b7b8ea16303e4ad82dd8af667bf40839ad3a5d37b164", size = 177478, upload-time = "2026-07-10T06:31:11.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/12/5b85b4e75d697e548a94962ce5c036b05dd21cb9545759d555c5586422fc/websockets-16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2bd3e12cd9afbe2baedae0b1eeade8ba64329b60fe2f9abdc966bd10fd2c2ef5", size = 177746, upload-time = "2026-07-10T06:31:12.386Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/62/79b1c8f0cee0da648b4899e1c5b0dbd3aa59846985136a54854db6827ab4/websockets-16.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f41979c8623df9bd30d949d82010a8fda5c56ff12cd8508a5b7272b6d4b53a", size = 187345, upload-time = "2026-07-10T06:31:13.754Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/34/b7c5c52c2f24280e1c017acb7ad491a566750a5cceca7f3cf999373bba21/websockets-16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a24d1f35aef07d794a16c853c688e74956c50239bec37b4f2de080056046419b", size = 188581, upload-time = "2026-07-10T06:31:15.075Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/37/604193bebcbeffe96fdf795960b83a15d600880c64dc17ec9c31c5b3427d/websockets-16.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c64c024ddf7a35331b21fcddb562a039c275d2c82e8c2d12939e7da23997270", size = 191362, upload-time = "2026-07-10T06:31:16.395Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/b4/5ee27575b367d7110d4d13945e2a9de067ec84dc71e54b87f01e38550d9a/websockets-16.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3e99757f5baafe20fc598e202ea6f5b0b265186ad38d0a17bd8beca16296955", size = 189216, upload-time = "2026-07-10T06:31:17.776Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/22/3e2dcc78d85fc5d9d814895ce6d07d0dfacc0f6aaa1d151f2b8c8d772299/websockets-16.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:353f3bc6e058ac1ccab4b3588e8598837a8c04cfc8351233e6d523be675d844c", size = 187971, upload-time = "2026-07-10T06:31:19.152Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/2f/cd271717b93d5ee19626cb5e38a85baab745c86e33db7c31a3ac729b31b8/websockets-16.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0352f5b38b40e857b6428d468fa21dbb4dd4a567d933c26d9831b4efe1b92f43", size = 185381, upload-time = "2026-07-10T06:31:20.665Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/91/6ad6f2f1426317b5001bd490534208c7360636b35bac1dec2e0c22bfc40e/websockets-16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70bd789afab579602968c39f21cb925466505f3edff22f0ae852bca54978a4f9", size = 188015, upload-time = "2026-07-10T06:31:22.024Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/6d/533733132ab4c07540efd4a8f0b9a435d3a5059b2f26cc476ace1abf7f45/websockets-16.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d0fb4b46f121eccd539353baebd1083a8767a9a351109453d1d1caecd1ba40c2", size = 186619, upload-time = "2026-07-10T06:31:23.376Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/73/16c059f3d73b3331eba10793704afa4faa9939234fb08ef7dca35794e8f0/websockets-16.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c14b6634af01541e4efe2954fd8f263386f7aa6d37c01e55dd8109fd17661452", size = 188497, upload-time = "2026-07-10T06:31:25.024Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/89/9a8fae7dd2acdcfb1a8844c29fe42b518a04b64fce38a0923b6290e452f1/websockets-16.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a58532c49a851bcb481e58c1be23b315c17fe2fbbed509d75aeea12f543d2c15", size = 186051, upload-time = "2026-07-10T06:31:26.291Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/40/b240c7dd6a0e0c59c1f68377cc3015263521080c327c15f5e753c1f6d378/websockets-16.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4e969170c3b08e1d8dabd990fef1fa702c4233aeaabec33f871806e444f6a0e4", size = 187029, upload-time = "2026-07-10T06:31:27.605Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/35/524e3fac40e47d6fdcf6c4b2c95ef1bc8a97e01593c90eff86621df7b716/websockets-16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff9b000064b88787ba9f7a3cb2af2b68a658ca5aad76458a46469e7124b678a0", size = 187308, upload-time = "2026-07-10T06:31:28.927Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/13/56840cf62c8859af6ba22b9529da937332468c80f32b598753e8a66d3990/websockets-16.1-cp312-cp312-win32.whl", hash = "sha256:b9f5d83f80f4d7c4bba6d97f3755ac05850c784dce0fd2ab371c4e41172f53ff", size = 180161, upload-time = "2026-07-10T06:31:30.316Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/ff/87eb9eb44cb62424a8d729834f2b0515a47e2669fabec29820268f4d50a1/websockets-16.1-cp312-cp312-win_amd64.whl", hash = "sha256:6852c9f653966c16109d3b6f31181fd734f7914927e3f0fa1117af7a18c9aa21", size = 180462, upload-time = "2026-07-10T06:31:31.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/63/df158b155420b566f025e75613424ad9649a24bcb0e9f259321ab3d58bea/websockets-16.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b0232ed141cec3df2af5a3959a071c51f40036336b0d37e17faf9ef52fc73e47", size = 179791, upload-time = "2026-07-10T06:31:33.108Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/cf/00fe9414dfeafa6fe54eae9f5716c8c8e9ac59d192be3b893c096d395846/websockets-16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a71b73d143991714144e159f767b698f03c4a70b8a65ae1733b650cff488045b", size = 177472, upload-time = "2026-07-10T06:31:34.522Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/76/b10633424d40681b4e892ffd08ca5226322b2426e62d4ab71eae484c3a32/websockets-16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:187323204c3b2fc465e8fc2609e60437c521790cb9c1acb49c4c452a33e57f37", size = 177737, upload-time = "2026-07-10T06:31:35.964Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/61/d3bb03b2229bb1afd72008742d586cf1ea240dce64dd48c71c8c7fd3294c/websockets-16.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9dba74233c8c3ce368850818c98354dad2570f57231b3fd3bd00d7aa57628881", size = 187403, upload-time = "2026-07-10T06:31:37.496Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/16/cc2e80478f688fc3c39c67dc1fac6a0783858058914ebc2489917462cb42/websockets-16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63339bc8c63c86a463177775cb7c677691f5bcfac7b3b2f01b286d42acd41600", size = 188639, upload-time = "2026-07-10T06:31:38.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/d6/ad87b2507e57de1cbf897a56c963f2925962ed5e85fbe06aaa83ced27acd/websockets-16.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:23e545ea8ae4263e37cdfd4e22a217f519e48e432728bc461185bbf585f38a83", size = 190078, upload-time = "2026-07-10T06:31:40.218Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/1a/5b37b3fd335d5811f29fc829f2646a3e6d1463a4bf09c3100708684c766e/websockets-16.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2237081454846fb40403a80ba86d82e2038b9c45865ab96af0abe7d002a91045", size = 189267, upload-time = "2026-07-10T06:31:41.523Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/98/06afc33e9450d4230f94c664db78875d90f5f6a5fb77f0bc6ec15ae74e1c/websockets-16.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5f5218de1ed047385ca53744caba9435d65f75d008364970a3fae95a05812cf9", size = 188022, upload-time = "2026-07-10T06:31:42.838Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/bf/42fef5d5887c18cf2d148b02debf56cecb9cfbffc68027cde9b12c8f432c/websockets-16.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75c98e3920039d0edff03b74478ada504b7ce3a1bc406db2cabfca84320f7baf", size = 185435, upload-time = "2026-07-10T06:31:44.219Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/9b/8021c133add5fe40ed40312553a6cd1408c069d7efe3444ad483d4973ed3/websockets-16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1facd189d8190af30487a55b4c3688484dd50801628a3b5b2ccd26db08e67057", size = 188080, upload-time = "2026-07-10T06:31:45.986Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/54/1e37384f395eaa127383aab15c1c45e200890a7d7b99db5c312233d193e0/websockets-16.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cc0c6a6eef613c7da32d4fb068f82ef834b58134f6a16b54e6c1e5bf9529ab3d", size = 186678, upload-time = "2026-07-10T06:31:47.449Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/79/1caeacab5bc2081e4519288d248bc8bd2de30652e6eaa94be6be09a1fe5b/websockets-16.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ad9411eded8988b879be6038206698bf7106c85a78f642c004485bcb95be17eb", size = 188554, upload-time = "2026-07-10T06:31:48.886Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/83/b3dca5fad71487b726e31cb0acf56f226792c1cc34e6ab18cbf146bd2d74/websockets-16.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd68f0914f3b64694895bc5e9b14e8b447e41d7bf5ffaf989bb8dcb5e2dfdce7", size = 186109, upload-time = "2026-07-10T06:31:50.508Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/0b/8f246c3712f07f207b52ea5fb47f3b2b66fafec7303162644c74aed51c6a/websockets-16.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fef2debfe7f7ebdda12176f26166f95b7af17af05ba06150fcf889032e0213e9", size = 187061, upload-time = "2026-07-10T06:31:51.861Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/eb/27d6c92a01696b6495386af4fc941d7d0a13f2eab2bf9c336111d7321491/websockets-16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3cd6c9b798218798f4bb7b2e71c38f0e744bb94ca537b13376f88019d46384d", size = 187347, upload-time = "2026-07-10T06:31:53.246Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/d5/eeee439921f55d5eaeabcea18d0f7ce32cdc39cb8fc1e185431a094c5c7b/websockets-16.1-cp313-cp313-win32.whl", hash = "sha256:84c170c6869633536921e4474b1cce7254c0c9b0053ef5725f966cee47e718e4", size = 180149, upload-time = "2026-07-10T06:31:55.058Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/03/971e98d4a4864cf263f9e94c5b2b7c9a9b7682d77bfbba4e732c55ee85a9/websockets-16.1-cp313-cp313-win_amd64.whl", hash = "sha256:bef52d327d70fa75dad93ee61ea2cb1d1489aca9f35c188833563f5a3b4df0a5", size = 180458, upload-time = "2026-07-10T06:31:56.767Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/e6/da1dc11507f8118145a81c751fe0c77e5e1c11b8554496addb39389e2dc2/websockets-16.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f881fca0a45dd6789939bd6637cd98169b92f1c3fdc78262f2cb9ec2cb1f324e", size = 179833, upload-time = "2026-07-10T06:31:58.19Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/ac/c0d46f62e31e232487b2c123bc3cfd9a4e45684ca7dc0c37f0987f29baae/websockets-16.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c379d5b207d3a7f0ba4c2e4602a895b0bcc63fb5f5371a4ae7fbddb03b672b", size = 177524, upload-time = "2026-07-10T06:31:59.563Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4a/33/abd966074b34a51e4f134e0aaed80f5a4a0a35163ea5ac58a1bc5a076d23/websockets-16.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:98ab58a4faa72b46da0127ccc1931dcbfc0985b0778892300a092185910c4cbe", size = 177743, upload-time = "2026-07-10T06:32:00.959Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/30/646e47b8a8dff04e227bdab512e6dde60663a647eeac7bbd6edddd92bbc5/websockets-16.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e9c4e369fc181b2d41a99e01477215cecdc8546a39f7d41a59cc0a7065a0b09", size = 187474, upload-time = "2026-07-10T06:32:02.54Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/72/890ab9d77494af93ea65268230bfbc0a90ba789401ed7a44356a44785644/websockets-16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0704df094b2d5fa7f6f410925a594c2a5c9a09167731a76292e5410934208209", size = 188717, upload-time = "2026-07-10T06:32:04.156Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/aa/baedbbaa6bf9ed6029617ed5e8976535bd805f483ca9b3484e7ad9ee08bf/websockets-16.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b22b1f4950f6ab7126623329c3b47b3b90a14c05db517f2db2a026ad6c928352", size = 190090, upload-time = "2026-07-10T06:32:05.822Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/4f/d813ec94e18002571ef4959d87a630eff6e01b72a51bcb0832b75ae8c51a/websockets-16.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1ae4a686a662964a6671069f84f7f908cc3475e782227726b0c622c715962105", size = 189320, upload-time = "2026-07-10T06:32:07.223Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/3c/8ec52a6662f3df64090fba28cd521d405d54759268d8e820477037e8c80d/websockets-16.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:856bdd638f8277f86465057bfdd4da097c73058fb0f9d2bd5baea29e2bf2d367", size = 188068, upload-time = "2026-07-10T06:32:08.586Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/7f/f0ae6042b14f86fa5f996c6563ea4cf107adc036ccbedc9d4f418d0095f9/websockets-16.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9003a1fde1c21a322a3ca3fa0c4bda8c639da81dbc925162766086643b05ba87", size = 185493, upload-time = "2026-07-10T06:32:09.968Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/ad/5ffc53af9939c49fd653d147fa5b8f78ced1f6bce6c49a7446860945b0ce/websockets-16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e947b1f5fdab045174306e3916785bf3ed537648acc1549827c08c33b10953", size = 188141, upload-time = "2026-07-10T06:32:11.434Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/62/729206c0ee577a4db8eae6dd06e0eef725a1287c6df11b2ef831d003df31/websockets-16.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5dd0e666b5931c0509cf65714686a1c5126771e663a79ac5d40da4f58b1f9502", size = 186653, upload-time = "2026-07-10T06:32:12.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/86/e8806a99ec4589914f255e6b658853fe537bf359c05e6ba5762ad9c27917/websockets-16.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a0285df7925657ad65a65fb8dc330808bce082827538fd50ef45fa12d1fc5bca", size = 188614, upload-time = "2026-07-10T06:32:14.236Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/38/ac554e2fc6ff0b8deeff9798b92e7abd8f99e2bd9731532e7033de208220/websockets-16.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:82d1c2cab3c133e9d059b3a5420bed9376bd30e21c185c63dda4ddadf6ddda47", size = 186165, upload-time = "2026-07-10T06:32:15.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/c5/4ef4d8e53342f94f3c49e1ae089b32c1e8b3878e15e0022c7708c647f351/websockets-16.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c39907f1eaf11f6277def65aa02d68f30576b693d0c1ca332aafa3caa723ac6d", size = 187119, upload-time = "2026-07-10T06:32:17.114Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/33/4788b1dd417bd97eeb2698af3b9df6775ac656f96e9987da0419a067602f/websockets-16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45c5ea55446171949eb99fd34b771ceddd511ca21958d40d0197ced33159e5ee", size = 187411, upload-time = "2026-07-10T06:32:18.629Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/38/00d37aad6dc3244ce349e2864815362e50b3cfc00cac28d216db20efe40f/websockets-16.1-cp314-cp314-win32.whl", hash = "sha256:b8ef8b1c8d6bd029a475ac432e730fba2dfd456715d26c473e2a82291024b99c", size = 179822, upload-time = "2026-07-10T06:32:20.233Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/37/2a8cb0eaddee5eaebda47a90a3ba0898d1ce3d866b02a4857fea17d82e5b/websockets-16.1-cp314-cp314-win_amd64.whl", hash = "sha256:7358ff21632b5d062707f73e859c824f1c3807e73d8ca25e71caca7c4cdcf145", size = 180167, upload-time = "2026-07-10T06:32:21.749Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/5a/262ad5fcaef4198997b165060f09a63f861e76939b1786ab546ccc3f8120/websockets-16.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d0f38f4c3e9b359e257c339c2cc1967ccaeedb102e57c1c986bdce4bf4f32268", size = 180166, upload-time = "2026-07-10T06:32:23.278Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/c7/36377db690f4292826e4501a6dec2801dc55fd1cf0405923b04937e478df/websockets-16.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3c3d2cbd1602593bad49bd86fa3fbb25407d87a3b4bf8857c0ac5ac4914e1901", size = 177697, upload-time = "2026-07-10T06:32:25.164Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/c7/07171abce1e39799a76f473608580fe98bd43a1230f5146159622c02bccf/websockets-16.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:36069b74671e7e667f48a7484249f84c45a825a134c8b1bdc01875d0daa10d79", size = 177902, upload-time = "2026-07-10T06:32:26.564Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/17/c831f48e250bc4749f57c00dcce73337c41cd32f6d59a64567b84e782601/websockets-16.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:587f83c2ce8a5d628e166384d77fa7f0ac69b9007d515ab442123e6615aa8da3", size = 187766, upload-time = "2026-07-10T06:32:27.981Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/2e/4dfe63e245b0ecfaf470cf082d25c6ce35808159135fd88c82653a6b11ab/websockets-16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6db7972d52bc1b66cefe2246902e256cbaebc9ba8a45eac09343d7eb6671b2", size = 188939, upload-time = "2026-07-10T06:32:29.365Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/e5/5faf65aebd9562f6b4bc473d24ce38cc56f84eb5f5bee66ed9b86733f93c/websockets-16.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e7d6014888a0632e1ed7a4095248bb3095232999447f2d83bfb1900987dd9ed9", size = 191081, upload-time = "2026-07-10T06:32:30.868Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/cd/2634f2f2c0556c1aae6501ed6840019cc569dd6fdbcac6494378daea4dc0/websockets-16.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cb074d150e4ad2a77aa8a332c2be85f3f64f2681519d2570c1225c12c9821ff", size = 189513, upload-time = "2026-07-10T06:32:32.399Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/bb/2c700b51196104f09715b326b1f092ed25326bdf79a03e00a4842e503743/websockets-16.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d19c9067e1fe9490f974bffbc0e443b80a7674c5efb4980c429cc00771f07c5a", size = 188240, upload-time = "2026-07-10T06:32:33.897Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/20/86283636e499a1a357fa9441f690ba34f255e731f2fea174132b3b762b57/websockets-16.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d440ff0c6c7469ad59c0a412c383c235935b43635e89425e3f6a0c36de90c31b", size = 185955, upload-time = "2026-07-10T06:32:35.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/23/d7fb734b0095d43bc7f1c9f68afd50adb4176e7e513403e8c70ad7daa4fa/websockets-16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8613129a2533f08de24505e69a3e403cedaadae49abdb043c4d170ca71b7e4bd", size = 188491, upload-time = "2026-07-10T06:32:36.673Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/5e/168a192689db468405ecf3b8e4a2c18811936b0724d017ad7e6d252734f0/websockets-16.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:a5bf9c23f197b4ec88290fd5463f33db67362a1bb10f85fc2e8e7627f0ddab97", size = 186983, upload-time = "2026-07-10T06:32:38.207Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/9b/66795fa91ebe49019ebe4fa910282172252e37046b80e08fc52e0c365150/websockets-16.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:520b0fd0395f075febb283c76755af724ab9fd19dffa4f3bfd18cb4e622790a3", size = 188890, upload-time = "2026-07-10T06:32:39.545Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/32/126bbc844be5afb3613fd43211dac10a9645f4cf39741d04acaa2ec7030c/websockets-16.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:7143aa09a67e1c013be44e81a88dfe90fc6244198ab86c7edd064152cf619805", size = 186583, upload-time = "2026-07-10T06:32:41.038Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/b9/0b5db9cbcf6e4970db4496893244a8d92e07f71a8ef27cf34b08aa02fef1/websockets-16.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:7acb811fad08e611755800d1560e395c67e11a6bd563598ea6abb319afb86938", size = 187353, upload-time = "2026-07-10T06:32:42.501Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/2e/254b2131a10d831b76e2c18dfe7add9729c6292c674a8085bf8de01ad151/websockets-16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c5cf88e3faa2f7931bc6baeee7599c97656a3f6ac7f831f4fccba233e141783a", size = 187784, upload-time = "2026-07-10T06:32:43.929Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/dc/e7288aa8e3ac5a88a0924619984d663c1abf2a87d0ea98290c66fdaee0ec/websockets-16.1-cp314-cp314t-win32.whl", hash = "sha256:589f8842521c8307684ce0b40ce4ad70c5e0aa46484c6f1225a94ef4b8970341", size = 179947, upload-time = "2026-07-10T06:32:45.495Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/de/37edf1260ff0fbbd2f82433489c4cfbe799ac2ff21355331609879329fe6/websockets-16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c0e0857c30bbbc2bb5c30687508f0b7ec19aa026cd9f2ff8424d0fee42dcc07", size = 180291, upload-time = "2026-07-10T06:32:47.119Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/58/bd83247f39ddc26ffc2c24eb05087a3b749e00cb4509fc6d19daa23c8495/websockets-16.1-py3-none-any.whl", hash = "sha256:c5149dfe490ec7e5ee5dbf624c642fb725f93a5575c7f00ab594ca9eddb8dd81", size = 174031, upload-time = "2026-07-10T06:32:56.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/a2/ba78a164eeea4620df4a4df4bd2ed6017438c4655cc0f36f2c0bc0432355/websockets-16.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8", size = 179635, upload-time = "2026-07-17T22:50:05.001Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/08/d26d7a7628cd4ac34cbbdb63ac80914ca842ed8e42938c40a53567806df3/websockets-16.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293", size = 177320, upload-time = "2026-07-17T22:50:06.427Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/45/ebec83e6269536aa5932533c67b0af5c781f3e73fdbcd68672dcf43f4f44/websockets-16.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051", size = 177544, upload-time = "2026-07-17T22:50:07.834Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/d5/abc614d2297f6c1c3e01e61260364457a47c25cc1cf6a879038902bc6aa8/websockets-16.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1", size = 187270, upload-time = "2026-07-17T22:50:09.275Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/71/4c99af3b87dff1b2927981f6876607d4acb45338c665242168d3982f7758/websockets-16.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7", size = 188509, upload-time = "2026-07-17T22:50:10.722Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/b4/5c8ca14b0df7eb84ed0524165c5359150210140817a3312aee57bf62a1cf/websockets-16.1.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31", size = 189882, upload-time = "2026-07-17T22:50:12.293Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/c1/bedfba9e70557129cb8083748d167bdcc01483dedf0f0df143676df05cbe/websockets-16.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0", size = 189114, upload-time = "2026-07-17T22:50:13.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/09/aa835b2787835aebd839114be5de51b797cb480b63ba42b26d34dfe147cb/websockets-16.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3", size = 187861, upload-time = "2026-07-17T22:50:15.179Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/26/f6408330694dbc9830857d9d23bc14ac4f6875127a480cfdda8d5ca21198/websockets-16.1.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562", size = 185286, upload-time = "2026-07-17T22:50:16.741Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/9a/e0675e70dd8a80762cf35bb18799d3f290a4890ffe6439bc51d222796083/websockets-16.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b", size = 187935, upload-time = "2026-07-17T22:50:18.213Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/c1/3234cfb86afde01b81e9bddcc6e534c440975d60a13991259e833069ab3e/websockets-16.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a", size = 186444, upload-time = "2026-07-17T22:50:19.67Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/89/87/9c15206e1d778923d8daa9657de07aa62ea815e13448319c98458c37b281/websockets-16.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c", size = 188409, upload-time = "2026-07-17T22:50:21.28Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/00/cf5de5c67676de2d3eef8b2a518f168f6796595447a5b7161ba0d012915c/websockets-16.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499", size = 185958, upload-time = "2026-07-17T22:50:22.719Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/c0/731b6ddede2e4136912ec4cff2cffbda35af73546be4762c3d7bd3bd79af/websockets-16.1.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985", size = 186911, upload-time = "2026-07-17T22:50:24.108Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/7f/39c634472c4469a24a7c09cecddffb08fac6d0e74f73881a94ee8a40a196/websockets-16.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9", size = 187204, upload-time = "2026-07-17T22:50:25.548Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/89/9667c256c256dafcc62d21328ce7a40067da857969b68ee9af375b0aaf72/websockets-16.1.1-cp314-cp314-win32.whl", hash = "sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328", size = 179603, upload-time = "2026-07-17T22:50:27.086Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/dd/1c099d6c0fc5deb6b46ccdbb6981fdb4b12c917869cb3952408409dc18db/websockets-16.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc", size = 179948, upload-time = "2026-07-17T22:50:28.521Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/25/9956b2d5e0529d5d23924f21bba1440d4c5c88a562e4f08550871ffa97a7/websockets-16.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573", size = 179963, upload-time = "2026-07-17T22:50:29.982Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/06/55ffc976c488b6aee9ea05761ff7c4e88e7c1fd82818c8ca7b556ad2f90c/websockets-16.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999", size = 177497, upload-time = "2026-07-17T22:50:31.396Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/e8/f7dac2e980bacc92bdc26cebae4ae4d50cae5380732c50980598fc0bbae4/websockets-16.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe", size = 177698, upload-time = "2026-07-17T22:50:32.829Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/39/26762f734113e22da2b942c3aca85798e0c0405d64c256549540ff31e5a1/websockets-16.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d", size = 187561, upload-time = "2026-07-17T22:50:34.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/11/94/c3f330851806b9b02138b774d593478323e73c99238681b4b93efe64e02d/websockets-16.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392", size = 188732, upload-time = "2026-07-17T22:50:36.088Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/f2/eb2c450f052de334ae33cf200ece6e87b0e14d186807074e4eb1cd2cdea2/websockets-16.1.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7", size = 190872, upload-time = "2026-07-17T22:50:38.008Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/31/2ac8cecf3a74f7fed9132129fc3d90b3998a1554570c11a69b2a8c20332d/websockets-16.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499", size = 189305, upload-time = "2026-07-17T22:50:39.53Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/cf/8ab19650d3c0d4562c92e70ab47c257c4aa5c6a713ed87fe63766b31fefc/websockets-16.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43", size = 188033, upload-time = "2026-07-17T22:50:40.912Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/d7/a49a38a6127a4acb134fb1912b215d900cc657605cff32445bf519f3acc4/websockets-16.1.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458", size = 185748, upload-time = "2026-07-17T22:50:42.559Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/95/3e/ad1fa40388c7f2e0bb2c7930d0090b6c5498594bd1cdaec18864df3d9e97/websockets-16.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62", size = 188285, upload-time = "2026-07-17T22:50:43.974Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/b8/d5db28ca264b9104f82196f92dc8843e35fd391f763d42e4ad358f5bc97e/websockets-16.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb", size = 186777, upload-time = "2026-07-17T22:50:45.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/9c/726cb39d0cc43ae848dce4aa2acb04eecc6738b1264ec6d700bf6bcfb9f8/websockets-16.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51", size = 188682, upload-time = "2026-07-17T22:50:46.973Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/c7/1168704de8c2dd483edabe4a22cbe4465dd8be8dd95561d214f9fe092871/websockets-16.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0", size = 186377, upload-time = "2026-07-17T22:50:48.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/40/f9ff2d630ffce4e7dfea0b2288e1caf9ebbf9ff8a9ec9396136ce8b94935/websockets-16.1.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217", size = 187148, upload-time = "2026-07-17T22:50:49.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/71/e177c8299f78d7cbe2d14df228643c10c70c0e86e108e092056bbcc16e46/websockets-16.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737", size = 187578, upload-time = "2026-07-17T22:50:51.619Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/b2/b6987faf330f5af5c787a2610124c2e8403d51724f9001ec4fff6311fe7a/websockets-16.1.1-cp314-cp314t-win32.whl", hash = "sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7", size = 179729, upload-time = "2026-07-17T22:50:53.269Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/6e/fbac6ed878dd362fbad7d415fa4f84d38e3e33fed8cde45c64e783acf826/websockets-16.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231", size = 180072, upload-time = "2026-07-17T22:50:54.969Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1807,39 +1485,6 @@ version = "2.2.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" },
|
||||
|
||||
Reference in New Issue
Block a user