# svcforge CI. # # The contract, in one line: merge to master -> three images built and scanned -> the # chart's image digests bumped -> ArgoCD syncs. CI never touches the cluster. There is no # kubeconfig here and there must never be one; the pipeline's last act is a git commit. # # Rules this file exists to enforce: # - Build once, promote the artifact. The digest that trivy scanned is the digest that # lands in values.yaml is the digest that runs. # - Deploy by digest, never a mutable tag. # - Everything pinned: actions by SHA, tool images by digest, deps by uv.lock + --frozen. # - Every gate required. None advisory. Fail the PR, not prod. # # Stage order is deliberate and matches the module: cheapest and most likely to fail first, # so a formatting mistake costs 20 seconds instead of three minutes of image builds. name: ci on: [pull_request, push] concurrency: # A second push to the same branch makes the first run's answer irrelevant. Cancel it — # except on master, where the run ends in a commit and must not be interrupted midway. group: ci-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} env: REGISTRY: gitea.oci-oci.duckdns.org IMAGE_NS: gitea_admin UV_VERSION: "0.11.29" GITLEAKS_VERSION: "8.30.1" GITLEAKS_SHA256: "e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080" jobs: # --- stage 1: lint -- fast, fails first --------------------------------------------- lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: ${{ env.UV_VERSION }} enable-cache: true # The uv store is keyed on the lockfile: same lock, same wheels, cache hit. cache-dependency-glob: uv.lock - run: uv sync --frozen - name: ruff run: uv run ruff check . && uv run ruff format --check . # --- stage 2: types -- your compiler ------------------------------------------------ types: runs-on: ubuntu-latest needs: [lint] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: ${{ env.UV_VERSION }} enable-cache: true cache-dependency-glob: uv.lock - run: uv sync --frozen - name: mypy --strict run: uv run mypy --strict . # --- stage 3: unit -- domain only, milliseconds, coverage gate ----------------------- unit: runs-on: ubuntu-latest needs: [lint] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: ${{ env.UV_VERSION }} enable-cache: true cache-dependency-glob: uv.lock - run: uv sync --frozen - name: pytest unit # The gate is on domain/ alone, and only domain/. It is pure, has no I/O, and needs # no mocks — there is no excuse for a gap there. Pointing this at the whole repo # would let untested SQL be paid for by well-tested pure functions. # # `--cov=svcforge_core.domain` — the MODULE, not a path. `--cov=libs/svcforge_core/domain` # is a path that does not exist (the package nests one level deeper, at # libs/svcforge_core/svcforge_core/domain), so coverage measured nothing and reported # 0.00%. A path-based --cov that misses silently reports 0 rather than erroring, so # without a --cov-fail-under this reads as a passing coverage gate over no code at all. run: uv run pytest tests/unit --cov=svcforge_core.domain --cov-fail-under=90 # --- stages 4+5: migrate, then integration against that schema ----------------------- integration: runs-on: ubuntu-latest needs: [unit] services: postgres: image: postgres:18@sha256:32ca0af8e77bfb8c6610c488e4691f83f972a3e9e64d3b02facf3ab111ad5500 env: POSTGRES_PASSWORD: postgres POSTGRES_DB: svcforge ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 10 env: # A scratch Postgres, so a literal password is correct here: it is not a secret, it # is a fixture. Real DSNs live in Vault and reach the pods via external-secrets. SVCFORGE_PG_DSN: postgresql://postgres:postgres@postgres:5432/svcforge SVCFORGE_PG_DSN_SESSION: postgresql://postgres:postgres@postgres:5432/svcforge # What tests/integration/conftest.py actually reads. Without it the fixture falls # back to testcontainers and starts a SECOND Postgres inside the runner's docker, # while the service container above sits unused — slower, and a different database # to the one `migrate` just ran against. SVCFORGE_TEST_DSN: postgresql://postgres:postgres@postgres:5432/svcforge steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: ${{ env.UV_VERSION }} enable-cache: true cache-dependency-glob: uv.lock - run: uv sync --frozen - name: migrate # The same entrypoint the chart's pre-upgrade hook runs. If migrations only ever # ran under testcontainers, CI would be testing a code path production never takes. run: uv run python -m svcforge_core.migrate - name: pytest integration run: uv run pytest tests/integration # --- stages 6+7+8: SAST, secrets, dependency CVEs ------------------------------------ # One job, three independent gates. They share a checkout and nothing else; each `run` # step fails the job on its own. security: runs-on: ubuntu-latest needs: [lint] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # gitleaks scans history, not just the tip. A secret committed and then reverted # is still a leaked secret, and a shallow clone cannot see it. fetch-depth: 0 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: ${{ env.UV_VERSION }} enable-cache: true cache-dependency-glob: uv.lock - run: uv sync --frozen - name: bandit (SAST) # `--with`, not a dev dependency: bandit is a CI tool, not something the project # imports, and ruff's S ruleset already runs its checks in the lint stage. This is # the belt to that suspenders — -ll reports medium severity and above only. # # `bandit[toml]` + `-c pyproject.toml`: without the toml extra bandit cannot read # its own config and silently ignores it, which looks identical to a clean run. run: uv run --with 'bandit[toml]' bandit -c pyproject.toml -r libs services -ll - name: gitleaks (secret scan) # A downloaded binary, not `docker run -v "$PWD:/repo"`. # # The container form was scanning NOTHING and passing. `-v "$PWD:/repo"` is # interpreted by the dind sidecar's daemon, which cannot see the job container's # checkout, so gitleaks got an empty mount and logged: # # ERR [git] fatal: not a git repository (or any parent up to mount point /) # ERR failed to scan Git repository error="stderr is not empty" # INF scan completed in 35.2ms # INF no leaks found <- exit 0 # # It reported success without looking, which is worse than having no scanner: the # gate was green and meaningless. The 35ms runtime was the tell — a real history # scan of this repo takes seconds. # # The `--log-opts` + commit-count assertion below is the guard against that class # of failure returning. A scanner that cannot fail is not a gate. run: | set -euo pipefail curl -fsSL -o /tmp/gitleaks.tgz \ "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_arm64.tar.gz" echo "${GITLEAKS_SHA256} /tmp/gitleaks.tgz" | sha256sum -c - tar -xzf /tmp/gitleaks.tgz -C /tmp gitleaks install -m 0755 /tmp/gitleaks /usr/local/bin/gitleaks # Prove there is a repository to scan before trusting the verdict. commits=$(git rev-list --count HEAD) echo "scanning $commits commits" test "$commits" -gt 0 gitleaks detect --no-banner --source . --redact --exit-code 1 - name: pip-audit (dependency CVEs) # --strict fails on an audit error rather than shrugging and reporting clean. # # Audits the LOCKED dependency set, not the installed environment. Auditing the env # means auditing `svcforge` and `svcforge-core` too, which are ours, are installed # editable, and are not on PyPI — under --strict that is a hard error ("distribution # marked as editable"), so the choice was to drop --strict or to stop asking PyPI # about packages it has never heard of. This asks about the 56 that actually came # from PyPI, and keeps --strict. run: | uv export --frozen --no-dev \ --no-emit-project --no-emit-package svcforge-core \ -o /tmp/requirements-audit.txt uv run --with pip-audit pip-audit --strict -r /tmp/requirements-audit.txt # --- stage 9: hadolint --------------------------------------------------------------- dockerfile: runs-on: ubuntu-latest needs: [lint] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: hadolint uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 with: recursive: true dockerfile: "services/*/Dockerfile" failure-threshold: warning # --- stage 9b: the chart must render -------------------------------------------------- # Without this, a chart that does not template reaches ArgoCD and fails in the cluster, # where the error surfaces as a sync failure with no PR attached to it. `helm template` # is the real gate: it is what ArgoCD does, and _helpers.tpl's image helper calls `fail` # on anything that is not a full sha256 digest. chart: runs-on: ubuntu-latest needs: [lint] env: # The same helm version the worker and reconciler images carry, so CI renders with # the version that ships. HELM_VERSION: "3.21.3" # sha256 of helm-v3.21.3-linux-arm64.tar.gz, from https://get.helm.sh/*.sha256sum. # Pinned for the same reason every image here is pinned by digest: a tarball fetched # over HTTPS is still a tarball whoever controls the bucket chose to serve. HELM_SHA256: "67f58155079ff9ffab98ba5c88daff0ed9b542f3a4732f5dd426dde7dd0f5244" steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: install helm # A downloaded binary rather than `docker run alpine/helm`. The container form # cannot see the checkout: `docker run -v "$PWD:/repo"` is interpreted by the dind # SIDECAR's daemon, and the chart path resolved to nothing inside it — # "stat deploy/chart/Chart.yaml: no such file or directory" while the file plainly # exists in the job. A binary on PATH has no such boundary to cross. run: | set -euo pipefail curl -fsSL -o /tmp/helm.tgz \ "https://get.helm.sh/helm-v${HELM_VERSION}-linux-arm64.tar.gz" echo "${HELM_SHA256} /tmp/helm.tgz" | sha256sum -c - tar -xzf /tmp/helm.tgz -C /tmp install -m 0755 /tmp/linux-arm64/helm /usr/local/bin/helm helm version --short - name: helm lint run: | helm lint deploy/chart - name: helm template (rejects unbumped digests) # values.yaml ships all-zeros placeholders, so a bare `helm template` MUST fail. # That is the guard working, not a broken chart — asserting the failure here is # what stops the guard silently regressing into a prefix check again. run: | set -euo pipefail if helm template svcforge deploy/chart >/dev/null 2>&1; then echo "FAIL: chart rendered against the placeholder digests in values.yaml." >&2 echo "The digest guard in _helpers.tpl is not guarding." >&2 exit 1 fi echo "ok: placeholder digests rejected" - name: helm template (renders with real digests) # Dummy but well-formed digests: this checks the templates themselves render, with # the two values-gated monitoring blocks explicitly on so they are covered too. run: | set -euo pipefail A="sha256:$(printf 'a%.0s' $(seq 64))" B="sha256:$(printf 'b%.0s' $(seq 64))" C="sha256:$(printf 'c%.0s' $(seq 64))" helm template svcforge deploy/chart \ --set image.api.digest="$A" \ --set image.worker.digest="$B" \ --set image.reconciler.digest="$C" \ --set serviceMonitor.enabled=true \ --set prometheusRule.enabled=true \ >/dev/null echo "ok: chart renders" # --- stage 10: build -> trivy -> push by digest -------------------------------------- image: runs-on: ubuntu-latest # Every gate above is required. An image is not built until all of them are green, # which is what makes "the digest CI pushed is a digest that passed everything" true. needs: [types, unit, integration, security, dockerfile, chart] permissions: contents: read strategy: fail-fast: false matrix: svc: [api, worker, reconciler] # Deliberately no `outputs:` here. Matrix legs share one outputs map and clobber each # other — the merge is not per-key and not ordered, so two of the three digests would # arrive empty or stale, intermittently. The bump job resolves the digests from the # registry instead, which is a read, not a rebuild. steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: registry login # Not gated to master any more: the build step now reads AND writes the layer cache # in the registry, so every run needs credentials. Pushing the release image is # still master-only — that gate lives on the `push by digest` step, where it belongs. uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: build # Context is the repo root and the Dockerfile is addressed with -f. It cannot be # otherwise: the image needs pyproject.toml, uv.lock and libs/, all of which live # above services//, and COPY ../.. is illegal. # # Loaded locally, not pushed. Trivy scans this exact image next; only then does it # get pushed. The alternative — push, scan, and hope nobody pulled meanwhile — is # how a CRITICAL ends up in the registry with a green checkmark next to it. # # The layer cache is `type=registry`, NOT `type=gha`. Two reasons, both hard: # # 1. act_runner's cache server is backed by a 1Gi PVC that also holds `.runner`, # the runner's own registration file. `mode=max` stores every intermediate layer # of three images — several GB. Filling that volume does not merely lose the # cache: the runner cannot write its state and has to be re-registered by hand. # Trading "slow CI" for "broken CI" is not a trade. # 2. act_runner evicts by AGE, with no size cap in its config. It will fill whatever # it is given and then wedge. The registry has no such limit and already holds # the images anyway. # # The uv/pip cache still uses the runner's cache service — that one is a few hundred # MB and fits. run: | docker buildx build \ -f services/${{ matrix.svc }}/Dockerfile \ --build-arg BUILD_SHA=${{ github.sha }} \ --cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \ --cache-to type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache,mode=max \ --load \ -t svcforge/${{ matrix.svc }}:ci \ . - name: trivy # Run trivy directly rather than via aquasecurity/trivy-action, for the same reason # gitleaks is run directly above: the command is the documented one, pinned by # digest, with no nested action resolution. # # It is also the only thing that works here. trivy-action internally does # `uses: aquasecurity/setup-trivy@v0.2.2`, and that tag no longer exists upstream — # the earliest published tag today is v0.2.6. The runner clones it and fails with # "Unable to resolve v0.2.2: reference not found". A third-party action pinned by # SHA still resolves ITS OWN dependencies by mutable tag, so pinning the outer # action bought nothing. # # --ignore-unfixed: a CVE with no fix available is not something this PR can act # on, and failing on it only teaches people to add ignore entries. Rebuilding on a # new base image picks the fix up the day it exists. # # The mounted cache dir keeps the vuln DB (~50MB) across the three matrix legs on # this runner instead of re-downloading it for each. run: | docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$PWD/.trivycache:/root/.cache/trivy" \ aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f \ image \ --severity HIGH,CRITICAL \ --ignore-unfixed \ --exit-code 1 \ --format table \ --no-progress \ svcforge/${{ matrix.svc }}:ci - name: push by digest if: github.ref == 'refs/heads/master' && github.event_name == 'push' # Re-running buildx here is a cache hit on every layer, not a second build: the # image is byte-identical to the one trivy just cleared. buildx cannot --load and # --push in one invocation, which is the only reason this step exists. # # The commit-SHA tag is a handle for the bump job to resolve, not something anything # deploys. What deploys is the digest that tag resolves to. run: | set -euo pipefail IMAGE="${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}" docker buildx build \ -f services/${{ matrix.svc }}/Dockerfile \ --build-arg BUILD_SHA=${{ github.sha }} \ --cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \ --push \ -t "${IMAGE}:${GITHUB_SHA}" \ . docker buildx imagetools inspect "${IMAGE}:${GITHUB_SHA}" \ --format '{{.Manifest.Digest}}' # --- stage 11: bump the chart's digests. CI's last act. ------------------------------ bump: runs-on: ubuntu-latest needs: [image] # The master+push guard lives on the STEPS, not on the job. # # A job-level `if` here evaluated false and skipped `bump` at 0s — before `image` had # even started — on a genuine push to master (`event: push`, `head_branch: master`). # The identical expression on the `push by digest` STEP inside the matrix job evaluates # true and runs. Gitea appears not to resolve a job-level `if` correctly when `needs` # points at a matrix job. # # `needs: [image]` still does the ordering and still gates on all three legs passing. # The steps below carry the branch guard, in the form this runner is known to evaluate # correctly. On a PR the job starts and every step no-ops, which costs a few seconds # and is the price of a guard that actually fires. permissions: contents: write steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.ref == 'refs/heads/master' && github.event_name == 'push' with: # A bot token with contents:write on this repo and nothing else: no kubeconfig, # no cluster credential, no ArgoCD API token. CI's maximum blast radius is a bad # commit, which is revertable. token: ${{ secrets.CI_BOT_TOKEN }} ref: master - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 if: github.ref == 'refs/heads/master' && github.event_name == 'push' with: registry: ${{ env.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: bump image digests in the chart if: github.ref == 'refs/heads/master' && github.event_name == 'push' env: REGISTRY: ${{ env.REGISTRY }} IMAGE_NS: ${{ env.IMAGE_NS }} IMAGE_TAG: ${{ github.sha }} # And then it stops. No kubectl, no helm upgrade, no argocd app sync. ArgoCD is # watching master and will have this within a minute. run: ./scripts/bump-digests.sh