svcforge/.gitea/workflows/ci.yaml

# svcforge CI.
#
# The contract, in one line: merge to master -> three images built and scanned -> the
# chart's image digests bumped -> ArgoCD syncs. CI never touches the cluster. There is no
# kubeconfig here and there must never be one; the pipeline's last act is a git commit.
#
# Rules this file exists to enforce:
#   - Build once, promote the artifact. The digest that trivy scanned is the digest that
#     lands in values.yaml is the digest that runs.
#   - Deploy by digest, never a mutable tag.
#   - Everything pinned: actions by SHA, tool images by digest, deps by uv.lock + --frozen.
#   - Every gate required. None advisory. Fail the PR, not prod.
#
# Stage order is deliberate and matches the module: cheapest and most likely to fail first,
# so a formatting mistake costs 20 seconds instead of three minutes of image builds.

name: ci

on: [pull_request, push]

concurrency:
  # A second push to the same branch makes the first run's answer irrelevant. Cancel it —
  # except on master, where the run ends in a commit and must not be interrupted midway.
  group: ci-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

env:
  REGISTRY: gitea.oci-oci.duckdns.org
  IMAGE_NS: gitea_admin
  UV_VERSION: "0.11.29"
  GITLEAKS_VERSION: "8.30.1"
  GITLEAKS_SHA256: "e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080"

jobs:
  # --- stage 1: lint -- fast, fails first ---------------------------------------------
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true
          # The uv store is keyed on the lockfile: same lock, same wheels, cache hit.
          cache-dependency-glob: uv.lock
      - run: uv sync --frozen
      - name: ruff
        run: uv run ruff check . && uv run ruff format --check .

  # --- stage 2: types -- your compiler ------------------------------------------------
  types:
    runs-on: ubuntu-latest
    needs: [lint]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true
          cache-dependency-glob: uv.lock
      - run: uv sync --frozen
      - name: mypy --strict
        run: uv run mypy --strict .

  # --- stage 3: unit -- domain only, milliseconds, coverage gate -----------------------
  unit:
    runs-on: ubuntu-latest
    needs: [lint]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true
          cache-dependency-glob: uv.lock
      - run: uv sync --frozen
      - name: pytest unit
        # The gate is on domain/ alone, and only domain/. It is pure, has no I/O, and needs
        # no mocks — there is no excuse for a gap there. Pointing this at the whole repo
        # would let untested SQL be paid for by well-tested pure functions.
        #
        # `--cov=svcforge_core.domain` — the MODULE, not a path. `--cov=libs/svcforge_core/domain`
        # is a path that does not exist (the package nests one level deeper, at
        # libs/svcforge_core/svcforge_core/domain), so coverage measured nothing and reported
        # 0.00%. A path-based --cov that misses silently reports 0 rather than erroring, so
        # without a --cov-fail-under this reads as a passing coverage gate over no code at all.
        run: uv run pytest tests/unit --cov=svcforge_core.domain --cov-fail-under=90

  # --- stages 4+5: migrate, then integration against that schema -----------------------
  integration:
    runs-on: ubuntu-latest
    needs: [unit]
    services:
      postgres:
        image: postgres:18@sha256:32ca0af8e77bfb8c6610c488e4691f83f972a3e9e64d3b02facf3ab111ad5500
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: svcforge
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
    env:
      # A scratch Postgres, so a literal password is correct here: it is not a secret, it
      # is a fixture. Real DSNs live in Vault and reach the pods via external-secrets.
      SVCFORGE_PG_DSN: postgresql://postgres:postgres@postgres:5432/svcforge
      SVCFORGE_PG_DSN_SESSION: postgresql://postgres:postgres@postgres:5432/svcforge
      # What tests/integration/conftest.py actually reads. Without it the fixture falls
      # back to testcontainers and starts a SECOND Postgres inside the runner's docker,
      # while the service container above sits unused — slower, and a different database
      # to the one `migrate` just ran against.
      SVCFORGE_TEST_DSN: postgresql://postgres:postgres@postgres:5432/svcforge
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true
          cache-dependency-glob: uv.lock
      - run: uv sync --frozen
      - name: migrate
        # The same entrypoint the chart's pre-upgrade hook runs. If migrations only ever
        # ran under testcontainers, CI would be testing a code path production never takes.
        run: uv run python -m svcforge_core.migrate
      - name: pytest integration
        run: uv run pytest tests/integration

  # --- stages 6+7+8: SAST, secrets, dependency CVEs ------------------------------------
  # One job, three independent gates. They share a checkout and nothing else; each `run`
  # step fails the job on its own.
  security:
    runs-on: ubuntu-latest
    needs: [lint]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          # gitleaks scans history, not just the tip. A secret committed and then reverted
          # is still a leaked secret, and a shallow clone cannot see it.
          fetch-depth: 0
      - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: ${{ env.UV_VERSION }}
          enable-cache: true
          cache-dependency-glob: uv.lock
      - run: uv sync --frozen

      - name: bandit (SAST)
        # `--with`, not a dev dependency: bandit is a CI tool, not something the project
        # imports, and ruff's S ruleset already runs its checks in the lint stage. This is
        # the belt to that suspenders — -ll reports medium severity and above only.
        #
        # `bandit[toml]` + `-c pyproject.toml`: without the toml extra bandit cannot read
        # its own config and silently ignores it, which looks identical to a clean run.
        run: uv run --with 'bandit[toml]' bandit -c pyproject.toml -r libs services -ll

      - name: gitleaks (secret scan)
        # A downloaded binary, not `docker run -v "$PWD:/repo"`.
        #
        # The container form was scanning NOTHING and passing. `-v "$PWD:/repo"` is
        # interpreted by the dind sidecar's daemon, which cannot see the job container's
        # checkout, so gitleaks got an empty mount and logged:
        #
        #     ERR [git] fatal: not a git repository (or any parent up to mount point /)
        #     ERR failed to scan Git repository error="stderr is not empty"
        #     INF scan completed in 35.2ms
        #     INF no leaks found          <- exit 0
        #
        # It reported success without looking, which is worse than having no scanner: the
        # gate was green and meaningless. The 35ms runtime was the tell — a real history
        # scan of this repo takes seconds.
        #
        # The `--log-opts` + commit-count assertion below is the guard against that class
        # of failure returning. A scanner that cannot fail is not a gate.
        run: |
          set -euo pipefail
          curl -fsSL -o /tmp/gitleaks.tgz \
            "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_arm64.tar.gz"
          echo "${GITLEAKS_SHA256}  /tmp/gitleaks.tgz" | sha256sum -c -
          tar -xzf /tmp/gitleaks.tgz -C /tmp gitleaks
          install -m 0755 /tmp/gitleaks /usr/local/bin/gitleaks

          # Prove there is a repository to scan before trusting the verdict.
          commits=$(git rev-list --count HEAD)
          echo "scanning $commits commits"
          test "$commits" -gt 0

          gitleaks detect --no-banner --source . --redact --exit-code 1

      - name: pip-audit (dependency CVEs)
        # --strict fails on an audit error rather than shrugging and reporting clean.
        #
        # Audits the LOCKED dependency set, not the installed environment. Auditing the env
        # means auditing `svcforge` and `svcforge-core` too, which are ours, are installed
        # editable, and are not on PyPI — under --strict that is a hard error ("distribution
        # marked as editable"), so the choice was to drop --strict or to stop asking PyPI
        # about packages it has never heard of. This asks about the 56 that actually came
        # from PyPI, and keeps --strict.
        run: |
          uv export --frozen --no-dev \
            --no-emit-project --no-emit-package svcforge-core \
            -o /tmp/requirements-audit.txt
          uv run --with pip-audit pip-audit --strict -r /tmp/requirements-audit.txt

  # --- stage 9: hadolint ---------------------------------------------------------------
  dockerfile:
    runs-on: ubuntu-latest
    needs: [lint]
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - name: hadolint
        uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
        with:
          recursive: true
          dockerfile: "services/*/Dockerfile"
          failure-threshold: warning

  # --- stage 9b: the chart must render --------------------------------------------------
  # Without this, a chart that does not template reaches ArgoCD and fails in the cluster,
  # where the error surfaces as a sync failure with no PR attached to it. `helm template`
  # is the real gate: it is what ArgoCD does, and _helpers.tpl's image helper calls `fail`
  # on anything that is not a full sha256 digest.
  chart:
    runs-on: ubuntu-latest
    needs: [lint]
    env:
      # The same helm version the worker and reconciler images carry, so CI renders with
      # the version that ships.
      HELM_VERSION: "3.21.3"
      # sha256 of helm-v3.21.3-linux-arm64.tar.gz, from https://get.helm.sh/*.sha256sum.
      # Pinned for the same reason every image here is pinned by digest: a tarball fetched
      # over HTTPS is still a tarball whoever controls the bucket chose to serve.
      HELM_SHA256: "67f58155079ff9ffab98ba5c88daff0ed9b542f3a4732f5dd426dde7dd0f5244"
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: install helm
        # A downloaded binary rather than `docker run alpine/helm`. The container form
        # cannot see the checkout: `docker run -v "$PWD:/repo"` is interpreted by the dind
        # SIDECAR's daemon, and the chart path resolved to nothing inside it —
        # "stat deploy/chart/Chart.yaml: no such file or directory" while the file plainly
        # exists in the job. A binary on PATH has no such boundary to cross.
        run: |
          set -euo pipefail
          curl -fsSL -o /tmp/helm.tgz \
            "https://get.helm.sh/helm-v${HELM_VERSION}-linux-arm64.tar.gz"
          echo "${HELM_SHA256}  /tmp/helm.tgz" | sha256sum -c -
          tar -xzf /tmp/helm.tgz -C /tmp
          install -m 0755 /tmp/linux-arm64/helm /usr/local/bin/helm
          helm version --short

      - name: helm lint
        run: |
          helm lint deploy/chart

      - name: helm template (the digest guard rejects bad digests)
        # Feed the guard bad digests explicitly with --set. The earlier version of this
        # step ran a bare `helm template` and asserted it FAILED, on the assumption that
        # values.yaml always holds all-zeros placeholders. That assumption dies the first
        # time the `bump` job runs: bump commits real digests into values.yaml, so the bare
        # render then succeeds and the assertion reports "the guard is not guarding" about
        # a guard that is fine. A test whose expected result flips depending on whether CI
        # has run before is not a test.
        #
        # Each case below is a distinct way to get a digest wrong, and each must be
        # rejected on its own.
        run: |
          set -euo pipefail
          ZEROS="sha256:$(printf '0%.0s' $(seq 64))"
          for bad_desc in \
            "all-zeros placeholder|${ZEROS}" \
            "not a digest at all|latest" \
            "right prefix, wrong length|sha256:abc123" \
            "empty|"; do
            desc="${bad_desc%%|*}"; bad="${bad_desc#*|}"
            if helm template svcforge deploy/chart \
                 --set image.api.digest="${bad}" >/dev/null 2>&1; then
              echo "FAIL: the chart rendered with a ${desc} digest (${bad@Q})." >&2
              echo "The digest guard in _helpers.tpl is not guarding." >&2
              exit 1
            fi
            echo "ok: rejected ${desc}"
          done

      - name: helm template (renders with real digests)
        # Dummy but well-formed digests: this checks the templates themselves render, with
        # the two values-gated monitoring blocks explicitly on so they are covered too.
        run: |
          set -euo pipefail
          A="sha256:$(printf 'a%.0s' $(seq 64))"
          B="sha256:$(printf 'b%.0s' $(seq 64))"
          C="sha256:$(printf 'c%.0s' $(seq 64))"
          helm template svcforge deploy/chart \
              --set image.api.digest="$A" \
              --set image.worker.digest="$B" \
              --set image.reconciler.digest="$C" \
              --set serviceMonitor.enabled=true \
              --set prometheusRule.enabled=true \
            >/dev/null
          echo "ok: chart renders"

  # --- stage 10: build -> trivy -> push by digest --------------------------------------
  image:
    runs-on: ubuntu-latest
    # Every gate above is required. An image is not built until all of them are green,
    # which is what makes "the digest CI pushed is a digest that passed everything" true.
    needs: [types, unit, integration, security, dockerfile, chart]
    permissions:
      contents: read
    strategy:
      fail-fast: false
      # One image at a time. The runner has capacity 2, so without this two buildx builds
      # run concurrently on a 2-core burstable node that already sits near 80% CPU. Both
      # then died with:
      #
      #   ERROR: failed to solve: DeadlineExceeded: no active session for <id>:
      #   context deadline exceeded
      #
      # That is the buildkit session heartbeat between buildx in the job container and
      # buildkitd in the dind sidecar missing its deadline under CPU starvation — not a
      # Dockerfile problem. The third leg, which had the node to itself, kept building.
      #
      # Serial costs wall-clock and buys builds that finish. Raise it when the runner has
      # cores to spare, not before.
      max-parallel: 1
      matrix:
        svc: [api, worker, reconciler]
    # Deliberately no `outputs:` here. Matrix legs share one outputs map and clobber each
    # other — the merge is not per-key and not ordered, so two of the three digests would
    # arrive empty or stale, intermittently. The bump job resolves the digests from the
    # registry instead, which is a read, not a rebuild.
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: registry login
        # Not gated to master any more: the build step now reads AND writes the layer cache
        # in the registry, so every run needs credentials. Pushing the release image is
        # still master-only — that gate lives on the `push by digest` step, where it belongs.
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_TOKEN }}

      - name: build
        # Context is the repo root and the Dockerfile is addressed with -f. It cannot be
        # otherwise: the image needs pyproject.toml, uv.lock and libs/, all of which live
        # above services/<svc>/, and COPY ../.. is illegal.
        #
        # Loaded locally, not pushed. Trivy scans this exact image next; only then does it
        # get pushed. The alternative — push, scan, and hope nobody pulled meanwhile — is
        # how a CRITICAL ends up in the registry with a green checkmark next to it.
        #
        # The layer cache is `type=registry`, NOT `type=gha`. Two reasons, both hard:
        #
        # 1. act_runner's cache server is backed by a 1Gi PVC that also holds `.runner`,
        #    the runner's own registration file. `mode=max` stores every intermediate layer
        #    of three images — several GB. Filling that volume does not merely lose the
        #    cache: the runner cannot write its state and has to be re-registered by hand.
        #    Trading "slow CI" for "broken CI" is not a trade.
        # 2. act_runner evicts by AGE, with no size cap in its config. It will fill whatever
        #    it is given and then wedge. The registry has no such limit and already holds
        #    the images anyway.
        #
        # The uv/pip cache still uses the runner's cache service — that one is a few hundred
        # MB and fits.
        run: |
          docker buildx build \
            -f services/${{ matrix.svc }}/Dockerfile \
            --build-arg BUILD_SHA=${{ github.sha }} \
            --cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \
            --cache-to type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache,mode=max \
            --load \
            -t svcforge/${{ matrix.svc }}:ci \
            .

      - name: trivy
        # Run trivy directly rather than via aquasecurity/trivy-action, for the same reason
        # gitleaks is run directly above: the command is the documented one, pinned by
        # digest, with no nested action resolution.
        #
        # It is also the only thing that works here. trivy-action internally does
        # `uses: aquasecurity/setup-trivy@v0.2.2`, and that tag no longer exists upstream —
        # the earliest published tag today is v0.2.6. The runner clones it and fails with
        # "Unable to resolve v0.2.2: reference not found". A third-party action pinned by
        # SHA still resolves ITS OWN dependencies by mutable tag, so pinning the outer
        # action bought nothing.
        #
        # --ignore-unfixed: a CVE with no fix available is not something this PR can act
        # on, and failing on it only teaches people to add ignore entries. Rebuilding on a
        # new base image picks the fix up the day it exists.
        #
        # The mounted cache dir keeps the vuln DB (~50MB) across the three matrix legs on
        # this runner instead of re-downloading it for each.
        run: |
          docker run --rm \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v "$PWD/.trivycache:/root/.cache/trivy" \
            aquasec/trivy:0.72.0@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f \
            image \
              --severity HIGH,CRITICAL \
              --ignore-unfixed \
              --exit-code 1 \
              --format table \
              --no-progress \
              svcforge/${{ matrix.svc }}:ci

      - name: push by digest
        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
        # Re-running buildx here is a cache hit on every layer, not a second build: the
        # image is byte-identical to the one trivy just cleared. buildx cannot --load and
        # --push in one invocation, which is the only reason this step exists.
        #
        # The commit-SHA tag is a handle for the bump job to resolve, not something anything
        # deploys. What deploys is the digest that tag resolves to.
        run: |
          set -euo pipefail
          IMAGE="${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}"
          docker buildx build \
            -f services/${{ matrix.svc }}/Dockerfile \
            --build-arg BUILD_SHA=${{ github.sha }} \
            --cache-from type=registry,ref=${REGISTRY}/${IMAGE_NS}/svcforge-${{ matrix.svc }}:buildcache \
            --push \
            -t "${IMAGE}:${GITHUB_SHA}" \
            .
          docker buildx imagetools inspect "${IMAGE}:${GITHUB_SHA}" \
            --format '{{.Manifest.Digest}}'

  # --- stage 11: bump the chart's digests. CI's last act. ------------------------------
  bump:
    runs-on: ubuntu-latest
    needs: [image]
    # The master+push guard lives on the STEPS, not on the job.
    #
    # A job-level `if` here evaluated false and skipped `bump` at 0s — before `image` had
    # even started — on a genuine push to master (`event: push`, `head_branch: master`).
    # The identical expression on the `push by digest` STEP inside the matrix job evaluates
    # true and runs. Gitea appears not to resolve a job-level `if` correctly when `needs`
    # points at a matrix job.
    #
    # `needs: [image]` still does the ordering and still gates on all three legs passing.
    # The steps below carry the branch guard, in the form this runner is known to evaluate
    # correctly. On a PR the job starts and every step no-ops, which costs a few seconds
    # and is the price of a guard that actually fires.
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
        with:
          # A bot token with contents:write on this repo and nothing else: no kubeconfig,
          # no cluster credential, no ArgoCD API token. CI's maximum blast radius is a bad
          # commit, which is revertable.
          token: ${{ secrets.CI_BOT_TOKEN }}
          ref: master
      - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_TOKEN }}
      - name: bump image digests in the chart
        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
        env:
          REGISTRY: ${{ env.REGISTRY }}
          IMAGE_NS: ${{ env.IMAGE_NS }}
          IMAGE_TAG: ${{ github.sha }}
        # And then it stops. No kubectl, no helm upgrade, no argocd app sync. ArgoCD is
        # watching master and will have this within a minute.
        run: ./scripts/bump-digests.sh