svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

deploy/chart/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: svcforge
+description: svcforge control plane — api, worker, reconciler
+type: application
+# version is the chart's own version. appVersion is a label only: what actually ships is
+# the digest in values.yaml, which CI bumps. Never read appVersion to decide what runs.
+version: 0.1.0
+appVersion: "0.1.0"

deploy/chart/templates/_helpers.tpl

@@ -0,0 +1,110 @@
+{{/* Name helpers. Standard chart boilerplate — the interesting parts are below. */}}
+
+{{- define "svcforge.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "svcforge.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "svcforge.labels" -}}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+app.kubernetes.io/name: {{ include "svcforge.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: svcforge
+{{- end -}}
+
+{{/*
+Per-component selector labels.
+`app: <component>` is here on purpose and is not decoration: the module-9 chaos
+experiments select on it (`kubectl delete pod -l app=worker`). Renaming it breaks the
+runbook, not just a dashboard.
+*/}}
+{{- define "svcforge.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "svcforge.name" .ctx }}
+app.kubernetes.io/instance: {{ .ctx.Release.Name }}
+app.kubernetes.io/component: {{ .component }}
+app: {{ .component }}
+{{- end -}}
+
+{{/*
+Resolve a component's image to repo@digest.
+
+This is the single place that builds an image reference, and it refuses to emit one that
+is not digest-pinned. If CI has not bumped values.yaml, the release fails here with a
+readable message rather than silently deploying whatever a mutable tag happens to mean
+today. (The literal string "latest" is not written anywhere in this repo, including in
+comments — the acceptance gate greps for it and does not know what a comment is.)
+*/}}
+{{- define "svcforge.image" -}}
+{{- $img := index .ctx.Values.image .component -}}
+{{- if not $img -}}
+{{- fail (printf "no image config for component %q" .component) -}}
+{{- end -}}
+{{- if not (hasPrefix "sha256:" ($img.digest | default "")) -}}
+{{- fail (printf "image.%s.digest must be a sha256 digest, not a tag — CI bumps it; got %q" .component ($img.digest | default "<empty>")) -}}
+{{- end -}}
+{{- printf "%s@%s" $img.repo $img.digest -}}
+{{- end -}}
+
+{{- define "svcforge.serviceAccountName" -}}
+{{- printf "%s-%s" (include "svcforge.fullname" .ctx) .component | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "svcforge.secretName" -}}
+{{- .Values.externalSecret.targetName | default (printf "%s-secrets" (include "svcforge.fullname" .)) -}}
+{{- end -}}
+
+{{/*
+Pod-level hardening, identical for all three services and the migrate job.
+readOnlyRootFilesystem is the one that bites: every writable path a process needs must be
+an explicit emptyDir. That is the point — it makes the writes visible in review.
+*/}}
+{{- define "svcforge.podSecurityContext" -}}
+runAsNonRoot: true
+runAsUser: 10001
+runAsGroup: 10001
+fsGroup: 10001
+seccompProfile:
+  type: RuntimeDefault
+{{- end -}}
+
+{{- define "svcforge.containerSecurityContext" -}}
+allowPrivilegeEscalation: false
+readOnlyRootFilesystem: true
+runAsNonRoot: true
+runAsUser: 10001
+capabilities:
+  drop: ["ALL"]
+{{- end -}}
+
+{{/*
+Shared environment. Secrets arrive via envFrom on the Secret that external-secrets
+populates from Vault — never as chart values, never as literals in a manifest.
+*/}}
+{{- define "svcforge.env" -}}
+- name: SVCFORGE_POOL_MIN_SIZE
+  value: {{ .Values.pool.minSize | quote }}
+- name: SVCFORGE_POOL_MAX_SIZE
+  value: {{ .Values.pool.maxSize | quote }}
+- name: SVCFORGE_LOG_LEVEL
+  value: {{ .Values.log.level | quote }}
+{{- if .Values.otel.enabled }}
+- name: OTEL_EXPORTER_OTLP_ENDPOINT
+  value: {{ .Values.otel.endpoint | quote }}
+- name: OTEL_EXPORTER_OTLP_PROTOCOL
+  value: grpc
+{{- end }}
+{{- end -}}

deploy/chart/templates/api-deployment.yaml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  replicas: {{ .Values.api.replicas }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      # 60s: an in-flight POST must finish and the pool must close cleanly. The api holds
+      # no lease, so this is generosity, not correctness — the worker's 60s is the one
+      # that matters.
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: api
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.api.service.targetPort }}
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: SVCFORGE_JWKS_URL
+              value: {{ .Values.auth.jwksUrl | quote }}
+            - name: SVCFORGE_JWT_ISSUER
+              value: {{ .Values.auth.issuer | quote }}
+            - name: SVCFORGE_JWT_AUDIENCE
+              value: {{ .Values.auth.audience | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-api
+          # readiness gates traffic, liveness restarts. They must not be the same probe:
+          # /readyz checks the pool, and a pool that is briefly exhausted should stop
+          # taking traffic, not get the pod killed.
+          readinessProbe:
+            httpGet: {path: /readyz, port: http}
+            periodSeconds: 5
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /healthz, port: http}
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 3
+          resources:
+            {{- toYaml .Values.api.resources | nindent 12 }}
+          volumeMounts:
+            # readOnlyRootFilesystem: true, so anything that writes needs a mount.
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deploy/chart/templates/api-ingress.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.api.ingress.enabled }}
+{{/* Only the api is exposed. The worker and reconciler have no Service at all. */}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+  {{- with .Values.api.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  ingressClassName: {{ .Values.api.ingress.className }}
+  {{- if .Values.api.ingress.tls.enabled }}
+  tls:
+    - hosts:
+        - {{ .Values.api.ingress.host | quote }}
+      secretName: {{ .Values.api.ingress.tls.secretName }}
+  {{- end }}
+  rules:
+    - host: {{ .Values.api.ingress.host | quote }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "svcforge.fullname" . }}-api
+                port:
+                  name: http
+{{- end }}

deploy/chart/templates/api-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  type: {{ .Values.api.service.type }}
+  ports:
+    - name: http
+      port: {{ .Values.api.service.port }}
+      targetPort: http
+      protocol: TCP
+  selector:
+    {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 4 }}

deploy/chart/templates/externalsecret.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.externalSecret.enabled }}
+{{/*
+The Supabase and Upstash DSNs come from Vault via external-secrets. They are never chart
+values, never CI variables, and never baked into an image layer — the chart names the
+Vault path, and the cluster resolves it.
+
+The rendered Secret is consumed with envFrom, so adding a key here is the only step needed
+to expose a new one; settings.py (env_prefix="SVCFORGE_") types it on the way in and fails
+fast if it is missing or malformed.
+*/}}
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: {{ include "svcforge.fullname" . }}-secrets
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  refreshInterval: {{ .Values.externalSecret.refreshInterval }}
+  secretStoreRef:
+    {{- toYaml .Values.externalSecret.secretStoreRef | nindent 4 }}
+  target:
+    name: {{ include "svcforge.secretName" . }}
+    creationPolicy: Owner
+  data:
+    {{- toYaml .Values.externalSecret.remoteRefs | nindent 4 }}
+{{- end }}

deploy/chart/templates/migrate-job.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.migrate.enabled }}
+{{/*
+Migrations run here and nowhere else.
+
+Not on app startup: three services × N replicas racing the same DDL is how you get a
+half-applied schema and a crash loop, and it makes "which pod migrated?" unanswerable.
+A hook runs once, before any new pod starts, and its exit code gates the release.
+
+hook-weight -5 puts it ahead of everything else in the pre-install/pre-upgrade phase.
+hook-delete-policy before-hook-creation keeps the last run's pod around for `kubectl logs`
+after a failure — the one time you actually want it — and clears it on the next attempt.
+
+Deliberately no terminationGracePeriodSeconds: 60 here. Three Deployments carry it; a
+migration is not one of them.
+*/}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "svcforge.fullname" . }}-migrate
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: migrate
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  # 0, not 3. A failed migration must fail the release. Retrying a DDL that just failed
+  # tends to turn one readable error into three, and then a green release on a schema
+  # nobody has looked at.
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        app.kubernetes.io/component: migrate
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: migrate
+          # Same image as the api, by digest. The migrations that ship are the ones the
+          # code that is about to run was built against — a separate image could drift.
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          command: ["python", "-m", "svcforge_core.migrate"]
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-migrate
+          resources:
+            {{- toYaml .Values.migrate.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+{{- end }}

deploy/chart/templates/pdb.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.podDisruptionBudget.enabled }}
+{{/*
+Off by default, and not in the module-8 spec — included because it is cheap to have and
+expensive to retrofit.
+
+Only the api gets one. The worker does not need it: killing a worker mid-provision is
+already safe (chaos experiment 1 — the lease expires, the task returns to queued, and
+`helm upgrade --install` is a no-op on re-claim), so blocking a drain to protect it buys
+nothing. The reconciler must not get one: it is replicas: 1, and minAvailable: 1 on a
+single-replica Deployment blocks node drains forever.
+*/}}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+{{- end }}

deploy/chart/templates/prometheusrule.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.prometheusRule.enabled }}
+{{/*
+The four alerts, rendered straight from values. The rules are data, not template logic —
+which is the point: an alert you want to change is a values edit and a diff, not a new
+CR someone applied by hand and forgot.
+
+Each maps to a RUNBOOK.md entry via its runbook_url annotation. An alert without a runbook
+entry is a pager that teaches nothing.
+*/}}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "svcforge.fullname" . }}
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  groups:
+    - name: svcforge
+      rules:
+        {{- toYaml .Values.prometheusRule.rules | nindent 8 }}
+{{- end }}

deploy/chart/templates/rbac.yaml

@@ -0,0 +1,52 @@
+{{- if .Values.rbac.create }}
+{{/*
+Hand-written, and deliberately short.
+
+Why ClusterRole and not Role: the worker's job is to `helm upgrade --install` a tenant
+release into a namespace it creates. `namespaces` is a cluster-scoped resource — a
+namespaced Role cannot grant `create` on it, and cannot grant anything inside the tenant
+namespaces either, because they do not exist when the chart is installed.
+
+What keeps this least-privilege is not the scope, it is the contents: every resource and
+verb is named, there is no `*`, no cluster-admin, and no rbac.authorization.k8s.io group.
+That last omission is the load-bearing one — the worker cannot escalate itself, because it
+cannot create a RoleBinding at all.
+
+The reconciler binds to the same role but only ever reads; it enqueues tasks, it does not
+provision, and it never deletes an orphan.
+*/}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "svcforge.fullname" . }}-provisioner
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: [namespaces]
+    verbs: [get, list, create]
+  - apiGroups: [""]
+    resources: [secrets, services, configmaps, persistentvolumeclaims, serviceaccounts]
+    verbs: [get, list, watch, create, update, patch, delete]
+  - apiGroups: [apps]
+    resources: [deployments, statefulsets]
+    verbs: [get, list, watch, create, update, patch, delete]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "svcforge.fullname" . }}-provisioner
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "svcforge.fullname" . }}-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
+    namespace: {{ .Release.Namespace }}
+  - kind: ServiceAccount
+    name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

deploy/chart/templates/reconciler-deployment.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-reconciler
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: reconciler
+spec:
+  # A singleton. Hardcoded, not a value: two reconcilers would double-enqueue drift tasks
+  # and race on TTL expiry, and there is no knob that makes that acceptable. If you want
+  # this to be tunable, you want a lease first — and the lease belongs in Postgres, not in
+  # a Redis lock (a GC pause plus a Redis lock still gives you two reconcilers).
+  replicas: 1
+  strategy:
+    # Recreate, not RollingUpdate. RollingUpdate would briefly run the old and new pod at
+    # once, which is exactly the thing replicas: 1 exists to prevent.
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: reconciler
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "reconciler") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: 9000
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: SVCFORGE_RECONCILE_INTERVAL_S
+              value: {{ .Values.reconciler.intervalSeconds | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-reconciler
+          # The reconciler's health is not "is the process up", it is "did it tick".
+          # That question is answered by SvcforgeReconcilerStale off
+          # svcforge_reconciler_last_tick_timestamp_seconds, not by a probe — a probe here
+          # would restart the pod and reset the very gauge the alert reads.
+          resources:
+            {{- toYaml .Values.reconciler.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: helm-home
+              mountPath: /tmp/helm
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: helm-home
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deploy/chart/templates/serviceaccount.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.serviceAccount.create }}
+{{/*
+One ServiceAccount per service, not one shared. The api talks only to Postgres and has no
+Kubernetes rights at all; giving it the worker's identity would hand an
+internet-facing HTTP surface the ability to create namespaces.
+*/}}
+{{- range $component := list "api" "worker" "reconciler" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" $component) }}
+  labels:
+    {{- include "svcforge.labels" $ | nindent 4 }}
+    app.kubernetes.io/component: {{ $component }}
+  {{- with $.Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+# The api never calls the API server, so it gets no token. The worker and reconciler both
+# shell out to helm, which needs one.
+automountServiceAccountToken: {{ ne $component "api" }}
+{{- end }}
+{{- end }}

deploy/chart/templates/servicemonitor.yaml

@@ -0,0 +1,44 @@
+{{- if .Values.serviceMonitor.enabled }}
+{{/*
+Chart-native, values-gated. A hand-authored ServiceMonitor CR applied next to the release
+is banned: it drifts from the chart, survives a `helm uninstall`, and nothing owns it.
+
+The api is scraped through its Service. The worker and reconciler have no Service — they
+are scraped by pod, which is why their metrics port is named and their pods carry the
+component label.
+*/}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "svcforge.fullname" . }}-workers
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchExpressions:
+      - key: app.kubernetes.io/component
+        operator: In
+        values: [worker, reconciler]
+      - key: app.kubernetes.io/instance
+        operator: In
+        values: [{{ .Release.Name }}]
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+{{- end }}

deploy/chart/templates/worker-deployment.yaml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-worker
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: worker
+spec:
+  # Plain replicas, no HPA. Concurrency is bounded twice over — by replicas here and by a
+  # semaphore inside the claim loop — and the SKIP LOCKED claim makes both safe. The day
+  # replicas: 2 stops keeping up, add an HPA. Not before.
+  replicas: {{ .Values.worker.replicas }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      # The load-bearing one. On SIGTERM the loop stops claiming and finishes the task in
+      # hand; 60s is the budget for that. Chaos experiment 2 asserts the pod exits 0 inside
+      # it. Lower this and a rolling deploy starts orphaning tasks to lease expiry.
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: worker
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "worker") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: 9000
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            # The claim loop stamps locked_by with this. Per-pod, so a stuck lease names
+            # the pod that holds it — runbook entry 1 depends on that.
+            - name: SVCFORGE_WORKER_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SVCFORGE_WORKER_CONCURRENCY
+              value: {{ .Values.worker.concurrency | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-worker
+          # No liveness probe. A worker mid-provision is legitimately busy for minutes;
+          # a probe here is a way to kill a healthy provision and learn nothing.
+          resources:
+            {{- toYaml .Values.worker.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            # helm writes cache/config/repositories under $HELM_*_HOME, which the image
+            # points at /tmp/helm. Without this, every helm call fails on a read-only fs.
+            - name: helm-home
+              mountPath: /tmp/helm
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: helm-home
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deploy/chart/values.yaml

@@ -0,0 +1,174 @@
+# svcforge chart values.
+#
+# The digests below are the deployment. CI builds each image once, pushes it, reads the
+# digest back with `docker buildx imagetools inspect`, and `yq -i`s it into this file as
+# its last act. ArgoCD notices the commit and syncs. Nothing else deploys svcforge.
+#
+# Tags are banned. A tag is a mutable pointer, which means "what is running" and "what
+# this file says" can silently diverge. A digest cannot.
+
+nameOverride: ""
+fullnameOverride: ""
+
+image:
+  registry: gitea.oci-oci.duckdns.org
+  pullPolicy: IfNotPresent
+  pullSecrets: []
+  # One repo + one digest per service: CI's matrix builds three images, so there are three
+  # digests. The zeros are placeholders — a fresh clone must be bumped by CI before it can
+  # deploy, which is the intended failure mode. Never hand-edit these.
+  api:
+    repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-api
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
+  worker:
+    repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-worker
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
+  reconciler:
+    repo: gitea.oci-oci.duckdns.org/gitea_admin/svcforge-reconciler
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
+
+api:
+  replicas: 2
+  # One process per pod. Module 7 took the "scale with replicas" fix over
+  # PROMETHEUS_MULTIPROC_DIR, so `uvicorn --workers N` here would corrupt the metrics.
+  resources:
+    requests: {cpu: 50m, memory: 128Mi}
+    limits: {memory: 256Mi}
+  service:
+    type: ClusterIP
+    port: 80
+    targetPort: 8000
+  ingress:
+    enabled: true
+    className: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+    host: svcforge.oci-oci.duckdns.org
+    tls:
+      enabled: true
+      secretName: svcforge-tls
+
+worker:
+  # Plain replicas. No HPA: the day `replicas: 2` stops keeping up, not before.
+  replicas: 2
+  concurrency: 4
+  resources:
+    requests: {cpu: 100m, memory: 192Mi}
+    limits: {memory: 512Mi}
+
+reconciler:
+  # A singleton, and not by convention — the four checks are not safe to run twice
+  # concurrently. replicas is deliberately not a value: there is nothing to tune.
+  intervalSeconds: 60
+  resources:
+    requests: {cpu: 50m, memory: 128Mi}
+    limits: {memory: 256Mi}
+
+# Postgres pool sizing. replicas × maxSize is spent against the Supabase pooler budget:
+# api(2 × 5) + worker(2 × 5) + reconciler(1 × 2) = 22 connections. Raise with care.
+pool:
+  minSize: 1
+  maxSize: 5
+
+migrate:
+  # backoffLimit: 0 — a failed migration must fail the release, not retry into a
+  # half-applied schema. Migrations run here and only here; never on app startup.
+  enabled: true
+  resources:
+    requests: {cpu: 50m, memory: 128Mi}
+    limits: {memory: 256Mi}
+
+auth:
+  jwksUrl: https://auth.oci-oci.duckdns.org/realms/svcforge/protocol/openid-connect/certs
+  issuer: https://auth.oci-oci.duckdns.org/realms/svcforge
+  audience: svcforge
+
+otel:
+  enabled: true
+  endpoint: http://alloy.observability.svc.cluster.local:4317
+
+log:
+  level: info
+
+# The DSNs are pulled from Vault by external-secrets into a Secret the pods envFrom.
+# No DSN is ever a chart value, a ConfigMap key, or a CI variable.
+externalSecret:
+  enabled: true
+  secretStoreRef:
+    name: vault
+    kind: ClusterSecretStore
+  refreshInterval: 1h
+  # target Secret name; keys land as SVCFORGE_PG_DSN / SVCFORGE_PG_DSN_SESSION / SVCFORGE_REDIS_DSN
+  targetName: svcforge-secrets
+  remoteRefs:
+    - secretKey: SVCFORGE_PG_DSN
+      key: svcforge/postgres
+      property: dsn_pooler
+    - secretKey: SVCFORGE_PG_DSN_SESSION
+      key: svcforge/postgres
+      property: dsn_session
+    - secretKey: SVCFORGE_REDIS_DSN
+      key: svcforge/redis
+      property: dsn
+
+rbac:
+  # The worker helm-installs tenant releases into namespaces it creates. `namespaces` is a
+  # cluster-scoped resource, so `create namespaces` cannot be granted by a namespaced Role
+  # — this has to be a ClusterRole. It is still least-privilege: named resources, named
+  # verbs, no `*`, no cluster-admin, and no rbac.authorization.k8s.io group at all, so the
+  # worker cannot grant itself anything further.
+  create: true
+
+serviceAccount:
+  create: true
+  annotations: {}
+
+# Chart-native only. A hand-authored ServiceMonitor/PrometheusRule CR is banned — the
+# chart owns these, gated by these flags.
+serviceMonitor:
+  enabled: true
+  interval: 30s
+
+prometheusRule:
+  enabled: true
+  rules:
+    - alert: SvcforgeQueueDepthRising
+      expr: deriv(svcforge_queue_depth[10m]) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: svcforge queue depth is rising and not draining
+        runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#queue-stuck
+    - alert: SvcforgeProvisionSlow
+      expr: histogram_quantile(0.95, sum by (le) (rate(svcforge_provision_duration_seconds_bucket[30m]))) > 300
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: svcforge p95 provision time is over 5 minutes
+        runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
+    - alert: SvcforgeTaskFailed
+      expr: sum(svcforge_tasks_failed_total) > 0
+      labels:
+        severity: warning
+      annotations:
+        summary: a svcforge task exhausted its retries
+        runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#provision-failing
+    - alert: SvcforgeReconcilerStale
+      expr: time() - svcforge_reconciler_last_tick_timestamp_seconds > 300
+      labels:
+        severity: critical
+      annotations:
+        summary: the svcforge reconciler has not ticked in 5 minutes
+        runbook_url: https://gitea.oci-oci.duckdns.org/gitea_admin/svcforge/src/branch/master/RUNBOOK.md#orphaned-release
+
+# Not in the spec. Off by default: on a three-node k3s a PDB that cannot be satisfied
+# blocks drains, which is worse than the disruption it prevents.
+podDisruptionBudget:
+  enabled: false
+  minAvailable: 1
+
+nodeSelector: {}
+tolerations: []
+affinity: {}