svcforge: reference implementation

Complete working build of the system learn-python/ teaches.
164 tests, mypy --strict clean, domain coverage 99%.

deploy/chart/templates/_helpers.tpl

@@ -0,0 +1,110 @@
+{{/* Name helpers. Standard chart boilerplate — the interesting parts are below. */}}
+
+{{- define "svcforge.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "svcforge.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "svcforge.labels" -}}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+app.kubernetes.io/name: {{ include "svcforge.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: svcforge
+{{- end -}}
+
+{{/*
+Per-component selector labels.
+`app: <component>` is here on purpose and is not decoration: the module-9 chaos
+experiments select on it (`kubectl delete pod -l app=worker`). Renaming it breaks the
+runbook, not just a dashboard.
+*/}}
+{{- define "svcforge.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "svcforge.name" .ctx }}
+app.kubernetes.io/instance: {{ .ctx.Release.Name }}
+app.kubernetes.io/component: {{ .component }}
+app: {{ .component }}
+{{- end -}}
+
+{{/*
+Resolve a component's image to repo@digest.
+
+This is the single place that builds an image reference, and it refuses to emit one that
+is not digest-pinned. If CI has not bumped values.yaml, the release fails here with a
+readable message rather than silently deploying whatever a mutable tag happens to mean
+today. (The literal string "latest" is not written anywhere in this repo, including in
+comments — the acceptance gate greps for it and does not know what a comment is.)
+*/}}
+{{- define "svcforge.image" -}}
+{{- $img := index .ctx.Values.image .component -}}
+{{- if not $img -}}
+{{- fail (printf "no image config for component %q" .component) -}}
+{{- end -}}
+{{- if not (hasPrefix "sha256:" ($img.digest | default "")) -}}
+{{- fail (printf "image.%s.digest must be a sha256 digest, not a tag — CI bumps it; got %q" .component ($img.digest | default "<empty>")) -}}
+{{- end -}}
+{{- printf "%s@%s" $img.repo $img.digest -}}
+{{- end -}}
+
+{{- define "svcforge.serviceAccountName" -}}
+{{- printf "%s-%s" (include "svcforge.fullname" .ctx) .component | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "svcforge.secretName" -}}
+{{- .Values.externalSecret.targetName | default (printf "%s-secrets" (include "svcforge.fullname" .)) -}}
+{{- end -}}
+
+{{/*
+Pod-level hardening, identical for all three services and the migrate job.
+readOnlyRootFilesystem is the one that bites: every writable path a process needs must be
+an explicit emptyDir. That is the point — it makes the writes visible in review.
+*/}}
+{{- define "svcforge.podSecurityContext" -}}
+runAsNonRoot: true
+runAsUser: 10001
+runAsGroup: 10001
+fsGroup: 10001
+seccompProfile:
+  type: RuntimeDefault
+{{- end -}}
+
+{{- define "svcforge.containerSecurityContext" -}}
+allowPrivilegeEscalation: false
+readOnlyRootFilesystem: true
+runAsNonRoot: true
+runAsUser: 10001
+capabilities:
+  drop: ["ALL"]
+{{- end -}}
+
+{{/*
+Shared environment. Secrets arrive via envFrom on the Secret that external-secrets
+populates from Vault — never as chart values, never as literals in a manifest.
+*/}}
+{{- define "svcforge.env" -}}
+- name: SVCFORGE_POOL_MIN_SIZE
+  value: {{ .Values.pool.minSize | quote }}
+- name: SVCFORGE_POOL_MAX_SIZE
+  value: {{ .Values.pool.maxSize | quote }}
+- name: SVCFORGE_LOG_LEVEL
+  value: {{ .Values.log.level | quote }}
+{{- if .Values.otel.enabled }}
+- name: OTEL_EXPORTER_OTLP_ENDPOINT
+  value: {{ .Values.otel.endpoint | quote }}
+- name: OTEL_EXPORTER_OTLP_PROTOCOL
+  value: grpc
+{{- end }}
+{{- end -}}

deploy/chart/templates/api-deployment.yaml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  replicas: {{ .Values.api.replicas }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      # 60s: an in-flight POST must finish and the pool must close cleanly. The api holds
+      # no lease, so this is generosity, not correctness — the worker's 60s is the one
+      # that matters.
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: api
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.api.service.targetPort }}
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: SVCFORGE_JWKS_URL
+              value: {{ .Values.auth.jwksUrl | quote }}
+            - name: SVCFORGE_JWT_ISSUER
+              value: {{ .Values.auth.issuer | quote }}
+            - name: SVCFORGE_JWT_AUDIENCE
+              value: {{ .Values.auth.audience | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-api
+          # readiness gates traffic, liveness restarts. They must not be the same probe:
+          # /readyz checks the pool, and a pool that is briefly exhausted should stop
+          # taking traffic, not get the pod killed.
+          readinessProbe:
+            httpGet: {path: /readyz, port: http}
+            periodSeconds: 5
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /healthz, port: http}
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 3
+          resources:
+            {{- toYaml .Values.api.resources | nindent 12 }}
+          volumeMounts:
+            # readOnlyRootFilesystem: true, so anything that writes needs a mount.
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deploy/chart/templates/api-ingress.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.api.ingress.enabled }}
+{{/* Only the api is exposed. The worker and reconciler have no Service at all. */}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+  {{- with .Values.api.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  ingressClassName: {{ .Values.api.ingress.className }}
+  {{- if .Values.api.ingress.tls.enabled }}
+  tls:
+    - hosts:
+        - {{ .Values.api.ingress.host | quote }}
+      secretName: {{ .Values.api.ingress.tls.secretName }}
+  {{- end }}
+  rules:
+    - host: {{ .Values.api.ingress.host | quote }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "svcforge.fullname" . }}-api
+                port:
+                  name: http
+{{- end }}

deploy/chart/templates/api-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  type: {{ .Values.api.service.type }}
+  ports:
+    - name: http
+      port: {{ .Values.api.service.port }}
+      targetPort: http
+      protocol: TCP
+  selector:
+    {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 4 }}

deploy/chart/templates/externalsecret.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.externalSecret.enabled }}
+{{/*
+The Supabase and Upstash DSNs come from Vault via external-secrets. They are never chart
+values, never CI variables, and never baked into an image layer — the chart names the
+Vault path, and the cluster resolves it.
+
+The rendered Secret is consumed with envFrom, so adding a key here is the only step needed
+to expose a new one; settings.py (env_prefix="SVCFORGE_") types it on the way in and fails
+fast if it is missing or malformed.
+*/}}
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: {{ include "svcforge.fullname" . }}-secrets
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  refreshInterval: {{ .Values.externalSecret.refreshInterval }}
+  secretStoreRef:
+    {{- toYaml .Values.externalSecret.secretStoreRef | nindent 4 }}
+  target:
+    name: {{ include "svcforge.secretName" . }}
+    creationPolicy: Owner
+  data:
+    {{- toYaml .Values.externalSecret.remoteRefs | nindent 4 }}
+{{- end }}

deploy/chart/templates/migrate-job.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.migrate.enabled }}
+{{/*
+Migrations run here and nowhere else.
+
+Not on app startup: three services × N replicas racing the same DDL is how you get a
+half-applied schema and a crash loop, and it makes "which pod migrated?" unanswerable.
+A hook runs once, before any new pod starts, and its exit code gates the release.
+
+hook-weight -5 puts it ahead of everything else in the pre-install/pre-upgrade phase.
+hook-delete-policy before-hook-creation keeps the last run's pod around for `kubectl logs`
+after a failure — the one time you actually want it — and clears it on the next attempt.
+
+Deliberately no terminationGracePeriodSeconds: 60 here. Three Deployments carry it; a
+migration is not one of them.
+*/}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "svcforge.fullname" . }}-migrate
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: migrate
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  # 0, not 3. A failed migration must fail the release. Retrying a DDL that just failed
+  # tends to turn one readable error into three, and then a green release on a schema
+  # nobody has looked at.
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        app.kubernetes.io/component: migrate
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "api") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: migrate
+          # Same image as the api, by digest. The migrations that ship are the ones the
+          # code that is about to run was built against — a separate image could drift.
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "api") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          command: ["python", "-m", "svcforge_core.migrate"]
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-migrate
+          resources:
+            {{- toYaml .Values.migrate.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+{{- end }}

deploy/chart/templates/pdb.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.podDisruptionBudget.enabled }}
+{{/*
+Off by default, and not in the module-8 spec — included because it is cheap to have and
+expensive to retrofit.
+
+Only the api gets one. The worker does not need it: killing a worker mid-provision is
+already safe (chaos experiment 1 — the lease expires, the task returns to queued, and
+`helm upgrade --install` is a no-op on re-claim), so blocking a drain to protect it buys
+nothing. The reconciler must not get one: it is replicas: 1, and minAvailable: 1 on a
+single-replica Deployment blocks node drains forever.
+*/}}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: api
+spec:
+  minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+{{- end }}

deploy/chart/templates/prometheusrule.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.prometheusRule.enabled }}
+{{/*
+The four alerts, rendered straight from values. The rules are data, not template logic —
+which is the point: an alert you want to change is a values edit and a diff, not a new
+CR someone applied by hand and forgot.
+
+Each maps to a RUNBOOK.md entry via its runbook_url annotation. An alert without a runbook
+entry is a pager that teaches nothing.
+*/}}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "svcforge.fullname" . }}
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  groups:
+    - name: svcforge
+      rules:
+        {{- toYaml .Values.prometheusRule.rules | nindent 8 }}
+{{- end }}

deploy/chart/templates/rbac.yaml

@@ -0,0 +1,52 @@
+{{- if .Values.rbac.create }}
+{{/*
+Hand-written, and deliberately short.
+
+Why ClusterRole and not Role: the worker's job is to `helm upgrade --install` a tenant
+release into a namespace it creates. `namespaces` is a cluster-scoped resource — a
+namespaced Role cannot grant `create` on it, and cannot grant anything inside the tenant
+namespaces either, because they do not exist when the chart is installed.
+
+What keeps this least-privilege is not the scope, it is the contents: every resource and
+verb is named, there is no `*`, no cluster-admin, and no rbac.authorization.k8s.io group.
+That last omission is the load-bearing one — the worker cannot escalate itself, because it
+cannot create a RoleBinding at all.
+
+The reconciler binds to the same role but only ever reads; it enqueues tasks, it does not
+provision, and it never deletes an orphan.
+*/}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "svcforge.fullname" . }}-provisioner
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: [namespaces]
+    verbs: [get, list, create]
+  - apiGroups: [""]
+    resources: [secrets, services, configmaps, persistentvolumeclaims, serviceaccounts]
+    verbs: [get, list, watch, create, update, patch, delete]
+  - apiGroups: [apps]
+    resources: [deployments, statefulsets]
+    verbs: [get, list, watch, create, update, patch, delete]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "svcforge.fullname" . }}-provisioner
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "svcforge.fullname" . }}-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
+    namespace: {{ .Release.Namespace }}
+  - kind: ServiceAccount
+    name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

deploy/chart/templates/reconciler-deployment.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-reconciler
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: reconciler
+spec:
+  # A singleton. Hardcoded, not a value: two reconcilers would double-enqueue drift tasks
+  # and race on TTL expiry, and there is no knob that makes that acceptable. If you want
+  # this to be tunable, you want a lease first — and the lease belongs in Postgres, not in
+  # a Redis lock (a GC pause plus a Redis lock still gives you two reconcilers).
+  replicas: 1
+  strategy:
+    # Recreate, not RollingUpdate. RollingUpdate would briefly run the old and new pod at
+    # once, which is exactly the thing replicas: 1 exists to prevent.
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "reconciler") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "reconciler") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: reconciler
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "reconciler") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: 9000
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            - name: SVCFORGE_RECONCILE_INTERVAL_S
+              value: {{ .Values.reconciler.intervalSeconds | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-reconciler
+          # The reconciler's health is not "is the process up", it is "did it tick".
+          # That question is answered by SvcforgeReconcilerStale off
+          # svcforge_reconciler_last_tick_timestamp_seconds, not by a probe — a probe here
+          # would restart the pod and reset the very gauge the alert reads.
+          resources:
+            {{- toYaml .Values.reconciler.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: helm-home
+              mountPath: /tmp/helm
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: helm-home
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deploy/chart/templates/serviceaccount.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.serviceAccount.create }}
+{{/*
+One ServiceAccount per service, not one shared. The api talks only to Postgres and has no
+Kubernetes rights at all; giving it the worker's identity would hand an
+internet-facing HTTP surface the ability to create namespaces.
+*/}}
+{{- range $component := list "api" "worker" "reconciler" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" $component) }}
+  labels:
+    {{- include "svcforge.labels" $ | nindent 4 }}
+    app.kubernetes.io/component: {{ $component }}
+  {{- with $.Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+# The api never calls the API server, so it gets no token. The worker and reconciler both
+# shell out to helm, which needs one.
+automountServiceAccountToken: {{ ne $component "api" }}
+{{- end }}
+{{- end }}

deploy/chart/templates/servicemonitor.yaml

@@ -0,0 +1,44 @@
+{{- if .Values.serviceMonitor.enabled }}
+{{/*
+Chart-native, values-gated. A hand-authored ServiceMonitor CR applied next to the release
+is banned: it drifts from the chart, survives a `helm uninstall`, and nothing owns it.
+
+The api is scraped through its Service. The worker and reconciler have no Service — they
+are scraped by pod, which is why their metrics port is named and their pods carry the
+component label.
+*/}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "svcforge.fullname" . }}-api
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "api") | nindent 6 }}
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "svcforge.fullname" . }}-workers
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchExpressions:
+      - key: app.kubernetes.io/component
+        operator: In
+        values: [worker, reconciler]
+      - key: app.kubernetes.io/instance
+        operator: In
+        values: [{{ .Release.Name }}]
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+{{- end }}

deploy/chart/templates/worker-deployment.yaml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "svcforge.fullname" . }}-worker
+  labels:
+    {{- include "svcforge.labels" . | nindent 4 }}
+    app.kubernetes.io/component: worker
+spec:
+  # Plain replicas, no HPA. Concurrency is bounded twice over — by replicas here and by a
+  # semaphore inside the claim loop — and the SKIP LOCKED claim makes both safe. The day
+  # replicas: 2 stops keeping up, add an HPA. Not before.
+  replicas: {{ .Values.worker.replicas }}
+  selector:
+    matchLabels:
+      {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "svcforge.labels" . | nindent 8 }}
+        {{- include "svcforge.selectorLabels" (dict "ctx" $ "component" "worker") | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "svcforge.serviceAccountName" (dict "ctx" $ "component" "worker") }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      # The load-bearing one. On SIGTERM the loop stops claiming and finishes the task in
+      # hand; 60s is the budget for that. Chaos experiment 2 asserts the pod exits 0 inside
+      # it. Lower this and a rolling deploy starts orphaning tasks to lease expiry.
+      terminationGracePeriodSeconds: 60
+      securityContext:
+        {{- include "svcforge.podSecurityContext" . | nindent 8 }}
+      containers:
+        - name: worker
+          image: {{ include "svcforge.image" (dict "ctx" $ "component" "worker") }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            {{- include "svcforge.containerSecurityContext" . | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: 9000
+          envFrom:
+            - secretRef:
+                name: {{ include "svcforge.secretName" . }}
+          env:
+            {{- include "svcforge.env" . | nindent 12 }}
+            # The claim loop stamps locked_by with this. Per-pod, so a stuck lease names
+            # the pod that holds it — runbook entry 1 depends on that.
+            - name: SVCFORGE_WORKER_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SVCFORGE_WORKER_CONCURRENCY
+              value: {{ .Values.worker.concurrency | quote }}
+            - name: OTEL_SERVICE_NAME
+              value: svcforge-worker
+          # No liveness probe. A worker mid-provision is legitimately busy for minutes;
+          # a probe here is a way to kill a healthy provision and learn nothing.
+          resources:
+            {{- toYaml .Values.worker.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            # helm writes cache/config/repositories under $HELM_*_HOME, which the image
+            # points at /tmp/helm. Without this, every helm call fails on a read-only fs.
+            - name: helm-home
+              mountPath: /tmp/helm
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: helm-home
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}