worker: create namespaces with helm, drop the kubectl binary
ci / lint (push) Successful in 25s
ci / unit (push) Successful in 59s
ci / types (push) Successful in 1m8s
ci / dockerfile (push) Successful in 13s
ci / chart (push) Successful in 8s
ci / security (push) Successful in 1m2s
ci / integration (push) Successful in 54s
ci / image (api) (push) Successful in 1m6s
ci / image (reconciler) (push) Successful in 3m6s
ci / image (worker) (push) Successful in 2m28s
ci / bump (push) Has been cancelled

trivy took the worker image 39 -> 18 -> 5 findings across two version bumps, and
the last 5 (4x golang.org/x/net, 1x Go stdlib) live in kubectl v1.36.2 — the
newest kubectl that exists. No release clears them; upstream has not rebuilt
against the patched Go yet. Chasing the version further has no end.

kubectl was in that image for exactly one call: ensure_namespace before helm.
`helm upgrade --install --create-namespace` does the same thing, idempotently,
as part of the install it already runs. So the binary goes, and its vendored CVEs
go with it. read_secret had no callers.

Tradeoff recorded: the namespace no longer gets an svcforge.io/team label, since
--create-namespace makes a bare one. Nothing reads that label today.
This commit is contained in:
Nguyen Minh Phuc
2026-07-18 13:05:22 +00:00
parent a59d3848ae
commit 4a426dbe50
7 changed files with 9 additions and 60 deletions
+1 -2
View File
@@ -128,8 +128,7 @@ sequenceDiagram
Note over W,P: every 5s, while a semaphore slot is free Note over W,P: every 5s, while a semaphore slot is free
W->>P: UPDATE ... FOR UPDATE SKIP LOCKED W->>P: UPDATE ... FOR UPDATE SKIP LOCKED
P-->>W: task (attempts now 1, locked_by=me) P-->>W: task (attempts now 1, locked_by=me)
W->>K: kubectl apply namespace (idempotent) W->>K: helm upgrade --install --create-namespace --wait
W->>K: helm upgrade --install --wait
K-->>W: release ready K-->>W: release ready
W->>P: CAS provisioning -> ready, set endpoint W->>P: CAS provisioning -> ready, set endpoint
W->>P: complete(task, worker_id) W->>P: complete(task, worker_id)
@@ -199,6 +199,13 @@ class HelmProvisioner:
entry.chart, entry.chart,
"--namespace", "--namespace",
ns, ns,
# `--namespace X` does not create X. Every tenant's first provision targets
# a namespace that does not exist yet, and helm fails with "namespaces not
# found". helm creates it here rather than a separate `kubectl apply` step,
# which keeps kubectl out of the worker image entirely — one fewer binary,
# and one fewer set of vendored Go CVEs to track. Idempotent: existing
# namespaces are left alone.
"--create-namespace",
"--version", "--version",
entry.chart_version, entry.chart_version,
"--values", "--values",
+1 -16
View File
@@ -3,7 +3,7 @@
# svcforge worker. Build from the REPO ROOT: # svcforge worker. Build from the REPO ROOT:
# docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev . # docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
# #
# The only service that shells out to helm/kubectl, so the only one carrying those two # The only service that shells out to helm, so the only one carrying that binary
# binaries. They are copied from pinned images rather than curl'd, so the version is a # binaries. They are copied from pinned images rather than curl'd, so the version is a
# reviewable line in a Dockerfile instead of a network call at build time. # reviewable line in a Dockerfile instead of a network call at build time.
@@ -46,21 +46,6 @@ COPY --from=builder --chown=10001:10001 /app /app
# (spdystream, fixed in 0.5.1) — trivy fails the build on them and is right to. # (spdystream, fixed in 0.5.1) — trivy fails the build on them and is right to.
# Deliberately 3.x: helm 4 is a breaking change and is not a CVE fix. # Deliberately 3.x: helm 4 is a breaking change and is not a CVE fix.
COPY --from=alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm COPY --from=alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm
# kubectl 1.36.2 from the OFFICIAL registry.k8s.io image, replacing
# bitnamilegacy/kubectl:1.31.2. Two reasons, either sufficient:
# - CVEs: the bitnami image is Go 1.22.9 and ships containerd < 1.7.29 (HIGH
# CVE-2024-25621) along with the same crypto/tls and grpc CRITICALs as helm above.
# - Skew: the target cluster runs v1.35.3. 1.31.2 is four minors behind, well outside
# kubectl's supported +/-1 window, so it was unsupported against this cluster.
# bitnamilegacy publishes no 1.35 tag. registry.k8s.io/kubectl is the upstream-maintained
# image, is a manifest list with linux/arm64 (this cluster is Ampere), and puts the binary
# at /bin/kubectl rather than bitnami's /opt/bitnami path.
#
# 1.36.2 rather than 1.36.x-matching-the-cluster: k8s v1.35.x vendors
# github.com/moby/spdystream v0.5.0, which carries CVE-2026-35469 (HIGH, fixed in
# 0.5.1). v1.36.2 vendors 0.5.1. trivy blocks the release on it, correctly. One
# minor ahead of the v1.35.3 API server is inside kubectl's supported +/-1 window.
COPY --from=registry.k8s.io/kubectl:v1.36.2@sha256:b0d792e0d8dfb9bb1b922b78b23137e2a34bb6f9667640353a9d2aadd1fd7761 /bin/kubectl /usr/local/bin/kubectl
ENV PATH="/app/.venv/bin:$PATH" \ ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \ PYTHONUNBUFFERED=1 \
-19
View File
@@ -8,7 +8,6 @@ run in milliseconds against a FakeProvisioner instead of needing a cluster.
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from typing import Protocol
from svcforge_core.adapters.clock import Clock from svcforge_core.adapters.clock import Clock
from svcforge_core.adapters.helm import Provisioner from svcforge_core.adapters.helm import Provisioner
@@ -20,20 +19,6 @@ from svcforge_core.repo.tasks import TaskRepo
from svcforge_core.settings import Settings from svcforge_core.settings import Settings
class NamespaceEnsurer(Protocol):
"""The one thing the worker needs from kubectl.
Narrower than `KubectlClient` on purpose: the handler creates namespaces and does
nothing else with the cluster, so that is the whole interface. Satisfied by
`KubectlClient` in production and by a fake in tests, which is what keeps the worker
suite running in milliseconds with no cluster.
"""
async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
"""Create the namespace if absent. Idempotent — safe on every retry."""
...
@dataclass(frozen=True) @dataclass(frozen=True)
class WorkerDeps: class WorkerDeps:
"""Everything a handler is allowed to touch.""" """Everything a handler is allowed to touch."""
@@ -42,10 +27,6 @@ class WorkerDeps:
instances: InstanceRepo instances: InstanceRepo
tasks: TaskRepo tasks: TaskRepo
provisioner: Provisioner provisioner: Provisioner
# Creates the tenant namespace before helm is pointed at it. `helm --namespace X`
# does NOT create X, so without this the very first provision for a new team fails
# with "namespace not found" — the one path that is guaranteed untested by a fake.
namespaces: NamespaceEnsurer
notifier: Notifier notifier: Notifier
clock: Clock clock: Clock
catalog: dict[str, CatalogEntry] catalog: dict[str, CatalogEntry]
-9
View File
@@ -63,15 +63,6 @@ async def handle_provision(task: Task, deps: WorkerDeps) -> None:
# below is idempotent either way, so this is bookkeeping, not a lock. # below is idempotent either way, so this is bookkeeping, not a lock.
await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING) await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
# `helm --namespace X` does not create X. Every tenant's first provision lands in a
# namespace that does not exist yet, so this has to happen before helm runs or the
# install fails with "namespaces not found". Idempotent (kubectl apply of a Namespace
# manifest), so it costs one no-op API call on every subsequent provision.
await deps.namespaces.ensure_namespace(
inst.namespace,
labels={"svcforge.io/team": inst.team},
)
await deps.provisioner.install( await deps.provisioner.install(
release=inst.release_name, release=inst.release_name,
ns=inst.namespace, ns=inst.namespace,
-2
View File
@@ -22,7 +22,6 @@ from services.worker.handlers import HANDLERS
from svcforge_core import obs from svcforge_core import obs
from svcforge_core.adapters.clock import SystemClock from svcforge_core.adapters.clock import SystemClock
from svcforge_core.adapters.helm import HelmProvisioner from svcforge_core.adapters.helm import HelmProvisioner
from svcforge_core.adapters.k8s import KubectlClient
from svcforge_core.adapters.notify import LogNotifier from svcforge_core.adapters.notify import LogNotifier
from svcforge_core.domain.catalog import load_catalog from svcforge_core.domain.catalog import load_catalog
from svcforge_core.domain.models import Task, TaskKind from svcforge_core.domain.models import Task, TaskKind
@@ -185,7 +184,6 @@ async def _amain() -> None:
instances=InstanceRepo(pool), instances=InstanceRepo(pool),
tasks=TaskRepo(pool), tasks=TaskRepo(pool),
provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)), provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
namespaces=KubectlClient(kubectl_bin=settings.kubectl_bin),
notifier=LogNotifier(), notifier=LogNotifier(),
clock=SystemClock(), clock=SystemClock(),
catalog=load_catalog(settings.catalog_path), catalog=load_catalog(settings.catalog_path),
-12
View File
@@ -41,21 +41,10 @@ def _settings(**over: object) -> Settings:
return Settings(**base) # type: ignore[arg-type] return Settings(**base) # type: ignore[arg-type]
class FakeNamespaceEnsurer:
"""Records the namespaces it was asked to create. Satisfies NamespaceEnsurer."""
def __init__(self) -> None:
self.ensured: list[str] = []
async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
self.ensured.append(ns)
def _deps( def _deps(
pool: DictPool, pool: DictPool,
prov: FakeProvisioner, prov: FakeProvisioner,
notifier: FakeNotifier | None = None, notifier: FakeNotifier | None = None,
namespaces: FakeNamespaceEnsurer | None = None,
**over: object, **over: object,
) -> WorkerDeps: ) -> WorkerDeps:
return WorkerDeps( return WorkerDeps(
@@ -63,7 +52,6 @@ def _deps(
instances=InstanceRepo(pool), instances=InstanceRepo(pool),
tasks=TaskRepo(pool), tasks=TaskRepo(pool),
provisioner=prov, provisioner=prov,
namespaces=namespaces or FakeNamespaceEnsurer(),
notifier=notifier or FakeNotifier(), notifier=notifier or FakeNotifier(),
clock=SystemClock(), clock=SystemClock(),
catalog=CATALOG, catalog=CATALOG,