worker: create namespaces with helm, drop the kubectl binary
ci / lint (push) Successful in 25s
ci / unit (push) Successful in 59s
ci / types (push) Successful in 1m8s
ci / dockerfile (push) Successful in 13s
ci / chart (push) Successful in 8s
ci / security (push) Successful in 1m2s
ci / integration (push) Successful in 54s
ci / image (api) (push) Successful in 1m6s
ci / image (reconciler) (push) Successful in 3m6s
ci / image (worker) (push) Successful in 2m28s
ci / bump (push) Has been cancelled
ci / lint (push) Successful in 25s
ci / unit (push) Successful in 59s
ci / types (push) Successful in 1m8s
ci / dockerfile (push) Successful in 13s
ci / chart (push) Successful in 8s
ci / security (push) Successful in 1m2s
ci / integration (push) Successful in 54s
ci / image (api) (push) Successful in 1m6s
ci / image (reconciler) (push) Successful in 3m6s
ci / image (worker) (push) Successful in 2m28s
ci / bump (push) Has been cancelled
trivy took the worker image 39 -> 18 -> 5 findings across two version bumps, and the last 5 (4x golang.org/x/net, 1x Go stdlib) live in kubectl v1.36.2 — the newest kubectl that exists. No release clears them; upstream has not rebuilt against the patched Go yet. Chasing the version further has no end. kubectl was in that image for exactly one call: ensure_namespace before helm. `helm upgrade --install --create-namespace` does the same thing, idempotently, as part of the install it already runs. So the binary goes, and its vendored CVEs go with it. read_secret had no callers. Tradeoff recorded: the namespace no longer gets an svcforge.io/team label, since --create-namespace makes a bare one. Nothing reads that label today.
This commit is contained in:
+1
-2
@@ -128,8 +128,7 @@ sequenceDiagram
|
|||||||
Note over W,P: every 5s, while a semaphore slot is free
|
Note over W,P: every 5s, while a semaphore slot is free
|
||||||
W->>P: UPDATE ... FOR UPDATE SKIP LOCKED
|
W->>P: UPDATE ... FOR UPDATE SKIP LOCKED
|
||||||
P-->>W: task (attempts now 1, locked_by=me)
|
P-->>W: task (attempts now 1, locked_by=me)
|
||||||
W->>K: kubectl apply namespace (idempotent)
|
W->>K: helm upgrade --install --create-namespace --wait
|
||||||
W->>K: helm upgrade --install --wait
|
|
||||||
K-->>W: release ready
|
K-->>W: release ready
|
||||||
W->>P: CAS provisioning -> ready, set endpoint
|
W->>P: CAS provisioning -> ready, set endpoint
|
||||||
W->>P: complete(task, worker_id)
|
W->>P: complete(task, worker_id)
|
||||||
|
|||||||
@@ -199,6 +199,13 @@ class HelmProvisioner:
|
|||||||
entry.chart,
|
entry.chart,
|
||||||
"--namespace",
|
"--namespace",
|
||||||
ns,
|
ns,
|
||||||
|
# `--namespace X` does not create X. Every tenant's first provision targets
|
||||||
|
# a namespace that does not exist yet, and helm fails with "namespaces not
|
||||||
|
# found". helm creates it here rather than a separate `kubectl apply` step,
|
||||||
|
# which keeps kubectl out of the worker image entirely — one fewer binary,
|
||||||
|
# and one fewer set of vendored Go CVEs to track. Idempotent: existing
|
||||||
|
# namespaces are left alone.
|
||||||
|
"--create-namespace",
|
||||||
"--version",
|
"--version",
|
||||||
entry.chart_version,
|
entry.chart_version,
|
||||||
"--values",
|
"--values",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# svcforge worker. Build from the REPO ROOT:
|
# svcforge worker. Build from the REPO ROOT:
|
||||||
# docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
|
# docker buildx build -f services/worker/Dockerfile -t svcforge/worker:dev .
|
||||||
#
|
#
|
||||||
# The only service that shells out to helm/kubectl, so the only one carrying those two
|
# The only service that shells out to helm, so the only one carrying that binary
|
||||||
# binaries. They are copied from pinned images rather than curl'd, so the version is a
|
# binaries. They are copied from pinned images rather than curl'd, so the version is a
|
||||||
# reviewable line in a Dockerfile instead of a network call at build time.
|
# reviewable line in a Dockerfile instead of a network call at build time.
|
||||||
|
|
||||||
@@ -46,21 +46,6 @@ COPY --from=builder --chown=10001:10001 /app /app
|
|||||||
# (spdystream, fixed in 0.5.1) — trivy fails the build on them and is right to.
|
# (spdystream, fixed in 0.5.1) — trivy fails the build on them and is right to.
|
||||||
# Deliberately 3.x: helm 4 is a breaking change and is not a CVE fix.
|
# Deliberately 3.x: helm 4 is a breaking change and is not a CVE fix.
|
||||||
COPY --from=alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm
|
COPY --from=alpine/helm:3.21.3@sha256:35da09ba0716fc7c3cd63b6b31ee380a9c7662e95f29ab0e4ae962420afd315b /usr/bin/helm /usr/local/bin/helm
|
||||||
# kubectl 1.36.2 from the OFFICIAL registry.k8s.io image, replacing
|
|
||||||
# bitnamilegacy/kubectl:1.31.2. Two reasons, either sufficient:
|
|
||||||
# - CVEs: the bitnami image is Go 1.22.9 and ships containerd < 1.7.29 (HIGH
|
|
||||||
# CVE-2024-25621) along with the same crypto/tls and grpc CRITICALs as helm above.
|
|
||||||
# - Skew: the target cluster runs v1.35.3. 1.31.2 is four minors behind, well outside
|
|
||||||
# kubectl's supported +/-1 window, so it was unsupported against this cluster.
|
|
||||||
# bitnamilegacy publishes no 1.35 tag. registry.k8s.io/kubectl is the upstream-maintained
|
|
||||||
# image, is a manifest list with linux/arm64 (this cluster is Ampere), and puts the binary
|
|
||||||
# at /bin/kubectl rather than bitnami's /opt/bitnami path.
|
|
||||||
#
|
|
||||||
# 1.36.2 rather than 1.36.x-matching-the-cluster: k8s v1.35.x vendors
|
|
||||||
# github.com/moby/spdystream v0.5.0, which carries CVE-2026-35469 (HIGH, fixed in
|
|
||||||
# 0.5.1). v1.36.2 vendors 0.5.1. trivy blocks the release on it, correctly. One
|
|
||||||
# minor ahead of the v1.35.3 API server is inside kubectl's supported +/-1 window.
|
|
||||||
COPY --from=registry.k8s.io/kubectl:v1.36.2@sha256:b0d792e0d8dfb9bb1b922b78b23137e2a34bb6f9667640353a9d2aadd1fd7761 /bin/kubectl /usr/local/bin/kubectl
|
|
||||||
|
|
||||||
ENV PATH="/app/.venv/bin:$PATH" \
|
ENV PATH="/app/.venv/bin:$PATH" \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ run in milliseconds against a FakeProvisioner instead of needing a cluster.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Protocol
|
|
||||||
|
|
||||||
from svcforge_core.adapters.clock import Clock
|
from svcforge_core.adapters.clock import Clock
|
||||||
from svcforge_core.adapters.helm import Provisioner
|
from svcforge_core.adapters.helm import Provisioner
|
||||||
@@ -20,20 +19,6 @@ from svcforge_core.repo.tasks import TaskRepo
|
|||||||
from svcforge_core.settings import Settings
|
from svcforge_core.settings import Settings
|
||||||
|
|
||||||
|
|
||||||
class NamespaceEnsurer(Protocol):
|
|
||||||
"""The one thing the worker needs from kubectl.
|
|
||||||
|
|
||||||
Narrower than `KubectlClient` on purpose: the handler creates namespaces and does
|
|
||||||
nothing else with the cluster, so that is the whole interface. Satisfied by
|
|
||||||
`KubectlClient` in production and by a fake in tests, which is what keeps the worker
|
|
||||||
suite running in milliseconds with no cluster.
|
|
||||||
"""
|
|
||||||
|
|
||||||
async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
|
|
||||||
"""Create the namespace if absent. Idempotent — safe on every retry."""
|
|
||||||
...
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class WorkerDeps:
|
class WorkerDeps:
|
||||||
"""Everything a handler is allowed to touch."""
|
"""Everything a handler is allowed to touch."""
|
||||||
@@ -42,10 +27,6 @@ class WorkerDeps:
|
|||||||
instances: InstanceRepo
|
instances: InstanceRepo
|
||||||
tasks: TaskRepo
|
tasks: TaskRepo
|
||||||
provisioner: Provisioner
|
provisioner: Provisioner
|
||||||
# Creates the tenant namespace before helm is pointed at it. `helm --namespace X`
|
|
||||||
# does NOT create X, so without this the very first provision for a new team fails
|
|
||||||
# with "namespace not found" — the one path that is guaranteed untested by a fake.
|
|
||||||
namespaces: NamespaceEnsurer
|
|
||||||
notifier: Notifier
|
notifier: Notifier
|
||||||
clock: Clock
|
clock: Clock
|
||||||
catalog: dict[str, CatalogEntry]
|
catalog: dict[str, CatalogEntry]
|
||||||
|
|||||||
@@ -63,15 +63,6 @@ async def handle_provision(task: Task, deps: WorkerDeps) -> None:
|
|||||||
# below is idempotent either way, so this is bookkeeping, not a lock.
|
# below is idempotent either way, so this is bookkeeping, not a lock.
|
||||||
await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
|
await deps.instances.update_state(inst.id, InstanceState.REQUESTED, InstanceState.PROVISIONING)
|
||||||
|
|
||||||
# `helm --namespace X` does not create X. Every tenant's first provision lands in a
|
|
||||||
# namespace that does not exist yet, so this has to happen before helm runs or the
|
|
||||||
# install fails with "namespaces not found". Idempotent (kubectl apply of a Namespace
|
|
||||||
# manifest), so it costs one no-op API call on every subsequent provision.
|
|
||||||
await deps.namespaces.ensure_namespace(
|
|
||||||
inst.namespace,
|
|
||||||
labels={"svcforge.io/team": inst.team},
|
|
||||||
)
|
|
||||||
|
|
||||||
await deps.provisioner.install(
|
await deps.provisioner.install(
|
||||||
release=inst.release_name,
|
release=inst.release_name,
|
||||||
ns=inst.namespace,
|
ns=inst.namespace,
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ from services.worker.handlers import HANDLERS
|
|||||||
from svcforge_core import obs
|
from svcforge_core import obs
|
||||||
from svcforge_core.adapters.clock import SystemClock
|
from svcforge_core.adapters.clock import SystemClock
|
||||||
from svcforge_core.adapters.helm import HelmProvisioner
|
from svcforge_core.adapters.helm import HelmProvisioner
|
||||||
from svcforge_core.adapters.k8s import KubectlClient
|
|
||||||
from svcforge_core.adapters.notify import LogNotifier
|
from svcforge_core.adapters.notify import LogNotifier
|
||||||
from svcforge_core.domain.catalog import load_catalog
|
from svcforge_core.domain.catalog import load_catalog
|
||||||
from svcforge_core.domain.models import Task, TaskKind
|
from svcforge_core.domain.models import Task, TaskKind
|
||||||
@@ -185,7 +184,6 @@ async def _amain() -> None:
|
|||||||
instances=InstanceRepo(pool),
|
instances=InstanceRepo(pool),
|
||||||
tasks=TaskRepo(pool),
|
tasks=TaskRepo(pool),
|
||||||
provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
|
provisioner=HelmProvisioner(helm_bin=settings.helm_bin, timeout_s=int(settings.helm_timeout_s)),
|
||||||
namespaces=KubectlClient(kubectl_bin=settings.kubectl_bin),
|
|
||||||
notifier=LogNotifier(),
|
notifier=LogNotifier(),
|
||||||
clock=SystemClock(),
|
clock=SystemClock(),
|
||||||
catalog=load_catalog(settings.catalog_path),
|
catalog=load_catalog(settings.catalog_path),
|
||||||
|
|||||||
@@ -41,21 +41,10 @@ def _settings(**over: object) -> Settings:
|
|||||||
return Settings(**base) # type: ignore[arg-type]
|
return Settings(**base) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
class FakeNamespaceEnsurer:
|
|
||||||
"""Records the namespaces it was asked to create. Satisfies NamespaceEnsurer."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.ensured: list[str] = []
|
|
||||||
|
|
||||||
async def ensure_namespace(self, ns: str, labels: dict[str, str] | None = None) -> None:
|
|
||||||
self.ensured.append(ns)
|
|
||||||
|
|
||||||
|
|
||||||
def _deps(
|
def _deps(
|
||||||
pool: DictPool,
|
pool: DictPool,
|
||||||
prov: FakeProvisioner,
|
prov: FakeProvisioner,
|
||||||
notifier: FakeNotifier | None = None,
|
notifier: FakeNotifier | None = None,
|
||||||
namespaces: FakeNamespaceEnsurer | None = None,
|
|
||||||
**over: object,
|
**over: object,
|
||||||
) -> WorkerDeps:
|
) -> WorkerDeps:
|
||||||
return WorkerDeps(
|
return WorkerDeps(
|
||||||
@@ -63,7 +52,6 @@ def _deps(
|
|||||||
instances=InstanceRepo(pool),
|
instances=InstanceRepo(pool),
|
||||||
tasks=TaskRepo(pool),
|
tasks=TaskRepo(pool),
|
||||||
provisioner=prov,
|
provisioner=prov,
|
||||||
namespaces=namespaces or FakeNamespaceEnsurer(),
|
|
||||||
notifier=notifier or FakeNotifier(),
|
notifier=notifier or FakeNotifier(),
|
||||||
clock=SystemClock(),
|
clock=SystemClock(),
|
||||||
catalog=CATALOG,
|
catalog=CATALOG,
|
||||||
|
|||||||
Reference in New Issue
Block a user