From 29ca381114346a8bed7d9ee8f0caf2aa5d446eeb Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 19 Nov 2026 09:37:41 +0100 Subject: [PATCH] images: slim OCI images of the swarm daemons from nix --- docs/4-implementation/k8s/images.md | 75 +++++++++++++++++++++++++++++ nix/checks.nix | 44 +++++++++++++++++ nix/images/default.nix | 68 ++++++++++++++++++++++++++ nix/packages/default.nix | 37 +++++++++++--- 4 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 docs/4-implementation/k8s/images.md create mode 100644 nix/images/default.nix diff --git a/docs/4-implementation/k8s/images.md b/docs/4-implementation/k8s/images.md new file mode 100644 index 000000000..1dfa6753a --- /dev/null +++ b/docs/4-implementation/k8s/images.md @@ -0,0 +1,75 @@ +# Service images + +The swarm daemons that run in the k8s cluster ship as OCI images built by nix. +`nix/images/default.nix` holds the one function that builds them, +`mkServiceImage`, and the set of daemons that use it. + +## What an image holds + +One image runs one daemon. Its contents are: + +- the daemon's per-bin package from `nix/packages/default.nix`, +- the CA roots from `pkgs.cacert`, with `SSL_CERT_FILE` pointing at them, +- the binary's own closure, which is glibc and libgcc. + +The image has no shell and no init. The kubelet restarts a crashed container, so a +service manager inside it would only disagree with the kubelet. A missing +shell also leaves nothing to `exec` into. The entrypoint is `/bin/`, and +the process runs as the `serviceUser` uid and gid that `nix/images/default.nix` +defines. A pod's `securityContext.runAsUser` and `runAsGroup` read that binding +too, so the number lives in one place. + +`mkServiceImage` takes `name`, `package`, and optionally `bin` (the binary +name, default `name`), `extraContents` and `env` (an attribute set added to +the container environment). + +## The image set + +Each of these is a flake package named `image-`, and the image name is +`hyperhive/`: + +- `swarm-controller` +- `swarm-nats-auth` +- `swarm-logs-auth` +- `swarm-metrics-auth` +- `swarm-notify-router` +- `swarm-ci-metrics` + +`nix flake check` evaluates all six and builds none of them, so the workspace +compile stays out of CI. `swarm-claude-login`, `swarm-stt` and +`swarm-review-classifier` need claude-code, whisper or similar tools in their +image and aren't in the set yet. + +## Tags + +An image has no fixed tag. The tag is the content hash, so a changed binary is +a changed reference. A manifest reads the full reference from the package's +`imageName` and `imageTag` attributes: + +```sh +nix eval --raw .#image-swarm-logs-auth.imageName +nix eval --raw .#image-swarm-logs-auth.imageTag +``` + +## Loading an image + +`nix build` produces a `.tar.gz` that `podman load`, `skopeo` and the k3s image +directory all read. Use `podman`, not docker: + +```sh +podman load -i "$(nix build --no-link --print-out-paths .#image-swarm-logs-auth)" +``` + +For a containerd node, stream the file into the node's image store: + +```sh +k3s ctr images import - < "$(nix build --no-link --print-out-paths .#image-swarm-logs-auth)" +``` + +## The `image-fn-smoke` check + +`checks..image-fn-smoke` builds `mkServiceImage` around `hello` and +reads the tarball back. It asserts the entrypoint, the user, the CA env var, a +layer listing without `bash` or `systemd` and a size under 50 MB. The listing +must also contain `hello` and `glibc`, so an empty listing can't pass the +shell arm. The check runs without compiling any hyperhive binary. diff --git a/nix/checks.nix b/nix/checks.nix index 75e916a91..081c8336f 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -297,6 +297,50 @@ in shared = moduleEvalShared; }; + # Builds `mkServiceImage` around `hello` and reads the tarball back, so the + # function is tested without compiling a hyperhive binary. The two `grep -q` + # arms for `hello` and `glibc` prove the layer listing finds store paths, + # which keeps the "no bash, no systemd" arm from passing on an empty list. + image-fn-smoke = + let + imageSet = import ./images { + inherit pkgs; + inherit (pkgs) lib; + }; + inherit (imageSet) serviceUser; + image = imageSet.mkServiceImage { + name = "smoke"; + package = pkgs.hello; + bin = "hello"; + }; + in + pkgs.runCommand "image-fn-smoke" { nativeBuildInputs = [ pkgs.jq ]; } '' + mkdir unpacked + tar -xzf ${image} -C unpacked + cd unpacked + + config=$(jq -r '.[0].Config' manifest.json) + test "$(jq -c '.config.Entrypoint' "$config")" = '["/bin/hello"]' + test "$(jq -r '.config.User' "$config")" = '${toString serviceUser.uid}:${toString serviceUser.gid}' + jq -e '.config.Env | index("SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt")' "$config" >/dev/null + test "${image.imageName}" = hyperhive/smoke + + for layer in $(jq -r '.[0].Layers[]' manifest.json); do + tar -tf "$layer" + done > paths + grep -q -- '-hello-' paths + grep -q -- '-glibc-' paths + if grep -E -- '-(systemd|bash)-' paths; then + echo "image carries a shell or systemd" >&2 + exit 1 + fi + + size=$(stat -c %s ${image}) + test "$size" -lt 50000000 + + touch $out + ''; + # Clippy via crane's first-class `cargoClippy` builder. Reuses the # shared `cargoArtifacts` (deps already built) and runs # `cargo clippy --workspace --all-targets` directly. diff --git a/nix/images/default.nix b/nix/images/default.nix new file mode 100644 index 000000000..adde321fb --- /dev/null +++ b/nix/images/default.nix @@ -0,0 +1,68 @@ +# Slim OCI images of the swarm daemons. Imported from ../packages/default.nix +# with the per-bin packages, and from ../checks.nix for `mkServiceImage` alone. +{ + pkgs, + lib, + packages ? { }, +}: +let + # The conventional nonroot id. An OCI image has no `config.ids` to take it + # from, so it is defined once here: the image's `User`, the smoke check and + # a pod's `securityContext.runAsUser`/`runAsGroup` must all read this. + serviceUser = { + uid = 65532; + gid = 65532; + }; + + # One daemon per image: the binary, CA roots and the binary's own closure. + # No shell and no init, so the image has no command to exec into and no + # service manager to disagree with the kubelet about restarts. + # + # `buildLayeredImage` writes a `.tar.gz`, which `podman load`, `skopeo` and + # k3s's image directory read. `streamLayeredImage` writes a script instead. + # `tag` stays unset, so it is the content hash and a changed binary is a + # changed reference; `imageName` and `imageTag` are on the result. + mkServiceImage = + { + name, + package, + bin ? name, + extraContents ? [ ], + env ? { }, + }: + pkgs.dockerTools.buildLayeredImage { + name = "hyperhive/${name}"; + contents = [ + package + pkgs.cacert + ] + ++ extraContents; + config = { + Entrypoint = [ "/bin/${bin}" ]; + Env = [ + "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + ] + ++ lib.mapAttrsToList (k: v: "${k}=${v}") env; + User = "${toString serviceUser.uid}:${toString serviceUser.gid}"; + }; + }; + + daemon = + name: + mkServiceImage { + inherit name; + package = packages.${name}; + }; +in +{ + inherit mkServiceImage serviceUser; + + images = lib.genAttrs [ + "swarm-controller" + "swarm-nats-auth" + "swarm-logs-auth" + "swarm-metrics-auth" + "swarm-notify-router" + "swarm-ci-metrics" + ] daemon; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 5ffc3abd4..652ec24cd 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -129,6 +129,29 @@ let binPkgs = lib.mapAttrs mkBinPackage daemonBins // { hivectl = hivectlPkg; }; + + # The daemons that get an OCI image (../images). Bound here so the image + # set can read them; the attributes below `inherit` them. + swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon"; + swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder"; + swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder"; + swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder"; + swarm-notify-router = mkBinPackage "swarm-notify-router" "hyperhive swarm notification router"; + swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller"; + + imageSet = import ../images { + inherit pkgs lib; + packages = { + inherit + swarm-controller + swarm-nats-auth + swarm-logs-auth + swarm-metrics-auth + swarm-notify-router + swarm-ci-metrics + ; + }; + }; in { # All workspace binaries in one derivation — a symlinkJoin over the @@ -155,21 +178,21 @@ in # `package` option, and one hive in a swarm runs it, so folding it # into the core bundle would put it in every hive's closure to no end. # Uses the same per-bin extractor, just bound on its own. - swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon"; + inherit swarm-controller; # The queue's auth-callout responder. Out of `daemonBins` for the same # reason as the two above and one more: it runs *inside* the swarm-nats # container, not on the host, so it belongs in that container's closure # rather than every hive's. - swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder"; + inherit swarm-nats-auth; # The log store gateway's `auth_request` responder for agents. Out of # `daemonBins` like the responder above: it runs on the log store's host only. - swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder"; + inherit swarm-logs-auth; # The metrics store gateway's `auth_request` responder for Grafana's bearer. Out # of `daemonBins` like the two above: it runs on the metrics store's host only. - swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder"; + inherit swarm-metrics-auth; # The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for # "runs *inside* a container, not on the host" reason as the responder @@ -182,9 +205,9 @@ in # reason: it runs inside its own container, not on the host. swarm-review-classifier = mkBinPackage "swarm-review-classifier" "hyperhive review classifier try-out daemon"; swarm-builder = mkBinPackage "swarm-builder" "hyperhive swarm builder: agent config build checks"; - swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller"; + inherit swarm-ci-metrics; swarm-claude-login = mkBinPackage "swarm-claude-login" "hyperhive claude login minter"; - swarm-notify-router = mkBinPackage "swarm-notify-router" "hyperhive swarm notification router"; + inherit swarm-notify-router; swarm-stt = mkBinPackage "swarm-stt" "hyperhive swarm speech-to-text service"; # The only process allowed to write swarm-authelia's users database — @@ -306,3 +329,5 @@ in docs-deploy = docsAttrs.deploy; docs-agent = docsAttrs.agent; } +# One `image-` per entry of ../images. CI evaluates them and builds none. +// lib.mapAttrs' (n: v: lib.nameValuePair "image-${n}" v) imageSet.images