images: slim OCI images of the swarm daemons from nix
This commit is contained in:
parent
ffbd2f4dea
commit
29ca381114
4 changed files with 218 additions and 6 deletions
75
docs/4-implementation/k8s/images.md
Normal file
75
docs/4-implementation/k8s/images.md
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
# Service images
|
||||||
|
|
||||||
|
The swarm daemons that run in the k8s cluster ship as OCI images built by nix.
|
||||||
|
`nix/images/default.nix` holds the one function that builds them,
|
||||||
|
`mkServiceImage`, and the set of daemons that use it.
|
||||||
|
|
||||||
|
## What an image holds
|
||||||
|
|
||||||
|
One image runs one daemon. Its contents are:
|
||||||
|
|
||||||
|
- the daemon's per-bin package from `nix/packages/default.nix`,
|
||||||
|
- the CA roots from `pkgs.cacert`, with `SSL_CERT_FILE` pointing at them,
|
||||||
|
- the binary's own closure, which is glibc and libgcc.
|
||||||
|
|
||||||
|
The image has no shell and no init. The kubelet restarts a crashed container, so a
|
||||||
|
service manager inside it would only disagree with the kubelet. A missing
|
||||||
|
shell also leaves nothing to `exec` into. The entrypoint is `/bin/<bin>`, and
|
||||||
|
the process runs as the `serviceUser` uid and gid that `nix/images/default.nix`
|
||||||
|
defines. A pod's `securityContext.runAsUser` and `runAsGroup` read that binding
|
||||||
|
too, so the number lives in one place.
|
||||||
|
|
||||||
|
`mkServiceImage` takes `name`, `package`, and optionally `bin` (the binary
|
||||||
|
name, default `name`), `extraContents` and `env` (an attribute set added to
|
||||||
|
the container environment).
|
||||||
|
|
||||||
|
## The image set
|
||||||
|
|
||||||
|
Each of these is a flake package named `image-<daemon>`, and the image name is
|
||||||
|
`hyperhive/<daemon>`:
|
||||||
|
|
||||||
|
- `swarm-controller`
|
||||||
|
- `swarm-nats-auth`
|
||||||
|
- `swarm-logs-auth`
|
||||||
|
- `swarm-metrics-auth`
|
||||||
|
- `swarm-notify-router`
|
||||||
|
- `swarm-ci-metrics`
|
||||||
|
|
||||||
|
`nix flake check` evaluates all six and builds none of them, so the workspace
|
||||||
|
compile stays out of CI. `swarm-claude-login`, `swarm-stt` and
|
||||||
|
`swarm-review-classifier` need claude-code, whisper or similar tools in their
|
||||||
|
image and aren't in the set yet.
|
||||||
|
|
||||||
|
## Tags
|
||||||
|
|
||||||
|
An image has no fixed tag. The tag is the content hash, so a changed binary is
|
||||||
|
a changed reference. A manifest reads the full reference from the package's
|
||||||
|
`imageName` and `imageTag` attributes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nix eval --raw .#image-swarm-logs-auth.imageName
|
||||||
|
nix eval --raw .#image-swarm-logs-auth.imageTag
|
||||||
|
```
|
||||||
|
|
||||||
|
## Loading an image
|
||||||
|
|
||||||
|
`nix build` produces a `.tar.gz` that `podman load`, `skopeo` and the k3s image
|
||||||
|
directory all read. Use `podman`, not docker:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
podman load -i "$(nix build --no-link --print-out-paths .#image-swarm-logs-auth)"
|
||||||
|
```
|
||||||
|
|
||||||
|
For a containerd node, stream the file into the node's image store:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
k3s ctr images import - < "$(nix build --no-link --print-out-paths .#image-swarm-logs-auth)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## The `image-fn-smoke` check
|
||||||
|
|
||||||
|
`checks.<system>.image-fn-smoke` builds `mkServiceImage` around `hello` and
|
||||||
|
reads the tarball back. It asserts the entrypoint, the user, the CA env var, a
|
||||||
|
layer listing without `bash` or `systemd` and a size under 50 MB. The listing
|
||||||
|
must also contain `hello` and `glibc`, so an empty listing can't pass the
|
||||||
|
shell arm. The check runs without compiling any hyperhive binary.
|
||||||
|
|
@ -297,6 +297,50 @@ in
|
||||||
shared = moduleEvalShared;
|
shared = moduleEvalShared;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Builds `mkServiceImage` around `hello` and reads the tarball back, so the
|
||||||
|
# function is tested without compiling a hyperhive binary. The two `grep -q`
|
||||||
|
# arms for `hello` and `glibc` prove the layer listing finds store paths,
|
||||||
|
# which keeps the "no bash, no systemd" arm from passing on an empty list.
|
||||||
|
image-fn-smoke =
|
||||||
|
let
|
||||||
|
imageSet = import ./images {
|
||||||
|
inherit pkgs;
|
||||||
|
inherit (pkgs) lib;
|
||||||
|
};
|
||||||
|
inherit (imageSet) serviceUser;
|
||||||
|
image = imageSet.mkServiceImage {
|
||||||
|
name = "smoke";
|
||||||
|
package = pkgs.hello;
|
||||||
|
bin = "hello";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
pkgs.runCommand "image-fn-smoke" { nativeBuildInputs = [ pkgs.jq ]; } ''
|
||||||
|
mkdir unpacked
|
||||||
|
tar -xzf ${image} -C unpacked
|
||||||
|
cd unpacked
|
||||||
|
|
||||||
|
config=$(jq -r '.[0].Config' manifest.json)
|
||||||
|
test "$(jq -c '.config.Entrypoint' "$config")" = '["/bin/hello"]'
|
||||||
|
test "$(jq -r '.config.User' "$config")" = '${toString serviceUser.uid}:${toString serviceUser.gid}'
|
||||||
|
jq -e '.config.Env | index("SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt")' "$config" >/dev/null
|
||||||
|
test "${image.imageName}" = hyperhive/smoke
|
||||||
|
|
||||||
|
for layer in $(jq -r '.[0].Layers[]' manifest.json); do
|
||||||
|
tar -tf "$layer"
|
||||||
|
done > paths
|
||||||
|
grep -q -- '-hello-' paths
|
||||||
|
grep -q -- '-glibc-' paths
|
||||||
|
if grep -E -- '-(systemd|bash)-' paths; then
|
||||||
|
echo "image carries a shell or systemd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
size=$(stat -c %s ${image})
|
||||||
|
test "$size" -lt 50000000
|
||||||
|
|
||||||
|
touch $out
|
||||||
|
'';
|
||||||
|
|
||||||
# Clippy via crane's first-class `cargoClippy` builder. Reuses the
|
# Clippy via crane's first-class `cargoClippy` builder. Reuses the
|
||||||
# shared `cargoArtifacts` (deps already built) and runs
|
# shared `cargoArtifacts` (deps already built) and runs
|
||||||
# `cargo clippy --workspace --all-targets` directly.
|
# `cargo clippy --workspace --all-targets` directly.
|
||||||
|
|
|
||||||
68
nix/images/default.nix
Normal file
68
nix/images/default.nix
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
# Slim OCI images of the swarm daemons. Imported from ../packages/default.nix
|
||||||
|
# with the per-bin packages, and from ../checks.nix for `mkServiceImage` alone.
|
||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
packages ? { },
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
# The conventional nonroot id. An OCI image has no `config.ids` to take it
|
||||||
|
# from, so it is defined once here: the image's `User`, the smoke check and
|
||||||
|
# a pod's `securityContext.runAsUser`/`runAsGroup` must all read this.
|
||||||
|
serviceUser = {
|
||||||
|
uid = 65532;
|
||||||
|
gid = 65532;
|
||||||
|
};
|
||||||
|
|
||||||
|
# One daemon per image: the binary, CA roots and the binary's own closure.
|
||||||
|
# No shell and no init, so the image has no command to exec into and no
|
||||||
|
# service manager to disagree with the kubelet about restarts.
|
||||||
|
#
|
||||||
|
# `buildLayeredImage` writes a `.tar.gz`, which `podman load`, `skopeo` and
|
||||||
|
# k3s's image directory read. `streamLayeredImage` writes a script instead.
|
||||||
|
# `tag` stays unset, so it is the content hash and a changed binary is a
|
||||||
|
# changed reference; `imageName` and `imageTag` are on the result.
|
||||||
|
mkServiceImage =
|
||||||
|
{
|
||||||
|
name,
|
||||||
|
package,
|
||||||
|
bin ? name,
|
||||||
|
extraContents ? [ ],
|
||||||
|
env ? { },
|
||||||
|
}:
|
||||||
|
pkgs.dockerTools.buildLayeredImage {
|
||||||
|
name = "hyperhive/${name}";
|
||||||
|
contents = [
|
||||||
|
package
|
||||||
|
pkgs.cacert
|
||||||
|
]
|
||||||
|
++ extraContents;
|
||||||
|
config = {
|
||||||
|
Entrypoint = [ "/bin/${bin}" ];
|
||||||
|
Env = [
|
||||||
|
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||||
|
]
|
||||||
|
++ lib.mapAttrsToList (k: v: "${k}=${v}") env;
|
||||||
|
User = "${toString serviceUser.uid}:${toString serviceUser.gid}";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
daemon =
|
||||||
|
name:
|
||||||
|
mkServiceImage {
|
||||||
|
inherit name;
|
||||||
|
package = packages.${name};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
inherit mkServiceImage serviceUser;
|
||||||
|
|
||||||
|
images = lib.genAttrs [
|
||||||
|
"swarm-controller"
|
||||||
|
"swarm-nats-auth"
|
||||||
|
"swarm-logs-auth"
|
||||||
|
"swarm-metrics-auth"
|
||||||
|
"swarm-notify-router"
|
||||||
|
"swarm-ci-metrics"
|
||||||
|
] daemon;
|
||||||
|
}
|
||||||
|
|
@ -129,6 +129,29 @@ let
|
||||||
binPkgs = lib.mapAttrs mkBinPackage daemonBins // {
|
binPkgs = lib.mapAttrs mkBinPackage daemonBins // {
|
||||||
hivectl = hivectlPkg;
|
hivectl = hivectlPkg;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# The daemons that get an OCI image (../images). Bound here so the image
|
||||||
|
# set can read them; the attributes below `inherit` them.
|
||||||
|
swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
|
||||||
|
swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder";
|
||||||
|
swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder";
|
||||||
|
swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder";
|
||||||
|
swarm-notify-router = mkBinPackage "swarm-notify-router" "hyperhive swarm notification router";
|
||||||
|
swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller";
|
||||||
|
|
||||||
|
imageSet = import ../images {
|
||||||
|
inherit pkgs lib;
|
||||||
|
packages = {
|
||||||
|
inherit
|
||||||
|
swarm-controller
|
||||||
|
swarm-nats-auth
|
||||||
|
swarm-logs-auth
|
||||||
|
swarm-metrics-auth
|
||||||
|
swarm-notify-router
|
||||||
|
swarm-ci-metrics
|
||||||
|
;
|
||||||
|
};
|
||||||
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
# All workspace binaries in one derivation — a symlinkJoin over the
|
# All workspace binaries in one derivation — a symlinkJoin over the
|
||||||
|
|
@ -155,21 +178,21 @@ in
|
||||||
# `package` option, and one hive in a swarm runs it, so folding it
|
# `package` option, and one hive in a swarm runs it, so folding it
|
||||||
# into the core bundle would put it in every hive's closure to no end.
|
# into the core bundle would put it in every hive's closure to no end.
|
||||||
# Uses the same per-bin extractor, just bound on its own.
|
# Uses the same per-bin extractor, just bound on its own.
|
||||||
swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
|
inherit swarm-controller;
|
||||||
|
|
||||||
# The queue's auth-callout responder. Out of `daemonBins` for the same
|
# The queue's auth-callout responder. Out of `daemonBins` for the same
|
||||||
# reason as the two above and one more: it runs *inside* the swarm-nats
|
# reason as the two above and one more: it runs *inside* the swarm-nats
|
||||||
# container, not on the host, so it belongs in that container's closure
|
# container, not on the host, so it belongs in that container's closure
|
||||||
# rather than every hive's.
|
# rather than every hive's.
|
||||||
swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder";
|
inherit swarm-nats-auth;
|
||||||
|
|
||||||
# The log store gateway's `auth_request` responder for agents. Out of
|
# The log store gateway's `auth_request` responder for agents. Out of
|
||||||
# `daemonBins` like the responder above: it runs on the log store's host only.
|
# `daemonBins` like the responder above: it runs on the log store's host only.
|
||||||
swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder";
|
inherit swarm-logs-auth;
|
||||||
|
|
||||||
# The metrics store gateway's `auth_request` responder for Grafana's bearer. Out
|
# The metrics store gateway's `auth_request` responder for Grafana's bearer. Out
|
||||||
# of `daemonBins` like the two above: it runs on the metrics store's host only.
|
# of `daemonBins` like the two above: it runs on the metrics store's host only.
|
||||||
swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder";
|
inherit swarm-metrics-auth;
|
||||||
|
|
||||||
# The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for
|
# The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for
|
||||||
# "runs *inside* a container, not on the host" reason as the responder
|
# "runs *inside* a container, not on the host" reason as the responder
|
||||||
|
|
@ -182,9 +205,9 @@ in
|
||||||
# reason: it runs inside its own container, not on the host.
|
# reason: it runs inside its own container, not on the host.
|
||||||
swarm-review-classifier = mkBinPackage "swarm-review-classifier" "hyperhive review classifier try-out daemon";
|
swarm-review-classifier = mkBinPackage "swarm-review-classifier" "hyperhive review classifier try-out daemon";
|
||||||
swarm-builder = mkBinPackage "swarm-builder" "hyperhive swarm builder: agent config build checks";
|
swarm-builder = mkBinPackage "swarm-builder" "hyperhive swarm builder: agent config build checks";
|
||||||
swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller";
|
inherit swarm-ci-metrics;
|
||||||
swarm-claude-login = mkBinPackage "swarm-claude-login" "hyperhive claude login minter";
|
swarm-claude-login = mkBinPackage "swarm-claude-login" "hyperhive claude login minter";
|
||||||
swarm-notify-router = mkBinPackage "swarm-notify-router" "hyperhive swarm notification router";
|
inherit swarm-notify-router;
|
||||||
swarm-stt = mkBinPackage "swarm-stt" "hyperhive swarm speech-to-text service";
|
swarm-stt = mkBinPackage "swarm-stt" "hyperhive swarm speech-to-text service";
|
||||||
|
|
||||||
# The only process allowed to write swarm-authelia's users database —
|
# The only process allowed to write swarm-authelia's users database —
|
||||||
|
|
@ -306,3 +329,5 @@ in
|
||||||
docs-deploy = docsAttrs.deploy;
|
docs-deploy = docsAttrs.deploy;
|
||||||
docs-agent = docsAttrs.agent;
|
docs-agent = docsAttrs.agent;
|
||||||
}
|
}
|
||||||
|
# One `image-<daemon>` per entry of ../images. CI evaluates them and builds none.
|
||||||
|
// lib.mapAttrs' (n: v: lib.nameValuePair "image-${n}" v) imageSet.images
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue