# Shared trust-bundle plumbing for containers that must trust the # self-signed gateway/forge leaf for *outbound* TLS (webhook delivery, # CI artifact upload, …). The anchors exist only at runtime — the host # `hive-tls-ca.service` (see `hive-tls.nix`) assembles them into # `trust-bundle.pem` — so they can't be baked into a derivation. Each such # container binds that bundle read-only and orders its `container@` # unit after `hive-tls-ca.service` so the bind source exists before nspawn # sets the mount up. # # `bindMount` + `containerOrdering` are the language-agnostic half. The # *consumption* differs per runtime: an additive variable (Node's # `NODE_EXTRA_CA_CERTS`, swarm-ci-runner) points straight at `caContainerPath` from # the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs # the system-CAs+bundle concat that `trustBundle` below does for it. # # Pure function — NOT a NixOS module (don't add it to the host-modules # aggregator). Call it from a module's `let`: # # caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # # then, in the container: # # bindMounts = { … } // caTrust.bindMount; # # systemd.services."container@swarm-ci-runner" = lib.mkMerge [ caTrust.containerOrdering … ]; # # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller # # `tlsCfg` = config.services.hyperhive.deploy.hive-controller.tls # `gatewayCfg` = config.services.hyperhive.gateway { lib, tlsCfg, gatewayCfg, }: let # `gateway.useSelfSigned` is the single source of truth for the # self-signed condition — no duplicated derivation. useSelfSigned = gatewayCfg.useSelfSigned; # The bundle `hive-tls.nix` writes: the store's `pki` root, which the # gateway's leaf chains to. caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem"; caContainerPath = "/run/hive-ca/trust-bundle.pem"; # One definition, used by `trustBundle` to WRITE the bundle and published # below so a caller can NAME it. Two copies of this path would be two # things to keep in step, and the one that drifts is the reader. bundleDirFor = name: "/run/${name}-ca"; bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem"; in { inherit useSelfSigned caContainerPath; # Where `trustBundle` below puts the assembled bundle, for the callers # that must NAME it rather than just have it exported. `SSL_CERT_FILE` is # set for you and needs no path here; a consumer that takes its own CA # argument (a client's `--cacert`) does, and the alternative is copying # `/run/-ca/…` to the call site. That copy breaks silently: the # bundle keeps being written, the consumer keeps reading a path that no # longer exists, and the failure surfaces as a TLS error naming the peer # rather than the file. # # ⛔ **Do not hand this path to a consumer that reads only ONE # certificate from it** — an OIDC verifier's `issuer_ca_path` is # exactly such a reader, and takes the swarm collector down. The bundle is # `system CAs ++ hive anchors`, so the anchor is ~123rd, and a # first-certificate-only reader gets whichever public CA sorts first and # can verify nothing of ours — with a full, valid, 125-certificate file # on disk and no check able to see it. # # 🔑 The general trust store (`SSL_CERT_FILE`, set by `trustBundle`) # reads every certificate and is order-independent, which is why a # consumer that can use the process trust store should simply be left to # do so. Before naming this path, check how that consumer parses it — # the requirement is a property of the *reader*, and nothing here can # enforce it. inherit bundlePathFor; # Fold into the container's `bindMounts` via `//`. Binds ONLY the public # CA cert (never the `hive-tls` state dir — it holds the CA + leaf private # keys), read-only. Empty when not self-signed, so the whole trust path # drops out cleanly. bindMount = lib.optionalAttrs useSelfSigned { ${caContainerPath} = { hostPath = caHostPath; isReadOnly = true; }; }; # Fold into the caller's `container@` unit (via `lib.mkMerge` if the # caller adds its own keys). Orders the # container after the host `hive-tls-ca.service` so the bind source exists # before nspawn sets the mount up — a condition-skipped/late CA would # otherwise fail the container start. containerOrdering = lib.mkIf useSelfSigned { after = [ "hive-tls-ca.service" ]; requires = [ "hive-tls-ca.service" ]; }; # System CAs + the trust bundle in one file, with `SSL_CERT_FILE` set on each # consumer — for runtimes whose trust variable *replaces* the store (Go, # rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`, # swarm-ci-runner) needs no bundle and should not use this. # # imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats"; # consumers = [ "swarm-nats-auth" ]; }) ]; # # Three constraints, each earned: # - `requires` on the CONSUMER: `before` orders but does not gate, so a # failed assembly otherwise leaves it running and trusting *nothing*. # - assemble to a temp path, verify, then move: `cat` of an empty bind # exits 0, and a partial bundle must never appear under the final name. # - `consumers` are BARE unit names — they are `systemd.services` keys # (no suffix) *and* go in `before`/`requires` (suffixed). Reversed, the # edge names a unit that does not exist and systemd orders nothing. # # Returns a module, not bare services: a caller already writing # `systemd.services.` cannot also write `systemd.services`. # # The two host-consumer flags are documented at their use sites below. trustBundle = { name, consumers, pkgs, # A HOST systemd service rather than something inside a container. # `caContainerPath` is a bind mount that only exists in a container, so # a host consumer reads the host copy — which means waiting for the # unit that writes it. In a container that wait is the container's own # (`containerOrdering`); here nothing carries it. One flag, because a # host source without the ordering is a race. hostUnit ? false, # Whether the consumer exists at all. A container caller imports this # into the CONTAINER's module set, so it vanishes with the container. A # host caller imports it at the host's top level, where `imports` is # unconditional — without this, a hive with the consumer off still gets # a bundle oneshot *and* a `systemd.services.` conjured by # `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`. enable ? true, }: let dir = bundleDirFor name; bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; # The source as the bundle was last built from it. anchor = "${dir}/anchor.pem"; refresh = "${unit}-refresh"; in { _file = "hive-ca-trust.nix#trustBundle:${name}"; # `hive-tls-ca` rewrites the source in place, so a watch on it sees the # write from inside a container: inotify is per inode, and a bind mount # shares the inode. `PathChanged` fires on close-after-write and on # `chmod`, so `${refresh}` is idempotent. config.systemd.paths = lib.optionalAttrs (useSelfSigned && enable) { ${refresh} = { description = "trigger ${refresh} when the trust bundle source is written"; wantedBy = [ "multi-user.target" ]; pathConfig.PathChanged = source; }; }; config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) ( { ${unit} = { description = "assemble ${name} TLS trust bundle (system CAs + hive CA)"; wantedBy = [ "multi-user.target" ]; before = map (c: "${c}.service") consumers; # Only for a host consumer: the CA file is written at runtime by # `hive-tls-ca.service`, and reading it directly means waiting # for it. A container consumer reads a bind mount instead, and # its `container@` unit carries the equivalent wait. after = lib.optionals hostUnit [ "hive-tls-ca.service" ]; requires = lib.optionals hostUnit [ "hive-tls-ca.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = unit; }; path = [ pkgs.coreutils pkgs.gnugrep ]; script = '' set -euo pipefail install -d -m 0755 ${dir} tmp=${bundlePath}.tmp # One snapshot of the source feeds the count, the bundle and # the copy `${refresh}` compares the live source against, so # the three cannot describe different roots. cp -f ${source} ${anchor}.tmp 2>/dev/null || : > ${anchor}.tmp # Count the HIVE half on its own, before assembling. # # Inspecting the assembled file cannot work: the system store # always holds certificates, so "does the result contain a # certificate" passes unconditionally — including in the one case # worth catching, where this source contributed nothing. And the # public CAs are irrelevant to what this bundle is for: every name # the consumer verifies is issued by our own CA, so a bundle of # nothing but public CAs is, for this purpose, an empty one that # measures as full. # # One helper, because the safe form is not the obvious one and # three copies of a subtle idiom is three chances to get it wrong. # # `grep -c` has two different zero-ish outcomes and `set -e` is # on: an EMPTY file prints `0` and exits 1, a MISSING file prints # nothing and exits 2. So `|| true` leaves the variable empty on # the second — the `-eq` then dies with "integer expected" # instead of reporting — and `|| echo 0` appends a second zero on # the first, yielding `00`. Normalising afterwards is the only # form that survives both. certs() { n=$(grep -c 'BEGIN CERTIFICATE' "$1" 2>/dev/null || true) if [ -z "$n" ]; then n=0; fi printf '%s' "$n" } contributed=$(certs ${anchor}.tmp) if [ "$contributed" -eq 0 ]; then echo "${unit}: ${source} contributed no certificate to the bundle" >&2 exit 1 fi system=$(certs /etc/ssl/certs/ca-certificates.crt) cat /etc/ssl/certs/ca-certificates.crt ${anchor}.tmp > "$tmp" # `cat` of a source truncated between the count and the copy still # exits 0, so the result is checked against what both halves # brought rather than merely for being non-empty. total=$(certs "$tmp") if [ "$total" -ne $((system + contributed)) ]; then echo "${unit}: bundle has $total certificates, expected $system + $contributed" >&2 exit 1 fi # ⚠️ Scope: this proves the anchor was CONTRIBUTED, not that it is # USABLE. A consumer that reads only the first certificate of the # file ignores it anyway — that is what took the swarm collector # down — and no check on this file can see it. Only a real # handshake can. Do not read a green assembly as a working trust # store. chmod 0644 "$tmp" mv ${anchor}.tmp ${anchor} mv "$tmp" ${bundlePath} ''; }; # Re-runs the bundle unit, and with it every consumer, when the # source holds a root the bundle was not built from. Separate from # the bundle unit because that one is `Before=` its consumers: a # restart issued from inside it would wait on jobs that wait on it. # `--no-block` keeps this one from waiting on them too. # # A restart of `${unit}` restarts every active unit that # `Requires=` it; the explicit `start` is for the consumers that are # not active, which a first boot leaves failed on the dependency. ${refresh} = { description = "rebuild ${name} TLS trust bundle when its source changes"; wantedBy = [ "multi-user.target" ]; after = [ "${unit}.service" ]; serviceConfig = { Type = "oneshot"; SyslogIdentifier = refresh; }; path = [ pkgs.coreutils pkgs.diffutils pkgs.gnugrep pkgs.systemd ]; script = '' set -euo pipefail # No root yet: nothing to converge on. The write that fills the # source fires this unit again. if ! grep -q 'BEGIN CERTIFICATE' ${source}; then exit 0; fi if cmp -s ${source} ${anchor}; then exit 0; fi echo "${source} changed — rebuilding the ${name} trust bundle" systemctl restart --no-block ${unit}.service systemctl start --no-block ${lib.concatMapStringsSep " " (c: "${c}.service") consumers} ''; }; } // lib.genAttrs consumers (_: { requires = [ "${unit}.service" ]; after = [ "${unit}.service" ]; environment.SSL_CERT_FILE = bundlePath; }) ); }; }