# Shared hive-CA trust plumbing for containers that must trust the # self-signed gateway/forge leaf for *outbound* TLS (webhook delivery, # CI artifact upload, …). The hive CA is generated at runtime by the host # `hive-tls-ca.service` (see `hive-tls.nix`) — it can't be baked into a # derivation — so each such container binds the public `ca.pem` read-only # and orders its `container@` unit after `hive-tls-ca.service` so the # bind source exists before nspawn sets the mount up. # # `bindMount` + `containerOrdering` are the language-agnostic half. The # *consumption* differs per runtime: an additive variable (Node's # `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from # the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs # the system-CAs+hive-CA concat that `trustBundle` below does for it. # # Pure function — NOT a NixOS module (don't add it to the host-modules # aggregator). Call it from a module's `let`: # # caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # # then, in the container: # # bindMounts = { … } // caTrust.bindMount; # # systemd.services."container@hive-ci" = lib.mkMerge [ caTrust.containerOrdering … ]; # # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller # # `tlsCfg` = config.services.hyperhive.tls # `gatewayCfg` = config.services.hyperhive.gateway { lib, tlsCfg, gatewayCfg, }: let # `gateway.useSelfSigned` is the single source of truth for the # self-signed condition — no duplicated derivation. useSelfSigned = gatewayCfg.useSelfSigned; # The bundle, not `ca.pem`: the hive CA is an intermediate under the # swarm root, and openssl (which is what both consumers below sit on — # Node's `NODE_EXTRA_CA_CERTS`, Go's `SSL_CERT_FILE`) will not end a # chain at a trusted cert that isn't self-signed. `hive-tls.nix` writes # the bundle next to the CA and explains the split. caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem"; caContainerPath = "/run/hive-ca/trust-bundle.pem"; # One definition, used by `trustBundle` to WRITE the bundle and published # below so a caller can NAME it. Two copies of this path would be two # things to keep in step, and the one that drifts is the reader. bundleDirFor = name: "/run/${name}-ca"; bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem"; in { inherit useSelfSigned caContainerPath; # Where `trustBundle` below puts the assembled bundle, for the callers # that must NAME it rather than just have it exported. `SSL_CERT_FILE` is # set for you and needs no path here; a consumer that takes its own CA # argument (a client's `--cacert`) does, and the alternative is copying # `/run/-ca/…` to the call site. That copy breaks silently: the # bundle keeps being written, the consumer keeps reading a path that no # longer exists, and the failure surfaces as a TLS error naming the peer # rather than the file. # # ⛔ **Do not hand this path to a consumer that reads only ONE # certificate from it.** This comment used to offer an OIDC verifier's # `issuer_ca_path` as the motivating example; that is exactly what took # the swarm collector down for forty minutes once. The bundle is # `system CAs ++ hive anchors`, so the anchor is ~123rd, and a # first-certificate-only reader gets whichever public CA sorts first and # can verify nothing of ours — with a full, valid, 125-certificate file # on disk and no check able to see it. # # 🔑 The general trust store (`SSL_CERT_FILE`, set by `trustBundle`) # reads every certificate and is order-independent, which is why a # consumer that can use the process trust store should simply be left to # do so. Before naming this path, check how that consumer parses it — # the requirement is a property of the *reader*, and nothing here can # enforce it. inherit bundlePathFor; # Fold into the container's `bindMounts` via `//`. Binds ONLY the public # CA cert (never the `hive-tls` state dir — it holds the CA + leaf private # keys), read-only. Empty when not self-signed, so the whole trust path # drops out cleanly. bindMount = lib.optionalAttrs useSelfSigned { ${caContainerPath} = { hostPath = caHostPath; isReadOnly = true; }; }; # Fold into the caller's `container@` unit (via `lib.mkMerge` if the # caller adds its own keys, e.g. hive-ci's `TimeoutStartSec`). Orders the # container after the host `hive-tls-ca.service` so the bind source exists # before nspawn sets the mount up — a condition-skipped/late CA would # otherwise fail the container start. containerOrdering = lib.mkIf useSelfSigned { after = [ "hive-tls-ca.service" ]; requires = [ "hive-tls-ca.service" ]; }; # System CAs + hive CA in one bundle, with `SSL_CERT_FILE` set on each # consumer — for runtimes whose trust variable *replaces* the store (Go, # rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`, # hive-ci) needs no bundle and should not use this. # # imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats"; # consumers = [ "swarm-nats-auth" ]; }) ]; # # Three constraints, each earned: # - `requires` on the CONSUMER: `before` orders but does not gate, so a # failed assembly otherwise leaves it running and trusting *nothing*. # - assemble to a temp path, verify, then move: `cat` of an empty bind # exits 0, and a partial bundle must never appear under the final name. # - `consumers` are BARE unit names — they are `systemd.services` keys # (no suffix) *and* go in `before`/`requires` (suffixed). Reversed, the # edge names a unit that does not exist and systemd orders nothing. # # Returns a module, not bare services: a caller already writing # `systemd.services.` cannot also write `systemd.services`. # # The two host-consumer flags are documented at their use sites below. trustBundle = { name, consumers, pkgs, # A HOST systemd service rather than something inside a container. # `caContainerPath` is a bind mount that only exists in a container, so # a host consumer reads the host copy — which means waiting for the # unit that writes it. In a container that wait is the container's own # (`containerOrdering`); here nothing carries it. One flag, because a # host source without the ordering is a race. hostUnit ? false, # Whether the consumer exists at all. A container caller imports this # into the CONTAINER's module set, so it vanishes with the container. A # host caller imports it at the host's top level, where `imports` is # unconditional — without this, a hive with the consumer off still gets # a bundle oneshot *and* a `systemd.services.` conjured by # `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`. enable ? true, }: let dir = bundleDirFor name; bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; in { _file = "hive-ca-trust.nix#trustBundle:${name}"; config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) ( { ${unit} = { description = "assemble ${name} TLS trust bundle (system CAs + hive CA)"; wantedBy = [ "multi-user.target" ]; before = map (c: "${c}.service") consumers; # Only for a host consumer: the CA file is written at runtime by # `hive-tls-ca.service`, and reading it directly means waiting # for it. A container consumer reads a bind mount instead, and # its `container@` unit carries the equivalent wait. after = lib.optionals hostUnit [ "hive-tls-ca.service" ]; requires = lib.optionals hostUnit [ "hive-tls-ca.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = unit; }; path = [ pkgs.coreutils pkgs.gnugrep ]; script = '' set -euo pipefail install -d -m 0755 ${dir} tmp=${bundlePath}.tmp # Count the HIVE half on its own, before assembling. # # Inspecting the assembled file cannot work: the system store # always holds certificates, so "does the result contain a # certificate" passes unconditionally — including in the one case # worth catching, where this source contributed nothing. And the # public CAs are irrelevant to what this bundle is for: every name # the consumer verifies is issued by our own CA, so a bundle of # nothing but public CAs is, for this purpose, an empty one that # measures as full. # # One helper, because the safe form is not the obvious one and # three copies of a subtle idiom is three chances to get it wrong. # # `grep -c` has two different zero-ish outcomes and `set -e` is # on: an EMPTY file prints `0` and exits 1, a MISSING file prints # nothing and exits 2. So `|| true` leaves the variable empty on # the second — the `-eq` then dies with "integer expected" # instead of reporting — and `|| echo 0` appends a second zero on # the first, yielding `00`. Normalising afterwards is the only # form that survives both. certs() { n=$(grep -c 'BEGIN CERTIFICATE' "$1" 2>/dev/null || true) if [ -z "$n" ]; then n=0; fi printf '%s' "$n" } contributed=$(certs ${source}) if [ "$contributed" -eq 0 ]; then echo "${unit}: ${source} contributed no certificate to the bundle" >&2 exit 1 fi system=$(certs /etc/ssl/certs/ca-certificates.crt) cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp" # `cat` of a source truncated between the count and the copy still # exits 0, so the result is checked against what both halves # brought rather than merely for being non-empty. total=$(certs "$tmp") if [ "$total" -ne $((system + contributed)) ]; then echo "${unit}: bundle has $total certificates, expected $system + $contributed" >&2 exit 1 fi # ⚠️ Scope: this proves the anchor was CONTRIBUTED, not that it is # USABLE. A consumer that reads only the first certificate of the # file ignores it anyway — that is what took the swarm collector # down — and no check on this file can see it. Only a real # handshake can. Do not read a green assembly as a working trust # store. chmod 0644 "$tmp" mv "$tmp" ${bundlePath} ''; }; } // lib.genAttrs consumers (_: { requires = [ "${unit}.service" ]; after = [ "${unit}.service" ]; environment.SSL_CERT_FILE = bundlePath; }) ); }; }