diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 468b3bcb..9b8cd95a 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -62,4 +62,96 @@ in after = [ "hive-tls-ca.service" ]; requires = [ "hive-tls-ca.service" ]; }; + + # Assemble system CAs + the hive CA into one bundle, for runtimes whose + # trust variable *replaces* the default store (Go's `SSL_CERT_FILE`, + # rustls-native-certs) rather than adding to it — pointing those at + # `caContainerPath` alone would drop every public anchor. + # + # The header above says consumption stays at the call site, and the *env + # var* still does. The assembly does not: four containers were each + # hand-rolling this concat, which is how they came to share one defect — + # `wantedBy` + `before` express ordering but not success, so a failed + # assembly let the consumer start against a missing file and end up + # trusting **nothing**, which fails every outbound TLS call while the unit + # looks healthy. + # + # Two things this does that a hand-rolled version kept getting wrong: + # - `requires` on the CONSUMER, so a failed bundle stops it — and the + # dependency is visible in `systemctl status `, where someone + # debugging a TLS failure actually looks. + # - assemble to a temp path, check the result is non-empty and actually + # contains a certificate, and only then move it into place. `cat` of an + # empty bind exits 0, so `set -e` alone does not catch it, and a partial + # bundle must never appear under the final name. + # + # Returns a MODULE to import inside the container, not an attrset to splice + # in — same shape as `swarm-container-resolver.nix`, and for a concrete + # reason: a caller that already writes `systemd.services. = …` + # cannot also write `systemd.services = …` in the same attrset, so anything + # returning bare services forces the call site to be restructured. + # + # imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats"; consumers = [ "swarm-nats-auth" ]; }) ]; + # + # It sets `SSL_CERT_FILE` on each consumer itself. That is the right default + # for this helper's audience — runtimes whose trust variable *replaces* the + # store. A runtime with an additive variable (Node's `NODE_EXTRA_CA_CERTS`, + # hive-ci) needs no bundle at all and should not use this. + # + # `consumers` are BARE unit names (no `.service`): they are used both as + # `systemd.services` attribute keys, which must not carry the suffix, and + # inside `before`/`requires`, which must. Getting that backwards produces an + # edge to a unit that does not exist — which systemd accepts in silence, + # ordering nothing. + trustBundle = + { + name, + consumers, + pkgs, + }: + let + dir = "/run/${name}-ca"; + bundlePath = "${dir}/trust-bundle.pem"; + unit = "${name}-ca-bundle"; + in + { + _file = "hive-ca-trust.nix#trustBundle:${name}"; + config.systemd.services = lib.optionalAttrs useSelfSigned ( + { + ${unit} = { + description = "assemble ${name} TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "multi-user.target" ]; + before = map (c: "${c}.service") consumers; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = unit; + }; + path = [ + pkgs.coreutils + pkgs.gnugrep + ]; + script = '' + set -euo pipefail + install -d -m 0755 ${dir} + tmp=${bundlePath}.tmp + cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} > "$tmp" + # `cat` of an empty or missing-but-mounted source exits 0, so the + # result has to be inspected rather than the command trusted. + if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then + echo "${unit}: assembled bundle contains no certificate" >&2 + exit 1 + fi + chmod 0644 "$tmp" + mv "$tmp" ${bundlePath} + ''; + }; + } + // lib.genAttrs consumers (_: { + requires = [ "${unit}.service" ]; + after = [ "${unit}.service" ]; + environment.SSL_CERT_FILE = bundlePath; + }) + ); + }; } diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index bfb83083..a29121e5 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -23,6 +23,22 @@ let clientAccount = "APP"; machine = "swarm-nats"; + + tlsCfg = config.services.hyperhive.tls; + gatewayCfg = config.services.hyperhive.gateway; + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + # The responder introspects authelia over https BY NAME. Its HTTP client is + # reqwest/rustls, and `rustls-platform-verifier` resolves roots through + # `rustls-native-certs`, which reads `SSL_CERT_FILE` — so the same assembled + # bundle the Go containers use applies here. Without it the handshake fails + # `UnknownIssuer`, introspection fails, and the responder denies *every* + # client: one missing trust anchor surfacing as `authorization violation` at + # every would-be queue user. + caBundleModule = caTrust.trustBundle { + inherit pkgs; + name = machine; + consumers = [ "swarm-nats-auth" ]; + }; # Where the responder's credentials live *inside* the container, and the # host path that resolves to. Two names for one location, because the # host is the only place both filesystems are addressable. @@ -438,6 +454,9 @@ in # unauthenticated interim state would be a hole rather than a # rough edge. privateNetwork = false; + # Binds only the public trust bundle, read-only. Empty when the gateway + # is not self-signed, so the whole trust path drops out cleanly. + bindMounts = caTrust.bindMount; config = { ... }: { @@ -450,6 +469,7 @@ in # file exists. dnsConsumers = [ "swarm-nats-auth.service" ]; }) + caBundleModule ]; system.stateVersion = "26.05"; @@ -575,6 +595,11 @@ in # refuses to start when a bind source is missing, so one absent seed # would take down the **whole container including the queue**, not # merely the responder. A far larger blast radius than the fault. + # Order the container after the host CA generator, so the bind source + # exists before nspawn sets the mount up. Without it a late CA fails the + # container start outright rather than degrading. + systemd.services."container@${machine}" = caTrust.containerOrdering; + systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured { description = "deliver the swarm queue responder's credentials"; before = [ "container@swarm-nats.service" ];