From 850cc2c1d3bf82c16a6b1e3b57f6b8a5670bcf31 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 17 Aug 2026 19:40:31 +0200 Subject: [PATCH] fix(#3363): give the queue's auth responder the hive CA The responder introspects authelia over https by name. It had no CA trust at all, so the handshake failed UnknownIssuer, introspection failed, and it denied every client -- surfacing at the controller as a 60s authorization-violation loop, two layers from the cause. Adds a shared trustBundle helper to lib/hive-ca-trust.nix rather than a fifth hand-rolled concat. Four containers were each assembling this themselves, which is how they came to share one defect: wantedBy + before express ordering but not success, so a failed assembly let the consumer start against a missing file and trust nothing at all. The helper fixes both halves of that. requires goes on the consumer, so a failed bundle stops it and the dependency is visible in systemctl status where someone debugging a TLS failure looks. And the script assembles to a temp path, checks the result actually contains a certificate, and only then moves it into place -- cat of an empty bind exits 0, so set -e does not catch it and a partial bundle must never appear under the final name. Returns a module rather than bare services: a caller that already writes systemd.services. cannot also write systemd.services in the same attrset. --- nix/host-modules/lib/hive-ca-trust.nix | 92 ++++++++++++++++++++++++++ nix/host-modules/swarm-nats.nix | 25 +++++++ 2 files changed, 117 insertions(+) diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 468b3bcb..9b8cd95a 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -62,4 +62,96 @@ in after = [ "hive-tls-ca.service" ]; requires = [ "hive-tls-ca.service" ]; }; + + # Assemble system CAs + the hive CA into one bundle, for runtimes whose + # trust variable *replaces* the default store (Go's `SSL_CERT_FILE`, + # rustls-native-certs) rather than adding to it — pointing those at + # `caContainerPath` alone would drop every public anchor. + # + # The header above says consumption stays at the call site, and the *env + # var* still does. The assembly does not: four containers were each + # hand-rolling this concat, which is how they came to share one defect — + # `wantedBy` + `before` express ordering but not success, so a failed + # assembly let the consumer start against a missing file and end up + # trusting **nothing**, which fails every outbound TLS call while the unit + # looks healthy. + # + # Two things this does that a hand-rolled version kept getting wrong: + # - `requires` on the CONSUMER, so a failed bundle stops it — and the + # dependency is visible in `systemctl status `, where someone + # debugging a TLS failure actually looks. + # - assemble to a temp path, check the result is non-empty and actually + # contains a certificate, and only then move it into place. `cat` of an + # empty bind exits 0, so `set -e` alone does not catch it, and a partial + # bundle must never appear under the final name. + # + # Returns a MODULE to import inside the container, not an attrset to splice + # in — same shape as `swarm-container-resolver.nix`, and for a concrete + # reason: a caller that already writes `systemd.services. = …` + # cannot also write `systemd.services = …` in the same attrset, so anything + # returning bare services forces the call site to be restructured. + # + # imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats"; consumers = [ "swarm-nats-auth" ]; }) ]; + # + # It sets `SSL_CERT_FILE` on each consumer itself. That is the right default + # for this helper's audience — runtimes whose trust variable *replaces* the + # store. A runtime with an additive variable (Node's `NODE_EXTRA_CA_CERTS`, + # hive-ci) needs no bundle at all and should not use this. + # + # `consumers` are BARE unit names (no `.service`): they are used both as + # `systemd.services` attribute keys, which must not carry the suffix, and + # inside `before`/`requires`, which must. Getting that backwards produces an + # edge to a unit that does not exist — which systemd accepts in silence, + # ordering nothing. + trustBundle = + { + name, + consumers, + pkgs, + }: + let + dir = "/run/${name}-ca"; + bundlePath = "${dir}/trust-bundle.pem"; + unit = "${name}-ca-bundle"; + in + { + _file = "hive-ca-trust.nix#trustBundle:${name}"; + config.systemd.services = lib.optionalAttrs useSelfSigned ( + { + ${unit} = { + description = "assemble ${name} TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "multi-user.target" ]; + before = map (c: "${c}.service") consumers; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = unit; + }; + path = [ + pkgs.coreutils + pkgs.gnugrep + ]; + script = '' + set -euo pipefail + install -d -m 0755 ${dir} + tmp=${bundlePath}.tmp + cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} > "$tmp" + # `cat` of an empty or missing-but-mounted source exits 0, so the + # result has to be inspected rather than the command trusted. + if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then + echo "${unit}: assembled bundle contains no certificate" >&2 + exit 1 + fi + chmod 0644 "$tmp" + mv "$tmp" ${bundlePath} + ''; + }; + } + // lib.genAttrs consumers (_: { + requires = [ "${unit}.service" ]; + after = [ "${unit}.service" ]; + environment.SSL_CERT_FILE = bundlePath; + }) + ); + }; } diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index bfb83083..a29121e5 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -23,6 +23,22 @@ let clientAccount = "APP"; machine = "swarm-nats"; + + tlsCfg = config.services.hyperhive.tls; + gatewayCfg = config.services.hyperhive.gateway; + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + # The responder introspects authelia over https BY NAME. Its HTTP client is + # reqwest/rustls, and `rustls-platform-verifier` resolves roots through + # `rustls-native-certs`, which reads `SSL_CERT_FILE` — so the same assembled + # bundle the Go containers use applies here. Without it the handshake fails + # `UnknownIssuer`, introspection fails, and the responder denies *every* + # client: one missing trust anchor surfacing as `authorization violation` at + # every would-be queue user. + caBundleModule = caTrust.trustBundle { + inherit pkgs; + name = machine; + consumers = [ "swarm-nats-auth" ]; + }; # Where the responder's credentials live *inside* the container, and the # host path that resolves to. Two names for one location, because the # host is the only place both filesystems are addressable. @@ -438,6 +454,9 @@ in # unauthenticated interim state would be a hole rather than a # rough edge. privateNetwork = false; + # Binds only the public trust bundle, read-only. Empty when the gateway + # is not self-signed, so the whole trust path drops out cleanly. + bindMounts = caTrust.bindMount; config = { ... }: { @@ -450,6 +469,7 @@ in # file exists. dnsConsumers = [ "swarm-nats-auth.service" ]; }) + caBundleModule ]; system.stateVersion = "26.05"; @@ -575,6 +595,11 @@ in # refuses to start when a bind source is missing, so one absent seed # would take down the **whole container including the queue**, not # merely the responder. A far larger blast radius than the fault. + # Order the container after the host CA generator, so the bind source + # exists before nspawn sets the mount up. Without it a late CA fails the + # container start outright rather than degrading. + systemd.services."container@${machine}" = caTrust.containerOrdering; + systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured { description = "deliver the swarm queue responder's credentials"; before = [ "container@swarm-nats.service" ];