From 041779d910712d47732dc010eca5835598b6967a Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 12:58:48 +0200 Subject: [PATCH] tls: a CA root change no longer restarts the store or bounces containers --- docs/swarm/ca.md | 37 +++++++++++----- nix/host-modules/hive-tls.nix | 19 +++++---- nix/host-modules/lib/hive-ca-trust.nix | 58 +++++++++++++++++++++++++- nix/module-eval/hive-tls.nix | 41 ++++++++++++++++++ 4 files changed, 134 insertions(+), 21 deletions(-) diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index ebd9b050..e83ddc0e 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -81,23 +81,40 @@ the file every consumer anchors on. It holds one certificate, a copy of `swarm-services-root.pem`: the store's `pki` root, which the gateway leaf and the swarm-services leaf both chain to. It's the only root the swarm has. -**Without the store's root, the bundle is empty and its readers go red.** +**Without the store's root, the bundle is empty and its readers go red until +it lands.** While `/swarm-services-root.pem` is missing or empty, as on a first boot before the store answers, `hive-tls-ca.service` writes an empty bundle and logs that file's path. Every `-ca-bundle` unit then fails with ` contributed no certificate to the bundle`, and the consumers that `Requires=` it don't start. -`swarm-services-cert.service` restarts `hive-tls-ca.service` when it writes a -new root. +`swarm-services-cert.service` reloads `hive-tls-ca.service` when it writes a +new root. That rewrites the bundle in place and restarts no container, the +store's included. -A deploy that changes `hive-tls-ca.service` reloads it: the same script -rewrites the bundle in place, and nothing that `Requires=` the unit restarts, -including the store's container. A `-ca-bundle` unit keeps the copy it -made when it started. +**Each `-ca-bundle` unit converges on its own.** Beside it, a +`-ca-bundle-refresh.path` unit watches the file the bundle derives +from: inside a container the bound `/run/hive-ca/trust-bundle.pem`, on the host +`/trust-bundle.pem`. A write to it starts +`-ca-bundle-refresh.service`. That service does nothing while the file +holds no certificate, or while it matches the copy the last assembly took +(`/run/-ca/anchor.pem`). Otherwise it restarts the bundle unit and starts +its consumers without waiting. The restart reaches the running consumers +through their `Requires=`; the start picks up those a first boot left failed. +The same service runs once at boot, after the bundle unit, which covers a root +that landed before the watch began. A first boot needs no second apply, and a +root rotation restarts the consumers once and the container never. The watch +fires for the host's in-place rewrite because inotify keys on the inode and a +bind mount shares it, so `hive-tls-ca` must keep rewriting the file in place +and never rename over it. -⚠️ **A consumer reads its trust when it starts.** An agent or service -container that last started before the store's root reached the bundle -fails TLS to a gateway serving a store-issued leaf, until it restarts. +A deploy that changes `hive-tls-ca.service` reloads it too, with the same +effect. + +⚠️ **A consumer outside `trustBundle` reads its trust when it starts.** A +container whose nix predates the refresh units keeps the bundle it started +with until it restarts, and fails TLS to a gateway serving a store-issued leaf +in the meantime. ## The swarm's service names come from the secret store diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 16679cc7..786d79da 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -400,9 +400,9 @@ in # # What that costs: the trust bundle written at the end of this script # is the services root, so on a first boot it is written before that - # file exists and is empty until a restart. The `if [ -s + # file exists and is empty until the script runs again. The `if [ -s # "$svcroot" ]` guard is what makes that state legal rather than fatal, - # and `swarm-services-cert` restarts this unit once the root it wrote + # and `swarm-services-cert` reloads this unit once the root it wrote # is new — so the bundle converges instead of being a boot behind. # # ⚠️ A switch that changes this unit reloads it, running the same @@ -411,8 +411,8 @@ in # comes back sealed under the `shamir` seal, every container that # binds the bundle, and every `-ca-bundle` unit with its # consumers. A reload rewrites the bundle in place and touches none of - # them, so a `-ca-bundle` unit keeps the copy it made at start - # until it restarts. + # them; each `-ca-bundle` unit watches the file it is built from + # and rebuilds itself and its consumers (`lib/hive-ca-trust.nix`). reloadIfChanged = true; reload = config.systemd.services.hive-tls-ca.script; serviceConfig = { @@ -637,7 +637,7 @@ in # gateway's cert import down with it, and nginx never started at all — # so the swarm served nothing rather than serving the wrong name. The # bundle reaches the root the other way round, at the end of the script - # below: it restarts `hive-tls-ca` when the root CHANGED, which is the + # below: it reloads `hive-tls-ca` when the root CHANGED, which is the # same path that already covered a store coming up hours late. before = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; requiredBy = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; @@ -726,7 +726,7 @@ in chmod 0644 "$svcroot" if systemctl is-active --quiet hive-tls-ca.service; then echo "the services root changed — rebuilding the trust bundle" - systemctl restart --no-block hive-tls-ca.service + systemctl reload --no-block hive-tls-ca.service fi exit 0'' else @@ -896,12 +896,13 @@ in # The bundle is assembled by `hive-tls-ca`, which runs BEFORE this # unit — it has to, or the store's container cannot start and there # is nothing to ask for a leaf. So on a first boot it wrote a bundle - # without the root below, and this restart is what completes it + # without the root below, and this reload is what completes it # rather than a retry-path fallback. Only on a CHANGED root: every - # boot otherwise bounces a unit with nothing to do. + # boot otherwise reruns a unit with nothing to do. A reload, not a + # restart: a restart restarts every unit that `Requires=` it. if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then echo "the services root changed — rebuilding the trust bundle" - systemctl restart --no-block hive-tls-ca.service + systemctl reload --no-block hive-tls-ca.service fi ''; }; diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index f945bafa..a6594947 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -139,9 +139,23 @@ in bundlePath = bundlePathFor name; unit = "${name}-ca-bundle"; source = if hostUnit then caHostPath else caContainerPath; + # The source as the bundle was last built from it. + anchor = "${dir}/anchor.pem"; + refresh = "${unit}-refresh"; in { _file = "hive-ca-trust.nix#trustBundle:${name}"; + # `hive-tls-ca` rewrites the source in place, so a watch on it sees the + # write from inside a container: inotify is per inode, and a bind mount + # shares the inode. `PathChanged` fires on close-after-write and on + # `chmod`, so `${refresh}` is idempotent. + config.systemd.paths = lib.optionalAttrs (useSelfSigned && enable) { + ${refresh} = { + description = "trigger ${refresh} when the trust bundle source is written"; + wantedBy = [ "multi-user.target" ]; + pathConfig.PathChanged = source; + }; + }; config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) ( { ${unit} = { @@ -167,6 +181,10 @@ in set -euo pipefail install -d -m 0755 ${dir} tmp=${bundlePath}.tmp + # One snapshot of the source feeds the count, the bundle and + # the copy `${refresh}` compares the live source against, so + # the three cannot describe different roots. + cp -f ${source} ${anchor}.tmp 2>/dev/null || : > ${anchor}.tmp # Count the HIVE half on its own, before assembling. # # Inspecting the assembled file cannot work: the system store @@ -193,13 +211,13 @@ in if [ -z "$n" ]; then n=0; fi printf '%s' "$n" } - contributed=$(certs ${source}) + contributed=$(certs ${anchor}.tmp) if [ "$contributed" -eq 0 ]; then echo "${unit}: ${source} contributed no certificate to the bundle" >&2 exit 1 fi system=$(certs /etc/ssl/certs/ca-certificates.crt) - cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp" + cat /etc/ssl/certs/ca-certificates.crt ${anchor}.tmp > "$tmp" # `cat` of a source truncated between the count and the copy still # exits 0, so the result is checked against what both halves # brought rather than merely for being non-empty. @@ -215,9 +233,45 @@ in # handshake can. Do not read a green assembly as a working trust # store. chmod 0644 "$tmp" + mv ${anchor}.tmp ${anchor} mv "$tmp" ${bundlePath} ''; }; + + # Re-runs the bundle unit, and with it every consumer, when the + # source holds a root the bundle was not built from. Separate from + # the bundle unit because that one is `Before=` its consumers: a + # restart issued from inside it would wait on jobs that wait on it. + # `--no-block` keeps this one from waiting on them too. + # + # A restart of `${unit}` restarts every active unit that + # `Requires=` it; the explicit `start` is for the consumers that are + # not active, which a first boot leaves failed on the dependency. + ${refresh} = { + description = "rebuild ${name} TLS trust bundle when its source changes"; + wantedBy = [ "multi-user.target" ]; + after = [ "${unit}.service" ]; + serviceConfig = { + Type = "oneshot"; + SyslogIdentifier = refresh; + }; + path = [ + pkgs.coreutils + pkgs.diffutils + pkgs.gnugrep + pkgs.systemd + ]; + script = '' + set -euo pipefail + # No root yet: nothing to converge on. The write that fills the + # source fires this unit again. + if ! grep -q 'BEGIN CERTIFICATE' ${source}; then exit 0; fi + if cmp -s ${source} ${anchor}; then exit 0; fi + echo "${source} changed — rebuilding the ${name} trust bundle" + systemctl restart --no-block ${unit}.service + systemctl start --no-block ${lib.concatMapStringsSep " " (c: "${c}.service") consumers} + ''; + }; } // lib.genAttrs consumers (_: { requires = [ "${unit}.service" ]; diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 51ba656f..ed77a93a 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -460,6 +460,47 @@ let && lib.elem "hive-tls-ca.service" store.requires && lib.elem "hive-tls-ca.service" store.after; } + { + # The same holds at runtime: a restart of the unit propagates through + # `Requires=` exactly as a switch's stop would. Both the root-only + # fetch and the issue path rebuild the bundle on a changed root. + name = "a changed services root reloads the trust bundle's unit and never restarts it"; + ok = + let + reloads = lib.hasInfix "systemctl reload --no-block hive-tls-ca.service"; + restarts = lib.hasInfix "restart --no-block hive-tls-ca.service"; + in + reloads boot.script && !(restarts boot.script) && reloads noSvcFetch && !(restarts noSvcFetch); + } + { + # Nothing bounces a container on a root change, so a bundle unit that + # failed on an empty source has to recover by itself: a path unit on the + # bound file starts a refresh, which restarts the bundle unit and starts + # its consumers. The refresh is not `Before=` them (it would deadlock on + # its own restart) and `--no-block` keeps it from waiting. The controls + # are the bundle unit itself, still `Before=` its consumer, and a bare + # host, which has no poller and so none of its units. + name = "an in-container trust bundle rebuilds and restarts its consumers when the bound source is written"; + ok = + let + bao = allLocal.containers.swarm-bao.config; + refresh = bao.systemd.services.swarm-bao-ca-bundle-refresh; + path = bao.systemd.paths.swarm-bao-ca-bundle-refresh; + bundle = bao.systemd.services.swarm-bao-ca-bundle; + in + path.pathConfig.PathChanged == "/run/hive-ca/trust-bundle.pem" + && lib.elem "multi-user.target" path.wantedBy + && lib.hasInfix "systemctl restart --no-block swarm-bao-ca-bundle.service" refresh.script + && lib.hasInfix "systemctl start --no-block opentelemetry-collector.service" refresh.script + && lib.hasInfix "cmp -s /run/hive-ca/trust-bundle.pem /run/swarm-bao-ca/anchor.pem" refresh.script + && lib.elem "swarm-bao-ca-bundle.service" refresh.after + && !(lib.elem "opentelemetry-collector.service" (refresh.before or [ ])) + && lib.elem "opentelemetry-collector.service" bundle.before + && lib.hasInfix "mv /run/swarm-bao-ca/anchor.pem.tmp /run/swarm-bao-ca/anchor.pem" bundle.script + && allLocal.systemd.paths ? hive-c0re-ca-bundle-refresh + && shared.bareWithController.systemd.paths ? hive-c0re-ca-bundle-refresh + && !(shared.bareWithController.systemd.paths ? swarm-ci-metrics-ca-bundle-refresh); + } { # A missing root is the one case where the bundle is empty, and every # `-ca-bundle` unit fails on it; the log names the file.