vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

tls: a CA root change no longer restarts the store or bounces containers

Browse source
This commit is contained in:
atlas 2026-10-17 12:58:48 +02:00 • committed by mara
commit 041779d910
4 changed files with 136 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

19
nix/host-modules/hive-tls.nix
View file

@ -400,9 +400,9 @@ in
#
# What that costs: the trust bundle written at the end of this script
# is the services root, so on a first boot it is written before that
# file exists and is empty until a restart. The `if [ -s
# file exists and is empty until the script runs again. The `if [ -s
# "$svcroot" ]` guard is what makes that state legal rather than fatal,
# and `swarm-services-cert` restarts this unit once the root it wrote
# and `swarm-services-cert` reloads this unit once the root it wrote
# is new — so the bundle converges instead of being a boot behind.
#
# ⚠️ A switch that changes this unit reloads it, running the same
@ -411,8 +411,8 @@ in
# comes back sealed under the `shamir` seal, every container that
# binds the bundle, and every `<name>-ca-bundle` unit with its
# consumers. A reload rewrites the bundle in place and touches none of
# them, so a `<name>-ca-bundle` unit keeps the copy it made at start
# until it restarts.
# them; each `<name>-ca-bundle` unit watches the file it is built from
# and rebuilds itself and its consumers (`lib/hive-ca-trust.nix`).
reloadIfChanged = true;
reload = config.systemd.services.hive-tls-ca.script;
serviceConfig = {
@ -637,7 +637,7 @@ in
# gateway's cert import down with it, and nginx never started at all —
# so the swarm served nothing rather than serving the wrong name. The
# bundle reaches the root the other way round, at the end of the script
# below: it restarts `hive-tls-ca` when the root CHANGED, which is the
# below: it reloads `hive-tls-ca` when the root CHANGED, which is the
# same path that already covered a store coming up hours late.
before = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service";
requiredBy = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service";
@ -726,7 +726,7 @@ in
chmod 0644 "$svcroot"
if systemctl is-active --quiet hive-tls-ca.service; then
echo "the services root changed — rebuilding the trust bundle"
systemctl restart --no-block hive-tls-ca.service
systemctl reload --no-block hive-tls-ca.service
fi
exit 0''
else
@ -896,12 +896,13 @@ in
# The bundle is assembled by `hive-tls-ca`, which runs BEFORE this
# unit — it has to, or the store's container cannot start and there
# is nothing to ask for a leaf. So on a first boot it wrote a bundle
# without the root below, and this restart is what completes it
# without the root below, and this reload is what completes it
# rather than a retry-path fallback. Only on a CHANGED root: every
# boot otherwise bounces a unit with nothing to do.
# boot otherwise reruns a unit with nothing to do. A reload, not a
# restart: a restart restarts every unit that `Requires=` it.
if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then
echo "the services root changed — rebuilding the trust bundle"
systemctl restart --no-block hive-tls-ca.service
systemctl reload --no-block hive-tls-ca.service
fi
'';
};

58
nix/host-modules/lib/hive-ca-trust.nix
View file

@ -139,9 +139,23 @@ in
bundlePath = bundlePathFor name;
unit = "${name}-ca-bundle";
source = if hostUnit then caHostPath else caContainerPath;
# The source as the bundle was last built from it.
anchor = "${dir}/anchor.pem";
refresh = "${unit}-refresh";
in
{
_file = "hive-ca-trust.nix#trustBundle:${name}";
# `hive-tls-ca` rewrites the source in place, so a watch on it sees the
# write from inside a container: inotify is per inode, and a bind mount
# shares the inode. `PathChanged` fires on close-after-write and on
# `chmod`, so `${refresh}` is idempotent.
config.systemd.paths = lib.optionalAttrs (useSelfSigned && enable) {
${refresh} = {
description = "trigger ${refresh} when the trust bundle source is written";
wantedBy = [ "multi-user.target" ];
pathConfig.PathChanged = source;
};
};
config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) (
{
${unit} = {
@ -167,6 +181,10 @@ in
set -euo pipefail
install -d -m 0755 ${dir}
tmp=${bundlePath}.tmp
# One snapshot of the source feeds the count, the bundle and
# the copy `${refresh}` compares the live source against, so
# the three cannot describe different roots.
cp -f ${source} ${anchor}.tmp 2>/dev/null || : > ${anchor}.tmp
# Count the HIVE half on its own, before assembling.
#
# Inspecting the assembled file cannot work: the system store
@ -193,13 +211,13 @@ in
if [ -z "$n" ]; then n=0; fi
printf '%s' "$n"
}
contributed=$(certs ${source})
contributed=$(certs ${anchor}.tmp)
if [ "$contributed" -eq 0 ]; then
echo "${unit}: ${source} contributed no certificate to the bundle" >&2
exit 1
fi
system=$(certs /etc/ssl/certs/ca-certificates.crt)
cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp"
cat /etc/ssl/certs/ca-certificates.crt ${anchor}.tmp > "$tmp"
# `cat` of a source truncated between the count and the copy still
# exits 0, so the result is checked against what both halves
# brought rather than merely for being non-empty.
@ -215,9 +233,45 @@ in
# handshake can. Do not read a green assembly as a working trust
# store.
chmod 0644 "$tmp"
mv ${anchor}.tmp ${anchor}
mv "$tmp" ${bundlePath}
'';
};
# Re-runs the bundle unit, and with it every consumer, when the
# source holds a root the bundle was not built from. Separate from
# the bundle unit because that one is `Before=` its consumers: a
# restart issued from inside it would wait on jobs that wait on it.
# `--no-block` keeps this one from waiting on them too.
#
# A restart of `${unit}` restarts every active unit that
# `Requires=` it; the explicit `start` is for the consumers that are
# not active, which a first boot leaves failed on the dependency.
${refresh} = {
description = "rebuild ${name} TLS trust bundle when its source changes";
wantedBy = [ "multi-user.target" ];
after = [ "${unit}.service" ];
serviceConfig = {
Type = "oneshot";
SyslogIdentifier = refresh;
};
path = [
pkgs.coreutils
pkgs.diffutils
pkgs.gnugrep
pkgs.systemd
];
script = ''
set -euo pipefail
# No root yet: nothing to converge on. The write that fills the
# source fires this unit again.
if ! grep -q 'BEGIN CERTIFICATE' ${source}; then exit 0; fi
if cmp -s ${source} ${anchor}; then exit 0; fi
echo "${source} changed — rebuilding the ${name} trust bundle"
systemctl restart --no-block ${unit}.service
systemctl start --no-block ${lib.concatMapStringsSep " " (c: "${c}.service") consumers}
'';
};
}
// lib.genAttrs consumers (_: {
requires = [ "${unit}.service" ];