0befa4f36d
A hive whose CA predates the swarm root keeps a self-signed CA, and nothing re-roots it. That is what makes the hierarchy non-disruptive, and it also means the O(1)-trust payoff never arrives for that hive. Adoption now happens by itself where this host owns the root, and nowhere else. The split is the whole design: adoption invalidates an anchor consumers already trust, and they refresh on their own schedule — on one box that schedule is knowable, across hosts it is not. So the all-local case migrates itself and every other case gets a failure that names both files, gives the two-command recipe, says why it is not automatic, and offers the marker as a deliberate opt-out. The previous CA rides in the trust bundle afterwards. Consumers read the bundle rather than ca.pem, so adoption is additive before it is subtractive; agents pick up new trust only when their container restarts, which is a window even on a single host. Dropping the old anchor stays a separate, deliberate step.
453 lines
21 KiB
Nix
453 lines
21 KiB
Nix
{
|
|
lib,
|
|
config,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.tls;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
swarmCaCfg = config.services.hyperhive.swarm.ca;
|
|
domain = hyperhiveCfg.domain;
|
|
|
|
# The host-managed hive CA is the trust anchor for self-signed mode.
|
|
# It is only stood up when the gateway actually serves a self-signed
|
|
# cert: the gateway must be in self-signed mode. `domain` is required
|
|
# (asserted in hive-network.nix), so the leaf SANs always have a
|
|
# domain to derive from. The self-signed condition is the gateway
|
|
# module's single source of truth (`gateway.useSelfSigned`): true when
|
|
# neither an operator cert (`tls.certDir`) nor ACME is set.
|
|
active = hyperhiveCfg.enable && gatewayCfg.useSelfSigned;
|
|
|
|
# The leaf-signing action shared by the boot-time `hive-tls-ca`
|
|
# generation and the weekly `hive-tls-resign` renewal: fresh key +
|
|
# CSR, SAN ext-file, sign under the (stable) CA, tighten modes.
|
|
# Takes the TLS state dir as `$1`; each caller keeps its own
|
|
# when-to-sign condition. The leaf covers the bare hive domain plus
|
|
# `forge.`, `matrix.` and `*.<domain>` so all sub-domains validate
|
|
# under the same cert + the hive CA.
|
|
# How this hive's CA comes into existence when it is missing — and it
|
|
# is one of exactly two things, chosen by config rather than by what
|
|
# happens to be on disk.
|
|
#
|
|
# Issuing a sub-CA requires the swarm root's PRIVATE key, so it can
|
|
# only happen where that key legitimately lives: the single-host
|
|
# deployment that `swarm.ca.autoConfigure` describes. A host cannot
|
|
# infer that it is that host — a swarm's services and its hives can
|
|
# sit anywhere — so with the flag off this hive self-signs exactly as
|
|
# it always has, and an operator who wants it in the hierarchy
|
|
# installs the CA themselves. Falling back to self-signed rather than
|
|
# failing keeps a plain hive working out of the box; what it loses is
|
|
# membership of a swarm's trust, which is the correct thing to lose
|
|
# for a hive nobody has federated.
|
|
caGenScript =
|
|
if swarmCaCfg.autoConfigure then
|
|
''
|
|
# The root should exist — `swarm-ca.service` runs before this and
|
|
# is required by it. If it doesn't, something upstream failed and
|
|
# signing with a half-provisioned root would be worse than stopping.
|
|
if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then
|
|
echo "swarm.ca.autoConfigure is set but there is no root CA key at $rootk" >&2
|
|
echo "(swarm-ca.service should have generated it) — refusing to issue a hive CA." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "issuing fresh hive CA at $ca under the swarm root"
|
|
cacsr="$(mktemp "$d/ca.csr.XXXXXX")"
|
|
caext="$(mktemp "$d/ca.ext.XXXXXX")"
|
|
trap 'rm -f "$cacsr" "$caext"' EXIT
|
|
|
|
openssl req -newkey rsa:4096 -nodes -sha256 \
|
|
-keyout "$cak" -out "$cacsr" \
|
|
-subj "/CN=hive-ca ${domain}"
|
|
|
|
# printf (not a heredoc) so the ext-file lines carry no leading
|
|
# whitespace once nix has stripped the indented-string indent.
|
|
{
|
|
printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n'
|
|
printf 'keyUsage=critical,keyCertSign,cRLSign\n'
|
|
printf 'subjectKeyIdentifier=hash\n'
|
|
printf 'authorityKeyIdentifier=keyid:always\n'
|
|
# The constraint is the point of the hierarchy, not a
|
|
# flourish: without it a leaked hive CA mints any name in
|
|
# the swarm, and it is verifiers that enforce this, not our
|
|
# good behaviour. The IP exclusions are not redundant — a
|
|
# DNS constraint says nothing about an iPAddress SAN, and a
|
|
# name type nobody constrained is a name type this CA is
|
|
# unconstrained for.
|
|
printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \
|
|
${lib.escapeShellArg domain}
|
|
} > "$caext"
|
|
|
|
openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \
|
|
-CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \
|
|
-extfile "$caext" -out "$ca"
|
|
rm -f "$cacsr" "$caext"
|
|
trap - EXIT
|
|
''
|
|
else
|
|
''
|
|
echo "generating fresh self-signed hive CA at $ca"
|
|
openssl req -x509 -newkey rsa:4096 -nodes -sha256 \
|
|
-days ${toString cfg.caValidityDays} \
|
|
-keyout "$cak" -out "$ca" \
|
|
-subj "/CN=hive-ca ${domain}" \
|
|
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
|
-addext "keyUsage=critical,keyCertSign,cRLSign"
|
|
'';
|
|
|
|
signLeafScript = pkgs.writeShellScript "hive-tls-sign-leaf" ''
|
|
set -euo pipefail
|
|
d="$1"
|
|
ca="$d/ca.pem"
|
|
cak="$d/ca-key.pem"
|
|
leaf="$d/gateway.pem"
|
|
leafk="$d/gateway-key.pem"
|
|
csr="$(mktemp "$d/gateway.csr.XXXXXX")"
|
|
ext="$(mktemp "$d/leaf.ext.XXXXXX")"
|
|
only="$(mktemp "$d/gateway.leaf.XXXXXX")"
|
|
trap 'rm -f "$csr" "$ext" "$only"' EXIT
|
|
|
|
openssl req -newkey rsa:4096 -nodes -sha256 \
|
|
-keyout "$leafk" -out "$csr" \
|
|
-subj "/CN=${domain}"
|
|
|
|
# printf (not a heredoc) so the ext-file lines carry no leading
|
|
# whitespace once nix has stripped the indented-string indent.
|
|
{
|
|
printf 'subjectAltName=DNS:%s,DNS:forge.%s,DNS:matrix.%s,DNS:*.%s\n' \
|
|
${lib.escapeShellArg domain} ${lib.escapeShellArg domain} \
|
|
${lib.escapeShellArg domain} ${lib.escapeShellArg domain}
|
|
printf 'basicConstraints=critical,CA:FALSE\n'
|
|
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
|
|
printf 'extendedKeyUsage=serverAuth\n'
|
|
} > "$ext"
|
|
|
|
openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \
|
|
-CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \
|
|
-extfile "$ext" -out "$only"
|
|
|
|
# nginx serves this file verbatim, so it must carry the leaf AND its
|
|
# issuer: the hive CA is an intermediate under the swarm root, and a
|
|
# client that anchors on the root cannot build the middle of the
|
|
# chain by itself. Agents anchor on the hive CA directly and
|
|
# validated either way — the appended cert is what makes a swarm
|
|
# peer, or anything else holding only the root, work.
|
|
# Leaf first: both `ssl_certificate` and the `openssl x509 -in`
|
|
# expiry checks read the first cert in the file.
|
|
cat "$only" "$ca" > "$leaf"
|
|
chmod 0600 "$leafk"
|
|
chmod 0644 "$leaf"
|
|
'';
|
|
in
|
|
{
|
|
# Host-side TLS trust root for the self-signed gateway mode.
|
|
#
|
|
# A bare self-signed leaf would be its own trust anchor, so every
|
|
# regeneration would be a new anchor and every consumer (agents,
|
|
# federation peers) would have to re-trust on each rotation — and a
|
|
# runtime-generated, in-container cert can't be wired into an agent's
|
|
# build-time trust store at all.
|
|
#
|
|
# So the issuer is a long-lived **hive CA** held on the host. The
|
|
# gateway serves a **leaf** signed by that CA (via the `tls.certDir`
|
|
# bind-mount path); agents and federation peers trust the *CA* once,
|
|
# and leaf rotation never re-breaks them. See `docs/gateway.md`
|
|
# ("Self-signed TLS").
|
|
#
|
|
# That CA is self-signed by default. Under
|
|
# `swarm.ca.autoConfigure` — the all-on-one-host case — it is instead
|
|
# an intermediate issued under the swarm root (./swarm-ca.nix owns the
|
|
# why) and name-constrained to this hive's domain, so it stays the
|
|
# anchor agents pin while a peer holding only the root can validate
|
|
# everything this hive serves. Either way an existing CA is left
|
|
# alone; see the issuance comment below.
|
|
|
|
options.services.hyperhive.tls = {
|
|
stateDir = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/var/lib/hive-tls";
|
|
description = ''
|
|
Host directory holding the hive CA + gateway leaf cert for the
|
|
self-signed gateway mode. `ca.pem` (the anchor agents and
|
|
federation peers trust), `ca-key.pem` (0600, never leaves the
|
|
host), `gateway.pem` / `gateway-key.pem` (the leaf the gateway
|
|
container bind-mounts and nginx serves). Persistent so the CA
|
|
survives reboots — re-deriving it would re-break every consumer.
|
|
'';
|
|
};
|
|
|
|
caValidityDays = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 7300;
|
|
description = ''
|
|
Validity window of the hive CA in days (default ~20y). Kept long
|
|
and well beyond `leafValidityDays` so the CA outlives many leaf
|
|
rotations — the whole point of the CA is to be a stable anchor
|
|
that consumers trust once. The CA is regenerated only if missing
|
|
or already expired.
|
|
'';
|
|
};
|
|
|
|
leafValidityDays = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 30;
|
|
description = ''
|
|
Validity window of the gateway leaf cert in days (default 30).
|
|
Short-lived by design — ahead of the CA/Browser-Forum's move
|
|
toward ~47-day max lifetimes — which bounds the blast radius of a
|
|
leaf-key compromise. The leaf is re-signed by the (stable) CA
|
|
when it is missing or near expiry; because it shares the CA
|
|
anchor, a rotation does not disturb consumer trust. Agents and
|
|
federation peers validate against the CA, not browser CA/B-forum
|
|
limits. The weekly `hive-tls-resign` timer re-signs the leaf once
|
|
it is within half its validity of expiry and propagates the new
|
|
leaf into the running gateway, so a long-uptime host renews
|
|
automatically without a reboot.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf active {
|
|
# Generate (and rotate) the hive CA + gateway leaf before the gateway
|
|
# container starts. Idempotent: the CA is created once and reused; the
|
|
# leaf is re-signed on expiry under the same CA so the anchor is stable.
|
|
systemd.services.hive-tls-ca = {
|
|
description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Gateway nginx reads the leaf from the bind-mount, so the cert must
|
|
# exist before the container starts. Declarative nixos-containers are
|
|
# instances of the `container@.service` template.
|
|
before = [ "container@hive-gateway.service" ];
|
|
requiredBy = [ "container@hive-gateway.service" ];
|
|
# The issuance below needs the swarm root key on disk. When this
|
|
# host generates it (single-host swarm) that unit must have run;
|
|
# when the operator provides it there is no unit to wait for, so
|
|
# the dependency is conditional rather than a unit that exists and
|
|
# does nothing.
|
|
after = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
|
|
requires = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
|
|
path = [ pkgs.openssl ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
UMask = "0077";
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-tls-ca";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
d=${lib.escapeShellArg cfg.stateDir}
|
|
install -d -m 0755 "$d"
|
|
|
|
ca="$d/ca.pem"
|
|
cak="$d/ca-key.pem"
|
|
leaf="$d/gateway.pem"
|
|
leafk="$d/gateway-key.pem"
|
|
root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"}
|
|
rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"}
|
|
|
|
prev="$d/ca-previous.pem"
|
|
marker="$d/.swarm-ca-adopted"
|
|
|
|
# --- Adoption: a hive whose CA predates the swarm root.
|
|
#
|
|
# Runs ONLY where this host also owns the root (`autoConfigure`),
|
|
# because adoption invalidates an anchor that consumers already
|
|
# trust and they refresh on their own schedule. On one box that
|
|
# schedule is knowable; across hosts it is not, so there the
|
|
# operator does it and this unit only says so, loudly.
|
|
#
|
|
# Guarded by a marker rather than by the state of the world: the
|
|
# marker's ABSENCE is the trigger, so this fires once per hive
|
|
# instead of re-deciding every activation. That is the difference
|
|
# between a migration and a kicking machine.
|
|
if [ -s "$root" ] && [ -s "$ca" ] && [ ! -e "$marker" ] \
|
|
&& ! openssl verify -CAfile "$root" "$ca" >/dev/null 2>&1; then
|
|
${
|
|
if swarmCaCfg.autoConfigure then
|
|
''
|
|
echo "adopting the swarm CA: $ca does not chain to $root" >&2
|
|
# Keep the old CA as an anchor across the overlap. Consumers
|
|
# read the bundle, so adoption is additive before it is
|
|
# subtractive — agents pick up new trust only when their
|
|
# container restarts, which is a window even on one host.
|
|
cp "$ca" "$prev"
|
|
chmod 0644 "$prev"
|
|
rm -f "$ca" "$cak"
|
|
touch "$marker"''
|
|
else
|
|
''
|
|
echo "hive-tls: this hive's CA does not chain to the swarm root." >&2
|
|
echo " hive CA: $ca" >&2
|
|
echo " swarm root: $root" >&2
|
|
echo "Adopting the hierarchy is not automatic here: it invalidates an" >&2
|
|
echo "anchor that peers and agents on OTHER hosts still trust, and they" >&2
|
|
echo "refresh on their own schedule — only you know when that is safe." >&2
|
|
echo "To adopt: rm $ca $cak && systemctl restart hive-tls-ca.service" >&2
|
|
echo "To keep the current CA deliberately: touch $marker" >&2
|
|
exit 1''
|
|
}
|
|
fi
|
|
|
|
# --- CA: generated once and reused across leaf rotations, in
|
|
# whichever of the two shapes `caGenScript` selected.
|
|
# Regenerated only if missing or already expired (checkend 0).
|
|
# A new CA means every consumer must re-trust, so the leaf is
|
|
# dropped to force a re-sign under the fresh CA.
|
|
#
|
|
# An existing CA is never re-rooted here. A hive predating the
|
|
# swarm root carries a self-signed `ca.pem`, and swapping it for
|
|
# a swarm-issued one would break every consumer already trusting
|
|
# it — including agents, whose trust is refreshed only when their
|
|
# container restarts. Adopting the hierarchy on such a hive is
|
|
# therefore an operator step (drop the CA, let this unit
|
|
# re-issue), which is also what keeps this change non-disruptive
|
|
# to hives that never adopt it.
|
|
if [ ! -s "$ca" ] || [ ! -s "$cak" ] \
|
|
|| ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then
|
|
${caGenScript}
|
|
chmod 0600 "$cak"
|
|
chmod 0644 "$ca"
|
|
rm -f "$leaf" "$leafk"
|
|
fi
|
|
|
|
# --- Leaf: (re)sign when missing or within 30 days of expiry,
|
|
# always under the current (stable) CA.
|
|
if [ ! -s "$leaf" ] || [ ! -s "$leafk" ] \
|
|
|| ! openssl x509 -in "$leaf" -noout -checkend 2592000 >/dev/null 2>&1; then
|
|
echo "signing fresh gateway leaf at $leaf"
|
|
${signLeafScript} "$d"
|
|
fi
|
|
|
|
# --- Trust bundle: what a consumer must TRUST, as opposed to
|
|
# `ca.pem`, which is what this host SIGNS with. The two were the
|
|
# same file while the hive CA was self-signed, and stopped being
|
|
# the same file the moment it became an intermediate: openssl
|
|
# refuses to end a chain at a trusted cert that isn't
|
|
# self-signed (that's what `-partial_chain` is for), so an
|
|
# agent's curl handed only `ca.pem` fails with "unable to get
|
|
# issuer certificate". Verified in both directions before this
|
|
# was written — rustls and Go accept a trusted intermediate,
|
|
# which is what makes the breakage partial and easy to miss.
|
|
#
|
|
# Consumers therefore trust hive CA + swarm root; on a hive that
|
|
# still has a self-signed CA the bundle is just that CA, so the
|
|
# consumer side needs no condition at all. Trusting the root is
|
|
# also the point of the hierarchy — it is what lets a peer hive
|
|
# validate without being hand-pinned.
|
|
#
|
|
# Written IN PLACE, never renamed into position: containers bind
|
|
# -mount this file, and a bind mount follows the inode. A
|
|
# rename would leave every consumer holding the old one.
|
|
# `ca-previous.pem` rides along after an adoption for the same
|
|
# reason: a consumer that has not restarted yet is still pinned to
|
|
# the old anchor, and dropping it the instant the new one exists
|
|
# would break exactly the hosts the migration was supposed to
|
|
# spare. Removing it is a deliberate later step, not this unit's
|
|
# call — "long enough" is a deployment fact.
|
|
#
|
|
# Built as an explicit anchor ARRAY with `if` guards, not
|
|
# `[ -s f ] && anchors+=(f)`: under `set -e` an && list whose test
|
|
# fails IS a failing command and kills the unit, so the tidier
|
|
# form would break precisely on the hive where the optional file
|
|
# is legitimately absent. The `|| true` patch for that is worse —
|
|
# it would also swallow a real failure to read the hive CA.
|
|
bundle="$d/trust-bundle.pem"
|
|
anchors=("$ca")
|
|
if [ -s "$prev" ]; then anchors+=("$prev"); fi
|
|
if [ -s "$root" ]; then anchors+=("$root"); fi
|
|
cat "''${anchors[@]}" > "$bundle"
|
|
chmod 0644 "$bundle"
|
|
'';
|
|
};
|
|
|
|
# Weekly re-sign of the gateway leaf so short-lived leaves renew
|
|
# without depending on a reboot.
|
|
#
|
|
# `hive-tls-ca` only re-signs at service activation (boot/rebuild); a
|
|
# long-uptime host would otherwise let a 30-day leaf lapse silently.
|
|
# This service re-signs the leaf directly (not by bouncing hive-tls-ca)
|
|
# and propagates the new leaf into the running gateway container when
|
|
# the file actually changed.
|
|
#
|
|
# Propagation mechanism: nginx in the gateway container serves a *copy*
|
|
# of the leaf written by `hive-gateway-self-signed-cert` (which runs at
|
|
# container start). A host-side `systemctl -M hive-gateway` call
|
|
# triggers the re-import + reload, mirroring how hive-c0re reloads the
|
|
# gateway after each agents.conf write. A path unit *inside* the
|
|
# container cannot do this: IN_MOVED_TO from an atomic rename on the
|
|
# host does not propagate across the nspawn mount-namespace boundary.
|
|
#
|
|
# `|| true` on propagation so a stopped gateway never fails the unit —
|
|
# its next boot will import the already-rotated leaf anyway.
|
|
systemd.services.hive-tls-resign = {
|
|
description = "Re-sign the gateway TLS leaf and propagate it into the gateway container";
|
|
# hive-tls-ca must have run first so the CA key exists before we try
|
|
# to re-sign under it. On first boot `Persistent=true` on the weekly
|
|
# timer fires immediately; without this ordering the resign could race
|
|
# the CA initialisation and fail with "no such file" on the CA key.
|
|
after = [ "hive-tls-ca.service" ];
|
|
path = [
|
|
pkgs.openssl
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
UMask = "0077";
|
|
SyslogIdentifier = "hive-tls-resign";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
d=${lib.escapeShellArg cfg.stateDir}
|
|
leaf="$d/gateway.pem"
|
|
|
|
# Re-sign only when the leaf is within half its validity of expiry.
|
|
# The weekly cadence catches this window well before the leaf lapses.
|
|
halflife=$(( ${toString cfg.leafValidityDays} * 86400 / 2 ))
|
|
if [ -s "$leaf" ] && \
|
|
openssl x509 -in "$leaf" -noout -checkend "$halflife" >/dev/null 2>&1; then
|
|
echo "gateway leaf valid for more than half its lifetime — no resign needed"
|
|
exit 0
|
|
fi
|
|
|
|
echo "gateway leaf missing or near expiry — re-signing under current CA"
|
|
before="$(sha256sum "$leaf" 2>/dev/null || true)"
|
|
|
|
${signLeafScript} "$d"
|
|
|
|
after="$(sha256sum "$leaf" 2>/dev/null || true)"
|
|
if [ "$before" != "$after" ]; then
|
|
echo "gateway leaf rotated — propagating into hive-gateway"
|
|
systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true
|
|
systemctl -M hive-gateway reload nginx.service || true
|
|
else
|
|
echo "gateway leaf unchanged (already up to date)"
|
|
fi
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hive-tls-resign = {
|
|
description = "Weekly gateway-leaf re-sign and propagation";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
# Run weekly; Persistent=true fires a missed run on next boot if
|
|
# the timer was not active (e.g. the host was off on the scheduled
|
|
# day), preventing a dormant timer from letting the leaf lapse.
|
|
OnCalendar = "weekly";
|
|
Persistent = true;
|
|
};
|
|
};
|
|
|
|
# Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts
|
|
# this file (read-only, public certs ONLY — never a key) into each
|
|
# agent container so agents + their tools can trust the gateway's
|
|
# self-signed leaf, and the meta flake wires the per-agent trust
|
|
# bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason
|
|
# spelled out where the bundle is written above; no key path is ever
|
|
# exposed (an agent that could read one could mint trusted certs).
|
|
systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem";
|
|
};
|
|
}
|