06710e83b4
Cross-hive trust was O(n²) hand-pinning: every hive had to name every peer's CA. A swarm root makes it O(1) — trust the root once and every present and future peer validates. The root is generated by a new `swarm-ca` unit on a single-host swarm and operator-provided otherwise; `swarm.ca.autoConfigure` picks between them and derives its default from `swarm.peers` being empty, so "all on one host" is read off the deployment rather than remembered. Both modes produce the same artifacts in the same places, so splitting hosts later is moving the service dirs, not switching code paths. The root key never enters the nix store, and the root is never regenerated automatically — replacing it invalidates every peer at once. Each hive CA carries `nameConstraints` pinned to that hive's domain, so a leaked hive CA can only mint names inside its own subdomain, enforced by verifiers rather than by convention. `ca.pem` was serving as both the issuer and the anchor consumers trust; those are the same file only while it is self-signed. openssl will not terminate a chain at a trusted cert that isn't self-signed (rustls and Go will), so the promotion would have broken some consumers and not others. `hive-tls-ca` now also writes `trust-bundle.pem` — the hive CA plus whatever it is rooted at — and every anchor consumer reads that: agents, the CI and forge containers, and the peer-config recipe. On a hive with no swarm root the bundle is just that CA, so nothing consuming it needs a mode to branch on.
369 lines
17 KiB
Nix
369 lines
17 KiB
Nix
{
|
|
lib,
|
|
config,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.tls;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
swarmCaCfg = config.services.hyperhive.swarm.ca;
|
|
domain = hyperhiveCfg.domain;
|
|
|
|
# The host-managed hive CA is the trust anchor for self-signed mode.
|
|
# It is only stood up when the gateway actually serves a self-signed
|
|
# cert: the gateway must be in self-signed mode. `domain` is required
|
|
# (asserted in hive-network.nix), so the leaf SANs always have a
|
|
# domain to derive from. The self-signed condition is the gateway
|
|
# module's single source of truth (`gateway.useSelfSigned`): true when
|
|
# neither an operator cert (`tls.certDir`) nor ACME is set.
|
|
active = hyperhiveCfg.enable && gatewayCfg.useSelfSigned;
|
|
|
|
# The leaf-signing action shared by the boot-time `hive-tls-ca`
|
|
# generation and the weekly `hive-tls-resign` renewal: fresh key +
|
|
# CSR, SAN ext-file, sign under the (stable) CA, tighten modes.
|
|
# Takes the TLS state dir as `$1`; each caller keeps its own
|
|
# when-to-sign condition. The leaf covers the bare hive domain plus
|
|
# `forge.`, `matrix.` and `*.<domain>` so all sub-domains validate
|
|
# under the same cert + the hive CA.
|
|
signLeafScript = pkgs.writeShellScript "hive-tls-sign-leaf" ''
|
|
set -euo pipefail
|
|
d="$1"
|
|
ca="$d/ca.pem"
|
|
cak="$d/ca-key.pem"
|
|
leaf="$d/gateway.pem"
|
|
leafk="$d/gateway-key.pem"
|
|
csr="$(mktemp "$d/gateway.csr.XXXXXX")"
|
|
ext="$(mktemp "$d/leaf.ext.XXXXXX")"
|
|
only="$(mktemp "$d/gateway.leaf.XXXXXX")"
|
|
trap 'rm -f "$csr" "$ext" "$only"' EXIT
|
|
|
|
openssl req -newkey rsa:4096 -nodes -sha256 \
|
|
-keyout "$leafk" -out "$csr" \
|
|
-subj "/CN=${domain}"
|
|
|
|
# printf (not a heredoc) so the ext-file lines carry no leading
|
|
# whitespace once nix has stripped the indented-string indent.
|
|
{
|
|
printf 'subjectAltName=DNS:%s,DNS:forge.%s,DNS:matrix.%s,DNS:*.%s\n' \
|
|
${lib.escapeShellArg domain} ${lib.escapeShellArg domain} \
|
|
${lib.escapeShellArg domain} ${lib.escapeShellArg domain}
|
|
printf 'basicConstraints=critical,CA:FALSE\n'
|
|
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
|
|
printf 'extendedKeyUsage=serverAuth\n'
|
|
} > "$ext"
|
|
|
|
openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \
|
|
-CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \
|
|
-extfile "$ext" -out "$only"
|
|
|
|
# nginx serves this file verbatim, so it must carry the leaf AND its
|
|
# issuer: the hive CA is an intermediate under the swarm root, and a
|
|
# client that anchors on the root cannot build the middle of the
|
|
# chain by itself. Agents anchor on the hive CA directly and
|
|
# validated either way — the appended cert is what makes a swarm
|
|
# peer, or anything else holding only the root, work.
|
|
# Leaf first: both `ssl_certificate` and the `openssl x509 -in`
|
|
# expiry checks read the first cert in the file.
|
|
cat "$only" "$ca" > "$leaf"
|
|
chmod 0600 "$leafk"
|
|
chmod 0644 "$leaf"
|
|
'';
|
|
in
|
|
{
|
|
# Host-side TLS trust root for the self-signed gateway mode.
|
|
#
|
|
# A bare self-signed leaf would be its own trust anchor, so every
|
|
# regeneration would be a new anchor and every consumer (agents,
|
|
# federation peers) would have to re-trust on each rotation — and a
|
|
# runtime-generated, in-container cert can't be wired into an agent's
|
|
# build-time trust store at all.
|
|
#
|
|
# So the issuer is a long-lived **hive CA** held on the host. The
|
|
# gateway serves a **leaf** signed by that CA (via the `tls.certDir`
|
|
# bind-mount path); agents and federation peers trust the *CA* once,
|
|
# and leaf rotation never re-breaks them. See `docs/gateway.md`
|
|
# ("Self-signed TLS").
|
|
#
|
|
# The hive CA is itself an intermediate, issued under the swarm root
|
|
# (./swarm-ca.nix, which owns the why) and name-constrained to this
|
|
# hive's domain — so it stays the anchor agents pin, while a peer that
|
|
# trusts only the root can still validate everything this hive serves.
|
|
# Hives that predate the root keep their self-signed CA until an
|
|
# operator drops it; see the issuance comment below.
|
|
|
|
options.services.hyperhive.tls = {
|
|
stateDir = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/var/lib/hive-tls";
|
|
description = ''
|
|
Host directory holding the hive CA + gateway leaf cert for the
|
|
self-signed gateway mode. `ca.pem` (the anchor agents and
|
|
federation peers trust), `ca-key.pem` (0600, never leaves the
|
|
host), `gateway.pem` / `gateway-key.pem` (the leaf the gateway
|
|
container bind-mounts and nginx serves). Persistent so the CA
|
|
survives reboots — re-deriving it would re-break every consumer.
|
|
'';
|
|
};
|
|
|
|
caValidityDays = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 7300;
|
|
description = ''
|
|
Validity window of the hive CA in days (default ~20y). Kept long
|
|
and well beyond `leafValidityDays` so the CA outlives many leaf
|
|
rotations — the whole point of the CA is to be a stable anchor
|
|
that consumers trust once. The CA is regenerated only if missing
|
|
or already expired.
|
|
'';
|
|
};
|
|
|
|
leafValidityDays = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 30;
|
|
description = ''
|
|
Validity window of the gateway leaf cert in days (default 30).
|
|
Short-lived by design — ahead of the CA/Browser-Forum's move
|
|
toward ~47-day max lifetimes — which bounds the blast radius of a
|
|
leaf-key compromise. The leaf is re-signed by the (stable) CA
|
|
when it is missing or near expiry; because it shares the CA
|
|
anchor, a rotation does not disturb consumer trust. Agents and
|
|
federation peers validate against the CA, not browser CA/B-forum
|
|
limits. The weekly `hive-tls-resign` timer re-signs the leaf once
|
|
it is within half its validity of expiry and propagates the new
|
|
leaf into the running gateway, so a long-uptime host renews
|
|
automatically without a reboot.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf active {
|
|
# Generate (and rotate) the hive CA + gateway leaf before the gateway
|
|
# container starts. Idempotent: the CA is created once and reused; the
|
|
# leaf is re-signed on expiry under the same CA so the anchor is stable.
|
|
systemd.services.hive-tls-ca = {
|
|
description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Gateway nginx reads the leaf from the bind-mount, so the cert must
|
|
# exist before the container starts. Declarative nixos-containers are
|
|
# instances of the `container@.service` template.
|
|
before = [ "container@hive-gateway.service" ];
|
|
requiredBy = [ "container@hive-gateway.service" ];
|
|
# The issuance below needs the swarm root key on disk. When this
|
|
# host generates it (single-host swarm) that unit must have run;
|
|
# when the operator provides it there is no unit to wait for, so
|
|
# the dependency is conditional rather than a unit that exists and
|
|
# does nothing.
|
|
after = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
|
|
requires = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
|
|
path = [ pkgs.openssl ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
UMask = "0077";
|
|
# Pin the journal identity (else it's the `script` store-path wrapper).
|
|
SyslogIdentifier = "hive-tls-ca";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
d=${lib.escapeShellArg cfg.stateDir}
|
|
install -d -m 0755 "$d"
|
|
|
|
ca="$d/ca.pem"
|
|
cak="$d/ca-key.pem"
|
|
leaf="$d/gateway.pem"
|
|
leafk="$d/gateway-key.pem"
|
|
root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"}
|
|
rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"}
|
|
|
|
# --- CA: an intermediate under the swarm root, generated once
|
|
# and reused across leaf rotations. Regenerated only if missing
|
|
# or already expired (checkend 0). A new CA means every consumer
|
|
# must re-trust, so the leaf is dropped to force a re-sign under
|
|
# the fresh CA.
|
|
#
|
|
# An existing CA is never re-rooted here. A hive predating the
|
|
# swarm root carries a self-signed `ca.pem`, and swapping it for
|
|
# a swarm-issued one would break every consumer already trusting
|
|
# it — including agents, whose trust is refreshed only when their
|
|
# container restarts. Adopting the hierarchy on such a hive is
|
|
# therefore an operator step (drop the CA, let this unit
|
|
# re-issue), which is also what keeps this change non-disruptive
|
|
# to hives that never adopt it.
|
|
if [ ! -s "$ca" ] || [ ! -s "$cak" ] \
|
|
|| ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then
|
|
# Signing needs the root's private key, which on a multi-host
|
|
# swarm is deliberately somewhere else. There is nothing to
|
|
# fall back to: a self-signed CA here would still serve TLS
|
|
# and would silently not be part of the swarm's trust.
|
|
if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then
|
|
echo "no swarm root CA key at $rootk — cannot issue this hive's CA." >&2
|
|
echo "Either set services.hyperhive.swarm.ca.autoConfigure (single-host swarm)," >&2
|
|
echo "or install the operator-issued hive CA at $ca + $cak." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "issuing fresh hive CA at $ca under the swarm root"
|
|
cacsr="$(mktemp "$d/ca.csr.XXXXXX")"
|
|
caext="$(mktemp "$d/ca.ext.XXXXXX")"
|
|
trap 'rm -f "$cacsr" "$caext"' EXIT
|
|
|
|
openssl req -newkey rsa:4096 -nodes -sha256 \
|
|
-keyout "$cak" -out "$cacsr" \
|
|
-subj "/CN=hive-ca ${domain}"
|
|
|
|
# printf (not a heredoc) so the ext-file lines carry no leading
|
|
# whitespace once nix has stripped the indented-string indent.
|
|
{
|
|
printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n'
|
|
printf 'keyUsage=critical,keyCertSign,cRLSign\n'
|
|
printf 'subjectKeyIdentifier=hash\n'
|
|
printf 'authorityKeyIdentifier=keyid:always\n'
|
|
# The constraint is the point of the hierarchy, not a
|
|
# flourish: without it a leaked hive CA mints any name in
|
|
# the swarm, and it is verifiers that enforce this, not our
|
|
# good behaviour. The IP exclusions are not redundant — a
|
|
# DNS constraint says nothing about an iPAddress SAN, and a
|
|
# name type nobody constrained is a name type this CA is
|
|
# unconstrained for.
|
|
printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \
|
|
${lib.escapeShellArg domain}
|
|
} > "$caext"
|
|
|
|
openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \
|
|
-CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \
|
|
-extfile "$caext" -out "$ca"
|
|
rm -f "$cacsr" "$caext"
|
|
trap - EXIT
|
|
chmod 0600 "$cak"
|
|
chmod 0644 "$ca"
|
|
rm -f "$leaf" "$leafk"
|
|
fi
|
|
|
|
# --- Leaf: (re)sign when missing or within 30 days of expiry,
|
|
# always under the current (stable) CA.
|
|
if [ ! -s "$leaf" ] || [ ! -s "$leafk" ] \
|
|
|| ! openssl x509 -in "$leaf" -noout -checkend 2592000 >/dev/null 2>&1; then
|
|
echo "signing fresh gateway leaf at $leaf"
|
|
${signLeafScript} "$d"
|
|
fi
|
|
|
|
# --- Trust bundle: what a consumer must TRUST, as opposed to
|
|
# `ca.pem`, which is what this host SIGNS with. The two were the
|
|
# same file while the hive CA was self-signed, and stopped being
|
|
# the same file the moment it became an intermediate: openssl
|
|
# refuses to end a chain at a trusted cert that isn't
|
|
# self-signed (that's what `-partial_chain` is for), so an
|
|
# agent's curl handed only `ca.pem` fails with "unable to get
|
|
# issuer certificate". Verified in both directions before this
|
|
# was written — rustls and Go accept a trusted intermediate,
|
|
# which is what makes the breakage partial and easy to miss.
|
|
#
|
|
# Consumers therefore trust hive CA + swarm root; on a hive that
|
|
# still has a self-signed CA the bundle is just that CA, so the
|
|
# consumer side needs no condition at all. Trusting the root is
|
|
# also the point of the hierarchy — it is what lets a peer hive
|
|
# validate without being hand-pinned.
|
|
#
|
|
# Written IN PLACE, never renamed into position: containers bind
|
|
# -mount this file, and a bind mount follows the inode. A
|
|
# rename would leave every consumer holding the old one.
|
|
bundle="$d/trust-bundle.pem"
|
|
if [ -s "$root" ]; then
|
|
cat "$ca" "$root" > "$bundle"
|
|
else
|
|
cat "$ca" > "$bundle"
|
|
fi
|
|
chmod 0644 "$bundle"
|
|
'';
|
|
};
|
|
|
|
# Weekly re-sign of the gateway leaf so short-lived leaves renew
|
|
# without depending on a reboot.
|
|
#
|
|
# `hive-tls-ca` only re-signs at service activation (boot/rebuild); a
|
|
# long-uptime host would otherwise let a 30-day leaf lapse silently.
|
|
# This service re-signs the leaf directly (not by bouncing hive-tls-ca)
|
|
# and propagates the new leaf into the running gateway container when
|
|
# the file actually changed.
|
|
#
|
|
# Propagation mechanism: nginx in the gateway container serves a *copy*
|
|
# of the leaf written by `hive-gateway-self-signed-cert` (which runs at
|
|
# container start). A host-side `systemctl -M hive-gateway` call
|
|
# triggers the re-import + reload, mirroring how hive-c0re reloads the
|
|
# gateway after each agents.conf write. A path unit *inside* the
|
|
# container cannot do this: IN_MOVED_TO from an atomic rename on the
|
|
# host does not propagate across the nspawn mount-namespace boundary.
|
|
#
|
|
# `|| true` on propagation so a stopped gateway never fails the unit —
|
|
# its next boot will import the already-rotated leaf anyway.
|
|
systemd.services.hive-tls-resign = {
|
|
description = "Re-sign the gateway TLS leaf and propagate it into the gateway container";
|
|
# hive-tls-ca must have run first so the CA key exists before we try
|
|
# to re-sign under it. On first boot `Persistent=true` on the weekly
|
|
# timer fires immediately; without this ordering the resign could race
|
|
# the CA initialisation and fail with "no such file" on the CA key.
|
|
after = [ "hive-tls-ca.service" ];
|
|
path = [
|
|
pkgs.openssl
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
UMask = "0077";
|
|
SyslogIdentifier = "hive-tls-resign";
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
d=${lib.escapeShellArg cfg.stateDir}
|
|
leaf="$d/gateway.pem"
|
|
|
|
# Re-sign only when the leaf is within half its validity of expiry.
|
|
# The weekly cadence catches this window well before the leaf lapses.
|
|
halflife=$(( ${toString cfg.leafValidityDays} * 86400 / 2 ))
|
|
if [ -s "$leaf" ] && \
|
|
openssl x509 -in "$leaf" -noout -checkend "$halflife" >/dev/null 2>&1; then
|
|
echo "gateway leaf valid for more than half its lifetime — no resign needed"
|
|
exit 0
|
|
fi
|
|
|
|
echo "gateway leaf missing or near expiry — re-signing under current CA"
|
|
before="$(sha256sum "$leaf" 2>/dev/null || true)"
|
|
|
|
${signLeafScript} "$d"
|
|
|
|
after="$(sha256sum "$leaf" 2>/dev/null || true)"
|
|
if [ "$before" != "$after" ]; then
|
|
echo "gateway leaf rotated — propagating into hive-gateway"
|
|
systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true
|
|
systemctl -M hive-gateway reload nginx.service || true
|
|
else
|
|
echo "gateway leaf unchanged (already up to date)"
|
|
fi
|
|
'';
|
|
};
|
|
|
|
systemd.timers.hive-tls-resign = {
|
|
description = "Weekly gateway-leaf re-sign and propagation";
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
# Run weekly; Persistent=true fires a missed run on next boot if
|
|
# the timer was not active (e.g. the host was off on the scheduled
|
|
# day), preventing a dormant timer from letting the leaf lapse.
|
|
OnCalendar = "weekly";
|
|
Persistent = true;
|
|
};
|
|
};
|
|
|
|
# Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts
|
|
# this file (read-only, public certs ONLY — never a key) into each
|
|
# agent container so agents + their tools can trust the gateway's
|
|
# self-signed leaf, and the meta flake wires the per-agent trust
|
|
# bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason
|
|
# spelled out where the bundle is written above; no key path is ever
|
|
# exposed (an agent that could read one could mint trusted certs).
|
|
systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem";
|
|
};
|
|
}
|