vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(nix): issue each hive's CA under a swarm root CA

Browse source 
Cross-hive trust was O(n²) hand-pinning: every hive had to name every
peer's CA. A swarm root makes it O(1) — trust the root once and every
present and future peer validates.

The root is generated by a new `swarm-ca` unit on a single-host swarm
and operator-provided otherwise; `swarm.ca.autoConfigure` picks between
them and derives its default from `swarm.peers` being empty, so "all on
one host" is read off the deployment rather than remembered. Both modes
produce the same artifacts in the same places, so splitting hosts later
is moving the service dirs, not switching code paths. The root key never
enters the nix store, and the root is never regenerated automatically —
replacing it invalidates every peer at once.

Each hive CA carries `nameConstraints` pinned to that hive's domain, so
a leaked hive CA can only mint names inside its own subdomain, enforced
by verifiers rather than by convention.

`ca.pem` was serving as both the issuer and the anchor consumers trust;
those are the same file only while it is self-signed. openssl will not
terminate a chain at a trusted cert that isn't self-signed (rustls and
Go will), so the promotion would have broken some consumers and not
others. `hive-tls-ca` now also writes `trust-bundle.pem` — the hive CA
plus whatever it is rooted at — and every anchor consumer reads that:
agents, the CI and forge containers, and the peer-config recipe. On a
hive with no swarm root the bundle is just that CA, so nothing consuming
it needs a mode to branch on.
This commit is contained in:
atlas 2026-08-05 14:33:49 +02:00 • committed by mara
commit 06710e83b4
8 changed files with 366 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -21,6 +21,7 @@
./hive-priv.nix
./hive-tls.nix
./otel.nix
./swarm-ca.nix
./swarm-controller.nix
./swarm-snapshot-store.nix
./swarm-wireguard.nix

139
nix/host-modules/hive-tls.nix
View file

@ -8,6 +8,7 @@ let
cfg = config.services.hyperhive.tls;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = config.services.hyperhive.gateway;
swarmCaCfg = config.services.hyperhive.swarm.ca;
domain = hyperhiveCfg.domain;
# The host-managed hive CA is the trust anchor for self-signed mode.
@ -35,7 +36,8 @@ let
leafk="$d/gateway-key.pem"
csr="$(mktemp "$d/gateway.csr.XXXXXX")"
ext="$(mktemp "$d/leaf.ext.XXXXXX")"
trap 'rm -f "$csr" "$ext"' EXIT
only="$(mktemp "$d/gateway.leaf.XXXXXX")"
trap 'rm -f "$csr" "$ext" "$only"' EXIT
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout "$leafk" -out "$csr" \
@ -54,7 +56,17 @@ let
openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \
-CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \
-extfile "$ext" -out "$leaf"
-extfile "$ext" -out "$only"
# nginx serves this file verbatim, so it must carry the leaf AND its
# issuer: the hive CA is an intermediate under the swarm root, and a
# client that anchors on the root cannot build the middle of the
# chain by itself. Agents anchor on the hive CA directly and
# validated either way — the appended cert is what makes a swarm
# peer, or anything else holding only the root, work.
# Leaf first: both `ssl_certificate` and the `openssl x509 -in`
# expiry checks read the first cert in the file.
cat "$only" "$ca" > "$leaf"
chmod 0600 "$leafk"
chmod 0644 "$leaf"
'';
@ -68,11 +80,18 @@ in
# runtime-generated, in-container cert can't be wired into an agent's
# build-time trust store at all.
#
# So the anchor is a long-lived **hive CA** held on the host. The
# So the issuer is a long-lived **hive CA** held on the host. The
# gateway serves a **leaf** signed by that CA (via the `tls.certDir`
# bind-mount path); agents and federation peers trust the *CA* once,
# and leaf rotation never re-breaks them. See `docs/gateway.md`
# ("Self-signed TLS").
#
# The hive CA is itself an intermediate, issued under the swarm root
# (./swarm-ca.nix, which owns the why) and name-constrained to this
# hive's domain — so it stays the anchor agents pin, while a peer that
# trusts only the root can still validate everything this hive serves.
# Hives that predate the root keep their self-signed CA until an
# operator drops it; see the issuance comment below.
options.services.hyperhive.tls = {
stateDir = lib.mkOption {
@ -131,6 +150,13 @@ in
# instances of the `container@.service` template.
before = [ "container@hive-gateway.service" ];
requiredBy = [ "container@hive-gateway.service" ];
# The issuance below needs the swarm root key on disk. When this
# host generates it (single-host swarm) that unit must have run;
# when the operator provides it there is no unit to wait for, so
# the dependency is conditional rather than a unit that exists and
# does nothing.
after = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
requires = lib.optional swarmCaCfg.autoConfigure "swarm-ca.service";
path = [ pkgs.openssl ];
serviceConfig = {
Type = "oneshot";
@ -148,20 +174,68 @@ in
cak="$d/ca-key.pem"
leaf="$d/gateway.pem"
leafk="$d/gateway-key.pem"
root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"}
rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"}
# --- CA: generate once, reuse across leaf rotations. Regenerate
# only if missing or already expired (checkend 0). A new CA means
# every consumer must re-trust, so the leaf is dropped to force a
# re-sign under the fresh CA.
# --- CA: an intermediate under the swarm root, generated once
# and reused across leaf rotations. Regenerated only if missing
# or already expired (checkend 0). A new CA means every consumer
# must re-trust, so the leaf is dropped to force a re-sign under
# the fresh CA.
#
# An existing CA is never re-rooted here. A hive predating the
# swarm root carries a self-signed `ca.pem`, and swapping it for
# a swarm-issued one would break every consumer already trusting
# it — including agents, whose trust is refreshed only when their
# container restarts. Adopting the hierarchy on such a hive is
# therefore an operator step (drop the CA, let this unit
# re-issue), which is also what keeps this change non-disruptive
# to hives that never adopt it.
if [ ! -s "$ca" ] || [ ! -s "$cak" ] \
|| ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then
echo "generating fresh hive CA at $ca"
openssl req -x509 -newkey rsa:4096 -nodes -sha256 \
-days ${toString cfg.caValidityDays} \
-keyout "$cak" -out "$ca" \
-subj "/CN=hive-ca ${domain}" \
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
-addext "keyUsage=critical,keyCertSign,cRLSign"
# Signing needs the root's private key, which on a multi-host
# swarm is deliberately somewhere else. There is nothing to
# fall back to: a self-signed CA here would still serve TLS
# and would silently not be part of the swarm's trust.
if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then
echo "no swarm root CA key at $rootk — cannot issue this hive's CA." >&2
echo "Either set services.hyperhive.swarm.ca.autoConfigure (single-host swarm)," >&2
echo "or install the operator-issued hive CA at $ca + $cak." >&2
exit 1
fi
echo "issuing fresh hive CA at $ca under the swarm root"
cacsr="$(mktemp "$d/ca.csr.XXXXXX")"
caext="$(mktemp "$d/ca.ext.XXXXXX")"
trap 'rm -f "$cacsr" "$caext"' EXIT
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout "$cak" -out "$cacsr" \
-subj "/CN=hive-ca ${domain}"
# printf (not a heredoc) so the ext-file lines carry no leading
# whitespace once nix has stripped the indented-string indent.
{
printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n'
printf 'keyUsage=critical,keyCertSign,cRLSign\n'
printf 'subjectKeyIdentifier=hash\n'
printf 'authorityKeyIdentifier=keyid:always\n'
# The constraint is the point of the hierarchy, not a
# flourish: without it a leaked hive CA mints any name in
# the swarm, and it is verifiers that enforce this, not our
# good behaviour. The IP exclusions are not redundant — a
# DNS constraint says nothing about an iPAddress SAN, and a
# name type nobody constrained is a name type this CA is
# unconstrained for.
printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \
${lib.escapeShellArg domain}
} > "$caext"
openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \
-CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \
-extfile "$caext" -out "$ca"
rm -f "$cacsr" "$caext"
trap - EXIT
chmod 0600 "$cak"
chmod 0644 "$ca"
rm -f "$leaf" "$leafk"
@ -174,6 +248,34 @@ in
echo "signing fresh gateway leaf at $leaf"
${signLeafScript} "$d"
fi
# --- Trust bundle: what a consumer must TRUST, as opposed to
# `ca.pem`, which is what this host SIGNS with. The two were the
# same file while the hive CA was self-signed, and stopped being
# the same file the moment it became an intermediate: openssl
# refuses to end a chain at a trusted cert that isn't
# self-signed (that's what `-partial_chain` is for), so an
# agent's curl handed only `ca.pem` fails with "unable to get
# issuer certificate". Verified in both directions before this
# was written — rustls and Go accept a trusted intermediate,
# which is what makes the breakage partial and easy to miss.
#
# Consumers therefore trust hive CA + swarm root; on a hive that
# still has a self-signed CA the bundle is just that CA, so the
# consumer side needs no condition at all. Trusting the root is
# also the point of the hierarchy — it is what lets a peer hive
# validate without being hand-pinned.
#
# Written IN PLACE, never renamed into position: containers bind
# -mount this file, and a bind mount follows the inode. A
# rename would leave every consumer holding the old one.
bundle="$d/trust-bundle.pem"
if [ -s "$root" ]; then
cat "$ca" "$root" > "$bundle"
else
cat "$ca" > "$bundle"
fi
chmod 0644 "$bundle"
'';
};
@ -256,11 +358,12 @@ in
};
# Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts
# this file (read-only, the CA cert ONLY — never the key) into each
# this file (read-only, public certs ONLY — never a key) into each
# agent container so agents + their tools can trust the gateway's
# self-signed leaf, and the meta flake wires the per-agent trust
# bundle. Only the `ca.pem` path is exposed; `ca-key.pem` stays on the
# host (an agent that could read it could mint trusted certs).
systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/ca.pem";
# bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason
# spelled out where the bundle is written above; no key path is ever
# exposed (an agent that could read one could mint trusted certs).
systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem";
};
}

9
nix/host-modules/lib/hive-ca-trust.nix
View file

@ -31,8 +31,13 @@ let
# `gateway.useSelfSigned` is the single source of truth for the
# self-signed condition — no duplicated derivation.
useSelfSigned = gatewayCfg.useSelfSigned;
caHostPath = "${tlsCfg.stateDir}/ca.pem";
caContainerPath = "/run/hive-ca/ca.pem";
# The bundle, not `ca.pem`: the hive CA is an intermediate under the
# swarm root, and openssl (which is what both consumers below sit on —
# Node's `NODE_EXTRA_CA_CERTS`, Go's `SSL_CERT_FILE`) will not end a
# chain at a trusted cert that isn't self-signed. `hive-tls.nix` writes
# the bundle next to the CA and explains the split.
caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem";
caContainerPath = "/run/hive-ca/trust-bundle.pem";
in
{
inherit useSelfSigned caContainerPath;

161
nix/host-modules/swarm-ca.nix Normal file
View file

@ -0,0 +1,161 @@
# The swarm root CA: the anchor a whole swarm shares, and the issuer of
# each hive's own CA (which is where it gets used — see ./hive-tls.nix).
#
# Why a hierarchy at all: cross-hive trust is hand-pinned today
# (`swarm.peers.<d>.caCert`), so every hive must name every peer — O(n²)
# configuration that a new hive can only join by editing all the others.
# One root makes it O(1): trust the root once and every present *and
# future* peer validates.
#
# Two provisioning modes, ONE structure — what differs is who puts the
# artifacts on disk, never what the artifacts are:
#
# - autoconfigured (the default while the hive declares no peers): the
# unit below generates the root here on first boot.
# - operator-provided (multi-host): the operator installs the root cert
# — and, on a host that does not hold the root key, the hive CA too —
# into the state dirs, and this unit does nothing. Splitting a
# single-host swarm across hosts is then a matter of moving the
# service dirs and turning `autoConfigure` off, not of a second code
# path that has to be kept agreeing with the first.
#
# The root KEY is the reason this is a runtime file and not a nix option:
# the store is world-readable and content-addressed, so a key committed
# to a flake is a key *published* to every consumer of that flake. The
# root CERT has the opposite property, and is already distributed at
# build time via `swarm.peers.<d>.caCert`. That asymmetry is what makes
# the layering work at all.
{
lib,
config,
pkgs,
...
}:
let
cfg = config.services.hyperhive.swarm.ca;
hyperhiveCfg = config.services.hyperhive;
# The subject CN is a label for a human reading a chain, not an
# identity anything authenticates against. Fall through swarm name →
# hive domain → a constant so a hive that has set neither still
# evaluates; a missing `domain` is reported by its own assertion in
# hive-network.nix, and shouldn't also surface here as a null.
swarmLabel =
if hyperhiveCfg.swarm.name != null then
hyperhiveCfg.swarm.name
else if hyperhiveCfg.domain != null then
hyperhiveCfg.domain
else
"hyperhive";
in
{
options.services.hyperhive.swarm.ca = {
autoConfigure = lib.mkOption {
type = lib.types.bool;
default = hyperhiveCfg.swarm.peers == { };
defaultText = lib.literalExpression "services.hyperhive.swarm.peers == { }";
example = false;
description = ''
Generate the swarm root CA on this host when it is missing.
Defaults to true exactly while this hive declares no peers —
an all-on-one-host swarm has no cross-hive trust to establish,
so it should cost no configuration. Declaring a peer turns it
off, because a hive that has peers is by definition not the
only place a root could come from, and a second independently
generated root is not a swarm root at all. Set it explicitly
to `true` on the host that does hold the root, or to `false`
to require the operator to provide it.
Turning this off does not disable anything else: the root is
read from the same `stateDir` either way.
'';
};
stateDir = lib.mkOption {
type = lib.types.str;
default = "/var/lib/swarm-ca";
description = ''
Host directory holding the swarm root CA: `root.pem` (the
anchor, safe to distribute — this is what a peer hive is
pointed at via `swarm.peers.<d>.caCert`) and `root-key.pem`
(0600, the one file that must never reach the nix store or
another host). The directory itself is 0700: nothing reads
out of it but the hive CA issuance in `hive-tls.nix`.
Moving the swarm CA to its own host is a matter of moving this
directory and setting `autoConfigure = false` here.
'';
};
validityDays = lib.mkOption {
type = lib.types.int;
default = 10950;
description = ''
Validity window of the swarm root CA in days (default ~30y).
Deliberately longer than `services.hyperhive.tls.caValidityDays`:
the root must outlive the hive CAs it issues, or those chains
expire out from under hives that are still perfectly happy with
their own intermediate. Rotating a root is the one operation in
this system with no partial-failure mode — it invalidates every
peer at once, paced by the slowest peer's rebuild — so it is
never automatic and this window is meant to be uneventful.
'';
};
};
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
systemd.services.swarm-ca = {
description = "Generate the swarm root CA when absent";
wantedBy = [ "multi-user.target" ];
path = [ pkgs.openssl ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
UMask = "0077";
# Pin the journal identity (else it's the `script` store-path wrapper).
SyslogIdentifier = "swarm-ca";
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg cfg.stateDir}
install -d -m 0700 "$d"
root="$d/root.pem"
rootk="$d/root-key.pem"
# Note the asymmetry with the hive CA in hive-tls.nix, which
# regenerates itself once expired: a root is never replaced
# automatically, not even an expired one. Consumers hold this
# cert, so replacing it is a swarm-wide flag day that wants an
# operator running it deliberately, with both roots trusted
# across the overlap.
if [ -s "$root" ] && [ -s "$rootk" ]; then
echo "swarm root CA already present at $root — leaving it alone"
exit 0
fi
# Half a root is not a root. Generating a fresh key beside an
# already-distributed cert (or the reverse) leaves every
# consumer trusting an anchor that no longer signs anything —
# and it would look like it worked.
if [ -e "$root" ] || [ -e "$rootk" ]; then
echo "swarm root CA half-provisioned ($root / $rootk) — refusing to generate over it" >&2
exit 1
fi
echo "generating swarm root CA at $root"
# pathlen:1 — the root signs hive CAs, which sign leaves. One
# intermediate below the root and no deeper.
openssl req -x509 -newkey rsa:4096 -nodes -sha256 \
-days ${toString cfg.validityDays} \
-keyout "$rootk" -out "$root" \
-subj "/CN=swarm-ca ${swarmLabel}" \
-addext "basicConstraints=critical,CA:TRUE,pathlen:1" \
-addext "keyUsage=critical,keyCertSign,cRLSign"
chmod 0600 "$rootk"
chmod 0644 "$root"
'';
};
};
}