hyperhive/nix/host-modules/hive-tls.nix
2026-08-06 00:12:28 +02:00

504 lines
23 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
lib,
config,
pkgs,
...
}:
let
cfg = config.services.hyperhive.tls;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = config.services.hyperhive.gateway;
swarmCaCfg = config.services.hyperhive.swarm.ca;
domain = hyperhiveCfg.domain;
# Derived once in ./swarm.nix and read here + in ./swarm-ca.nix, so
# the names this leaf carries as SANs and the names the sub-CA is
# constrained to cannot disagree.
swarmServiceDomains = hyperhiveCfg.swarm.serviceDomains;
# The host-managed hive CA is the trust anchor for self-signed mode.
# It is only stood up when the gateway actually serves a self-signed
# cert: the gateway must be in self-signed mode. `domain` is required
# (asserted in hive-network.nix), so the leaf SANs always have a
# domain to derive from. The self-signed condition is the gateway
# module's single source of truth (`gateway.useSelfSigned`): true when
# neither an operator cert (`tls.certDir`) nor ACME is set.
active = hyperhiveCfg.enable && gatewayCfg.useSelfSigned;
# How this hive's CA comes into existence when it is missing — and it
# is one of exactly two things, chosen by config rather than by what
# happens to be on disk.
#
# Issuing a sub-CA requires the swarm root's PRIVATE key, so it can
# only happen where that key legitimately lives: the single-host
# deployment that `swarm.ca.autoConfigure` describes. A host cannot
# infer that it is that host — a swarm's services and its hives can
# sit anywhere — so with the flag off this hive self-signs exactly as
# it always has, and an operator who wants it in the hierarchy
# installs the CA themselves. Falling back to self-signed rather than
# failing keeps a plain hive working out of the box; what it loses is
# membership of a swarm's trust, which is the correct thing to lose
# for a hive nobody has federated.
caGenScript =
if swarmCaCfg.autoConfigure then
''
# The root should exist `swarm-ca.service` runs before this and
# is required by it. If it doesn't, something upstream failed and
# signing with a half-provisioned root would be worse than stopping.
if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then
echo "swarm.ca.autoConfigure is set but there is no root CA key at $rootk" >&2
echo "(swarm-ca.service should have generated it) refusing to issue a hive CA." >&2
exit 1
fi
echo "issuing fresh hive CA at $ca under the swarm root"
cacsr="$(mktemp "$d/ca.csr.XXXXXX")"
caext="$(mktemp "$d/ca.ext.XXXXXX")"
trap 'rm -f "$cacsr" "$caext"' EXIT
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout "$cak" -out "$cacsr" \
-subj "/CN=hive-ca ${domain}"
# printf (not a heredoc) so the ext-file lines carry no leading
# whitespace once nix has stripped the indented-string indent.
{
printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n'
printf 'keyUsage=critical,keyCertSign,cRLSign\n'
printf 'subjectKeyIdentifier=hash\n'
printf 'authorityKeyIdentifier=keyid:always\n'
# The constraint is the point of the hierarchy, not a
# flourish: without it a leaked hive CA mints any name in
# the swarm, and it is verifiers that enforce this, not our
# good behaviour. The IP exclusions are not redundant a
# DNS constraint says nothing about an iPAddress SAN, and a
# name type nobody constrained is a name type this CA is
# unconstrained for.
printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \
${lib.escapeShellArg domain}
} > "$caext"
openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \
-CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \
-extfile "$caext" -out "$ca"
rm -f "$cacsr" "$caext"
trap - EXIT
''
else
''
echo "generating fresh self-signed hive CA at $ca"
openssl req -x509 -newkey rsa:4096 -nodes -sha256 \
-days ${toString cfg.caValidityDays} \
-keyout "$cak" -out "$ca" \
-subj "/CN=hive-ca ${domain}" \
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
-addext "keyUsage=critical,keyCertSign,cRLSign"
'';
# Sign one leaf. Parameterised rather than hardcoded to `gateway.*`
# because there are now two: the hive's own leaf, issued by the hive
# CA, and the swarm-services leaf, issued by the services sub-CA that
# ./swarm-ca.nix maintains. Same ceremony, different issuer and names
# — and one script means the two cannot drift in how they are built.
#
# $1 stateDir $2 basename $3 CN $4 SAN list $5 issuer cert $6 issuer key
signLeafScript = pkgs.writeShellScript "hive-tls-sign-leaf" ''
set -euo pipefail
d="$1"
base="$2"
cn="$3"
sans="$4"
ca="$5"
cak="$6"
leaf="$d/$base.pem"
leafk="$d/$base-key.pem"
csr="$(mktemp "$d/$base.csr.XXXXXX")"
ext="$(mktemp "$d/$base.ext.XXXXXX")"
only="$(mktemp "$d/$base.leaf.XXXXXX")"
trap 'rm -f "$csr" "$ext" "$only"' EXIT
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout "$leafk" -out "$csr" \
-subj "/CN=$cn"
# printf (not a heredoc) so the ext-file lines carry no leading
# whitespace once nix has stripped the indented-string indent.
{
printf 'subjectAltName=%s\n' "$sans"
printf 'basicConstraints=critical,CA:FALSE\n'
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
printf 'extendedKeyUsage=serverAuth\n'
} > "$ext"
openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \
-CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \
-extfile "$ext" -out "$only"
# nginx serves this file verbatim, so it must carry the leaf AND its
# issuer: the hive CA is an intermediate under the swarm root, and a
# client that anchors on the root cannot build the middle of the
# chain by itself. Agents anchor on the hive CA directly and
# validated either way the appended cert is what makes a swarm
# peer, or anything else holding only the root, work.
# Leaf first: both `ssl_certificate` and the `openssl x509 -in`
# expiry checks read the first cert in the file.
cat "$only" "$ca" > "$leaf"
chmod 0600 "$leafk"
chmod 0644 "$leaf"
'';
# The hive's own leaf: signed by the hive CA, covering the hive domain
# and its sub-domains.
signHiveLeaf = ''
${signLeafScript} "$d" gateway ${lib.escapeShellArg domain} \
${lib.escapeShellArg "DNS:${domain},DNS:forge.${domain},DNS:matrix.${domain},DNS:*.${domain}"} \
"$d/ca.pem" "$d/ca-key.pem"
'';
# The swarm-services leaf: signed by the services sub-CA, covering the
# swarm's service names. Those are *siblings* of the hive domain, not
# children, so the hive CA is name-constrained out of them and cannot
# sign this however its SAN list is written.
#
# Skipped when the sub-CA isn't on disk: it only exists where the
# swarm CA is autoconfigured, and a hive that gets its certs from its
# operator has nothing for this to do.
signServicesLeaf = lib.optionalString (swarmServiceDomains != [ ]) ''
servicesCa=${lib.escapeShellArg "${swarmCaCfg.stateDir}/services-ca.pem"}
servicesCaKey=${lib.escapeShellArg "${swarmCaCfg.stateDir}/services-ca-key.pem"}
if [ -s "$servicesCa" ] && [ -s "$servicesCaKey" ]; then
${signLeafScript} "$d" swarm-services \
${lib.escapeShellArg (builtins.head swarmServiceDomains)} \
${lib.escapeShellArg (lib.concatMapStringsSep "," (n: "DNS:${n}") swarmServiceDomains)} \
"$servicesCa" "$servicesCaKey"
else
echo "no swarm-services sub-CA at $servicesCa skipping the services leaf"
fi
'';
in
{
# Host-side TLS trust root for the self-signed gateway mode.
#
# A bare self-signed leaf would be its own trust anchor, so every
# regeneration would be a new anchor and every consumer (agents,
# federation peers) would have to re-trust on each rotation — and a
# runtime-generated, in-container cert can't be wired into an agent's
# build-time trust store at all.
#
# So the issuer is a long-lived **hive CA** held on the host. The
# gateway serves a **leaf** signed by that CA (via the `tls.certDir`
# bind-mount path); agents and federation peers trust the *CA* once,
# and leaf rotation never re-breaks them. See `docs/gateway.md`
# ("Self-signed TLS").
#
# That CA is self-signed by default. Under
# `swarm.ca.autoConfigure` — the all-on-one-host case — it is instead
# an intermediate issued under the swarm root (./swarm-ca.nix owns the
# why) and name-constrained to this hive's domain, so it stays the
# anchor agents pin while a peer holding only the root can validate
# everything this hive serves. Either way an existing CA is left
# alone; see the issuance comment below.
options.services.hyperhive.tls = {
stateDir = lib.mkOption {
type = lib.types.str;
default = "/var/lib/hive-tls";
description = ''
Host directory holding the hive CA + gateway leaf cert for the
self-signed gateway mode. `ca.pem` (the anchor agents and
federation peers trust), `ca-key.pem` (0600, never leaves the
host), `gateway.pem` / `gateway-key.pem` (the leaf the gateway
container bind-mounts and nginx serves). Persistent so the CA
survives reboots re-deriving it would re-break every consumer.
'';
};
caValidityDays = lib.mkOption {
type = lib.types.int;
default = 7300;
description = ''
Validity window of the hive CA in days (default ~20y). Kept long
and well beyond `leafValidityDays` so the CA outlives many leaf
rotations the whole point of the CA is to be a stable anchor
that consumers trust once. The CA is regenerated only if missing
or already expired.
'';
};
leafValidityDays = lib.mkOption {
type = lib.types.int;
default = 30;
description = ''
Validity window of the gateway leaf cert in days (default 30).
Short-lived by design ahead of the CA/Browser-Forum's move
toward ~47-day max lifetimes which bounds the blast radius of a
leaf-key compromise. The leaf is re-signed by the (stable) CA
when it is missing or near expiry; because it shares the CA
anchor, a rotation does not disturb consumer trust. Agents and
federation peers validate against the CA, not browser CA/B-forum
limits. The weekly `hive-tls-resign` timer re-signs the leaf once
it is within half its validity of expiry and propagates the new
leaf into the running gateway, so a long-uptime host renews
automatically without a reboot.
'';
};
};
config = lib.mkIf active {
# Generate (and rotate) the hive CA + gateway leaf before the gateway
# container starts. Idempotent: the CA is created once and reused; the
# leaf is re-signed on expiry under the same CA so the anchor is stable.
systemd.services.hive-tls-ca = {
description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)";
wantedBy = [ "multi-user.target" ];
# Gateway nginx reads the leaf from the bind-mount, so the cert must
# exist before the container starts. Declarative nixos-containers are
# instances of the `container@.service` template.
before = [ "container@hive-gateway.service" ];
requiredBy = [ "container@hive-gateway.service" ];
# The issuance below needs the swarm root key on disk, and (for the
# services leaf) the services sub-CA it signs under. When this host
# generates them (single-host swarm) both units must have run first;
# when the operator provides the material there is no unit to wait
# for, so the dependency is conditional rather than a unit that
# exists and does nothing.
#
# Without waiting for swarm-services-ca specifically, this unit races
# it: if hive-tls-ca finishes first, it finds no services-ca.pem yet,
# silently skips signing the services leaf (the same as "operator
# hasn't set one up"), and the gateway comes up with a vhost pointed
# at a cert that was never written.
after = lib.optionals swarmCaCfg.autoConfigure [
"swarm-ca.service"
"swarm-services-ca.service"
];
requires = lib.optionals swarmCaCfg.autoConfigure [
"swarm-ca.service"
"swarm-services-ca.service"
];
path = [ pkgs.openssl ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
UMask = "0077";
# Pin the journal identity (else it's the `script` store-path wrapper).
SyslogIdentifier = "hive-tls-ca";
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg cfg.stateDir}
install -d -m 0755 "$d"
ca="$d/ca.pem"
cak="$d/ca-key.pem"
leaf="$d/gateway.pem"
leafk="$d/gateway-key.pem"
root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"}
rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"}
prev="$d/ca-previous.pem"
marker="$d/.swarm-ca-adopted"
# --- Adoption: a hive whose CA predates the swarm root.
#
# Runs ONLY where this host also owns the root (`autoConfigure`),
# because adoption invalidates an anchor that consumers already
# trust and they refresh on their own schedule. On one box that
# schedule is knowable; across hosts it is not, so there the
# operator does it and this unit only says so, loudly.
#
# Guarded by a marker rather than by the state of the world: the
# marker's ABSENCE is the trigger, so this fires once per hive
# instead of re-deciding every activation. That is the difference
# between a migration and a kicking machine.
if [ -s "$root" ] && [ -s "$ca" ] && [ ! -e "$marker" ] \
&& ! openssl verify -CAfile "$root" "$ca" >/dev/null 2>&1; then
${
if swarmCaCfg.autoConfigure then
''
echo "adopting the swarm CA: $ca does not chain to $root" >&2
# Keep the old CA as an anchor across the overlap. Consumers
# read the bundle, so adoption is additive before it is
# subtractive agents pick up new trust only when their
# container restarts, which is a window even on one host.
cp "$ca" "$prev"
chmod 0644 "$prev"
rm -f "$ca" "$cak"
touch "$marker"''
else
''
echo "hive-tls: this hive's CA does not chain to the swarm root." >&2
echo " hive CA: $ca" >&2
echo " swarm root: $root" >&2
echo "Adopting the hierarchy is not automatic here: it invalidates an" >&2
echo "anchor that peers and agents on OTHER hosts still trust, and they" >&2
echo "refresh on their own schedule only you know when that is safe." >&2
echo "To adopt: rm $ca $cak && systemctl restart hive-tls-ca.service" >&2
echo "To keep the current CA deliberately: touch $marker" >&2
exit 1''
}
fi
# --- CA: generated once and reused across leaf rotations, in
# whichever of the two shapes `caGenScript` selected.
# Regenerated only if missing or already expired (checkend 0).
# A new CA means every consumer must re-trust, so the leaf is
# dropped to force a re-sign under the fresh CA.
#
# An existing CA is never re-rooted here. A hive predating the
# swarm root carries a self-signed `ca.pem`, and swapping it for
# a swarm-issued one would break every consumer already trusting
# it including agents, whose trust is refreshed only when their
# container restarts. Adopting the hierarchy on such a hive is
# therefore an operator step (drop the CA, let this unit
# re-issue), which is also what keeps this change non-disruptive
# to hives that never adopt it.
if [ ! -s "$ca" ] || [ ! -s "$cak" ] \
|| ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then
${caGenScript}
chmod 0600 "$cak"
chmod 0644 "$ca"
rm -f "$leaf" "$leafk"
fi
# --- Leaf: (re)sign when missing or within 30 days of expiry,
# always under the current (stable) CA.
if [ ! -s "$leaf" ] || [ ! -s "$leafk" ] \
|| ! openssl x509 -in "$leaf" -noout -checkend 2592000 >/dev/null 2>&1; then
echo "signing fresh gateway leaf at $leaf"
${signHiveLeaf}
${signServicesLeaf}
fi
# --- Trust bundle: what a consumer must TRUST, as opposed to
# `ca.pem`, which is what this host SIGNS with. Why they stopped
# being the same file, and why `ca-previous.pem` stays in the set
# after an adoption, are in docs/swarm/ca.md. Three constraints
# the code cannot state:
#
# Written IN PLACE, never renamed into position containers
# bind-mount this file and a bind mount follows the inode, so a
# rename leaves every consumer holding the old one.
#
# Dropping `ca-previous.pem` is deliberately NOT done here:
# "long enough" is a deployment fact, not a unit's call.
#
# Built as an array with `if` guards, not
# `[ -s f ] && anchors+=(f)`: under `set -e` an && list whose test
# fails IS a failing command and kills the unit on exactly the
# hive where the optional file is legitimately absent. The
# reflexive `|| true` is worse; it also swallows a real failure to
# read the hive CA.
bundle="$d/trust-bundle.pem"
anchors=("$ca")
if [ -s "$prev" ]; then anchors+=("$prev"); fi
if [ -s "$root" ]; then anchors+=("$root"); fi
cat "''${anchors[@]}" > "$bundle"
chmod 0644 "$bundle"
'';
};
# Weekly re-sign of the gateway leaf so short-lived leaves renew
# without depending on a reboot.
#
# `hive-tls-ca` only re-signs at service activation (boot/rebuild); a
# long-uptime host would otherwise let a 30-day leaf lapse silently.
# This service re-signs the leaf directly (not by bouncing hive-tls-ca)
# and propagates the new leaf into the running gateway container when
# the file actually changed.
#
# Propagation mechanism: nginx in the gateway container serves a *copy*
# of the leaf written by `hive-gateway-self-signed-cert` (which runs at
# container start). A host-side `systemctl -M hive-gateway` call
# triggers the re-import + reload, mirroring how hive-c0re reloads the
# gateway after each agents.conf write. A path unit *inside* the
# container cannot do this: IN_MOVED_TO from an atomic rename on the
# host does not propagate across the nspawn mount-namespace boundary.
#
# `|| true` on propagation so a stopped gateway never fails the unit —
# its next boot will import the already-rotated leaf anyway.
systemd.services.hive-tls-resign = {
description = "Re-sign the gateway TLS leaf and propagate it into the gateway container";
# hive-tls-ca must have run first so the CA key exists before we try
# to re-sign under it. On first boot `Persistent=true` on the weekly
# timer fires immediately; without this ordering the resign could race
# the CA initialisation and fail with "no such file" on the CA key.
after = [ "hive-tls-ca.service" ];
path = [
pkgs.openssl
pkgs.coreutils
pkgs.systemd
];
serviceConfig = {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "hive-tls-resign";
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg cfg.stateDir}
leaf="$d/gateway.pem"
# EVERY leaf this host issues must be listed here. A leaf that
# first-boot issuance creates and this unit does not know about
# looks perfect for its entire validity and then expires with no
# warning the failure is invisible until it is total.
svcleaf="$d/swarm-services.pem"
# Re-sign only when a leaf is within half its validity of expiry.
# The weekly cadence catches this window well before one lapses.
halflife=$(( ${toString cfg.leafValidityDays} * 86400 / 2 ))
fresh() { # a leaf is fresh if it exists and is not near expiry
[ -s "$1" ] && openssl x509 -in "$1" -noout -checkend "$halflife" >/dev/null 2>&1
}
# The services leaf is only expected where the sub-CA exists;
# elsewhere its absence is the correct state, not a stale leaf.
want_svc=${if swarmServiceDomains == [ ] then "0" else "1"}
if [ ! -s ${lib.escapeShellArg "${swarmCaCfg.stateDir}/services-ca.pem"} ]; then
want_svc=0
fi
if fresh "$leaf" && { [ "$want_svc" = 0 ] || fresh "$svcleaf"; }; then
echo "leaves valid for more than half their lifetime no resign needed"
exit 0
fi
echo "a leaf is missing or near expiry re-signing under the current CAs"
before="$(sha256sum "$leaf" "$svcleaf" 2>/dev/null || true)"
${signHiveLeaf}
${signServicesLeaf}
after="$(sha256sum "$leaf" "$svcleaf" 2>/dev/null || true)"
if [ "$before" != "$after" ]; then
echo "gateway leaf rotated propagating into hive-gateway"
systemctl -M hive-gateway restart hive-gateway-self-signed-cert.service || true
systemctl -M hive-gateway reload nginx.service || true
else
echo "gateway leaf unchanged (already up to date)"
fi
'';
};
systemd.timers.hive-tls-resign = {
description = "Weekly gateway-leaf re-sign and propagation";
wantedBy = [ "timers.target" ];
timerConfig = {
# Run weekly; Persistent=true fires a missed run on next boot if
# the timer was not active (e.g. the host was off on the scheduled
# day), preventing a dormant timer from letting the leaf lapse.
OnCalendar = "weekly";
Persistent = true;
};
};
# Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts
# this file (read-only, public certs ONLY — never a key) into each
# agent container so agents + their tools can trust the gateway's
# self-signed leaf, and the meta flake wires the per-agent trust
# bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason
# spelled out where the bundle is written above; no key path is ever
# exposed (an agent that could read one could mint trusted certs).
systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem";
};
}