{ lib, config, pkgs, ... }: let cfg = config.services.hyperhive.deploy.hive-controller.tls; hyperhiveCfg = config.services.hyperhive; gatewayCfg = config.services.hyperhive.gateway; swarmCaCfg = config.services.hyperhive.swarm.ca; domain = hyperhiveCfg.domain; baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = hyperhiveCfg.deploy.bao; # The endpoint the services leaf comes out of. Read off the store's own # options rather than spelled here: ./swarm-bao.nix enables the mount, # generates its root and writes the grant that names this exact path, and a # second module composing its own is how the grant and the caller stop # agreeing. baoServicesPkiMount = baoDeploy.servicesPkiMountPath; baoServicesPkiRole = baoDeploy.servicesPkiRoleName; # Derived once in ./swarm.nix and read here + in ./swarm-bao.nix, so # the names this leaf carries as SANs and the names the store's # `pki/roles/swarm-services` is narrowed to cannot disagree. swarmServiceDomains = hyperhiveCfg.swarm.serviceDomains; # The host-managed hive CA is the trust anchor for self-signed mode. # It is only stood up when the gateway actually serves a self-signed # cert: the gateway must be in self-signed mode. `domain` is required # (asserted in hive-network.nix), so the leaf SANs always have a # domain to derive from. The self-signed condition is the gateway # module's single source of truth (`gateway.useSelfSigned`): true when # neither an operator cert (`tls.certDir`) nor ACME is set. active = hyperhiveCfg.enable && gatewayCfg.useSelfSigned; # How this hive's CA comes into existence when it is missing — and it # is one of exactly two things, chosen by config rather than by what # happens to be on disk. # # Issuing a sub-CA requires the swarm root's PRIVATE key, so it can # only happen where that key legitimately lives: the single-host # deployment that `swarm.ca.autoConfigure` describes. A host cannot # infer that it is that host — a swarm's services and its hives can # sit anywhere — so with the flag off this hive self-signs exactly as # it always has, and an operator who wants it in the hierarchy # installs the CA themselves. Falling back to self-signed rather than # failing keeps a plain hive working out of the box; what it loses is # membership of a swarm's trust, which is the correct thing to lose # for a hive nobody has federated. caGenScript = if swarmCaCfg.autoConfigure then '' # The root should exist — `swarm-ca.service` runs before this and # is required by it. If it doesn't, something upstream failed and # signing with a half-provisioned root would be worse than stopping. if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then echo "swarm.ca.autoConfigure is set but there is no root CA key at $rootk" >&2 echo "(swarm-ca.service should have generated it) — refusing to issue a hive CA." >&2 exit 1 fi echo "issuing fresh hive CA at $ca under the swarm root" cacsr="$(mktemp "$d/ca.csr.XXXXXX")" caext="$(mktemp "$d/ca.ext.XXXXXX")" trap 'rm -f "$cacsr" "$caext"' EXIT openssl req -newkey rsa:4096 -nodes -sha256 \ -keyout "$cak" -out "$cacsr" \ -subj "/CN=hive-ca ${domain}" # printf (not a heredoc) so the ext-file lines carry no leading # whitespace once nix has stripped the indented-string indent. { printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n' printf 'keyUsage=critical,keyCertSign,cRLSign\n' printf 'subjectKeyIdentifier=hash\n' printf 'authorityKeyIdentifier=keyid:always\n' # The constraint is the point of the hierarchy, not a # flourish: without it a leaked hive CA mints any name in # the swarm, and it is verifiers that enforce this, not our # good behaviour. The IP exclusions are not redundant — a # DNS constraint says nothing about an iPAddress SAN, and a # name type nobody constrained is a name type this CA is # unconstrained for. printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \ ${lib.escapeShellArg domain} } > "$caext" openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \ -CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \ -extfile "$caext" -out "$ca" rm -f "$cacsr" "$caext" trap - EXIT '' else '' echo "generating fresh self-signed hive CA at $ca" openssl req -x509 -newkey rsa:4096 -nodes -sha256 \ -days ${toString cfg.caValidityDays} \ -keyout "$cak" -out "$ca" \ -subj "/CN=hive-ca ${domain}" \ -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ -addext "keyUsage=critical,keyCertSign,cRLSign" ''; # Sign one leaf. Parameterised rather than hardcoded to `gateway.*` # because there are now two: the hive's own leaf, issued by the hive # CA, and the swarm-services leaf, issued by the services sub-CA that # ./swarm-ca.nix maintains. Same ceremony, different issuer and names # — and one script means the two cannot drift in how they are built. # # $1 stateDir $2 basename $3 CN $4 SAN list $5 issuer cert $6 issuer key signLeafScript = pkgs.writeShellScript "hive-tls-sign-leaf" '' set -euo pipefail d="$1" base="$2" cn="$3" sans="$4" ca="$5" cak="$6" leaf="$d/$base.pem" leafk="$d/$base-key.pem" csr="$(mktemp "$d/$base.csr.XXXXXX")" ext="$(mktemp "$d/$base.ext.XXXXXX")" only="$(mktemp "$d/$base.leaf.XXXXXX")" trap 'rm -f "$csr" "$ext" "$only"' EXIT openssl req -newkey rsa:4096 -nodes -sha256 \ -keyout "$leafk" -out "$csr" \ -subj "/CN=$cn" # printf (not a heredoc) so the ext-file lines carry no leading # whitespace once nix has stripped the indented-string indent. { printf 'subjectAltName=%s\n' "$sans" printf 'basicConstraints=critical,CA:FALSE\n' printf 'keyUsage=critical,digitalSignature,keyEncipherment\n' printf 'extendedKeyUsage=serverAuth\n' } > "$ext" openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \ -CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \ -extfile "$ext" -out "$only" # nginx serves this file verbatim, so it must carry the leaf AND its # issuer: the hive CA is an intermediate under the swarm root, and a # client that anchors on the root cannot build the middle of the # chain by itself. Agents anchor on the hive CA directly and # validated either way — the appended cert is what makes a swarm # peer, or anything else holding only the root, work. # Leaf first: both `ssl_certificate` and the `openssl x509 -in` # expiry checks read the first cert in the file. cat "$only" "$ca" > "$leaf" chmod 0600 "$leafk" chmod 0644 "$leaf" ''; # The hive's own leaf: signed by the hive CA, covering the hive domain # and its sub-domains. # # `forge.` and `matrix.` were listed here explicitly and both were # redundant with the wildcard beside them — a wildcard covers exactly # one label, and those are one label. Naming them read as policy # ("these two services live under this leaf"), which is the reason they # outlived it: a configured service name is no longer necessarily under # this domain, and when it isn't, this is the one list it cannot join. # The hive CA is `nameConstraints=permitted;DNS:`, and a # violation invalidates the CERTIFICATE rather than the offending name # — measured with openssl, not reasoned about — so one sibling name # added here takes the leaf down for the dashboard and every other # vhost sharing it. Configured service names get the swarm-services # leaf below, whose sub-CA is constrained to exactly those names. signHiveLeaf = '' ${signLeafScript} "$d" gateway ${lib.escapeShellArg domain} \ ${lib.escapeShellArg "DNS:${domain},DNS:*.${domain}"} \ "$d/ca.pem" "$d/ca-key.pem" ''; # Shared by every unit that decides whether to re-issue a leaf. It # lives here rather than in one of them because the guards have to # agree: they answer the same question at different times # (`hive-tls-ca` at service activation, i.e. on the deploy; # `hive-tls-resign` from a weekly timer for a host that stays up long # enough to drift; `swarm-services-cert` for the leaf the store # issues). A rule implemented in one and not the others is worse than # one implemented in none — it looks fixed and only fires on whichever # path you did not take, which is exactly how a corrected # `serviceDomains` still served a stale leaf. # # Expects `$d` (state dir) to be set; defines `$leaf`-adjacent names and # `covers`. leafCoverage = '' svcleaf="$d/swarm-services.pem" svcroot="$d/swarm-services-root.pem" hiveNames=${lib.escapeShellArg "${domain} *.${domain}"} svcNames=${lib.escapeShellArg (lib.concatStringsSep " " swarmServiceDomains)} # A hive with no configured service names has no services leaf to # hold, and its absence there is the correct state rather than a # stale one. Anywhere else it is expected: the leaf comes from the # secret store's `pki` mount now, not from a sub-CA that exists on # one host in the swarm, so "this host cannot issue it" is no longer # one of the answers. want_svc=${if swarmServiceDomains == [ ] then "0" else "1"} # Expiry is not the only way a leaf goes wrong. One signed when the # configured name set was smaller stays valid for its whole lifetime # while omitting every name added since — so a config change can # evaluate, build and deploy cleanly while the gateway keeps serving a # certificate that does not cover the new service. # # Reads the names out of the CERTIFICATE, not a sidecar file: the pem # is what nginx serves, and a bookkeeping file drifts from it the # moment anyone replaces a leaf by hand. covers() { # $1 = leaf, $2 = space-separated names it must carry [ -s "$1" ] || return 1 _have="$(openssl x509 -in "$1" -noout -ext subjectAltName 2>/dev/null \ | tr ',' '\n' | sed -n 's/^[[:space:]]*DNS:\(.*\)$/\1/p' || true)" [ -n "$_have" ] || return 1 _missing=0 # ⚠️ The hive leaf carries `*.`; without this the shell # expands it against the cwd and the comparison silently tests a # filename instead of a name. set -f for _n in $2; do printf '%s\n' "$_have" | grep -qxF "$_n" || _missing=1 done set +f [ "$_missing" = 0 ] } # True when the leaf this host SIGNS is present and carries its # configured names. Expiry is the callers' own business — they use # different windows. # # ⚠️ The services leaf is deliberately NOT part of this any more, and # the omission is load-bearing rather than a simplification. Its two # callers sign under the hive CA's key; the services leaf comes out # of the secret store, which neither of them can reach. A guard that # went on reporting it stale would make `hive-tls-ca` re-sign the # hive leaf forever over a file it has no way to fix. The # `swarm-services-cert` unit below holds the same rule for that leaf, # using the `covers` helper above so the two cannot drift. leavesCoverNames() { covers "$leaf" "$hiveNames" } ''; in { # Host-side TLS trust root for the self-signed gateway mode. # # A bare self-signed leaf would be its own trust anchor, so every # regeneration would be a new anchor and every consumer (agents, # federation peers) would have to re-trust on each rotation — and a # runtime-generated, in-container cert can't be wired into an agent's # build-time trust store at all. # # So the issuer is a long-lived **hive CA** held on the host. The # gateway serves a **leaf** signed by that CA (via the `tls.certDir` # bind-mount path); agents and federation peers trust the *CA* once, # and leaf rotation never re-breaks them. See `docs/networking/gateway.md` # ("Self-signed TLS"). # # That CA is self-signed by default. Under # `swarm.ca.autoConfigure` — the all-on-one-host case — it is instead # an intermediate issued under the swarm root (./swarm-ca.nix owns the # why) and name-constrained to this hive's domain, so it stays the # anchor agents pin while a peer holding only the root can validate # everything this hive serves. Either way an existing CA is left # alone; see the issuance comment below. options.services.hyperhive.deploy.hive-controller.tls = { stateDir = lib.mkOption { type = lib.types.str; default = "/var/lib/hive-tls"; description = '' Host directory holding the hive CA + gateway leaf cert for the self-signed gateway mode. `ca.pem` (the anchor agents and federation peers trust), `ca-key.pem` (0600, never leaves the host), `gateway.pem` / `gateway-key.pem` (the leaf the gateway container bind-mounts and nginx serves). Persistent so the CA survives reboots — re-deriving it would re-break every consumer. ''; }; caValidityDays = lib.mkOption { type = lib.types.int; default = 7300; description = '' Validity window of the hive CA in days (default ~20y). Kept long and well beyond `leafValidityDays` so the CA outlives many leaf rotations — the whole point of the CA is to be a stable anchor that consumers trust once. The CA is regenerated only if missing or already expired. ''; }; baoClientCertFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "/var/lib/swarm-bao-pki/services-issuer.pem"; description = '' Certificate `swarm-services-cert` presents to the secret store when it asks for the swarm-services leaf the gateway serves. Its subject has to be {option}`services.hyperhive.deploy.bao.servicesIssuerCommonName` — cert auth matches on the common name, and this is the one role whose policy permits `pki/issue/swarm-services`. A **separate** credential from {option}`services.hyperhive.deploy.bao.clientCertFile`, which is this hive's own reader identity and whose policy reads every secret the hive is entitled to. A unit that renews one server certificate has no business holding that. On a hive that mints its own store PKI a glue module supplies the leaf as a `mkDefault`; everywhere else it is the credential an operator places, for the reason the store's own client certificate is. A path, never a value. ''; }; baoClientKeyFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "/var/lib/swarm-bao-pki/services-issuer-key.pem"; description = '' Private key for {option}`services.hyperhive.deploy.hive-controller.tls.baoClientCertFile`. Both or neither — a certificate with no key authenticates nothing. ''; }; leafValidityDays = lib.mkOption { type = lib.types.int; default = 30; description = '' Validity window of the gateway leaf cert in days (default 30). Short-lived by design — ahead of the CA/Browser-Forum's move toward ~47-day max lifetimes — which bounds the blast radius of a leaf-key compromise. The leaf is re-signed by the (stable) CA when it is missing or near expiry; because it shares the CA anchor, a rotation does not disturb consumer trust. Agents and federation peers validate against the CA, not browser CA/B-forum limits. The weekly `hive-tls-resign` timer re-signs the leaf once it is within half its validity of expiry and propagates the new leaf into the running gateway, so a long-uptime host renews automatically without a reboot. ''; }; }; config = lib.mkIf active { # 🚫 The "these service names are outside this hive's domain and this # host cannot sign for them" warning is GONE, and its absence is the # deliverable rather than a tidy-up. It fired on # `!swarm.ca.autoConfigure` — i.e. "this host does not hold the swarm # root key" — which was the whole reason a hive could end up serving # its own leaf on a swarm-service name. The services leaf now comes # from the secret store's `pki` mount, which every hive reaches over # the network with its own identity, so holding the root key stopped # being the thing that decides. Left in place it would warn, on every # rebuild of every hive that is not the CA host, about a fallback # that no longer happens. # Generate (and rotate) the hive CA + gateway leaf before anything # serves it. Idempotent: the CA is created once and reused; the leaf # is re-signed on expiry under the same CA so the anchor is stable. systemd.services.hive-tls-ca = { description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)"; wantedBy = [ "multi-user.target" ]; # The consumer is `hive-gateway-self-signed-cert`, which copies the # leaf into the gateway's state dir at the mode nginx can read, and # which nginx in turn `Requires=`. So this must run first or that # copy fails under `set -eu` and takes nginx down with it — order # against the unit that reads the file. before = [ "hive-gateway-self-signed-cert.service" ]; requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The issuance below needs the swarm root key on disk. When this # host generates it (single-host swarm) that unit must have run # first; when the operator provides the material there is no unit # to wait for, so the dependency is conditional rather than a unit # that exists and does nothing. # # `swarm-services-cert` is ordered before this one rather than the # other way round, and not because this unit needs the leaf — it # never touches it. The trust bundle written at the end of this # script anchors on every root a consumer must hold, and the # services root is one of them, so the file has to be on disk # before the `cat`. Get it wrong and the bundle is a boot behind: # correct-looking, and missing the anchor for exactly the names the # gateway serves with it. after = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ]; requires = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ]; path = [ pkgs.openssl ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; UMask = "0077"; # Pin the journal identity (else it's the `script` store-path wrapper). SyslogIdentifier = "hive-tls-ca"; }; script = '' set -euo pipefail d=${lib.escapeShellArg cfg.stateDir} install -d -m 0755 "$d" ca="$d/ca.pem" cak="$d/ca-key.pem" leaf="$d/gateway.pem" leafk="$d/gateway-key.pem" root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"} rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"} prev="$d/ca-previous.pem" marker="$d/.swarm-ca-adopted" # --- Adoption: a hive whose CA predates the swarm root. # # Runs ONLY where this host also owns the root (`autoConfigure`), # because adoption invalidates an anchor that consumers already # trust and they refresh on their own schedule. On one box that # schedule is knowable; across hosts it is not, so there the # operator does it and this unit only says so, loudly. # # Guarded by a marker rather than by the state of the world: the # marker's ABSENCE is the trigger, so this fires once per hive # instead of re-deciding every activation. That is the difference # between a migration and a kicking machine. if [ -s "$root" ] && [ -s "$ca" ] && [ ! -e "$marker" ] \ && ! openssl verify -CAfile "$root" "$ca" >/dev/null 2>&1; then ${ if swarmCaCfg.autoConfigure then '' echo "adopting the swarm CA: $ca does not chain to $root" >&2 # Keep the old CA as an anchor across the overlap. Consumers # read the bundle, so adoption is additive before it is # subtractive — agents pick up new trust only when their # container restarts, which is a window even on one host. cp "$ca" "$prev" chmod 0644 "$prev" rm -f "$ca" "$cak" touch "$marker"'' else '' echo "hive-tls: this hive's CA does not chain to the swarm root." >&2 echo " hive CA: $ca" >&2 echo " swarm root: $root" >&2 echo "Adopting the hierarchy is not automatic here: it invalidates an" >&2 echo "anchor that peers and agents on OTHER hosts still trust, and they" >&2 echo "refresh on their own schedule — only you know when that is safe." >&2 echo "To adopt: rm $ca $cak && systemctl restart hive-tls-ca.service" >&2 echo "To keep the current CA deliberately: touch $marker" >&2 exit 1'' } fi # --- CA: generated once and reused across leaf rotations, in # whichever of the two shapes `caGenScript` selected. # Regenerated only if missing or already expired (checkend 0). # A new CA means every consumer must re-trust, so the leaf is # dropped to force a re-sign under the fresh CA. # # An existing CA is never re-rooted here. A hive predating the # swarm root carries a self-signed `ca.pem`, and swapping it for # a swarm-issued one would break every consumer already trusting # it — including agents, whose trust is refreshed only when their # container restarts. Adopting the hierarchy on such a hive is # therefore an operator step (drop the CA, let this unit # re-issue), which is also what keeps this change non-disruptive # to hives that never adopt it. if [ ! -s "$ca" ] || [ ! -s "$cak" ] \ || ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then ${caGenScript} chmod 0600 "$cak" chmod 0644 "$ca" rm -f "$leaf" "$leafk" fi # --- Leaf: (re)sign when missing, within 30 days of expiry, or no # longer covering the configured names, always under the current # (stable) CA. ${leafCoverage} # ⚠️ This is the guard that runs ON THE DEPLOY — `hive-tls-resign` # only fires from a weekly timer, so a rule enforced only there is # up to a week late and does nothing for the rebuild that changed # the names in the first place. # resign=0 { [ -s "$leaf" ] && [ -s "$leafk" ]; } || resign=1 openssl x509 -in "$leaf" -noout -checkend 2592000 >/dev/null 2>&1 || resign=1 leavesCoverNames || resign=1 if [ "$resign" = 1 ]; then echo "signing gateway leaf at $leaf (missing, near expiry, or missing a configured name)" ${signHiveLeaf} fi # --- Trust bundle: what a consumer must TRUST, as opposed to # `ca.pem`, which is what this host SIGNS with. Why they stopped # being the same file, and why `ca-previous.pem` stays in the set # after an adoption, are in docs/swarm/ca.md. Three constraints # the code cannot state: # # Written IN PLACE, never renamed into position — containers # bind-mount this file and a bind mount follows the inode, so a # rename leaves every consumer holding the old one. # # Dropping `ca-previous.pem` is deliberately NOT done here: # "long enough" is a deployment fact, not a unit's call. # # Built as an array with `if` guards, not # `[ -s f ] && anchors+=(f)`: under `set -e` an && list whose test # fails IS a failing command and kills the unit — on exactly the # hive where the optional file is legitimately absent. The # reflexive `|| true` is worse; it also swallows a real failure to # read the hive CA. # # `swarm-services-root.pem` joins the set for the reason the swarm # root is in it: it is an ANCHOR a consumer terminates at, not an # intermediate. It is also what keeps this change from being a # flag day — the swarm root stays in the bundle beside it, so a # leaf still chaining to the old services sub-CA and one issued by # the store both verify against the same file, and a hive can be # rebuilt before or after its peers. bundle="$d/trust-bundle.pem" anchors=("$ca") if [ -s "$prev" ]; then anchors+=("$prev"); fi if [ -s "$root" ]; then anchors+=("$root"); fi if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi cat "''${anchors[@]}" > "$bundle" chmod 0644 "$bundle" ''; }; # Weekly re-sign of the gateway leaf so short-lived leaves renew # without depending on a reboot. # # `hive-tls-ca` only re-signs at service activation (boot/rebuild); a # long-uptime host would otherwise let a 30-day leaf lapse silently. # This service re-signs the leaf directly (not by bouncing hive-tls-ca) # and propagates the new leaf to nginx when the file actually changed. # # Propagation mechanism: nginx serves a *copy* of the leaf, written by # `hive-gateway-self-signed-cert` at the mode nginx can read. Re-signing # the source therefore changes nothing on its own — the copy has to be # remade and nginx reloaded, which is what the two calls below do. # # ⚠️ Neither call is `|| true`: a swallowed propagation failure means # the leaf rotates on disk while nginx keeps serving the old copy # until it expires, with this unit reporting success the whole time. # A failure here must fail the timer. systemd.services.hive-tls-resign = { description = "Re-sign the gateway TLS leaf and reload nginx"; # hive-tls-ca must have run first so the CA key exists before we try # to re-sign under it. On first boot `Persistent=true` on the weekly # timer fires immediately; without this ordering the resign could race # the CA initialisation and fail with "no such file" on the CA key. after = [ "hive-tls-ca.service" ]; path = [ pkgs.openssl pkgs.coreutils pkgs.systemd ]; serviceConfig = { Type = "oneshot"; UMask = "0077"; SyslogIdentifier = "hive-tls-resign"; }; script = '' set -euo pipefail d=${lib.escapeShellArg cfg.stateDir} leaf="$d/gateway.pem" # ⚠️ EVERY leaf this host SIGNS must be covered here. A leaf that # first-boot issuance creates and this unit does not know about # looks perfect for its entire validity and then expires with no # warning — the failure is invisible until it is total. The name # checks come from the shared snippet, so this unit and # `hive-tls-ca` cannot disagree about what a good leaf is. # # The services leaf is not one of them any longer: it is issued # by the secret store, whose key this unit does not hold and # cannot re-sign under. Renewing it is `swarm-services-cert`'s # job, at boot, which is a cadence its own issue owns. ${leafCoverage} # Re-sign only when a leaf is within half its validity of expiry. # The weekly cadence catches this window well before one lapses. halflife=$(( ${toString cfg.leafValidityDays} * 86400 / 2 )) fresh() { # a leaf is fresh if it exists and is not near expiry [ -s "$1" ] && openssl x509 -in "$1" -noout -checkend "$halflife" >/dev/null 2>&1 } if fresh "$leaf" && leavesCoverNames; then echo "leaves valid, and covering the configured names — no resign needed" exit 0 fi echo "a leaf is missing, near expiry, or missing a configured name — re-signing" before="$(sha256sum "$leaf" 2>/dev/null || true)" ${signHiveLeaf} after="$(sha256sum "$leaf" 2>/dev/null || true)" if [ "$before" != "$after" ]; then echo "gateway leaf rotated — re-importing and reloading nginx" systemctl restart hive-gateway-self-signed-cert.service systemctl reload nginx.service else echo "gateway leaf unchanged (already up to date)" fi ''; }; # The swarm-services leaf, issued by the secret store's `pki` mount. # # ⚠️ This unit replaces a chain, not a command. What used to happen # was two openssl hops on whichever host held the swarm root key: a # name-constrained `swarm-services` sub-CA (../swarm-ca.nix), then a # leaf signed under it here. Both are gone. The store's # `pki/roles/swarm-services` carries the same narrowing as an # `allowed_domains` list enforced at issue time, so the intermediate # that used to encode it in x509 `nameConstraints` has nothing left # to express — and with the root inside the mount there is nothing # for it to be an intermediate OF. # # The names it may carry are not this unit's to assert: the role # refuses anything outside `allowed_domains`, which is read from the # same `swarm.serviceDomains` the SANs below are built from. A # mismatch is a refusal from the store naming the offending name, # not a certificate quietly issued for something nobody configured. systemd.services.swarm-services-cert = { description = "Issue the swarm-services TLS leaf from the secret store's PKI"; wantedBy = [ "multi-user.target" ]; # ⚠️ The ordering the old shape got by ACCIDENT, written down. # Nothing used to declare "no TLS until the store is up" — it held # because `swarm-services-ca.service` happened to block the gateway. # Now it is a real edge: the gateway's certificate comes out of the # store, so the store being up is a precondition rather than a # coincidence, and `requiredBy` is what makes a store that never # comes up an outage that says so instead of a gateway serving the # wrong name. before = [ "hive-tls-ca.service" "hive-gateway-self-signed-cert.service" ]; requiredBy = [ "hive-gateway-self-signed-cert.service" ]; # The store's container, where it runs here. On a hive that reads a # store hosted elsewhere no such unit exists and systemd ignores # the name, which is the correct behaviour rather than a gap: what # this unit actually needs is the store reachable, and on a remote # store nothing local can order against that. after = [ "container@${baoCfg.machine}.service" "swarm-bao-services-issuer-policy.service" ]; wants = [ "container@${baoCfg.machine}.service" ]; path = [ baoDeploy.package pkgs.jq pkgs.openssl pkgs.coreutils pkgs.systemd ]; # Sized like the store's own granting units, and for the same # reason: under `seal = "shamir"` an operator unseals BY HAND, and # the login below fails for as long as that takes. 2880 × 30s is # 24h inside a 25h window — `StartLimit*` are `[Unit]` settings, so # the window must exceed `RestartSec × burst` or it closes between # attempts and the burst is never reached. startLimitBurst = 2880; startLimitIntervalSec = 90000; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; UMask = "0077"; SyslogIdentifier = "swarm-services-cert"; Restart = "on-failure"; RestartSec = 30; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; } // lib.optionalAttrs (cfg.baoClientCertFile != null) { BAO_CLIENT_CERT = cfg.baoClientCertFile; } // lib.optionalAttrs (cfg.baoClientKeyFile != null) { BAO_CLIENT_KEY = cfg.baoClientKeyFile; } # Absent means the system trust store, which is what a deployment # with a real CA wants and what a self-signed one must not be left # with. // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; script = '' set -euo pipefail d=${lib.escapeShellArg cfg.stateDir} install -d -m 0755 "$d" ${leafCoverage} if [ "$want_svc" = 0 ]; then echo "no swarm service domains configured — nothing to issue" exit 0 fi # Same rule as the hive leaf's, at a different cadence: re-issue # when the file is missing, within 30 days of expiry, or no # longer carrying every configured name. The name check is what # makes adding a swarm service take effect on the rebuild that # added it rather than whenever the certificate happens to lapse. reissue=0 { [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1 openssl x509 -in "$svcleaf" -noout -checkend 2592000 >/dev/null 2>&1 || reissue=1 covers "$svcleaf" "$svcNames" || reissue=1 [ -s "$svcroot" ] || reissue=1 if [ "$reissue" = 0 ]; then echo "swarm-services leaf valid and covering the configured names — leaving it alone" exit 0 fi ${ if cfg.baoClientCertFile == null || cfg.baoClientKeyFile == null then '' echo "no bao client certificate configured for this host, so the gateway's" >&2 echo "swarm-services certificate cannot be requested from the store." >&2 echo "Set services.hyperhive.deploy.hive-controller.tls.baoClient{Cert,Key}File" >&2 echo "to a leaf the store's CA signed with CN=${baoDeploy.servicesIssuerCommonName}." >&2 exit 1'' else "" } # `bao`'s own message is the only thing separating an unreachable # store from a refused identity from a role that would not issue # these names, and this unit retries on all three — so it reports # which one rather than asserting all three in a sentence of ours. err="$(mktemp)" trap 'rm -f "$err"' EXIT # Cert auth is a login, not a transport setting: the # `BAO_CLIENT_*` variables only decide which certificate the # handshake presents. Without a token `bao` asks its token # helper, and that is a `sh` this unit's `path` does not carry — # `-token-only` answers on stdout and skips the helper. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2 cat "$err" >&2 exit 1 fi export BAO_TOKEN echo "requesting the swarm-services leaf from the store" resp="$(mktemp "$d/swarm-services.json.XXXXXX")" trap 'rm -f "$err" "$resp"' EXIT # ONE call, and the response carries all three artefacts — # certificate, private key and issuing CA. Splitting it into a # local CSR plus `pki/sign` would buy a key that never crosses # the wire, and cost the thing that makes this unit worth having: # the store, not this host, decides what the certificate says. if ! bao write -format=json \ ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${baoServicesPkiRole}"} \ common_name=${lib.escapeShellArg (builtins.head swarmServiceDomains)} \ alt_names=${lib.escapeShellArg (lib.concatStringsSep "," swarmServiceDomains)} \ > "$resp" 2>"$err"; then echo "the store refused to issue the swarm-services certificate." >&2 cat "$err" >&2 exit 1 fi # Three fields out of ONE issuance, which is why the response is # captured and then split rather than asked for three times with # `bao write -field=…`: each of those calls would mint a separate # certificate, and the key would belong to a different one than # the leaf. # # `jq -r` into a file the shell opened, never into a variable and # never into an argv — the private key does not become visible in # /proc on its way to disk. # # `// empty` on every one of them: `jq -r` renders a missing # field as the four characters `null`, which is a non-empty file # and would sail past the check below into a certificate slot. umask 077 jq -r '.data.private_key // empty' < "$resp" > "$d/swarm-services-key.pem.new" jq -r '.data.certificate // empty' < "$resp" > "$d/swarm-services-leaf.new" jq -r '.data.issuing_ca // empty' < "$resp" > "$svcroot.new" for f in "$d/swarm-services-key.pem.new" "$d/swarm-services-leaf.new" "$svcroot.new"; do if [ ! -s "$f" ]; then echo "the store's response was missing a field: $f is empty" >&2 exit 1 fi done # nginx serves this file verbatim. It carries the leaf ALONE and # not leaf-plus-issuer, which is the one visible consequence of # flattening the chain: the issuer here is a self-signed root, so # appending it would ship an anchor a client either already holds # — in which case it is redundant — or does not, in which case # shipping it proves nothing. The old file appended the services # sub-CA because that WAS an intermediate, and a client anchored # on the swarm root could not build the middle of the chain by # itself. mv -f "$d/swarm-services-leaf.new" "$svcleaf" mv -f "$d/swarm-services-key.pem.new" "$d/swarm-services-key.pem" chmod 0644 "$svcleaf" chmod 0600 "$d/swarm-services-key.pem" # The anchor, world-readable, beside the leaf it signed. This is # the file an operator adds to a browser's trust store, and the # one `hive-tls-ca` folds into `trust-bundle.pem`. Public # certificate material only — the private half never leaves the # store's `pki` mount, which is what `root/generate/internal` # buys. rootchanged=0 if ! cmp -s "$svcroot.new" "$svcroot"; then rootchanged=1 fi mv -f "$svcroot.new" "$svcroot" chmod 0644 "$svcroot" # Propagation, for the RETRY path only. Ordered before both of # these, so on a normal boot they have not run yet, `is-active` # is false, and ordering alone does the work. What this covers is # the store coming up hours after the gateway did: nginx serves a # *copy* of the leaf, so re-issuing the source changes nothing # until the copy is remade. # # ⚠️ `--no-block`, and it is not a preference. This unit declares # `Before=` both of these, so a blocking `systemctl restart` # enqueues a job that systemd will not start until this unit is # active — and this unit is not active until its ExecStart # returns, which is waiting on that job. A deadlock, held until # the 24h retry window's `TimeoutStartSec` fires. Queueing the # job and letting it run once we exit is the only ordering that # terminates. # # The cost is real and is the reason `hive-tls-resign` does NOT # do this: a propagation that fails after we return does not fail # us, so the leaf can rotate on disk while nginx keeps the old # copy. That unit blocks because nothing orders it before its # consumers; this one cannot. if systemctl is-active --quiet nginx.service; then echo "swarm-services leaf rotated — re-importing and reloading nginx" systemctl restart --no-block hive-gateway-self-signed-cert.service systemctl reload --no-block nginx.service fi # The bundle is assembled by `hive-tls-ca`, which is ordered # after this unit and therefore already holds the current root on # any normal boot. On the retry path it ran a long time ago, so a # CHANGED root has to reach it — and only a changed one, or every # boot would bounce a unit with nothing to do. if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then echo "the services root changed — rebuilding the trust bundle" systemctl restart --no-block hive-tls-ca.service fi ''; }; systemd.timers.hive-tls-resign = { description = "Weekly gateway-leaf re-sign and propagation"; wantedBy = [ "timers.target" ]; timerConfig = { # Run weekly; Persistent=true fires a missed run on next boot if # the timer was not active (e.g. the host was off on the scheduled # day), preventing a dormant timer from letting the leaf lapse. OnCalendar = "weekly"; Persistent = true; }; }; # Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts # this file (read-only, public certs ONLY — never a key) into each # agent container so agents + their tools can trust the gateway's # self-signed leaf, and the meta flake wires the per-agent trust # bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason # spelled out where the bundle is written above; no key path is ever # exposed (an agent that could read one could mint trusted certs). systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem"; # The same anchor, named for the swarm-queue clients that need it when # they mint a token from authelia over TLS. Declared HERE, beside the # bundle, rather than in each consumer's module: the path is this # module's fact, and two consumers re-deriving `${stateDir}/…` would be # two places to fix the day it moves. # # ⚠️ This is what was missing. `swarm-queue-client` built a bare # `reqwest::Client`, so it trusted only the platform roots and died at # `invalid peer certificate: UnknownIssuer` against a swarm whose # authelia is signed by the swarm CA — while this very bundle sat on # disk, already assembled, already handed to hive-c0re under a different # variable name. The anchor was never missing; nothing pointed the queue # client at it. # # Set unconditionally within this module's `active` guard, exactly like # the line above: where there is no hive CA this module contributes # nothing at all, and the clients then fall back to the platform roots — # which is correct for a swarm fronted by a public certificate. systemd.services.hive-c0re.environment.HIVE_C0RE_OIDC_CA_FILE = "${cfg.stateDir}/trust-bundle.pem"; # ⚠️ Gated, where the hive-c0re line above is not, and the asymmetry is # the point: hive-c0re runs on every hive, the controller runs on one. # Defining an environment key on a unit that does not exist CREATES a # unit fragment for it — inert (no `ExecStart`, empty `wantedBy`, never # activated) but present on every non-controller hive with a CA. Caught # in review on this PR; it evaluates and builds clean either way, which # is exactly why it needed a reviewer rather than a check. systemd.services.swarm-controller.environment.SWARM_CONTROLLER_OIDC_CA_FILE = lib.mkIf hyperhiveCfg.deploy.swarm-controller.enable "${cfg.stateDir}/trust-bundle.pem"; }; }