cf7d357f17
The docs, option descriptions and comments that named the swarm root, the swarm CA or the hive CA as an existing trust anchor now name the secret store's pki root or the trust bundle that holds it: - docs/swarm/README.md, secrets.md, bao.md - deploy.bao.clientCaFile's and gateway.verifiedProxyTo's descriptions - comments in hive-c0re/src/meta.rs, hive-agent/src/swarm_queue.rs, swarm-logs/src/auth.rs, swarm-controller/src/vcs_metrics.rs, hive-forge, swarm-authelia, swarm-ci-runner, glue-bao-tls, swarm-bao, swarm-grafana, swarm-otel, otel, hive-c0re, lib/hive-ca-trust.nix, hive-forge/Cargo.toml and the flake-update workflow Nothing behaves differently: every edit is a nix, Rust or TOML comment, an option description, or a test assertion message. The Rust edits change the workspace source, which this PR already rebuilds for hivectl. Refs #4588
280 lines
15 KiB
Nix
280 lines
15 KiB
Nix
# Glue: give the secret store a PKI of its own, and point it at it.
|
||
#
|
||
# ONE PAIRING PER FILE — `glue-<consumer>-<what>.nix`. A single module holding
|
||
# every co-location default becomes the file nobody dares change, because a
|
||
# reader cannot tell which of its rules their deployment is subject to. Each
|
||
# of these should be deletable on its own, and deleting this one leaves a
|
||
# store that takes operator-provided certificates and nothing else.
|
||
#
|
||
# ⚠️ Why the PKI lives HERE and not in ./swarm-bao.nix: the store must have no
|
||
# opinion about where its identity comes from. Minting is an opinion — the
|
||
# most consequential one available — so it belongs to the glue that decides
|
||
# this deployment self-signs, not to the service that merely serves what it is
|
||
# handed. A deployment with a real internal CA drops this file and names its
|
||
# own paths; nothing in the store changes.
|
||
#
|
||
# ⚠️ Not the store's `pki` root, which the store distributes. An authority
|
||
# you must already hold a certificate from cannot be one the store hands
|
||
# out — reach the store to get the CA material, need a cert from that CA to
|
||
# reach the store. This CA signs a fixed, short
|
||
# list of leaves and distributes nothing, so it cannot enter that cycle.
|
||
#
|
||
# ⚠️ Files like this are the only place a `deploy.<foo>` value may derive from
|
||
# a `deploy.<bar>.enable`. Everywhere else that is forbidden. The exception
|
||
# earns itself: the derivation happens either way, and the alternative is
|
||
# having it spread through the service modules where it is invisible.
|
||
#
|
||
# Everything is `mkDefault`. An operator naming their own paths wins.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
deployCfg = hyperhiveCfg.deploy;
|
||
cfg = hyperhiveCfg.swarm.bao;
|
||
|
||
# Host-side, outside the container's tree, for the same reason the raft data
|
||
# is: `nixos-container destroy` must not take it. Losing the CA key means
|
||
# re-issuing every client certificate in the swarm.
|
||
pkiDir = "/var/lib/swarm-bao-pki";
|
||
|
||
# What a reader calls itself to the store. The hive's name, because a bao
|
||
# cert-auth role matches on the CN — this is an interface, not a label.
|
||
# No fallback: `hiveName` is asserted set wherever `deploy.bao.enable` is
|
||
# (./hyperhive.nix), the same condition this file's `config` is gated on. A
|
||
# fallback here reads as a second supported spelling and there is no such
|
||
# thing.
|
||
clientCn = hyperhiveCfg.hiveName;
|
||
|
||
# $1 dir $2 basename $3 CN $4 SAN or "" $5 EKU
|
||
signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" ''
|
||
set -euo pipefail
|
||
d="$1"; base="$2"; cn="$3"; sans="$4"; eku="$5"
|
||
csr="$(mktemp "$d/$base.csr.XXXXXX")"
|
||
ext="$(mktemp "$d/$base.ext.XXXXXX")"
|
||
trap 'rm -f "$csr" "$ext"' EXIT
|
||
|
||
openssl req -newkey rsa:4096 -nodes -sha256 \
|
||
-keyout "$d/$base-key.pem" -out "$csr" -subj "/CN=$cn"
|
||
{
|
||
[ -n "$sans" ] && printf 'subjectAltName=%s\n' "$sans"
|
||
printf 'basicConstraints=critical,CA:FALSE\n'
|
||
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
|
||
printf 'extendedKeyUsage=%s\n' "$eku"
|
||
} > "$ext"
|
||
openssl x509 -req -in "$csr" -CA "$d/ca.pem" -CAkey "$d/ca-key.pem" \
|
||
-CAcreateserial -days 3650 -sha256 -extfile "$ext" -out "$d/$base.pem"
|
||
chmod 0600 "$d/$base-key.pem"
|
||
chmod 0644 "$d/$base.pem"
|
||
'';
|
||
in
|
||
{
|
||
config = lib.mkIf deployCfg.bao.enable {
|
||
services.hyperhive.deploy.bao = {
|
||
serverCertFile = lib.mkDefault "${pkiDir}/server.pem";
|
||
serverKeyFile = lib.mkDefault "${pkiDir}/server-key.pem";
|
||
clientCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
||
|
||
# A reader on this host, which happens to be the host that mints. Only
|
||
# these three are what a reader elsewhere needs placed by hand; that they
|
||
# collapse to the same CA file here is a property of self-signing, not of
|
||
# the pairing.
|
||
clientCertFile = lib.mkDefault "${pkiDir}/client.pem";
|
||
clientKeyFile = lib.mkDefault "${pkiDir}/client-key.pem";
|
||
serverCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
||
|
||
# 🩸 The readers below each present a leaf of their own rather than
|
||
# `client.pem` above. The leaf is what bao sees, so this pairing is the
|
||
# whole of what turns "one principal with the union of their grants" into
|
||
# one principal per grant — see ./swarm-bao.nix's `perHiveReaders` block
|
||
# for the grants themselves.
|
||
#
|
||
# Defaulted here rather than in `glue-<consumer>-bao-identity.nix` files:
|
||
# those exist where the consumer is a CONTAINER with a
|
||
# `deploy.<service>.*` namespace of its own to point at. These are host
|
||
# units reading the store, which is the pairing this file already is.
|
||
queueAgentClientCertFile = lib.mkDefault "${pkiDir}/queue-agent.pem";
|
||
queueAgentClientKeyFile = lib.mkDefault "${pkiDir}/queue-agent-key.pem";
|
||
grafanaOidcClientCertFile = lib.mkDefault "${pkiDir}/grafana-oidc.pem";
|
||
grafanaOidcClientKeyFile = lib.mkDefault "${pkiDir}/grafana-oidc-key.pem";
|
||
otelOidcClientCertFile = lib.mkDefault "${pkiDir}/otel-oidc.pem";
|
||
otelOidcClientKeyFile = lib.mkDefault "${pkiDir}/otel-oidc-key.pem";
|
||
# The store forwarder's.
|
||
forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem";
|
||
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
|
||
ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem";
|
||
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
|
||
|
||
# The granter: every `swarm-bao-*-policy` unit on this host logs in with
|
||
# it. ⚠️ Unlike every other leaf here, never the file an operator copies:
|
||
# its policy is root-equivalent and its only reader is this host.
|
||
granterClientCertFile = lib.mkDefault "${pkiDir}/granter.pem";
|
||
granterClientKeyFile = lib.mkDefault "${pkiDir}/granter-key.pem";
|
||
};
|
||
|
||
# Idempotent on ABSENCE, never on content. Re-issuing the CA invalidates
|
||
# every client certificate already trusting it, so a rebuild that
|
||
# "refreshed" it would lock every reader in the swarm out at once — the
|
||
# same rule the store's TPM PIN unit follows, for a sharper reason.
|
||
systemd.services.swarm-bao-pki = {
|
||
description = "mint the swarm secret store's own CA and leaves";
|
||
before = [ "swarm-bao-certs.service" ];
|
||
requiredBy = [ "swarm-bao-certs.service" ];
|
||
# 🩸 A target wants this, not only `swarm-bao-certs`, and the reason is
|
||
# what happens when the LIST of leaves grows. `requiredBy` alone is
|
||
# satisfied by a unit that already ran: this one is `RemainAfterExit`,
|
||
# so an activation that adds a leaf here finds it active, pulls nothing,
|
||
# and the new leaf is never minted — while every sibling granting unit,
|
||
# each of which carries this line, was re-run by the same activation.
|
||
# That is how `services-issuer.pem` came to be missing on a host whose
|
||
# config named it, leaving `swarm-services-cert` to fail on an absent
|
||
# file until someone restarted this by hand.
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${pkiDir}
|
||
|
||
if [ ! -s ${pkiDir}/ca.pem ]; then
|
||
openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \
|
||
-keyout ${pkiDir}/ca-key.pem -out ${pkiDir}/ca.pem \
|
||
-subj "/CN=swarm-bao-ca ${cfg.domain}" \
|
||
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
||
-addext "keyUsage=critical,keyCertSign,cRLSign"
|
||
chmod 0600 ${pkiDir}/ca-key.pem
|
||
chmod 0644 ${pkiDir}/ca.pem
|
||
fi
|
||
|
||
# The store's own identity, and the identity of a reader on this host.
|
||
# A reader elsewhere gets its leaf from this CA out of band — that is
|
||
# what makes the store reachable from another machine at all, and why
|
||
# the CA is a file rather than a service.
|
||
[ -s ${pkiDir}/server.pem ] || ${signLeaf} ${pkiDir} server \
|
||
${lib.escapeShellArg cfg.domain} ${lib.escapeShellArg "DNS:${cfg.domain}"} serverAuth
|
||
[ -s ${pkiDir}/client.pem ] || ${signLeaf} ${pkiDir} client \
|
||
${lib.escapeShellArg clientCn} "" clientAuth
|
||
|
||
# Minted whether or not a controller runs here, because the case it
|
||
# serves is the one where it does not: a controller elsewhere needs a
|
||
# leaf from this CA and has no way to sign one. Issuing it here turns
|
||
# "obtain a certificate out of band" into "copy this file".
|
||
#
|
||
# Its own CN rather than the reader's above: the controller's policy
|
||
# lets it create roles for every hive, and the reader's leaf carries
|
||
# this hive's name.
|
||
[ -s ${pkiDir}/controller.pem ] || ${signLeaf} ${pkiDir} controller \
|
||
${lib.escapeShellArg deployCfg.bao.controllerCommonName} "" clientAuth
|
||
|
||
# The secret publisher's, minted here for the reason the controller's
|
||
# line above gives — and it serves that case more often, not less: the
|
||
# publisher runs beside AUTHELIA, which is the one host guaranteed not
|
||
# to be this one whenever the store has a host of its own.
|
||
[ -s ${pkiDir}/secret-publisher.pem ] || ${signLeaf} ${pkiDir} secret-publisher \
|
||
${lib.escapeShellArg deployCfg.bao.secretPublisherCommonName} "" clientAuth
|
||
|
||
# The matrix container's `swarm-matrix-ctl`. Minted unconditionally like the two
|
||
# above, and for the third variant of the same reason: the homeserver
|
||
# is a swarm singleton, so on every hive but the one running it this
|
||
# leaf is the file an operator copies rather than a file anything
|
||
# local reads.
|
||
#
|
||
# Its own CN, not the reader's: the reader's leaf carries this hive's
|
||
# name and its policy reads the whole store, while this principal may
|
||
# only write one path — which is the entire point of giving the
|
||
# container an identity instead of lending it the hive's.
|
||
[ -s ${pkiDir}/matrix-ctl.pem ] || ${signLeaf} ${pkiDir} matrix-ctl \
|
||
${lib.escapeShellArg deployCfg.bao.matrixCtlCommonName} "" clientAuth
|
||
|
||
# 🩸 The readers' leaves, separate from `client.pem` above. Each carries
|
||
# its own subject, which is the entire mechanism: bao matches a
|
||
# cert-auth role on the CN, so units sharing one leaf would be one
|
||
# principal holding the union of their grants, and the union would
|
||
# include every agent credential in the swarm.
|
||
#
|
||
# ⚠️ The first carries THIS hive's name in the subject, unlike the three
|
||
# service leaves above. Its grant names one hive's path, because a queue
|
||
# credential lives under `swarm/hives/<name>/` and every hive runs a
|
||
# reader for its own; ./swarm-bao.nix writes one role per hive in the
|
||
# swarm directory to match. The OIDC readers need no such segment — a
|
||
# client is registered once per swarm.
|
||
#
|
||
# Minted whether or not the consumer runs here, for the reason the three
|
||
# leaves above give: on a hive that does not run the store this is the
|
||
# file an operator copies, and a leaf that only appears where its
|
||
# consumer does is one nobody can copy from anywhere.
|
||
[ -s ${pkiDir}/queue-agent.pem ] || ${signLeaf} ${pkiDir} queue-agent \
|
||
${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/grafana-oidc.pem ] || ${signLeaf} ${pkiDir} grafana-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.grafanaOidcCommonName} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/otel-oidc.pem ] || ${signLeaf} ${pkiDir} otel-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.otelOidcCommonName} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/forwarder-oidc.pem ] || ${signLeaf} ${pkiDir} forwarder-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.forwarderOidcCommonName} "" clientAuth
|
||
|
||
# The identity a hive presents to ask the store's `pki` mount for the
|
||
# swarm-services certificate its gateway serves. Minted here like the
|
||
# three above, and the reason is the sharpest of the four: this leaf
|
||
# is what OPENS the mint, so it cannot come out of it. A certificate
|
||
# authority that issues the credential admitting you to it is the
|
||
# cycle this file's header exists to keep out — which is exactly why
|
||
# the swarm-services root lives in the store and this one does not.
|
||
#
|
||
# Its own CN, not the reader's: the hive's own leaf reads every secret
|
||
# its policy names, while this principal may do one thing — `update`
|
||
# on `pki/issue/swarm-services` — and giving a renewal unit the wider
|
||
# credential would undo the separation the role was created for.
|
||
[ -s ${pkiDir}/services-issuer.pem ] || ${signLeaf} ${pkiDir} services-issuer \
|
||
${lib.escapeShellArg deployCfg.bao.servicesIssuerCommonName} "" clientAuth
|
||
|
||
# The queue's, which opens `pki/issue/swarm-nats` and nothing else.
|
||
# Minted here for the services issuer's reason: it opens the mint, so
|
||
# it cannot come out of it. On a queue host without the store this is
|
||
# the file an operator copies.
|
||
[ -s ${pkiDir}/nats.pem ] || ${signLeaf} ${pkiDir} nats \
|
||
${lib.escapeShellArg deployCfg.bao.natsCommonName} "" clientAuth
|
||
|
||
# The granter's, which writes every `swarm-*` grant. Minted here because
|
||
# it opens the store for the units that create the roles every other
|
||
# leaf logs in with. Stays on this host; see its default above.
|
||
[ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
|
||
${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
|
||
|
||
# The queue's auth-callout responder, which reads agent queue
|
||
# credentials. Minted here for the matrix-ctl leaf's reason: the queue
|
||
# is a swarm singleton, so elsewhere this is the file an operator copies.
|
||
[ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
|
||
${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
|
||
|
||
# The CI runner's token fetch. Minted here for the matrix-ctl leaf's
|
||
# reason: the runner is a swarm singleton, so on a runner host without
|
||
# the store this is the file an operator copies.
|
||
[ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \
|
||
${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth
|
||
|
||
${lib.optionalString deployCfg.hive-controller.enable ''
|
||
# This hive's gateway-leaf identity, which opens
|
||
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
||
# subject carries this hive's name, like the queue-agent leaf
|
||
# above, because ./swarm-bao.nix writes one role
|
||
# per hive. Only where a hive runs; hives on other hosts are issued
|
||
# theirs out of band.
|
||
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||
''}
|
||
'';
|
||
};
|
||
};
|
||
}
|