868c6cf96c
Every agent now connects to the queue and reads logs with its own secret from the store (#5181), so the per-hive `hive-<h>-agent` OIDC client has no reader left. Remove it end to end: - hive-c0re no longer forwards `hive-queue-agent-{secret,client-id}` into containers, and no longer forwards HIVE_AGENT_OIDC_TOKEN_ENDPOINT; the agent module that loaded them (nix/agent-modules/queue.nix) is gone, its options refused with a removal message. - The host reader (glue-queue-agent-credential.nix, swarm-bao-queue-agent), its store role and policy unit (swarm-bao-queue-agent-policy), its PKI leaf and the deploy.bao.queueAgent* / queue.agentCredentialDir options are gone; the options are refused with a removal message. - authelia mints no `hive-<h>-agent` client, and swarm-secret-publish no longer writes swarm/hives/<h>/queue/agent. - swarm-secret-client drops queue::agent_client_path / queue::Credential. - HIVE_AGENT_NATS_URL is now gated on the agents' queue address alone. The log store's authelia check stays for operators. swarm-nats-auth still refuses a `-agent` client id; that refusal and `agentClientSuffix` are left for a follow-up. Refs #4459 Refs #4765
362 lines
20 KiB
Nix
362 lines
20 KiB
Nix
# Glue: give the secret store a PKI of its own, and point it at it.
|
|
#
|
|
# ONE PAIRING PER FILE — `glue-<consumer>-<what>.nix`. A single module holding
|
|
# every co-location default becomes the file nobody dares change, because a
|
|
# reader cannot tell which of its rules their deployment is subject to. Each
|
|
# of these should be deletable on its own, and deleting this one leaves a
|
|
# store that takes operator-provided certificates and nothing else.
|
|
#
|
|
# ⚠️ Why the PKI lives HERE and not in ./swarm-bao.nix: the store must have no
|
|
# opinion about where its identity comes from. Minting is an opinion — the
|
|
# most consequential one available — so it belongs to the glue that decides
|
|
# this deployment self-signs, not to the service that merely serves what it is
|
|
# handed. A deployment with a real internal CA drops this file and names its
|
|
# own paths; nothing in the store changes.
|
|
#
|
|
# ⚠️ Not the store's `pki` root, which the store distributes. An authority
|
|
# you must already hold a certificate from cannot be one the store hands
|
|
# out — reach the store to get the CA material, need a cert from that CA to
|
|
# reach the store. This CA signs a fixed, short
|
|
# list of leaves and distributes nothing, so it cannot enter that cycle.
|
|
#
|
|
# ⚠️ Files like this are the only place a `deploy.<foo>` value may derive from
|
|
# a `deploy.<bar>.enable`. Everywhere else that is forbidden. The exception
|
|
# earns itself: the derivation happens either way, and the alternative is
|
|
# having it spread through the service modules where it is invisible.
|
|
#
|
|
# Everything is `mkDefault`. An operator naming their own paths wins.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
deployCfg = hyperhiveCfg.deploy;
|
|
cfg = hyperhiveCfg.swarm.bao;
|
|
|
|
# Host-side, outside the container's tree, for the same reason the raft data
|
|
# is: `nixos-container destroy` must not take it. Losing the CA key means
|
|
# re-issuing every client certificate in the swarm.
|
|
pkiDir = import ./lib/bao-pki-dir.nix;
|
|
|
|
# What a reader calls itself to the store. The hive's name, because a bao
|
|
# cert-auth role matches on the CN — this is an interface, not a label.
|
|
# No fallback: `hiveName` is asserted set wherever `deploy.bao.enable` is
|
|
# (./hyperhive.nix), the same condition this file's `config` is gated on. A
|
|
# fallback here reads as a second supported spelling and there is no such
|
|
# thing.
|
|
clientCn = hyperhiveCfg.hiveName;
|
|
|
|
# Every other hive in the swarm directory. This host's own hive is signed
|
|
# into the top of the directory, beside the leaves its units read.
|
|
otherHives = lib.filter (h: h != clientCn) (lib.attrNames hyperhiveCfg.swarm.hives);
|
|
|
|
# $1 dir $2 basename $3 CN $4 SAN or "" $5 EKU
|
|
# The CA is always the one at the top of `pkiDir`, whichever dir the leaf
|
|
# lands in.
|
|
signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" ''
|
|
set -euo pipefail
|
|
d="$1"; base="$2"; cn="$3"; sans="$4"; eku="$5"
|
|
csr="$(mktemp "$d/$base.csr.XXXXXX")"
|
|
ext="$(mktemp "$d/$base.ext.XXXXXX")"
|
|
trap 'rm -f "$csr" "$ext"' EXIT
|
|
|
|
openssl req -newkey rsa:4096 -nodes -sha256 \
|
|
-keyout "$d/$base-key.pem" -out "$csr" -subj "/CN=$cn"
|
|
{
|
|
[ -n "$sans" ] && printf 'subjectAltName=%s\n' "$sans"
|
|
printf 'basicConstraints=critical,CA:FALSE\n'
|
|
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
|
|
printf 'extendedKeyUsage=%s\n' "$eku"
|
|
} > "$ext"
|
|
openssl x509 -req -in "$csr" -CA ${pkiDir}/ca.pem -CAkey ${pkiDir}/ca-key.pem \
|
|
-CAcreateserial -days 3650 -sha256 -extfile "$ext" -out "$d/$base.pem"
|
|
chmod 0600 "$d/$base-key.pem"
|
|
chmod 0644 "$d/$base.pem"
|
|
'';
|
|
in
|
|
{
|
|
config = lib.mkIf deployCfg.bao.enable {
|
|
services.hyperhive.deploy.bao = {
|
|
serverCertFile = lib.mkDefault "${pkiDir}/server.pem";
|
|
serverKeyFile = lib.mkDefault "${pkiDir}/server-key.pem";
|
|
clientCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
|
|
|
# A reader on this host, which happens to be the host that mints. Only
|
|
# these three are what a reader elsewhere needs placed by hand; that they
|
|
# collapse to the same CA file here is a property of self-signing, not of
|
|
# the pairing.
|
|
clientCertFile = lib.mkDefault "${pkiDir}/client.pem";
|
|
clientKeyFile = lib.mkDefault "${pkiDir}/client-key.pem";
|
|
serverCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
|
|
|
# 🩸 The readers below each present a leaf of their own rather than
|
|
# `client.pem` above. The leaf is what bao sees, so this pairing is the
|
|
# whole of what turns "one principal with the union of their grants" into
|
|
# one principal per grant — see ./swarm-bao.nix's `readStanza` block for
|
|
# the grants themselves.
|
|
#
|
|
# Defaulted here rather than in `glue-<consumer>-bao-identity.nix` files:
|
|
# those exist where the consumer is a CONTAINER with a
|
|
# `deploy.<service>.*` namespace of its own to point at. These are host
|
|
# units reading the store, which is the pairing this file already is.
|
|
grafanaOidcClientCertFile = lib.mkDefault "${pkiDir}/grafana-oidc.pem";
|
|
grafanaOidcClientKeyFile = lib.mkDefault "${pkiDir}/grafana-oidc-key.pem";
|
|
otelOidcClientCertFile = lib.mkDefault "${pkiDir}/otel-oidc.pem";
|
|
otelOidcClientKeyFile = lib.mkDefault "${pkiDir}/otel-oidc-key.pem";
|
|
# The store forwarder's.
|
|
forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem";
|
|
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
|
|
ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem";
|
|
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
|
|
reviewClassifierClientCertFile = lib.mkDefault "${pkiDir}/review-classifier.pem";
|
|
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
|
|
builderClientCertFile = lib.mkDefault "${pkiDir}/builder.pem";
|
|
builderClientKeyFile = lib.mkDefault "${pkiDir}/builder-key.pem";
|
|
ciMetricsClientCertFile = lib.mkDefault "${pkiDir}/ci-metrics.pem";
|
|
ciMetricsClientKeyFile = lib.mkDefault "${pkiDir}/ci-metrics-key.pem";
|
|
claudeLoginClientCertFile = lib.mkDefault "${pkiDir}/claude-login.pem";
|
|
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
|
|
notifyRouterClientCertFile = lib.mkDefault "${pkiDir}/notify-router.pem";
|
|
notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem";
|
|
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
|
|
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
|
|
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
|
|
servicesIssuerClientKeyFile = lib.mkDefault "${pkiDir}/services-issuer-key.pem";
|
|
|
|
# The granter: every `swarm-bao-*-policy` unit on this host logs in with
|
|
# it. ⚠️ Unlike every other leaf here, never the file an operator copies:
|
|
# its policy is root-equivalent and its only reader is this host.
|
|
granterClientCertFile = lib.mkDefault "${pkiDir}/granter.pem";
|
|
granterClientKeyFile = lib.mkDefault "${pkiDir}/granter-key.pem";
|
|
};
|
|
|
|
# Idempotent on ABSENCE, never on content. Re-issuing the CA invalidates
|
|
# every client certificate already trusting it, so a rebuild that
|
|
# "refreshed" it would lock every reader in the swarm out at once — the
|
|
# same rule the store's TPM PIN unit follows, for a sharper reason.
|
|
#
|
|
# ⚠️ Not `RemainAfterExit`: the unit goes inactive after every run, so
|
|
# each start of a unit that wants or requires it, and each switch's start
|
|
# of `multi-user.target`, runs it again with the definition loaded at that
|
|
# moment. That is what mints a leaf a deploy adds, or a consumer it
|
|
# enables, on that deploy. An active `RemainAfterExit` unit satisfies all
|
|
# of those without running, and a switch only restarts it when its
|
|
# definition changes. A run where nothing is missing is one `[ -s ]` test
|
|
# per leaf.
|
|
systemd.services.swarm-bao-pki = {
|
|
description = "mint the swarm secret store's own CA and leaves";
|
|
before = [ "swarm-bao-certs.service" ];
|
|
requiredBy = [ "swarm-bao-certs.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
# No start limit: a mass container start pulls this in once per unit it
|
|
# starts, which outruns systemd's default of 5 starts in 10s. No
|
|
# `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
|
|
# `serviceConfig`.
|
|
startLimitIntervalSec = 0;
|
|
path = [
|
|
pkgs.openssl
|
|
pkgs.coreutils
|
|
];
|
|
serviceConfig.Type = "oneshot";
|
|
script = ''
|
|
set -euo pipefail
|
|
install -d -m 0700 ${pkiDir}
|
|
|
|
if [ ! -s ${pkiDir}/ca.pem ]; then
|
|
openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \
|
|
-keyout ${pkiDir}/ca-key.pem -out ${pkiDir}/ca.pem \
|
|
-subj "/CN=swarm-bao-ca ${cfg.domain}" \
|
|
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
|
-addext "keyUsage=critical,keyCertSign,cRLSign"
|
|
chmod 0600 ${pkiDir}/ca-key.pem
|
|
chmod 0644 ${pkiDir}/ca.pem
|
|
fi
|
|
|
|
# The store's own identity, and the identity of a reader on this host.
|
|
# A reader elsewhere gets its leaf from this CA too — that is what
|
|
# makes the store reachable from another machine at all, and why the
|
|
# CA is a file rather than a service.
|
|
[ -s ${pkiDir}/server.pem ] || ${signLeaf} ${pkiDir} server \
|
|
${lib.escapeShellArg cfg.domain} ${lib.escapeShellArg "DNS:${cfg.domain}"} serverAuth
|
|
[ -s ${pkiDir}/client.pem ] || ${signLeaf} ${pkiDir} client \
|
|
${lib.escapeShellArg clientCn} "" clientAuth
|
|
|
|
# Minted whether or not a controller runs here, because the case it
|
|
# serves is the one where it does not: a controller elsewhere needs a
|
|
# leaf from this CA and has no way to sign one. Issuing it here turns
|
|
# "obtain a certificate out of band" into "copy this file".
|
|
#
|
|
# Its own CN rather than the reader's above: the controller's policy
|
|
# lets it create roles for every hive, and the reader's leaf carries
|
|
# this hive's name.
|
|
[ -s ${pkiDir}/controller.pem ] || ${signLeaf} ${pkiDir} controller \
|
|
${lib.escapeShellArg deployCfg.bao.controllerCommonName} "" clientAuth
|
|
|
|
# The secret publisher's, minted here for the reason the controller's
|
|
# line above gives — and it serves that case more often, not less: the
|
|
# publisher runs in AUTHELIA's container, on the one host
|
|
# guaranteed not to be this one once the store has a host of its own.
|
|
[ -s ${pkiDir}/secret-publisher.pem ] || ${signLeaf} ${pkiDir} secret-publisher \
|
|
${lib.escapeShellArg deployCfg.bao.secretPublisherCommonName} "" clientAuth
|
|
|
|
# The matrix container's `swarm-matrix-ctl`. Minted unconditionally like the two
|
|
# above, and for the third variant of the same reason: the homeserver
|
|
# is a swarm singleton, so on every hive but the one running it this
|
|
# leaf is the file an operator copies rather than a file anything
|
|
# local reads.
|
|
#
|
|
# Its own CN, not the reader's: the reader's leaf carries this hive's
|
|
# name and its policy reads every agent credential in the swarm, while this
|
|
# principal may only write one path — which is the entire point of giving the
|
|
# container an identity instead of lending it the hive's.
|
|
[ -s ${pkiDir}/matrix-ctl.pem ] || ${signLeaf} ${pkiDir} matrix-ctl \
|
|
${lib.escapeShellArg deployCfg.bao.matrixCtlCommonName} "" clientAuth
|
|
|
|
# 🩸 The readers' leaves, separate from `client.pem` above. Each carries
|
|
# its own subject, which is the entire mechanism: bao matches a
|
|
# cert-auth role on the CN, so units sharing one leaf would be one
|
|
# principal holding the union of their grants, and the union would
|
|
# include every agent credential in the swarm.
|
|
#
|
|
# Minted whether or not the consumer runs here, for the reason the three
|
|
# leaves above give: on a hive that does not run the store this is the
|
|
# file an operator copies, and a leaf that only appears where its
|
|
# consumer does is one nobody can copy from anywhere.
|
|
[ -s ${pkiDir}/grafana-oidc.pem ] || ${signLeaf} ${pkiDir} grafana-oidc \
|
|
${lib.escapeShellArg deployCfg.bao.grafanaOidcCommonName} "" clientAuth
|
|
|
|
[ -s ${pkiDir}/otel-oidc.pem ] || ${signLeaf} ${pkiDir} otel-oidc \
|
|
${lib.escapeShellArg deployCfg.bao.otelOidcCommonName} "" clientAuth
|
|
|
|
[ -s ${pkiDir}/forwarder-oidc.pem ] || ${signLeaf} ${pkiDir} forwarder-oidc \
|
|
${lib.escapeShellArg deployCfg.bao.forwarderOidcCommonName} "" clientAuth
|
|
|
|
# The identity a gateway host presents to ask the store's `pki` mount
|
|
# for the swarm-services certificate it serves. Minted here like the
|
|
# three above, and the reason is the sharpest of the four: this leaf
|
|
# is what OPENS the mint, so it cannot come out of it. A certificate
|
|
# authority that issues the credential admitting you to it is the
|
|
# cycle this file's header exists to keep out — which is exactly why
|
|
# the swarm-services root lives in the store and this one does not.
|
|
#
|
|
# Its own CN, not the reader's: the hive's own leaf reads every secret
|
|
# its policy names, while this principal may do one thing — `update`
|
|
# on `pki/issue/swarm-services` — and giving a renewal unit the wider
|
|
# credential would undo the separation the role was created for.
|
|
[ -s ${pkiDir}/services-issuer.pem ] || ${signLeaf} ${pkiDir} services-issuer \
|
|
${lib.escapeShellArg deployCfg.bao.servicesIssuerCommonName} "" clientAuth
|
|
|
|
# The queue's, which opens `pki/issue/swarm-nats` and nothing else.
|
|
# Minted here for the services issuer's reason: it opens the mint, so
|
|
# it cannot come out of it. On a queue host without the store this is
|
|
# the file an operator copies.
|
|
[ -s ${pkiDir}/nats.pem ] || ${signLeaf} ${pkiDir} nats \
|
|
${lib.escapeShellArg deployCfg.bao.natsCommonName} "" clientAuth
|
|
|
|
# The granter's, which writes every `swarm-*` grant. Minted here because
|
|
# it opens the store for the units that create the roles every other
|
|
# leaf logs in with. Stays on this host; see its default above.
|
|
[ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
|
|
${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
|
|
|
|
# The queue's auth-callout responder, which reads agent queue
|
|
# credentials. Minted here for the matrix-ctl leaf's reason: the queue
|
|
# is a swarm singleton, so elsewhere this is the file an operator copies.
|
|
[ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
|
|
${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
|
|
|
|
# The log store's agent auth responder, which reads agent queue
|
|
# credentials. Minted here for the matrix-ctl leaf's reason: the log
|
|
# store is a swarm singleton, so elsewhere this is the file an operator
|
|
# copies.
|
|
[ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \
|
|
${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth
|
|
|
|
# The metrics store's auth responders, which read Grafana's metrics
|
|
# bearer and agents' queue secrets. Minted here for the matrix-ctl leaf's reason: the
|
|
# metrics store is a swarm singleton, so elsewhere this is the file an
|
|
# operator copies.
|
|
[ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \
|
|
${lib.escapeShellArg deployCfg.bao.metricsAuthCommonName} "" clientAuth
|
|
|
|
# The CI runner's token fetch. Minted here for the matrix-ctl leaf's
|
|
# reason: the runner is a swarm singleton, so on a runner host without
|
|
# the store this is the file an operator copies.
|
|
[ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \
|
|
${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth
|
|
|
|
# The review classifier's, which reads its API key and queue client
|
|
# secret. Minted here for the matrix-ctl leaf's reason: the classifier
|
|
# is a swarm service, so on a classifier host without the store this is
|
|
# the file an operator copies.
|
|
[ -s ${pkiDir}/review-classifier.pem ] || ${signLeaf} ${pkiDir} review-classifier \
|
|
${lib.escapeShellArg deployCfg.bao.reviewClassifierCommonName} "" clientAuth
|
|
|
|
# The swarm builder's, which reads its forge token and queue client
|
|
# secret. Minted here for the matrix-ctl leaf's reason: the builder is a
|
|
# swarm service, so on a builder host without the store this is the
|
|
# file an operator copies.
|
|
[ -s ${pkiDir}/builder.pem ] || ${signLeaf} ${pkiDir} builder \
|
|
${lib.escapeShellArg deployCfg.bao.builderCommonName} "" clientAuth
|
|
|
|
# The CI run poller's, which reads its forge token and OIDC client
|
|
# secret. Minted here for the matrix-ctl leaf's reason: the poller is a
|
|
# swarm singleton, so on a poller host without the store this is the
|
|
# file an operator copies.
|
|
[ -s ${pkiDir}/ci-metrics.pem ] || ${signLeaf} ${pkiDir} ci-metrics \
|
|
${lib.escapeShellArg deployCfg.bao.ciMetricsCommonName} "" clientAuth
|
|
|
|
# The claude login minter's, which writes agents' claude credentials
|
|
# and reads its queue client secret. Minted here for the matrix-ctl
|
|
# leaf's reason: the minter is a swarm service, so on a minter host
|
|
# without the store this is the file an operator copies.
|
|
[ -s ${pkiDir}/claude-login.pem ] || ${signLeaf} ${pkiDir} claude-login \
|
|
${lib.escapeShellArg deployCfg.bao.claudeLoginCommonName} "" clientAuth
|
|
|
|
# The notification router's, which reads its queue client secret.
|
|
# Minted here for the matrix-ctl leaf's reason: the router is a swarm
|
|
# service, so on a router host without the store this is the file an
|
|
# operator copies.
|
|
[ -s ${pkiDir}/notify-router.pem ] || ${signLeaf} ${pkiDir} notify-router \
|
|
${lib.escapeShellArg deployCfg.bao.notifyRouterCommonName} "" clientAuth
|
|
|
|
# The forge's token publisher. Minted here for the matrix-ctl leaf's
|
|
# reason: the forge is a swarm singleton, so on a forge host without the
|
|
# store this is the file an operator copies.
|
|
[ -s ${pkiDir}/forge-publisher.pem ] || ${signLeaf} ${pkiDir} forge-publisher \
|
|
${lib.escapeShellArg deployCfg.bao.forgePublisherCommonName} "" clientAuth
|
|
|
|
${lib.optionalString deployCfg.hive-controller.enable ''
|
|
# This hive's gateway-leaf identity, which opens
|
|
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
|
# subject carries this hive's name, because ./swarm-bao.nix writes
|
|
# one role per hive. Only where a hive runs; the other hives' are
|
|
# below.
|
|
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
|
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
|
''}
|
|
|
|
# The reader and gateway-leaf leaves of every other hive in
|
|
# the swarm directory: the files an operator copies to that hive's
|
|
# host. One directory per hive, with the basenames this hive's leaves
|
|
# use, so a hive's set is one directory to copy.
|
|
${lib.concatMapStrings (
|
|
h:
|
|
let
|
|
d = lib.escapeShellArg "${pkiDir}/hives/${h}";
|
|
in
|
|
''
|
|
install -d -m 0700 ${d}
|
|
[ -s ${d}/client.pem ] || ${signLeaf} ${d} client \
|
|
${lib.escapeShellArg h} "" clientAuth
|
|
[ -s ${d}/hive-leaf.pem ] || ${signLeaf} ${d} hive-leaf \
|
|
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${h}"} "" clientAuth
|
|
''
|
|
) otherHives}
|
|
'';
|
|
};
|
|
};
|
|
}
|