b072fbf4d6
A swarm-ci-metrics host unit on a 5-minute timer lists every forge repo's Actions runs as the new read-only ci-metrics account and pushes each finished run's duration and queue wait over OTLP, stamped at the run's stop time. The first pass backfills from run 1. swarm-controller mints the account's token into bao and keeps a ci-metrics team with read on the Actions unit in every organisation. Closes #5376
367 lines
20 KiB
Nix
367 lines
20 KiB
Nix
# Glue: give the secret store a PKI of its own, and point it at it.
|
||
#
|
||
# ONE PAIRING PER FILE — `glue-<consumer>-<what>.nix`. A single module holding
|
||
# every co-location default becomes the file nobody dares change, because a
|
||
# reader cannot tell which of its rules their deployment is subject to. Each
|
||
# of these should be deletable on its own, and deleting this one leaves a
|
||
# store that takes operator-provided certificates and nothing else.
|
||
#
|
||
# ⚠️ Why the PKI lives HERE and not in ./swarm-bao.nix: the store must have no
|
||
# opinion about where its identity comes from. Minting is an opinion — the
|
||
# most consequential one available — so it belongs to the glue that decides
|
||
# this deployment self-signs, not to the service that merely serves what it is
|
||
# handed. A deployment with a real internal CA drops this file and names its
|
||
# own paths; nothing in the store changes.
|
||
#
|
||
# ⚠️ Not the store's `pki` root, which the store distributes. An authority
|
||
# you must already hold a certificate from cannot be one the store hands
|
||
# out — reach the store to get the CA material, need a cert from that CA to
|
||
# reach the store. This CA signs a fixed, short
|
||
# list of leaves and distributes nothing, so it cannot enter that cycle.
|
||
#
|
||
# ⚠️ Files like this are the only place a `deploy.<foo>` value may derive from
|
||
# a `deploy.<bar>.enable`. Everywhere else that is forbidden. The exception
|
||
# earns itself: the derivation happens either way, and the alternative is
|
||
# having it spread through the service modules where it is invisible.
|
||
#
|
||
# Everything is `mkDefault`. An operator naming their own paths wins.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
deployCfg = hyperhiveCfg.deploy;
|
||
cfg = hyperhiveCfg.swarm.bao;
|
||
|
||
# Host-side, outside the container's tree, for the same reason the raft data
|
||
# is: `nixos-container destroy` must not take it. Losing the CA key means
|
||
# re-issuing every client certificate in the swarm.
|
||
pkiDir = import ./lib/bao-pki-dir.nix;
|
||
|
||
# What a reader calls itself to the store. The hive's name, because a bao
|
||
# cert-auth role matches on the CN — this is an interface, not a label.
|
||
# No fallback: `hiveName` is asserted set wherever `deploy.bao.enable` is
|
||
# (./hyperhive.nix), the same condition this file's `config` is gated on. A
|
||
# fallback here reads as a second supported spelling and there is no such
|
||
# thing.
|
||
clientCn = hyperhiveCfg.hiveName;
|
||
|
||
# Every other hive in the swarm directory. This host's own hive is signed
|
||
# into the top of the directory, beside the leaves its units read.
|
||
otherHives = lib.filter (h: h != clientCn) (lib.attrNames hyperhiveCfg.swarm.hives);
|
||
|
||
# $1 dir $2 basename $3 CN $4 SAN or "" $5 EKU
|
||
# The CA is always the one at the top of `pkiDir`, whichever dir the leaf
|
||
# lands in.
|
||
signLeaf = pkgs.writeShellScript "swarm-bao-sign-leaf" ''
|
||
set -euo pipefail
|
||
d="$1"; base="$2"; cn="$3"; sans="$4"; eku="$5"
|
||
csr="$(mktemp "$d/$base.csr.XXXXXX")"
|
||
ext="$(mktemp "$d/$base.ext.XXXXXX")"
|
||
trap 'rm -f "$csr" "$ext"' EXIT
|
||
|
||
openssl req -newkey rsa:4096 -nodes -sha256 \
|
||
-keyout "$d/$base-key.pem" -out "$csr" -subj "/CN=$cn"
|
||
{
|
||
[ -n "$sans" ] && printf 'subjectAltName=%s\n' "$sans"
|
||
printf 'basicConstraints=critical,CA:FALSE\n'
|
||
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
|
||
printf 'extendedKeyUsage=%s\n' "$eku"
|
||
} > "$ext"
|
||
openssl x509 -req -in "$csr" -CA ${pkiDir}/ca.pem -CAkey ${pkiDir}/ca-key.pem \
|
||
-CAcreateserial -days 3650 -sha256 -extfile "$ext" -out "$d/$base.pem"
|
||
chmod 0600 "$d/$base-key.pem"
|
||
chmod 0644 "$d/$base.pem"
|
||
'';
|
||
in
|
||
{
|
||
config = lib.mkIf deployCfg.bao.enable {
|
||
services.hyperhive.deploy.bao = {
|
||
serverCertFile = lib.mkDefault "${pkiDir}/server.pem";
|
||
serverKeyFile = lib.mkDefault "${pkiDir}/server-key.pem";
|
||
clientCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
||
|
||
# A reader on this host, which happens to be the host that mints. Only
|
||
# these three are what a reader elsewhere needs placed by hand; that they
|
||
# collapse to the same CA file here is a property of self-signing, not of
|
||
# the pairing.
|
||
clientCertFile = lib.mkDefault "${pkiDir}/client.pem";
|
||
clientKeyFile = lib.mkDefault "${pkiDir}/client-key.pem";
|
||
serverCaFile = lib.mkDefault "${pkiDir}/ca.pem";
|
||
|
||
# 🩸 The readers below each present a leaf of their own rather than
|
||
# `client.pem` above. The leaf is what bao sees, so this pairing is the
|
||
# whole of what turns "one principal with the union of their grants" into
|
||
# one principal per grant — see ./swarm-bao.nix's `perHiveReaders` block
|
||
# for the grants themselves.
|
||
#
|
||
# Defaulted here rather than in `glue-<consumer>-bao-identity.nix` files:
|
||
# those exist where the consumer is a CONTAINER with a
|
||
# `deploy.<service>.*` namespace of its own to point at. These are host
|
||
# units reading the store, which is the pairing this file already is.
|
||
queueAgentClientCertFile = lib.mkDefault "${pkiDir}/queue-agent.pem";
|
||
queueAgentClientKeyFile = lib.mkDefault "${pkiDir}/queue-agent-key.pem";
|
||
grafanaOidcClientCertFile = lib.mkDefault "${pkiDir}/grafana-oidc.pem";
|
||
grafanaOidcClientKeyFile = lib.mkDefault "${pkiDir}/grafana-oidc-key.pem";
|
||
otelOidcClientCertFile = lib.mkDefault "${pkiDir}/otel-oidc.pem";
|
||
otelOidcClientKeyFile = lib.mkDefault "${pkiDir}/otel-oidc-key.pem";
|
||
# The store forwarder's.
|
||
forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem";
|
||
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
|
||
ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem";
|
||
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
|
||
reviewClassifierClientCertFile = lib.mkDefault "${pkiDir}/review-classifier.pem";
|
||
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
|
||
builderClientCertFile = lib.mkDefault "${pkiDir}/builder.pem";
|
||
builderClientKeyFile = lib.mkDefault "${pkiDir}/builder-key.pem";
|
||
ciMetricsClientCertFile = lib.mkDefault "${pkiDir}/ci-metrics.pem";
|
||
ciMetricsClientKeyFile = lib.mkDefault "${pkiDir}/ci-metrics-key.pem";
|
||
claudeLoginClientCertFile = lib.mkDefault "${pkiDir}/claude-login.pem";
|
||
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
|
||
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
|
||
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
|
||
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
|
||
servicesIssuerClientKeyFile = lib.mkDefault "${pkiDir}/services-issuer-key.pem";
|
||
|
||
# The granter: every `swarm-bao-*-policy` unit on this host logs in with
|
||
# it. ⚠️ Unlike every other leaf here, never the file an operator copies:
|
||
# its policy is root-equivalent and its only reader is this host.
|
||
granterClientCertFile = lib.mkDefault "${pkiDir}/granter.pem";
|
||
granterClientKeyFile = lib.mkDefault "${pkiDir}/granter-key.pem";
|
||
};
|
||
|
||
# Idempotent on ABSENCE, never on content. Re-issuing the CA invalidates
|
||
# every client certificate already trusting it, so a rebuild that
|
||
# "refreshed" it would lock every reader in the swarm out at once — the
|
||
# same rule the store's TPM PIN unit follows, for a sharper reason.
|
||
#
|
||
# ⚠️ Not `RemainAfterExit`: the unit goes inactive after every run, so
|
||
# each start of a unit that wants or requires it, and each switch's start
|
||
# of `multi-user.target`, runs it again with the definition loaded at that
|
||
# moment. That is what mints a leaf a deploy adds, or a consumer it
|
||
# enables, on that deploy. An active `RemainAfterExit` unit satisfies all
|
||
# of those without running, and a switch only restarts it when its
|
||
# definition changes. A run where nothing is missing is one `[ -s ]` test
|
||
# per leaf.
|
||
systemd.services.swarm-bao-pki = {
|
||
description = "mint the swarm secret store's own CA and leaves";
|
||
before = [ "swarm-bao-certs.service" ];
|
||
requiredBy = [ "swarm-bao-certs.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
# No start limit: a mass container start pulls this in once per unit it
|
||
# starts, which outruns systemd's default of 5 starts in 10s. No
|
||
# `Restart=`, so nothing here loops. `[Unit]` setting, hence not in
|
||
# `serviceConfig`.
|
||
startLimitIntervalSec = 0;
|
||
path = [
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
];
|
||
serviceConfig.Type = "oneshot";
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${pkiDir}
|
||
|
||
if [ ! -s ${pkiDir}/ca.pem ]; then
|
||
openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \
|
||
-keyout ${pkiDir}/ca-key.pem -out ${pkiDir}/ca.pem \
|
||
-subj "/CN=swarm-bao-ca ${cfg.domain}" \
|
||
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
||
-addext "keyUsage=critical,keyCertSign,cRLSign"
|
||
chmod 0600 ${pkiDir}/ca-key.pem
|
||
chmod 0644 ${pkiDir}/ca.pem
|
||
fi
|
||
|
||
# The store's own identity, and the identity of a reader on this host.
|
||
# A reader elsewhere gets its leaf from this CA too — that is what
|
||
# makes the store reachable from another machine at all, and why the
|
||
# CA is a file rather than a service.
|
||
[ -s ${pkiDir}/server.pem ] || ${signLeaf} ${pkiDir} server \
|
||
${lib.escapeShellArg cfg.domain} ${lib.escapeShellArg "DNS:${cfg.domain}"} serverAuth
|
||
[ -s ${pkiDir}/client.pem ] || ${signLeaf} ${pkiDir} client \
|
||
${lib.escapeShellArg clientCn} "" clientAuth
|
||
|
||
# Minted whether or not a controller runs here, because the case it
|
||
# serves is the one where it does not: a controller elsewhere needs a
|
||
# leaf from this CA and has no way to sign one. Issuing it here turns
|
||
# "obtain a certificate out of band" into "copy this file".
|
||
#
|
||
# Its own CN rather than the reader's above: the controller's policy
|
||
# lets it create roles for every hive, and the reader's leaf carries
|
||
# this hive's name.
|
||
[ -s ${pkiDir}/controller.pem ] || ${signLeaf} ${pkiDir} controller \
|
||
${lib.escapeShellArg deployCfg.bao.controllerCommonName} "" clientAuth
|
||
|
||
# The secret publisher's, minted here for the reason the controller's
|
||
# line above gives — and it serves that case more often, not less: the
|
||
# publisher runs beside AUTHELIA, which is the one host guaranteed not
|
||
# to be this one whenever the store has a host of its own.
|
||
[ -s ${pkiDir}/secret-publisher.pem ] || ${signLeaf} ${pkiDir} secret-publisher \
|
||
${lib.escapeShellArg deployCfg.bao.secretPublisherCommonName} "" clientAuth
|
||
|
||
# The matrix container's `swarm-matrix-ctl`. Minted unconditionally like the two
|
||
# above, and for the third variant of the same reason: the homeserver
|
||
# is a swarm singleton, so on every hive but the one running it this
|
||
# leaf is the file an operator copies rather than a file anything
|
||
# local reads.
|
||
#
|
||
# Its own CN, not the reader's: the reader's leaf carries this hive's
|
||
# name and its policy reads every agent credential in the swarm, while this
|
||
# principal may only write one path — which is the entire point of giving the
|
||
# container an identity instead of lending it the hive's.
|
||
[ -s ${pkiDir}/matrix-ctl.pem ] || ${signLeaf} ${pkiDir} matrix-ctl \
|
||
${lib.escapeShellArg deployCfg.bao.matrixCtlCommonName} "" clientAuth
|
||
|
||
# 🩸 The readers' leaves, separate from `client.pem` above. Each carries
|
||
# its own subject, which is the entire mechanism: bao matches a
|
||
# cert-auth role on the CN, so units sharing one leaf would be one
|
||
# principal holding the union of their grants, and the union would
|
||
# include every agent credential in the swarm.
|
||
#
|
||
# ⚠️ The first carries THIS hive's name in the subject, unlike the three
|
||
# service leaves above. Its grant names one hive's path, because a queue
|
||
# credential lives under `swarm/hives/<name>/` and every hive runs a
|
||
# reader for its own; ./swarm-bao.nix writes one role per hive in the
|
||
# swarm directory to match. The OIDC readers need no such segment — a
|
||
# client is registered once per swarm.
|
||
#
|
||
# Minted whether or not the consumer runs here, for the reason the three
|
||
# leaves above give: on a hive that does not run the store this is the
|
||
# file an operator copies, and a leaf that only appears where its
|
||
# consumer does is one nobody can copy from anywhere.
|
||
[ -s ${pkiDir}/queue-agent.pem ] || ${signLeaf} ${pkiDir} queue-agent \
|
||
${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/grafana-oidc.pem ] || ${signLeaf} ${pkiDir} grafana-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.grafanaOidcCommonName} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/otel-oidc.pem ] || ${signLeaf} ${pkiDir} otel-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.otelOidcCommonName} "" clientAuth
|
||
|
||
[ -s ${pkiDir}/forwarder-oidc.pem ] || ${signLeaf} ${pkiDir} forwarder-oidc \
|
||
${lib.escapeShellArg deployCfg.bao.forwarderOidcCommonName} "" clientAuth
|
||
|
||
# The identity a gateway host presents to ask the store's `pki` mount
|
||
# for the swarm-services certificate it serves. Minted here like the
|
||
# three above, and the reason is the sharpest of the four: this leaf
|
||
# is what OPENS the mint, so it cannot come out of it. A certificate
|
||
# authority that issues the credential admitting you to it is the
|
||
# cycle this file's header exists to keep out — which is exactly why
|
||
# the swarm-services root lives in the store and this one does not.
|
||
#
|
||
# Its own CN, not the reader's: the hive's own leaf reads every secret
|
||
# its policy names, while this principal may do one thing — `update`
|
||
# on `pki/issue/swarm-services` — and giving a renewal unit the wider
|
||
# credential would undo the separation the role was created for.
|
||
[ -s ${pkiDir}/services-issuer.pem ] || ${signLeaf} ${pkiDir} services-issuer \
|
||
${lib.escapeShellArg deployCfg.bao.servicesIssuerCommonName} "" clientAuth
|
||
|
||
# The queue's, which opens `pki/issue/swarm-nats` and nothing else.
|
||
# Minted here for the services issuer's reason: it opens the mint, so
|
||
# it cannot come out of it. On a queue host without the store this is
|
||
# the file an operator copies.
|
||
[ -s ${pkiDir}/nats.pem ] || ${signLeaf} ${pkiDir} nats \
|
||
${lib.escapeShellArg deployCfg.bao.natsCommonName} "" clientAuth
|
||
|
||
# The granter's, which writes every `swarm-*` grant. Minted here because
|
||
# it opens the store for the units that create the roles every other
|
||
# leaf logs in with. Stays on this host; see its default above.
|
||
[ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
|
||
${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
|
||
|
||
# The queue's auth-callout responder, which reads agent queue
|
||
# credentials. Minted here for the matrix-ctl leaf's reason: the queue
|
||
# is a swarm singleton, so elsewhere this is the file an operator copies.
|
||
[ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
|
||
${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
|
||
|
||
# The log store's agent auth responder, which reads agent queue
|
||
# credentials. Minted here for the matrix-ctl leaf's reason: the log
|
||
# store is a swarm singleton, so elsewhere this is the file an operator
|
||
# copies.
|
||
[ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \
|
||
${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth
|
||
|
||
# The metrics store's Grafana auth responder, which reads Grafana's
|
||
# metrics bearer. Minted here for the matrix-ctl leaf's reason: the
|
||
# metrics store is a swarm singleton, so elsewhere this is the file an
|
||
# operator copies.
|
||
[ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \
|
||
${lib.escapeShellArg deployCfg.bao.metricsAuthCommonName} "" clientAuth
|
||
|
||
# The CI runner's token fetch. Minted here for the matrix-ctl leaf's
|
||
# reason: the runner is a swarm singleton, so on a runner host without
|
||
# the store this is the file an operator copies.
|
||
[ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \
|
||
${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth
|
||
|
||
# The review classifier's, which reads its API key and queue client
|
||
# secret. Minted here for the matrix-ctl leaf's reason: the classifier
|
||
# is a swarm service, so on a classifier host without the store this is
|
||
# the file an operator copies.
|
||
[ -s ${pkiDir}/review-classifier.pem ] || ${signLeaf} ${pkiDir} review-classifier \
|
||
${lib.escapeShellArg deployCfg.bao.reviewClassifierCommonName} "" clientAuth
|
||
|
||
# The swarm builder's, which reads its forge token and queue client
|
||
# secret. Minted here for the matrix-ctl leaf's reason: the builder is a
|
||
# swarm service, so on a builder host without the store this is the
|
||
# file an operator copies.
|
||
[ -s ${pkiDir}/builder.pem ] || ${signLeaf} ${pkiDir} builder \
|
||
${lib.escapeShellArg deployCfg.bao.builderCommonName} "" clientAuth
|
||
|
||
# The CI run poller's, which reads its forge token and OIDC client
|
||
# secret. Minted here for the matrix-ctl leaf's reason: the poller is a
|
||
# swarm singleton, so on a poller host without the store this is the
|
||
# file an operator copies.
|
||
[ -s ${pkiDir}/ci-metrics.pem ] || ${signLeaf} ${pkiDir} ci-metrics \
|
||
${lib.escapeShellArg deployCfg.bao.ciMetricsCommonName} "" clientAuth
|
||
|
||
# The claude login minter's, which writes agents' claude credentials
|
||
# and reads its queue client secret. Minted here for the matrix-ctl
|
||
# leaf's reason: the minter is a swarm service, so on a minter host
|
||
# without the store this is the file an operator copies.
|
||
[ -s ${pkiDir}/claude-login.pem ] || ${signLeaf} ${pkiDir} claude-login \
|
||
${lib.escapeShellArg deployCfg.bao.claudeLoginCommonName} "" clientAuth
|
||
|
||
# The forge's token publisher. Minted here for the matrix-ctl leaf's
|
||
# reason: the forge is a swarm singleton, so on a forge host without the
|
||
# store this is the file an operator copies.
|
||
[ -s ${pkiDir}/forge-publisher.pem ] || ${signLeaf} ${pkiDir} forge-publisher \
|
||
${lib.escapeShellArg deployCfg.bao.forgePublisherCommonName} "" clientAuth
|
||
|
||
${lib.optionalString deployCfg.hive-controller.enable ''
|
||
# This hive's gateway-leaf identity, which opens
|
||
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
||
# subject carries this hive's name, like the queue-agent leaf
|
||
# above, because ./swarm-bao.nix writes one role
|
||
# per hive. Only where a hive runs; the other hives' are below.
|
||
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||
''}
|
||
|
||
# The reader, queue-agent and gateway-leaf leaves of every other hive in
|
||
# the swarm directory: the files an operator copies to that hive's
|
||
# host. One directory per hive, with the basenames this hive's leaves
|
||
# use, so a hive's set is one directory to copy.
|
||
${lib.concatMapStrings (
|
||
h:
|
||
let
|
||
d = lib.escapeShellArg "${pkiDir}/hives/${h}";
|
||
in
|
||
''
|
||
install -d -m 0700 ${d}
|
||
[ -s ${d}/client.pem ] || ${signLeaf} ${d} client \
|
||
${lib.escapeShellArg h} "" clientAuth
|
||
[ -s ${d}/queue-agent.pem ] || ${signLeaf} ${d} queue-agent \
|
||
${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${h}"} "" clientAuth
|
||
[ -s ${d}/hive-leaf.pem ] || ${signLeaf} ${d} hive-leaf \
|
||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${h}"} "" clientAuth
|
||
''
|
||
) otherHives}
|
||
'';
|
||
};
|
||
};
|
||
}
|