drop the shared hive agent login
Every agent now connects to the queue and reads logs with its own secret from the store (#5181), so the per-hive `hive-<h>-agent` OIDC client has no reader left. Remove it end to end: - hive-c0re no longer forwards `hive-queue-agent-{secret,client-id}` into containers, and no longer forwards HIVE_AGENT_OIDC_TOKEN_ENDPOINT; the agent module that loaded them (nix/agent-modules/queue.nix) is gone, its options refused with a removal message. - The host reader (glue-queue-agent-credential.nix, swarm-bao-queue-agent), its store role and policy unit (swarm-bao-queue-agent-policy), its PKI leaf and the deploy.bao.queueAgent* / queue.agentCredentialDir options are gone; the options are refused with a removal message. - authelia mints no `hive-<h>-agent` client, and swarm-secret-publish no longer writes swarm/hives/<h>/queue/agent. - swarm-secret-client drops queue::agent_client_path / queue::Credential. - HIVE_AGENT_NATS_URL is now gated on the agents' queue address alone. The log store's authelia check stays for operators. swarm-nats-auth still refuses a `-agent` client id; that refusal and `agentClientSuffix` are left for a follow-up. Refs #4459 Refs #4765
This commit is contained in:
parent
d19bd83aa5
commit
868c6cf96c
41 changed files with 286 additions and 1410 deletions
|
|
@ -47,7 +47,6 @@
|
|||
./glue-nats-bao-identity.nix
|
||||
./glue-nats-oidc-client.nix
|
||||
./glue-nats-wireguard.nix
|
||||
./glue-queue-agent-credential.nix
|
||||
./glue-builder-oidc-client.nix
|
||||
./glue-ci-metrics-otel-oidc-client.nix
|
||||
./glue-ci-runner-otel-oidc-client.nix
|
||||
|
|
|
|||
|
|
@ -53,6 +53,14 @@ let
|
|||
puts it there. Remove this definition; the file it named is read by
|
||||
nothing now and can be deleted.
|
||||
'';
|
||||
|
||||
# Shared by the removed options of the hive's agent queue credential reader.
|
||||
queueAgentCredentialRemoved = ''
|
||||
A hive no longer fetches a shared queue credential for its agents: each
|
||||
agent reads its own secret from the swarm's secret store under its own
|
||||
certificate. Remove this definition; the leaf or directory it named is read
|
||||
by nothing now and can be deleted.
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
|
|
@ -95,6 +103,35 @@ in
|
|||
"statusPublish"
|
||||
"clientSecretFile"
|
||||
] statusSecretRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"bao"
|
||||
"queueAgentCommonNamePrefix"
|
||||
] queueAgentCredentialRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"bao"
|
||||
"queueAgentClientCertFile"
|
||||
] queueAgentCredentialRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"bao"
|
||||
"queueAgentClientKeyFile"
|
||||
] queueAgentCredentialRemoved)
|
||||
(lib.mkRemovedOptionModule [
|
||||
"services"
|
||||
"hyperhive"
|
||||
"deploy"
|
||||
"hive-controller"
|
||||
"queue"
|
||||
"agentCredentialDir"
|
||||
] queueAgentCredentialRemoved)
|
||||
|
||||
# Same type, same meaning, new path — so a rename carries it exactly
|
||||
# and existing configs keep evaluating with one warning naming both
|
||||
|
|
|
|||
|
|
@ -31,10 +31,6 @@ let
|
|||
|
||||
# Reader unit → the gate its own module defines it under.
|
||||
readersHere = {
|
||||
# ./glue-queue-agent-credential.nix
|
||||
swarm-bao-queue-agent =
|
||||
deployCfg.hive-controller.enable
|
||||
&& havePair baoDeploy.queueAgentClientCertFile baoDeploy.queueAgentClientKeyFile;
|
||||
# ./swarm-grafana.nix
|
||||
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
|
||||
# ./swarm-otel.nix
|
||||
|
|
|
|||
|
|
@ -95,15 +95,13 @@ in
|
|||
# 🩸 The readers below each present a leaf of their own rather than
|
||||
# `client.pem` above. The leaf is what bao sees, so this pairing is the
|
||||
# whole of what turns "one principal with the union of their grants" into
|
||||
# one principal per grant — see ./swarm-bao.nix's `perHiveReaders` block
|
||||
# for the grants themselves.
|
||||
# one principal per grant — see ./swarm-bao.nix's `readStanza` block for
|
||||
# the grants themselves.
|
||||
#
|
||||
# Defaulted here rather than in `glue-<consumer>-bao-identity.nix` files:
|
||||
# those exist where the consumer is a CONTAINER with a
|
||||
# `deploy.<service>.*` namespace of its own to point at. These are host
|
||||
# units reading the store, which is the pairing this file already is.
|
||||
queueAgentClientCertFile = lib.mkDefault "${pkiDir}/queue-agent.pem";
|
||||
queueAgentClientKeyFile = lib.mkDefault "${pkiDir}/queue-agent-key.pem";
|
||||
grafanaOidcClientCertFile = lib.mkDefault "${pkiDir}/grafana-oidc.pem";
|
||||
grafanaOidcClientKeyFile = lib.mkDefault "${pkiDir}/grafana-oidc-key.pem";
|
||||
otelOidcClientCertFile = lib.mkDefault "${pkiDir}/otel-oidc.pem";
|
||||
|
|
@ -223,20 +221,10 @@ in
|
|||
# principal holding the union of their grants, and the union would
|
||||
# include every agent credential in the swarm.
|
||||
#
|
||||
# ⚠️ The first carries THIS hive's name in the subject, unlike the three
|
||||
# service leaves above. Its grant names one hive's path, because a queue
|
||||
# credential lives under `swarm/hives/<name>/` and every hive runs a
|
||||
# reader for its own; ./swarm-bao.nix writes one role per hive in the
|
||||
# swarm directory to match. The OIDC readers need no such segment — a
|
||||
# client is registered once per swarm.
|
||||
#
|
||||
# Minted whether or not the consumer runs here, for the reason the three
|
||||
# leaves above give: on a hive that does not run the store this is the
|
||||
# file an operator copies, and a leaf that only appears where its
|
||||
# consumer does is one nobody can copy from anywhere.
|
||||
[ -s ${pkiDir}/queue-agent.pem ] || ${signLeaf} ${pkiDir} queue-agent \
|
||||
${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||||
|
||||
[ -s ${pkiDir}/grafana-oidc.pem ] || ${signLeaf} ${pkiDir} grafana-oidc \
|
||||
${lib.escapeShellArg deployCfg.bao.grafanaOidcCommonName} "" clientAuth
|
||||
|
||||
|
|
@ -344,14 +332,14 @@ in
|
|||
${lib.optionalString deployCfg.hive-controller.enable ''
|
||||
# This hive's gateway-leaf identity, which opens
|
||||
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
||||
# subject carries this hive's name, like the queue-agent leaf
|
||||
# above, because ./swarm-bao.nix writes one role
|
||||
# per hive. Only where a hive runs; the other hives' are below.
|
||||
# subject carries this hive's name, because ./swarm-bao.nix writes
|
||||
# one role per hive. Only where a hive runs; the other hives' are
|
||||
# below.
|
||||
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
||||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||||
''}
|
||||
|
||||
# The reader, queue-agent and gateway-leaf leaves of every other hive in
|
||||
# The reader and gateway-leaf leaves of every other hive in
|
||||
# the swarm directory: the files an operator copies to that hive's
|
||||
# host. One directory per hive, with the basenames this hive's leaves
|
||||
# use, so a hive's set is one directory to copy.
|
||||
|
|
@ -364,8 +352,6 @@ in
|
|||
install -d -m 0700 ${d}
|
||||
[ -s ${d}/client.pem ] || ${signLeaf} ${d} client \
|
||||
${lib.escapeShellArg h} "" clientAuth
|
||||
[ -s ${d}/queue-agent.pem ] || ${signLeaf} ${d} queue-agent \
|
||||
${lib.escapeShellArg "${deployCfg.bao.queueAgentCommonNamePrefix}-${h}"} "" clientAuth
|
||||
[ -s ${d}/hive-leaf.pem ] || ${signLeaf} ${d} hive-leaf \
|
||||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${h}"} "" clientAuth
|
||||
''
|
||||
|
|
|
|||
|
|
@ -1,284 +0,0 @@
|
|||
# Glue: this hive's agent queue credential comes out of the secret store.
|
||||
#
|
||||
# A cert login that fails LOUDLY because every state it fails on is one a
|
||||
# retry fixes, then a read that degrades QUIETLY because no retry turns "no
|
||||
# value there" into a value.
|
||||
#
|
||||
# ⚠️ Two files, not one, and that is the consumer's shape rather than a
|
||||
# preference. `swarm-logs` inside the agent reads the secret and the client id
|
||||
# each from its own file (`<prefix>_OIDC_CLIENT_SECRET_FILE`,
|
||||
# `<prefix>_OIDC_CLIENT_ID_FILE`), so splitting them here is what lets
|
||||
# hive-c0re hand both to an agent without parsing anything.
|
||||
#
|
||||
# ⚠️ An ABSENT file means this hive's agents do not connect to the queue, and
|
||||
# that is correct rather than degraded. The publisher runs on the authelia
|
||||
# host and authelia mints on its first boot, so "nothing at that path yet" is
|
||||
# the ordinary early state of a swarm. Nothing here writes a local stand-in:
|
||||
# there is no such thing as a locally valid OIDC client secret, so a placeholder would turn a hive that cannot connect
|
||||
# into one that is refused, which reaches the agent as a timeout.
|
||||
#
|
||||
# 📌 This runs wherever a client identity is configured, NOT only where the
|
||||
# store is. `deploy.bao.enable` would have been the co-location assumption
|
||||
# itself; the reader needs a certificate, not a neighbour. On the store's own
|
||||
# host ./glue-bao-tls.nix supplies one as a `mkDefault`; elsewhere an operator
|
||||
# places the leaf and names it, and the same unit works. It also needs this
|
||||
# host to run a hive (`deploy.hive-controller.enable`): the credential is for
|
||||
# the agents hive-c0re creates here at runtime, and the unit writes under
|
||||
# hive-c0re's own state directory.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
deployCfg = hyperhiveCfg.deploy;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
|
||||
baoDeploy = deployCfg.bao;
|
||||
|
||||
# What decides whether this unit exists at all. A reader is defined by holding
|
||||
# a certificate the store accepts, and that is true on the store's own host
|
||||
# and on a hive three networks away for exactly the same reason.
|
||||
#
|
||||
# 🩸 This principal's OWN leaf, not `clientCertFile` — the hive's. Bao
|
||||
# matches a cert-auth role on the CN, so one leaf shared by four readers
|
||||
# would be ONE principal holding the union of four grants: read on
|
||||
# `swarm/agents/*` AND `swarm/hives/<hive>/*`, when this unit reads one
|
||||
# queue credential and nothing else. Its own leaf carries
|
||||
# `<queueAgentCommonNamePrefix>-<hive>` and its role grants the single path
|
||||
# below — still this hive's own, so the narrowing costs no reach.
|
||||
haveClientIdentity =
|
||||
baoDeploy.queueAgentClientCertFile != null && baoDeploy.queueAgentClientKeyFile != null;
|
||||
|
||||
# Does this host read the store at all — the hive's own leaf, which is the
|
||||
# one thing a remote-store deployment has to place by hand. Only used to
|
||||
# decide whether a missing per-principal leaf is a mistake or a deployment
|
||||
# that has no store: a host holding neither is the supported no-store
|
||||
# shape, and one holding this pair but not the pair above reads the store
|
||||
# without a reader identity of its own.
|
||||
hiveReaderIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
|
||||
|
||||
credentialDir = toString deployCfg.hive-controller.queue.agentCredentialDir;
|
||||
secretFile = "${credentialDir}/secret";
|
||||
clientIdFile = "${credentialDir}/client_id";
|
||||
|
||||
# Where the credential lives in the store, spelled from the same pieces the
|
||||
# writer uses. Whoever writes it and whoever reads it must agree, and the
|
||||
# agreement belongs in one visible place.
|
||||
#
|
||||
# ⚠️ The `hives/<name>` segment is not decoration — it is what the reader's
|
||||
# own grant covers, so a path outside it is a 403 rather than a miss, however
|
||||
# correct it looks. `swarm-secret-client`'s `queue::agent_client_path` builds
|
||||
# the same string from the same pieces; this literal is the nix half of that
|
||||
# one agreement.
|
||||
#
|
||||
# `hiveName` has no fallback here for the reason ./glue-bao-tls.nix gives at
|
||||
# its own use of it: it is asserted set wherever
|
||||
# `deploy.hive-controller.enable` is.
|
||||
credentialPath = "secret/swarm/hives/${hyperhiveCfg.hiveName}/queue/agent";
|
||||
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.deploy.hive-controller.queue = {
|
||||
agentCredentialDir = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = "/var/lib/hyperhive/queue-agent";
|
||||
description = ''
|
||||
Host directory holding this hive's agent queue credential: `secret`
|
||||
(the OIDC client secret, `0600`) and `client_id` (the client that
|
||||
secret authenticates, `0644` — it is sent to the token endpoint on
|
||||
every connection and is not itself a secret).
|
||||
|
||||
Both files are written by `swarm-bao-queue-agent.service`, which also
|
||||
creates this directory. Named here because nothing else creates it:
|
||||
the parent `/var/lib/hyperhive` belongs to `hive-c0re.service`'s
|
||||
`StateDirectory=`, which re-applies its own mode on every start, and a
|
||||
second declaration of *that* path is the two-mechanisms-one-path trap
|
||||
./hive-gateway/default.nix records — so this directory is the unit's
|
||||
to make and the parent stays c0re's.
|
||||
|
||||
Absent files mean this hive has no queue credential yet, which is what
|
||||
a swarm looks like before the publisher in authelia's container has run.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkMerge [
|
||||
# ⚠️ A SEPARATE arm from the unit below, and that separation is the whole
|
||||
# mechanism: the unit's arm is gated on `haveClientIdentity`, so an
|
||||
# assertion written inside it could never be reached in the state it
|
||||
# exists to report.
|
||||
#
|
||||
# Shaped after ./swarm-grafana.nix's `haveClientIdentity` assertion — the
|
||||
# same refusal, named to this principal's own pair. Where Grafana's gate
|
||||
# is `deploy.grafana.enable`, this reader's is the hive's own: every hive
|
||||
# runs one for its own agents, so on a hive, reading the store at all is
|
||||
# what asks for it. Hence `hiveReaderIdentity` — a host holding no hive
|
||||
# leaf has no store to read and nothing is missing.
|
||||
(lib.mkIf (deployCfg.hive-controller.enable && hiveReaderIdentity) {
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
This host reads the swarm secret store (services.hyperhive.deploy.bao.clientCertFile
|
||||
is set), so it needs the agent queue credential reader's own client
|
||||
identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.queueAgentClientCertFile
|
||||
services.hyperhive.deploy.bao.queueAgentClientKeyFile
|
||||
|
||||
swarm-bao-queue-agent.service fetches this hive's agent queue
|
||||
credential out of the store, and without these it is not rendered
|
||||
at all — leaving hive-c0re with no queue credential and no unit
|
||||
that would ever have written one.
|
||||
|
||||
Every hive runs this reader for its own agents, so unlike the other
|
||||
three principals there is no per-service toggle that turns it off:
|
||||
a hive that reads the store at all is a hive that needs this pair.
|
||||
|
||||
⚠️ This reader's OWN leaf, not deploy.bao.clientCertFile. That one
|
||||
is the hive's, and its grant reads every agent credential in the
|
||||
swarm and this hive's own secrets; this role reads the one
|
||||
queue-credential path. Pointing this option at the hive's leaf
|
||||
would evaluate, deploy and log in — and undo the split.
|
||||
|
||||
On a hive that runs the store, glue-bao-tls.nix supplies both as
|
||||
defaults and there is nothing to do. Elsewhere copy the pair from
|
||||
/var/lib/swarm-bao-pki/hives/<hive>/ on the store's host and name it
|
||||
here — see docs/swarm/bao.md.
|
||||
'';
|
||||
}
|
||||
];
|
||||
})
|
||||
|
||||
(lib.mkIf (deployCfg.hive-controller.enable && haveClientIdentity) {
|
||||
systemd.services.swarm-bao-queue-agent = {
|
||||
description = "fetch this hive's agent queue credential from the swarm secret store";
|
||||
# Every one of these names a unit that exists only where the store runs,
|
||||
# so the ordering is conditional even though the read is not: off-host
|
||||
# there is nothing local to wait for, and the timeout below is what
|
||||
# bounds the attempt instead. `Wants=` pki, for
|
||||
# ./glue-bao-readers-store-order.nix's reason.
|
||||
after = lib.optionals baoDeploy.enable [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
wants = lib.optionals baoDeploy.enable [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
# Ordered before hive-c0re, so no agent container renders ahead of an
|
||||
# attempt at its credential. `Wants=`, not `Requires=`: a store this
|
||||
# unit can't reach delays hive-c0re's start by one attempt
|
||||
# (`TimeoutStartSec` below) rather than failing it — hive-c0re starts
|
||||
# after that attempt with whatever credential is on disk, while
|
||||
# ./lib/store-retry.nix keeps retrying in the background.
|
||||
before = [ "hive-c0re.service" ];
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
"hive-c0re.service"
|
||||
];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
];
|
||||
# ./lib/store-retry.nix.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
# What actually bounds each attempt at the reads below. Stated here
|
||||
# rather than left to systemd's default, so the number a boot waits on
|
||||
# is in the file that waits.
|
||||
TimeoutStartSec = 30;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.queueAgentClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.queueAgentClientKeyFile;
|
||||
}
|
||||
# Absent means the system trust store, which is what a deployment with a
|
||||
# real CA wants and what a self-signed one must not be left with.
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
|
||||
# `bao`'s own message is the only thing separating a missing value from
|
||||
# a refused identity from an unreachable host. This unit's degraded
|
||||
# mode is correct for all three, so it reports which one rather than
|
||||
# asserting all three in a sentence of ours.
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
|
||||
# variables above only decide which certificate the TLS handshake
|
||||
# presents; without a token `bao` asks its token helper instead, and
|
||||
# that is a `sh` this unit's `path` does not carry. `-token-only`
|
||||
# answers on stdout and skips the helper on both sides.
|
||||
#
|
||||
# Fails LOUDLY, unlike the reads below: the three states a login
|
||||
# failure covers — store not up, sealed, role not written yet — are all
|
||||
# things a retry fixes, and `Restart=on-failure` above is what retries.
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to swarm-bao with this host's certificate; leaving the queue credential in ${credentialDir} as it is." >&2
|
||||
if [ -s "$err" ]; then
|
||||
cat "$err" >&2
|
||||
else
|
||||
echo "bao failed without writing a diagnostic." >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Two reads of one object rather than one `-format=json` parsed with
|
||||
# `jq`: no sibling unit carries `jq` on its `path`, and the pair cannot
|
||||
# actually disagree — the client id is derived from this hive's name, so
|
||||
# a rotation landing between these two calls changes the secret and
|
||||
# rewrites the same id.
|
||||
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg credentialPath} 2>"$err")"; then
|
||||
echo "swarm-bao did not return ${credentialPath}; this hive's agents have no queue credential yet." >&2
|
||||
if [ -s "$err" ]; then
|
||||
cat "$err" >&2
|
||||
else
|
||||
echo "bao failed without writing a diagnostic." >&2
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! client_id="$(bao kv get -field=client_id ${lib.escapeShellArg credentialPath} 2>"$err")"; then
|
||||
echo "${credentialPath} holds no client_id; the secret alone is not a usable credential, so nothing is written." >&2
|
||||
if [ -s "$err" ]; then
|
||||
cat "$err" >&2
|
||||
else
|
||||
echo "bao failed without writing a diagnostic." >&2
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Both or neither: a client that finds one of the two comes up "fine"
|
||||
# and never connects.
|
||||
if [ -z "$secret" ] || [ -z "$client_id" ]; then
|
||||
echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
install -d -m 0755 ${lib.escapeShellArg credentialDir}
|
||||
|
||||
atomic_write_secret 0600 "" ${lib.escapeShellArg secretFile} "$secret"
|
||||
|
||||
# `0644` on purpose: an OIDC client id is presented to the token
|
||||
# endpoint on every connection and is public by construction.
|
||||
atomic_write_secret 0644 "" ${lib.escapeShellArg clientIdFile} "$client_id"
|
||||
'';
|
||||
};
|
||||
})
|
||||
];
|
||||
}
|
||||
|
|
@ -333,8 +333,7 @@ in
|
|||
# from an unprivileged daemon without widening either.
|
||||
# The gate is the identity, never `deploy.bao.enable`: a hive that
|
||||
# reads a store on another machine holds one of these and runs no
|
||||
# store. ./glue-queue-agent-credential.nix gates its own reader the
|
||||
# same way.
|
||||
# store. ../swarm-otel.nix gates its own reader the same way.
|
||||
lib.optionals baoClientIdentity [
|
||||
"bao-client.pem:${baoDeploy.clientCertFile}"
|
||||
"bao-client-key.pem:${baoDeploy.clientKeyFile}"
|
||||
|
|
|
|||
|
|
@ -11,9 +11,8 @@ let
|
|||
cfg = config.services.hyperhive.c0re;
|
||||
baoDeploy = config.services.hyperhive.deploy.bao;
|
||||
baoCfg = config.services.hyperhive.swarm.bao;
|
||||
# The same test ../glue-queue-agent-credential.nix applies, and for the same
|
||||
# reason: a reader is defined by holding a certificate the store accepts,
|
||||
# not by sharing a host with the store. Deriving this from
|
||||
# A reader is defined by holding a certificate the store accepts, not by
|
||||
# sharing a host with the store. Deriving this from
|
||||
# `deploy.bao.enable` would be the co-location assumption itself.
|
||||
haveBaoClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
|
||||
in
|
||||
|
|
@ -222,31 +221,15 @@ in
|
|||
# the one the roster gave it.
|
||||
HIVE_C0RE_OIDC_CLIENT_ID = "hive-${config.services.hyperhive.hiveName}";
|
||||
}
|
||||
// {
|
||||
# Where ../glue-queue-agent-credential.nix lands the AGENTS' queue
|
||||
# credential. Read by `hive_c0re::lifecycle::host_config`, which stats the
|
||||
# two files and forwards them into each container as systemd credentials.
|
||||
# Never read for its contents here: the secret is `0600` root-owned and
|
||||
# this daemon is `hive-core`, which is exactly why the transport is a
|
||||
# credential rather than a bind mount.
|
||||
HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR = toString config.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
||||
}
|
||||
//
|
||||
# The agents' half of the same queue, forwarded by `hive_c0re::meta` into
|
||||
# every container. A pair, gated together, because an agent that got one of
|
||||
# them would report a half-configured queue instead of none.
|
||||
# every container.
|
||||
#
|
||||
# ⚠️ By name, like `HIVE_C0RE_NATS_URL` above, and never a loopback
|
||||
# address: inside a container `127.0.0.1` is the agent itself.
|
||||
lib.optionalAttrs
|
||||
(
|
||||
config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null
|
||||
&& config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null
|
||||
)
|
||||
{
|
||||
HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl;
|
||||
HIVE_AGENT_OIDC_TOKEN_ENDPOINT = config.services.hyperhive.swarm.statusPublish.tokenEndpoint;
|
||||
}
|
||||
lib.optionalAttrs (config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null) {
|
||||
HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl;
|
||||
}
|
||||
//
|
||||
# Where an agent reads the swarm's logs. Gated on the store identity below,
|
||||
# because `swarm-logs` authenticates with the agent's own queue secret, read
|
||||
|
|
|
|||
|
|
@ -352,22 +352,14 @@ in
|
|||
readOnly = true;
|
||||
default = "-agent";
|
||||
description = ''
|
||||
Suffix appended to a hive's own client id to name the client its
|
||||
*agent containers* present — agents on hive `alpha` all present
|
||||
`${config.services.hyperhive.swarm.authelia.hiveClientPrefix}alpha${config.services.hyperhive.swarm.authelia.agentClientSuffix}`.
|
||||
Read-only for the same reason as `hiveClientPrefix`, and read by the
|
||||
same consumer with the same failure mode: a split spelling denies
|
||||
every agent as a timeout.
|
||||
Suffix the queue's auth-callout responder refuses on a hive's client
|
||||
id: `${config.services.hyperhive.swarm.authelia.hiveClientPrefix}alpha${config.services.hyperhive.swarm.authelia.agentClientSuffix}`
|
||||
is denied, never read as hive `alpha-agent`. No client is minted with
|
||||
it; an agent authenticates to the queue with its own secret.
|
||||
|
||||
A suffix on the hive's id rather than a prefix of its own, because
|
||||
`agent-alpha` reads as *the agent named alpha* — which is precisely
|
||||
what this identity does not say.
|
||||
|
||||
One id per hive rather than per agent, because agents are created at
|
||||
runtime and a per-agent client would make creating one a config
|
||||
change plus a reload. The consequence is that this identity says
|
||||
*which hive* an agent belongs to and never *which agent* — the
|
||||
broker cannot tell two agents on one hive apart.
|
||||
Read-only for the same reason as `hiveClientPrefix`. A hive named with
|
||||
this suffix is refused at evaluation, since the queue would refuse its
|
||||
own client id.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -230,8 +230,8 @@ let
|
|||
# The secret publisher runs in this container, beside the mint whose secrets
|
||||
# it copies. It needs a store identity, and a reader is defined by holding a
|
||||
# certificate the store accepts, never by standing next to it — the rule
|
||||
# ./glue-queue-agent-credential.nix states in full. `deploy.bao.enable` here
|
||||
# would be the co-location assumption itself.
|
||||
# ./swarm-bao.nix's `deploy.bao` options block states in full.
|
||||
# `deploy.bao.enable` here would be the co-location assumption itself.
|
||||
publisherCfg = deployCfg.swarm-secret-publisher;
|
||||
publishes =
|
||||
publisherCfg.enable
|
||||
|
|
@ -300,71 +300,6 @@ let
|
|||
accessTokenSignedResponseAlg = "RS256";
|
||||
}) hyperhiveCfg.swarm.hives;
|
||||
|
||||
# The one URL an agent's token has to be authorised at beyond the queue:
|
||||
# the log store's machine query route. Read off `swarm.victorialogs.domain`
|
||||
# — an unconditional option, declared whether or not this host runs the
|
||||
# store — rather than restated, because it is also the string `swarm-logs`
|
||||
# requests as its audience and the string `hive-c0re/environment.nix` hands
|
||||
# an agent as its endpoint. Three readers, one formula; two spellings
|
||||
# present as a valid token refused at the store.
|
||||
logsQueryUrl = "https://${hyperhiveCfg.swarm.victorialogs.domain}/select/logsql/query";
|
||||
|
||||
# The identity an agent container presents to the swarm's queue. One per
|
||||
# HIVE, not per agent, and that is the load-bearing choice rather than a
|
||||
# shortcut: agents are created at **runtime**, so anything minted per agent
|
||||
# would make creating one a config change plus an authelia reload. Keyed on
|
||||
# the hive, this list's length tracks the roster above it — deploy-time, like
|
||||
# every other entry here.
|
||||
#
|
||||
# ⚠️ The cost, stated rather than discovered later: every agent on a hive
|
||||
# presents the SAME client id, so the broker can tell *hives* apart and not
|
||||
# *agents*. Deliberate and deferred — closing it requires the client list to
|
||||
# stop being static, which is the same problem the users-database writer
|
||||
# already solves for identities.
|
||||
#
|
||||
# 🏷️ The id EXTENDS the hive's own (`hive-<name>-agent`) rather than taking a
|
||||
# prefix of its own (`agent-<name>`), because that reads as *the agent called
|
||||
# `<name>`* — which is the one thing this identity does not carry. The cost is
|
||||
# that the two ids are not distinguishable by prefix, so the responder's
|
||||
# agent rule must be tried BEFORE its hive rule; `policy.rs` says so at the
|
||||
# match site, and the assertion below covers the case that ordering cannot.
|
||||
#
|
||||
# Mirrors `hiveClients` field for field so the two stay comparable. The
|
||||
# signing algorithm is not load-bearing here — this client's consumers are
|
||||
# the queue's auth-callout responder, which *introspects* rather than
|
||||
# verifying offline, and authelia's own authz endpoint — but it matches its
|
||||
# sibling rather than inventing a second answer to a question nobody asked.
|
||||
agentClients = lib.mapAttrsToList (name: _: {
|
||||
id = "${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}";
|
||||
description = "HyperHive agents on hive ${name}";
|
||||
kind = "machine";
|
||||
redirectUris = [ ];
|
||||
# ⚠️ **Two** audiences, where `hiveClients` has one, because an agent
|
||||
# presents this token at two different kinds of consumer. The queue
|
||||
# introspects it and reads the client id; the log store's gateway
|
||||
# location runs `auth_request` against authelia, which authorises a
|
||||
# bearer token **by the URL being requested** and refuses one whose
|
||||
# audience does not name it. Measured, not assumed: without the URL
|
||||
# registered here the token endpoint answers `invalid_target` for it,
|
||||
# and a token minted with no audience at all is answered by the gateway
|
||||
# with a bare 401 that names no cause.
|
||||
audience = [
|
||||
"${cfg.hiveClientPrefix}${name}${cfg.agentClientSuffix}"
|
||||
logsQueryUrl
|
||||
];
|
||||
# Without it the authz endpoint refuses an otherwise valid token and
|
||||
# blames the token rather than the missing grant — the same note
|
||||
# `glue-swarm-otel-oidc-client.nix` carries over the collector's client.
|
||||
bearerAuthz = true;
|
||||
# Stated rather than left on authelia's default, for
|
||||
# `glue-swarm-otel-oidc-client.nix`'s reason: authelia permits only
|
||||
# basic / JWT methods for a confidential client holding this scope, and
|
||||
# enforces it in the startup validator. `swarm-queue-client` sends the
|
||||
# credential as basic auth, so this is also what the minter already does.
|
||||
tokenEndpointAuthMethod = "client_secret_basic";
|
||||
accessTokenSignedResponseAlg = "RS256";
|
||||
}) hyperhiveCfg.swarm.hives;
|
||||
|
||||
# `swarm-authelia-bridge`'s own identity — distinct from
|
||||
# `swarm-controller`'s (`swarm-controller.nix`'s `queueClientId`). A
|
||||
# resource server introspecting a token proves its OWN identity to the
|
||||
|
|
@ -706,7 +641,7 @@ in
|
|||
# wherever this module is, so `oidc.clients` is never actually empty
|
||||
# — see `oidcEnabled`'s comment above.
|
||||
services.hyperhive.swarm.authelia.oidc.clients =
|
||||
lib.optionals cfg.oidc.hiveIdentities (hiveClients ++ agentClients)
|
||||
lib.optionals cfg.oidc.hiveIdentities hiveClients
|
||||
++ [
|
||||
bridgeClient
|
||||
forwarderClient
|
||||
|
|
@ -749,18 +684,11 @@ in
|
|||
}
|
||||
{
|
||||
# A hive whose name ENDS with the agent suffix mints an id the
|
||||
# queue's responder reads as somebody else's agents:
|
||||
# queue's responder reads as a hive's agent client, which it refuses:
|
||||
# `hive-foo-agent` parses as *the agents of hive foo* before it
|
||||
# parses as *the hive foo-agent*, because the agent rule is the
|
||||
# more specific one and therefore runs first.
|
||||
#
|
||||
# ⚠️ NOT covered by the duplicate-id assertion above, and the gap is
|
||||
# the interesting half. That one fires only when BOTH `foo` and
|
||||
# `foo-agent` are on the roster, because only then are two clients
|
||||
# actually named the same string. With `foo-agent` alone there is no
|
||||
# duplicate and nothing to see — the hive simply receives its
|
||||
# agents' grant instead of its own, and a NATS denial arrives as a
|
||||
# timeout, so the symptom names nothing.
|
||||
# more specific one and therefore runs first. A NATS denial arrives
|
||||
# as a timeout, so the symptom names nothing.
|
||||
#
|
||||
# Checkable here and nowhere downstream: the responder runs in a
|
||||
# container with no view of the roster (`policy.rs`'s `hive_name`
|
||||
|
|
@ -774,10 +702,9 @@ in
|
|||
+ lib.concatMapStringsSep ", " (h: "'${h}'") (
|
||||
lib.filter (h: lib.hasSuffix cfg.agentClientSuffix h) (lib.attrNames hyperhiveCfg.swarm.hives)
|
||||
)
|
||||
+ " — names ending with '${cfg.agentClientSuffix}', the suffix that "
|
||||
+ "marks a hive's agent containers. Such a hive's own client id is "
|
||||
+ "indistinguishable from another hive's agent client, and the "
|
||||
+ "queue resolves it as the agents. Rename the hive.";
|
||||
+ " — names ending with '${cfg.agentClientSuffix}', the suffix the "
|
||||
+ "queue refuses as a hive's agent client. Such a hive's own client "
|
||||
+ "id reads as one, and the queue refuses it. Rename the hive.";
|
||||
}
|
||||
# The two obligations `authelia.bearer.authz` carries. Authelia
|
||||
# enforces both itself — but in its `preStart` validator, so a
|
||||
|
|
|
|||
|
|
@ -808,9 +808,9 @@ let
|
|||
}
|
||||
'';
|
||||
|
||||
# One gateway-leaf issuer per hive in the swarm directory, for the reason
|
||||
# `perHiveReaders` below is per hive. One name for the pki role, the policy
|
||||
# and the cert-auth role, as for the readers.
|
||||
# One gateway-leaf issuer per hive in the swarm directory: one subject for
|
||||
# every hive could ask for any hive's certificate. One name for the pki role,
|
||||
# the policy and the cert-auth role, as for the readers below.
|
||||
hiveLeafIssuers = lib.mapAttrsToList (hiveName: hive: {
|
||||
name = "${baoDeploy.hiveLeafPkiRolePrefix}-${hiveName}";
|
||||
cn = "${baoDeploy.hiveLeafCommonNamePrefix}-${hiveName}";
|
||||
|
|
@ -897,48 +897,21 @@ let
|
|||
# `swarm/agents/*` AND `swarm/hives/<hive>/*`, and nothing under
|
||||
# `swarm/services/`.
|
||||
# Principals presenting one certificate are ONE principal to bao, so sharing
|
||||
# the hive's leaf would give each of them everything that leaf reads: the
|
||||
# unit that fetches this hive's queue credential could fetch every agent
|
||||
# credential in the swarm, and a unit that fetches a service's OIDC client
|
||||
# secret could not fetch it at all, as that path is not the hive's to read.
|
||||
# the hive's leaf would give each of them everything that leaf reads: a unit
|
||||
# that fetches a service's OIDC client secret could fetch every agent
|
||||
# credential in the swarm, and could not fetch its own secret at all, as
|
||||
# that path is not the hive's to read.
|
||||
#
|
||||
# Each gets its own leaf, its own cert-auth role and a policy holding the
|
||||
# one `secret/data/…` path that principal's script actually names — spelled
|
||||
# to the LEAF rather than to a prefix, for the reason `matrixCtlPolicyText`
|
||||
# above gives for doing the same.
|
||||
#
|
||||
# ⚠️ The queue-credential reader is PER HIVE and the OIDC readers are not,
|
||||
# and that asymmetry is the shape of the paths rather than a preference. A
|
||||
# queue credential lives under `swarm/hives/<name>/`, and every hive runs a
|
||||
# reader for its own; an OIDC client secret lives under
|
||||
# `swarm/services/<client-id>/` and a swarm registers each of those exactly
|
||||
# once. A single role for a per-hive reader would have to name `hives/*` to
|
||||
# serve every hive — which would let one hive read another's queue
|
||||
# credential, a reach no hive needs.
|
||||
readStanza = path: ''
|
||||
path "${path}" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
'';
|
||||
|
||||
# One reader per hive in the swarm directory. Written from the directory
|
||||
# rather than from this host's own name because the policy is written where
|
||||
# the STORE is and the reader runs where its hive is, so this host's name is
|
||||
# not the reader's.
|
||||
perHiveReaders =
|
||||
{ rolePrefix, cnPrefix, ... }@spec:
|
||||
lib.mapAttrsToList (hiveName: _: {
|
||||
name = "${rolePrefix}-${hiveName}";
|
||||
cn = "${cnPrefix}-${hiveName}";
|
||||
policyText = readStanza (spec.path hiveName);
|
||||
}) hyperhiveCfg.swarm.hives;
|
||||
|
||||
queueAgentReaders = perHiveReaders {
|
||||
rolePrefix = "swarm-queue-agent";
|
||||
cnPrefix = baoDeploy.queueAgentCommonNamePrefix;
|
||||
path = h: "${credentialMountPath}/data/swarm/hives/${h}/queue/agent";
|
||||
};
|
||||
|
||||
# Singletons, so a list of one rather than a second shape: the granting unit
|
||||
# below takes a list either way and the units stay identical apart from
|
||||
# the objects they write.
|
||||
|
|
@ -1257,9 +1230,9 @@ let
|
|||
|
||||
# One unit per PRINCIPAL, not one unit for all of them: these names are
|
||||
# operator-facing strings, the same reason `swarm-bao-secret-publisher-policy`
|
||||
# below gives for not being more script inside the controller's unit. A hive
|
||||
# whose queue credential never arrives is diagnosed by the state of the unit named
|
||||
# after it.
|
||||
# below gives for not being more script inside the controller's unit. A
|
||||
# reader whose secret never arrives is diagnosed by the state of the unit
|
||||
# named after it.
|
||||
readerPolicyUnit =
|
||||
description: objects:
|
||||
granterPolicyUnit description (
|
||||
|
|
@ -1589,7 +1562,7 @@ in
|
|||
# `serverCaFile` live here too, and they are set by whoever authenticates TO
|
||||
# the store — which includes a host that runs none of it. A reader is defined
|
||||
# by holding a certificate the store accepts, never by sharing a host with
|
||||
# one; ./glue-queue-agent-credential.nix gates on exactly that and not on
|
||||
# one; ./swarm-otel.nix's store reader gates on exactly that and not on
|
||||
# `enable`.
|
||||
# Reading this block as store-runner-only is what makes an off-host reader
|
||||
# look inexpressible when it is already supported.
|
||||
|
|
@ -2102,28 +2075,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
queueAgentCommonNamePrefix = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-bao-queue-agent";
|
||||
example = "swarm-bao-queue-agent.svc";
|
||||
description = ''
|
||||
Prefix of the subject the store's per-hive queue-credential roles
|
||||
accept. The role for hive `h` accepts `<prefix>-h` and grants read on
|
||||
exactly `swarm/hives/h/queue/agent` — the client credential that hive's
|
||||
agent containers authenticate to the swarm queue with.
|
||||
|
||||
A **prefix** rather than one subject because every hive runs its own
|
||||
reader and the path it reads carries that hive's name. One subject for
|
||||
all of them would need a `hives/*` grant, which would let one hive read
|
||||
another hive's queue credential.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, per hive, for the reason its
|
||||
fixed-subject siblings are: cert auth trusts the CA and
|
||||
`allowed_common_names` is the whole narrowing, so a hive whose own leaf
|
||||
carried one of these spellings would receive that grant.
|
||||
'';
|
||||
};
|
||||
|
||||
hiveLeafCommonNamePrefix = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-bao-hive-leaf";
|
||||
|
|
@ -2134,9 +2085,8 @@ in
|
|||
`<servicesPkiMountPath>/issue/<hiveLeafPkiRolePrefix>-h`, which issues
|
||||
for `h`'s domain and `*.` that domain and nothing else.
|
||||
|
||||
Per hive for the reason
|
||||
{option}`services.hyperhive.deploy.bao.queueAgentCommonNamePrefix` is:
|
||||
one subject for every hive could ask for any hive's certificate.
|
||||
Per hive because one subject for every hive could ask for any hive's
|
||||
certificate.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, per hive, like its siblings.
|
||||
'';
|
||||
|
|
@ -2163,7 +2113,7 @@ in
|
|||
|
||||
Its grant is one path,
|
||||
`swarm/services/<grafana client id>/oidc/client`, and read only. Not
|
||||
per hive, unlike the two prefixes above: an OIDC client is registered
|
||||
per hive, unlike the prefix above: an OIDC client is registered
|
||||
once per swarm, so the path names the service and never a hive.
|
||||
|
||||
⚠️ Same collision as its siblings, and the same answer: ./swarm.nix
|
||||
|
|
@ -2206,37 +2156,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
queueAgentClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/queue-agent.pem";
|
||||
description = ''
|
||||
Certificate the unit that fetches this hive's queue client credential
|
||||
presents to the store. Its subject must be
|
||||
`<queueAgentCommonNamePrefix>-<this hive's name>`; cert auth matches on
|
||||
the CN and the role accepts nothing else.
|
||||
|
||||
⚠️ **Not** {option}`services.hyperhive.deploy.bao.clientCertFile`.
|
||||
Pointing this at the hive's own leaf would evaluate, deploy and log in —
|
||||
and hand a unit that reads one credential a certificate that reads every
|
||||
agent credential in the swarm and this hive's own secrets.
|
||||
|
||||
No default here: ./glue-bao-tls.nix points it at the leaf it mints,
|
||||
wherever this host mints its own PKI. A deployment that reads a store on
|
||||
another host names it by hand, alongside the leaf itself.
|
||||
'';
|
||||
};
|
||||
|
||||
queueAgentClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/queue-agent-key.pem";
|
||||
description = ''
|
||||
Private key for
|
||||
{option}`services.hyperhive.deploy.bao.queueAgentClientCertFile`.
|
||||
'';
|
||||
};
|
||||
|
||||
grafanaOidcClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -2246,8 +2165,10 @@ in
|
|||
to the store. Its subject must be
|
||||
{option}`services.hyperhive.deploy.bao.grafanaOidcCommonName`.
|
||||
|
||||
⚠️ Not the hive's own leaf, for the reason
|
||||
{option}`services.hyperhive.deploy.bao.queueAgentClientCertFile` gives.
|
||||
⚠️ **Not** {option}`services.hyperhive.deploy.bao.clientCertFile`.
|
||||
Pointing this at the hive's own leaf would evaluate, deploy and log in —
|
||||
and hand a unit that reads one secret a certificate that reads every
|
||||
agent credential in the swarm and this hive's own secrets.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -2271,7 +2192,7 @@ in
|
|||
{option}`services.hyperhive.deploy.bao.otelOidcCommonName`.
|
||||
|
||||
⚠️ Not the hive's own leaf, for the reason
|
||||
{option}`services.hyperhive.deploy.bao.queueAgentClientCertFile` gives.
|
||||
{option}`services.hyperhive.deploy.bao.grafanaOidcClientCertFile` gives.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -2295,7 +2216,7 @@ in
|
|||
{option}`services.hyperhive.deploy.bao.forwarderOidcCommonName`.
|
||||
|
||||
⚠️ Not the hive's own leaf, for the reason
|
||||
{option}`services.hyperhive.deploy.bao.queueAgentClientCertFile` gives.
|
||||
{option}`services.hyperhive.deploy.bao.grafanaOidcClientCertFile` gives.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -3543,12 +3464,10 @@ in
|
|||
'';
|
||||
|
||||
# 🩸 The principals that each log in with a leaf of their own, not as the
|
||||
# hive. See the block around `perHiveReaders` for what each grant is and
|
||||
# why the first is per hive; the units are identical apart from the
|
||||
# objects they write, so the shape lives in `readerPolicyUnit` and the
|
||||
# difference is visible here in one line each.
|
||||
systemd.services.swarm-bao-queue-agent-policy = readerPolicyUnit "write the per-hive queue-credential bao policies and cert-auth roles" queueAgentReaders;
|
||||
|
||||
# hive. See the block around `readStanza` for what each grant is; the
|
||||
# units are identical apart from the objects they write, so the shape
|
||||
# lives in `readerPolicyUnit` and the difference is visible here in one
|
||||
# line each.
|
||||
systemd.services.swarm-bao-grafana-oidc-policy = readerPolicyUnit "write Grafana's OIDC-secret-reader bao policy and cert-auth role" grafanaOidcReaders;
|
||||
|
||||
systemd.services.swarm-bao-otel-oidc-policy = readerPolicyUnit "write the collector's OIDC-secret-reader bao policy and cert-auth role" otelOidcReaders;
|
||||
|
|
@ -3693,7 +3612,6 @@ in
|
|||
"swarm-bao-controller-policy.service"
|
||||
"swarm-bao-secret-publisher-policy.service"
|
||||
"swarm-bao-matrix-ctl-policy.service"
|
||||
"swarm-bao-queue-agent-policy.service"
|
||||
"swarm-bao-grafana-oidc-policy.service"
|
||||
"swarm-bao-otel-oidc-policy.service"
|
||||
"swarm-bao-forwarder-oidc-policy.service"
|
||||
|
|
|
|||
|
|
@ -282,8 +282,8 @@ let
|
|||
);
|
||||
|
||||
# A reader of the store is defined by holding a certificate the store
|
||||
# accepts, never by standing next to it — the rule
|
||||
# ./glue-queue-agent-credential.nix states in full.
|
||||
# accepts, never by standing next to it — the rule ./swarm-bao.nix's
|
||||
# `deploy.bao` options block states in full.
|
||||
#
|
||||
# Also asserted rather than gating anything, and that is the ruling this
|
||||
# module's second delivery route was deleted under: the store exists so that
|
||||
|
|
@ -722,10 +722,10 @@ in
|
|||
# to be wrong, and a gate whose false arm is a Grafana with no login.
|
||||
# Recorded in docs/swarm/secrets.md.
|
||||
#
|
||||
# Shaped after ./glue-queue-agent-credential.nix, because it is a store
|
||||
# reader and this is what the store's readers do: a cert login that fails
|
||||
# LOUDLY, since every state it fails on is one a retry fixes, then a read
|
||||
# that degrades QUIETLY, since no retry turns "no value there" into a value.
|
||||
# It is a store reader, and this is what the store's readers do: a cert
|
||||
# login that fails LOUDLY, since every state it fails on is one a retry
|
||||
# fixes, then a read that degrades QUIETLY, since no retry turns "no value
|
||||
# there" into a value.
|
||||
#
|
||||
# ⚠️ An ABSENT secret is "not published yet", not a fault of this host's.
|
||||
# The publisher runs in authelia's container and authelia mints on its
|
||||
|
|
|
|||
|
|
@ -873,7 +873,7 @@ in
|
|||
# terminal rows and its turn-state header
|
||||
# (`hive-agent::swarm_agent_state`), two subject families so a
|
||||
# subscriber following one agent's terminal is not also handed
|
||||
# every header. The hive's shared agent client gets nothing.
|
||||
# every header.
|
||||
#
|
||||
# `$$`, not `$`: systemd substitutes `$NAME` in `ExecStart`
|
||||
# whether or not it is quoted, so a single dollar reaches the
|
||||
|
|
|
|||
|
|
@ -232,12 +232,11 @@ let
|
|||
haveCollectorSecret = deployCfg.swarm-otel.clientSecretFile != null;
|
||||
|
||||
# A reader of the store is defined by holding a certificate the store
|
||||
# accepts, never by standing next to it — the rule
|
||||
# ./glue-queue-agent-credential.nix states in full. Unlike Grafana's identical-
|
||||
# accepts, never by standing next to it — the rule ./swarm-bao.nix's
|
||||
# `deploy.bao` options block states in full. Unlike Grafana's identical-
|
||||
# looking flag, this one does not gate an assertion on its own: the
|
||||
# store-reading unit below simply does not render without it, the same choice
|
||||
# ./glue-queue-agent-credential.nix makes for its own reader, because a collector with no client identity is
|
||||
# `haveCollectorSecret = false` above, and that is already a supported,
|
||||
# store-reading unit below simply does not render without it, because a
|
||||
# collector with no client identity is `haveCollectorSecret = false` above, and that is already a supported,
|
||||
# merely degraded shape rather than a service with no way in at all. What
|
||||
# IS asserted is narrower and lives in `assertions` below — see
|
||||
# `hiveReaderIdentity`.
|
||||
|
|
@ -539,7 +538,7 @@ in
|
|||
# exists so a host holds ONE out-of-band secret — its client certificate —
|
||||
# and reads everything else with it. Recorded in docs/swarm/secrets.md.
|
||||
#
|
||||
# Shaped after ./glue-queue-agent-credential.nix: a cert login that fails
|
||||
# Shaped after ./swarm-grafana.nix's store reader: a cert login that fails
|
||||
# LOUDLY, since every state it fails on is one a retry fixes, then a read
|
||||
# that degrades QUIETLY, since no retry turns "no value there" into a
|
||||
# value.
|
||||
|
|
|
|||
|
|
@ -12,15 +12,9 @@ let
|
|||
hyperhiveCfg = config.services.hyperhive;
|
||||
hiveNames = lib.attrNames hyperhiveCfg.swarm.hives;
|
||||
|
||||
# The client id agent containers present, per hive — composed exactly as
|
||||
# ./swarm-authelia.nix composes it, from the same two read-only options, so a
|
||||
# rename there cannot leave this spelling behind.
|
||||
agentClientId =
|
||||
hive:
|
||||
"${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}${hyperhiveCfg.swarm.authelia.agentClientSuffix}";
|
||||
|
||||
# The hive's own client id, which hive-c0re publishes the hive's status with.
|
||||
# Composed as ./swarm-authelia.nix composes it, for the reason above.
|
||||
# Composed exactly as ./swarm-authelia.nix composes it, from the same
|
||||
# read-only option, so a rename there cannot leave this spelling behind.
|
||||
hiveClientId = hive: "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}";
|
||||
|
||||
# The swarm's own services, as opposed to its hives. One client for the whole
|
||||
|
|
@ -83,22 +77,14 @@ let
|
|||
controllerSecretPath = "secret/swarm/controller/swarm-controller/oidc/client";
|
||||
|
||||
secrets =
|
||||
# `client_id` beside the value: an agent's id is derived per hive and has
|
||||
# to be reconstructable from the store alone.
|
||||
map (hive: {
|
||||
clientId = agentClientId hive;
|
||||
path = "secret/swarm/hives/${hive}/queue/agent";
|
||||
withClientId = true;
|
||||
}) hiveNames
|
||||
# Each hive's own client, which that hive's hive-c0re reads under its own
|
||||
# certificate. The nix half of
|
||||
# `swarm_secret_client::queue::hive_client_path`.
|
||||
++ map (hive: {
|
||||
map (hive: {
|
||||
clientId = hiveClientId hive;
|
||||
path = "secret/swarm/hives/${hive}/oidc/client";
|
||||
}) hiveNames
|
||||
# No `client_id`: a service's client id is the swarm-wide option both
|
||||
# ends already read.
|
||||
# A service's client id is the swarm-wide option both ends already read.
|
||||
++ map (id: {
|
||||
clientId = id;
|
||||
path = "secret/swarm/services/${id}/oidc/client";
|
||||
|
|
@ -120,9 +106,8 @@ in
|
|||
default = false;
|
||||
description = ''
|
||||
Publish the OIDC client secrets this host mints into the swarm's
|
||||
secret store, so a hive that does not run authelia can read its
|
||||
own and its agents' credentials and a swarm service or the
|
||||
controller can read its own — from wherever it runs, this host
|
||||
secret store, so a hive that does not run authelia, a swarm service
|
||||
or the controller can read its own — from wherever it runs, this host
|
||||
included. The unit, `swarm-secret-publish`, runs inside authelia's
|
||||
container, so it exists only where authelia runs.
|
||||
|
||||
|
|
@ -137,10 +122,9 @@ in
|
|||
publishing nothing. Whether the wiring is complete is the client
|
||||
identity's job (see `baoClientCertFile`), not this option's.
|
||||
|
||||
Turning it off leaves every hive but this one without its agents'
|
||||
credential, every hive unable to publish its status, the swarm
|
||||
controller unable to start, the swarm's Grafana without any login at
|
||||
all, its collector pushing unauthenticated, the queue's responder
|
||||
Turning it off leaves every hive unable to publish its status, the
|
||||
swarm controller unable to start, the swarm's Grafana without any login
|
||||
at all, its collector pushing unauthenticated, the queue's responder
|
||||
unable to start, the homeserver unable to start, and the forge's login
|
||||
source without its secret.
|
||||
Each secret has exactly one route and this is the producer's
|
||||
|
|
@ -185,9 +169,8 @@ in
|
|||
internal = true;
|
||||
default = secrets;
|
||||
description = ''
|
||||
Every secret the publisher copies: the minted client's id, the store
|
||||
path its reader fetches, and whether `client_id` is stored beside the
|
||||
value. Derived from `swarm.*` alone, so it is the same list on every
|
||||
Every secret the publisher copies: the minted client's id and the store
|
||||
path its reader fetches. Derived from `swarm.*` alone, so it is the same list on every
|
||||
host; ./swarm-authelia.nix hands it to the unit in its container.
|
||||
'';
|
||||
};
|
||||
|
|
|
|||
|
|
@ -64,17 +64,14 @@ let
|
|||
deployCfg.bao.notifyRouterCommonName
|
||||
deployCfg.bao.forgePublisherCommonName
|
||||
]
|
||||
# The per-hive subjects, spelled out per hive rather than as the prefix.
|
||||
# The per-hive subject, spelled out per hive rather than as the prefix.
|
||||
# The prefix alone would reserve the wrong string: the role for hive
|
||||
# `h` accepts `<prefix>-h`, so the name a hive must not BE is that composed
|
||||
# spelling, and a hive named it would present a leaf that role accepts.
|
||||
#
|
||||
# Every spelling is reserved for every declared hive, including the one the
|
||||
# CN belongs to — a hive cannot be named after its own reader either.
|
||||
++ lib.concatMap (hive: [
|
||||
"${deployCfg.bao.queueAgentCommonNamePrefix}-${hive}"
|
||||
"${deployCfg.bao.hiveLeafCommonNamePrefix}-${hive}"
|
||||
]) hiveNames;
|
||||
# CN belongs to — a hive cannot be named after its own leaf issuer either.
|
||||
++ map (hive: "${deployCfg.bao.hiveLeafCommonNamePrefix}-${hive}") hiveNames;
|
||||
|
||||
# Public hostnames of the swarm's own services, in declaration order.
|
||||
# `serviceDomains` below is this set sorted + deduplicated.
|
||||
|
|
@ -477,9 +474,8 @@ in
|
|||
# its own network namespace and resolver, so this is its own option, even
|
||||
# though by name it defaults to the same URL as `statusPublish.natsUrl`.
|
||||
#
|
||||
# Only the address lives here. The credential does not: it is published per
|
||||
# hive and read out of the store by ./glue-queue-agent-credential.nix, which
|
||||
# owns `queue.agentCredentialDir` in the same namespace.
|
||||
# Only the address lives here. Each agent reads its own credential from the
|
||||
# store (`nix/agent-modules/queue-identity.nix`).
|
||||
options.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = "tls://${swarmCfg.nats.domain}:${toString swarmCfg.nats.port}";
|
||||
|
|
@ -496,10 +492,8 @@ in
|
|||
agent's network namespace `127.0.0.1` is the agent, not this host,
|
||||
and the queue's certificate names no address anyway.
|
||||
|
||||
Null means this hive's agents have not been given the queue's address.
|
||||
Together with
|
||||
{option}`services.hyperhive.swarm.statusPublish.tokenEndpoint` it is
|
||||
what decides whether the harness is handed queue coordinates at all.
|
||||
Null means this hive's agents have not been given the queue's address,
|
||||
and the harness is handed none.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue