Watch
0
0
Fork
You've already forked hyperhive
0

swarm-nats: require the responder's store identity, and drop its no-store mode

The responder reads the callout xkey before it connects, so a responder
with no secret store no longer starts, and its "deny every agent token,
serve OIDC" mode could not run. That mode is gone: `agent_token::Store`
is built from the settings the xkey load already read, and `authorize`
takes a source rather than an `Option` of one.

On the nix side, `deploy.nats.authBaoClientCertFile` and
`authBaoClientKeyFile` are now asserted at eval when the queue is
enabled. The units' no-identity branches, the copy unit's empty-file
fallback and its "denies agent tokens" message are removed; an absent
leaf file fails `swarm-nats-auth-secrets` like an absent seed does.
This commit is contained in:
atlas 2026-10-05 22:13:58 +02:00 • committed by mara
commit 52c931c3a7
5 changed files with 193 additions and 248 deletions

View file

@ -1,8 +1,8 @@
# Glue: point the queue's auth-callout responder at the bao leaf minted for it.
#
# ONE PAIRING PER FILE — the swarm-nats-auth principal ← bao, and nothing else.
# Deleting this leaves a responder with no store identity unless the operator
# names one: every agent token is then denied, and OIDC clients are unaffected.
# Deleting this leaves a queue with no responder identity unless the operator
# names one, which ./swarm-nats.nix refuses at eval.
#
# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
# leaf. What belongs here is the pairing: which paths the responder presents.

View file

@ -297,23 +297,21 @@ let
tlsKeyCredential = "tls-key";
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
# The responder's own store identity, for reading agent queue credentials.
# Without it the responder denies every agent token; agents then fall back to
# their hive's OIDC client.
# The responder's own store identity, for the callout xkey, agent queue
# credentials and its OIDC client secret. Required: an assertion below refuses
# a queue without it.
authStoreActive =
deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
# The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
authCertRole = "swarm-nats-auth";
# Store identity files the copy unit delivers, as credential id → host source.
authStoreFiles = lib.optionalAttrs authStoreActive (
{
authStoreFiles = {
"bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
"bao-ca.pem" = baoDeploy.serverCaFile;
}
);
};
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
@ -516,17 +514,19 @@ in
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
description = ''
Client certificate the auth-callout responder presents to the swarm's
secret store to read agent queue credentials, and that
`swarm-bao-nats-auth-oidc` presents to read the responder's own OIDC
client secret. Its subject must be
secret store to read the callout xkey and agent queue credentials, and
that `swarm-bao-nats-auth-oidc` and `swarm-bao-nats-callout-xkey`
present to read the responder's own OIDC client secret and the xkey's
public half. Its subject must be
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
Required when `deploy.nats.enable` is set; unset fails at eval.
./glue-nats-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints, where this host mints one; on any other
queue host it is a file the operator places. Unset, or set to a file
that does not exist, `swarm-bao-nats-auth-oidc` fails; a responder
with no OIDC client secret on disk cannot start, and the queue then
refuses every client.
queue host it is a file the operator places. Set to a file that does
not exist, `swarm-nats-auth-secrets`, `swarm-bao-nats-auth-oidc` and
`swarm-bao-nats-callout-xkey` fail, and the queue's server does not
start without the xkey.
A path, never a value.
'';
@ -620,6 +620,24 @@ in
to have this host mint all four.
'';
}
{
# The callout xkey and the responder's OIDC client secret are read
# from the store under this identity; without either the server or
# the responder does not start.
assertion = authStoreActive;
message = ''
services.hyperhive.deploy.nats.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
The queue's callout xkey and its responder's OIDC client secret
are read from the swarm's secret store with this certificate, and
neither the server nor the responder starts without them. They
default to the leaf the store mints when it runs on this host;
elsewhere, issue a leaf whose CN is
services.hyperhive.deploy.bao.natsAuthCommonName and name it in
both options.
'';
}
];
# Declared here rather than in the collector's module: an entry then
@ -929,18 +947,16 @@ in
RestartSec = "5s";
SyslogIdentifier = "swarm-nats-auth";
};
# The store the responder reads agent credentials from. Absent
# without an identity, which the responder reads as "no store".
environment = lib.optionalAttrs authStoreActive (
{
# The store the responder reads the callout xkey and agent
# credentials from.
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = authCredential "bao-client.pem";
BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
}
// lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
BAO_CACERT = authCredential "bao-ca.pem";
}
);
};
};
# The server binary, so an operator with a shell in here can
@ -973,7 +989,7 @@ in
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
# Where the store's PKI is minted on this host, the responder's leaf is
# one of its files. Ordering only: elsewhere the unit does not exist.
++ lib.optional authStoreActive "swarm-bao-pki.service";
++ [ "swarm-bao-pki.service" ];
requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
serviceConfig = {
Type = "oneshot";
@ -989,17 +1005,9 @@ in
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
${lib.escapeShellArg (hostPath "issuer.seed")}
''
# The store identity is optional to the responder's agent-token path, so
# an absent source is delivered as an empty file rather than failing this
# unit: a missing `LoadCredential` source stops the responder starting.
+ lib.concatStrings (
lib.mapAttrsToList (id: source: ''
if [ -s ${lib.escapeShellArg source} ]; then
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
else
echo "${source} is absent; the queue responder denies agent tokens until it exists" >&2
install -m 0400 /dev/null ${lib.escapeShellArg (hostPath id)}
fi
'') authStoreFiles
);
};
@ -1013,8 +1021,7 @@ in
#
# ⚠️ Without the secret the responder does not start (its `LoadCredential`
# source is missing), and `auth_callout` with no responder refuses every
# client. A host with no responder leaf therefore fails here, naming the
# options, rather than rendering no unit.
# client.
systemd.services.swarm-bao-nats-auth-oidc = {
description = "fetch the swarm queue responder's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
@ -1050,22 +1057,13 @@ in
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so it cannot read its OIDC client secret: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
script = ''
set -euo pipefail
${atomicWriteSecret}
@ -1157,22 +1155,13 @@ in
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so the callout xkey can be neither created nor fetched and the queue does not start: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
script = ''
set -euo pipefail
${atomicWriteSecret}

View file

@ -82,6 +82,12 @@ let
responderOf = h: h.containers.swarm-nats.config.systemd.services.swarm-nats-auth;
refusedWithoutIdentity =
m:
lib.any (
a: !a.assertion && lib.hasInfix "deploy.nats.authBaoClientCertFile and" a.message
) m.assertions;
# A hive running NOTHING of the swarm's own services — no IdP here, no
# `swarm.authelia.url` set by hand. The whole point of the fixture is what it
# does *not* say: no IdP address is given and none is derived from
@ -91,6 +97,17 @@ let
autheliaColocated = hive { deploy.authelia.enable = true; };
cases = [
{
# The xkey and the responder's OIDC client secret are read only from
# the store, so a queue with no responder leaf cannot start. Refused at
# eval; the controls are a leaf named by hand and the store host's
# minted one.
name = "a queue with no responder store identity is refused at eval, one with a leaf is not";
ok =
refusedWithoutIdentity natsOldPath
&& !(refusedWithoutIdentity natsWithStoreIdentity)
&& !(refusedWithoutIdentity natsOnStoreHost);
}
{
# Reads the RENDERED settings, not the option: `calloutBlocks {…} // {
# … }` is a shallow merge, and a future edit that dropped or shadowed
@ -182,17 +199,6 @@ let
&& !(lib.hasInfix "exit 0" s)
&& u.serviceConfig.Restart == "on-failure";
}
{
name = "a queue host with no responder leaf renders an xkey fetch that fails naming the options";
ok =
let
u = natsOldPath.systemd.services.swarm-bao-nats-callout-xkey;
in
!(u.environment ? BAO_CLIENT_CERT)
&& lib.hasInfix "deploy.nats.authBaoClientCertFile" u.script
&& lib.hasInfix "exit 1" u.script
&& !(lib.hasInfix "ensure-xkey" u.script);
}
{
# Its creates are what the responder's role was granted; on the store's
# host the role's unit has to have run first.
@ -281,16 +287,16 @@ let
&& lib.hasPrefix "https://" u.environment.BAO_ADDR;
}
{
# An absent leaf must not stop the responder's agent-token path: it is
# delivered empty, beside the two seeds.
name = "the copy unit delivers the store identity, and an absent one as an empty file";
# Copied as the seeds are, so an absent leaf fails the unit rather than
# reaching the responder as a file it cannot log in with.
name = "the copy unit delivers the store identity beside the two seeds, and nothing in its place when absent";
ok =
let
s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script;
in
lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s
&& lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s
&& lib.hasInfix "install -m 0400 /dev/null " s
&& !(lib.hasInfix "/dev/null" s)
&& lib.hasInfix "callout-user.seed" s
&& lib.hasInfix "issuer.seed" s;
}
@ -318,31 +324,6 @@ let
&& !(lib.hasInfix "oidc-client" copy)
&& !(lib.hasInfix "authelia" copy);
}
{
# No responder leaf is a responder with no OIDC secret, which refuses
# every client. The unit still renders, and fails naming the options.
name = "a queue host with no responder leaf renders a secret fetch that fails naming the options";
ok =
let
fetch = natsOldPath.systemd.services.swarm-bao-nats-auth-oidc;
in
!(fetch.environment ? BAO_CLIENT_CERT)
&& lib.hasInfix "deploy.nats.authBaoClientCertFile" fetch.script
&& lib.hasInfix "exit 1" fetch.script
&& !(lib.hasInfix "bao login" fetch.script);
}
{
# The control: no identity, no store wiring, and the responder's
# credentials are exactly the three it cannot run without.
name = "a responder with no store identity is not pointed at a store";
ok =
let
u = responderOf natsOldPath;
in
!(u.environment ? BAO_ADDR)
&& !(u.environment ? BAO_CLIENT_CERT)
&& lib.length u.serviceConfig.LoadCredential == 3;
}
{
# On the store's host the fetch logs in under the responder's leaf, so it
# waits for the unit writing that role.

View file

@ -58,19 +58,11 @@ pub struct Store {
}
impl Store {
/// The store named by the `BAO_*` environment, or `None` when that is
/// unset: a responder with no store identity denies every agent token and
/// serves the OIDC path unchanged.
pub fn from_env(cert_role: String) -> Option<Self> {
match Settings::from_env() {
Ok(settings) => Some(Self {
/// The store at `settings`, logged in to as `cert_role`.
pub fn new(settings: Settings, cert_role: String) -> Self {
Self {
settings,
cert_role,
}),
Err(e) => {
tracing::info!(reason = %e, "no secret store configured; agent tokens are denied");
None
}
}
}
}
@ -149,20 +141,11 @@ async fn verify(source: &impl CredentialSource, token: &AgentToken<'_>) -> bool
}
/// The grant for an agent token, or `None` for a denial.
///
/// `source` is `None` when this responder has no store identity, which denies.
pub async fn authorize<S: CredentialSource>(
pub async fn authorize(
policy: &Policy,
source: Option<&S>,
source: &impl CredentialSource,
token: &AgentToken<'_>,
) -> Option<Permissions> {
let Some(source) = source else {
tracing::warn!(
agent = token.agent,
"agent token presented, but this responder has no secret store; denying"
);
return None;
};
if !verify(source, token).await {
return None;
}
@ -237,7 +220,7 @@ mod tests {
.expect("valid")
}
async fn grant(source: Option<&Fake>, token: &str) -> Option<Permissions> {
async fn grant(source: &Fake, token: &str) -> Option<Permissions> {
let Presented::Agent(token) = classify(token) else {
panic!("{token:?} must classify as an agent token");
};
@ -246,7 +229,7 @@ mod tests {
#[tokio::test]
async fn a_valid_agent_token_is_granted_exactly_its_own_subjects() {
let g = grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRET")
let g = grant(&atlas(), "swarm-agent.atlas.Ab9_-zSECRET")
.await
.expect("granted");
assert_eq!(
@ -262,12 +245,12 @@ mod tests {
#[tokio::test]
async fn a_wrong_secret_is_denied() {
assert!(
grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECREX")
grant(&atlas(), "swarm-agent.atlas.Ab9_-zSECREX")
.await
.is_none()
);
assert!(
grant(Some(&atlas()), "swarm-agent.atlas.Ab9_-zSECRE")
grant(&atlas(), "swarm-agent.atlas.Ab9_-zSECRE")
.await
.is_none()
);
@ -279,7 +262,7 @@ mod tests {
#[tokio::test]
async fn another_agents_name_with_this_agents_secret_is_denied() {
assert!(
grant(Some(&atlas()), "swarm-agent.argus.Ab9_-zSECRET")
grant(&atlas(), "swarm-agent.argus.Ab9_-zSECRET")
.await
.is_none()
);
@ -288,10 +271,7 @@ mod tests {
#[tokio::test]
async fn an_agent_with_nothing_stored_is_denied() {
assert!(
grant(
Some(&stored("argus", "argus")),
"swarm-agent.atlas.Ab9_-zSECRET"
)
grant(&stored("argus", "argus"), "swarm-agent.atlas.Ab9_-zSECRET")
.await
.is_none()
);
@ -300,7 +280,7 @@ mod tests {
#[tokio::test]
async fn a_failed_lookup_is_denied() {
assert!(
grant(Some(&Fake::Fails), "swarm-agent.atlas.Ab9_-zSECRET")
grant(&Fake::Fails, "swarm-agent.atlas.Ab9_-zSECRET")
.await
.is_none()
);
@ -313,7 +293,7 @@ mod tests {
let bound = crate::introspect::INTROSPECTION_TIMEOUT;
let started = std::time::Instant::now();
assert!(
grant(Some(&Fake::Hangs), "swarm-agent.atlas.Ab9_-zSECRET")
grant(&Fake::Hangs, "swarm-agent.atlas.Ab9_-zSECRET")
.await
.is_none()
);
@ -325,22 +305,10 @@ mod tests {
);
}
#[tokio::test]
async fn no_store_is_denied() {
assert!(
grant(None, "swarm-agent.atlas.Ab9_-zSECRET")
.await
.is_none()
);
}
#[tokio::test]
async fn a_stored_object_naming_another_agent_is_denied() {
assert!(
grant(
Some(&stored("argus", "atlas")),
"swarm-agent.argus.Ab9_-zSECRET"
)
grant(&stored("argus", "atlas"), "swarm-agent.argus.Ab9_-zSECRET")
.await
.is_none()
);
@ -359,7 +327,7 @@ mod tests {
let Presented::Agent(token) = classify("swarm-agent.atlas.Ab9_-zSECRET") else {
panic!("an agent token");
};
assert!(authorize(&bare, Some(&atlas()), &token).await.is_none());
assert!(authorize(&bare, &atlas(), &token).await.is_none());
}
/// Everything without the prefix reaches introspection as it arrived.

View file

@ -155,8 +155,9 @@ struct Args {
agent_token_publish_subjects: Vec<String>,
/// Role on the secret store's `cert` auth mount this responder logs in
/// as to read agent credentials. The store is found through the `BAO_*`
/// environment; with none, every agent token is denied.
/// as to read the callout xkey and agent credentials. The store is found
/// through the `BAO_*` environment, which the responder does not start
/// without.
#[arg(long, default_value = "swarm-nats-auth")]
store_cert_role: String,
}
@ -219,19 +220,25 @@ async fn serve(args: &Args) -> anyhow::Result<()> {
args.hive_publish_subjects.clone(),
)?
.with_agent_token_subjects(args.agent_token_publish_subjects.clone())?;
let store = agent_token::Store::from_env(args.store_cert_role.clone());
let http = reqwest::Client::new();
let issuer = nkeys::KeyPair::from_seed(&read_secret(&args.issuer_seed_file)?)
.context("parse the account signing seed")?;
let settings = swarm_secret_client::client::Settings::from_env()
.context("reading the secret store's address and this responder's identity")?;
// Before the connect: a responder that cannot open a sealed request must
// not start answering. Read, never created here; see `xkey`.
let callout_xkey = xkey::load(
&swarm_secret_client::SecretStore::from_env(&args.store_cert_role)
&swarm_secret_client::SecretStore::connect(
&settings,
&args.store_cert_role,
swarm_secret_client::client::DEFAULT_CERT_MOUNT,
)
.await
.context("logging in to the secret store for the callout xkey")?,
)
.await?;
tracing::info!(public_key = %callout_xkey.public_key(), "loaded the callout xkey");
let store = agent_token::Store::new(settings, args.store_cert_role.clone());
let user_seed = read_secret(&args.user_seed_file)?;
let client = async_nats::ConnectOptions::with_nkey(user_seed)
.name("swarm-nats-auth")
@ -266,7 +273,7 @@ async fn serve(args: &Args) -> anyhow::Result<()> {
// the secret proved it; `granted` says which.
Some(agent_token::Presented::Agent(token)) => (
Some(format!("agent:{}", token.agent)),
agent_token::authorize(&policy, store.as_ref(), &token).await,
agent_token::authorize(&policy, &store, &token).await,
),
Some(agent_token::Presented::Malformed(e)) => {
tracing::warn!(error = %e, "malformed agent token; denying");