vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-nats: require the responder's store identity, and drop its no-store mode

Browse source 
The responder reads the callout xkey before it connects, so a responder
with no secret store no longer starts, and its "deny every agent token,
serve OIDC" mode could not run. That mode is gone: `agent_token::Store`
is built from the settings the xkey load already read, and `authorize`
takes a source rather than an `Option` of one.

On the nix side, `deploy.nats.authBaoClientCertFile` and
`authBaoClientKeyFile` are now asserted at eval when the queue is
enabled. The units' no-identity branches, the copy unit's empty-file
fallback and its "denies agent tokens" message are removed; an absent
leaf file fails `swarm-nats-auth-secrets` like an absent seed does.
This commit is contained in:
atlas 2026-10-05 22:13:58 +02:00 • committed by mara
commit 52c931c3a7
5 changed files with 193 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

255
nix/host-modules/swarm-nats.nix
View file

@ -297,23 +297,21 @@ let
tlsKeyCredential = "tls-key";
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
# The responder's own store identity, for reading agent queue credentials.
# Without it the responder denies every agent token; agents then fall back to
# their hive's OIDC client.
# The responder's own store identity, for the callout xkey, agent queue
# credentials and its OIDC client secret. Required: an assertion below refuses
# a queue without it.
authStoreActive =
deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
# The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
authCertRole = "swarm-nats-auth";
# Store identity files the copy unit delivers, as credential id → host source.
authStoreFiles = lib.optionalAttrs authStoreActive (
{
"bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
"bao-ca.pem" = baoDeploy.serverCaFile;
}
);
authStoreFiles = {
"bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
"bao-ca.pem" = baoDeploy.serverCaFile;
};
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
@ -516,17 +514,19 @@ in
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
description = ''
Client certificate the auth-callout responder presents to the swarm's
secret store to read agent queue credentials, and that
`swarm-bao-nats-auth-oidc` presents to read the responder's own OIDC
client secret. Its subject must be
secret store to read the callout xkey and agent queue credentials, and
that `swarm-bao-nats-auth-oidc` and `swarm-bao-nats-callout-xkey`
present to read the responder's own OIDC client secret and the xkey's
public half. Its subject must be
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
No default. ./glue-nats-auth-bao-identity.nix points it at the leaf
Required when `deploy.nats.enable` is set; unset fails at eval.
./glue-nats-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints, where this host mints one; on any other
queue host it is a file the operator places. Unset, or set to a file
that does not exist, `swarm-bao-nats-auth-oidc` fails; a responder
with no OIDC client secret on disk cannot start, and the queue then
refuses every client.
queue host it is a file the operator places. Set to a file that does
not exist, `swarm-nats-auth-secrets`, `swarm-bao-nats-auth-oidc` and
`swarm-bao-nats-callout-xkey` fail, and the queue's server does not
start without the xkey.
A path, never a value.
'';
@ -620,6 +620,24 @@ in
to have this host mint all four.
'';
}
{
# The callout xkey and the responder's OIDC client secret are read
# from the store under this identity; without either the server or
# the responder does not start.
assertion = authStoreActive;
message = ''
services.hyperhive.deploy.nats.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
The queue's callout xkey and its responder's OIDC client secret
are read from the swarm's secret store with this certificate, and
neither the server nor the responder starts without them. They
default to the leaf the store mints when it runs on this host;
elsewhere, issue a leaf whose CN is
services.hyperhive.deploy.bao.natsAuthCommonName and name it in
both options.
'';
}
];
# Declared here rather than in the collector's module: an entry then
@ -929,18 +947,16 @@ in
RestartSec = "5s";
SyslogIdentifier = "swarm-nats-auth";
};
# The store the responder reads agent credentials from. Absent
# without an identity, which the responder reads as "no store".
environment = lib.optionalAttrs authStoreActive (
{
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = authCredential "bao-client.pem";
BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
}
// lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
BAO_CACERT = authCredential "bao-ca.pem";
}
);
# The store the responder reads the callout xkey and agent
# credentials from.
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = authCredential "bao-client.pem";
BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
}
// lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
BAO_CACERT = authCredential "bao-ca.pem";
};
};
# The server binary, so an operator with a shell in here can
@ -973,7 +989,7 @@ in
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
# Where the store's PKI is minted on this host, the responder's leaf is
# one of its files. Ordering only: elsewhere the unit does not exist.
++ lib.optional authStoreActive "swarm-bao-pki.service";
++ [ "swarm-bao-pki.service" ];
requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
serviceConfig = {
Type = "oneshot";
@ -989,17 +1005,9 @@ in
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
${lib.escapeShellArg (hostPath "issuer.seed")}
''
# The store identity is optional to the responder's agent-token path, so
# an absent source is delivered as an empty file rather than failing this
# unit: a missing `LoadCredential` source stops the responder starting.
+ lib.concatStrings (
lib.mapAttrsToList (id: source: ''
if [ -s ${lib.escapeShellArg source} ]; then
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
else
echo "${source} is absent; the queue responder denies agent tokens until it exists" >&2
install -m 0400 /dev/null ${lib.escapeShellArg (hostPath id)}
fi
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
'') authStoreFiles
);
};
@ -1013,8 +1021,7 @@ in
#
# ⚠️ Without the secret the responder does not start (its `LoadCredential`
# source is missing), and `auth_callout` with no responder refuses every
# client. A host with no responder leaf therefore fails here, naming the
# options, rather than rendering no unit.
# client.
systemd.services.swarm-bao-nats-auth-oidc = {
description = "fetch the swarm queue responder's OIDC client secret from the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
@ -1050,70 +1057,61 @@ in
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so it cannot read its OIDC client secret: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving its OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving its OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
exit 1
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; the queue responder has no OIDC client secret yet." >&2
exit 1
fi
# Root-owned 0400: `LoadCredential` reads it as root before the
# responder's `DynamicUser` exists. `printf` inside
# `atomic_write_secret` is a builtin, so the plaintext never
# becomes an argument in /proc.
install -d -m 0700 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"
fi
# Root-owned 0400: `LoadCredential` reads it as root before the
# responder's `DynamicUser` exists. `printf` inside
# `atomic_write_secret` is a builtin, so the plaintext never
# becomes an argument in /proc.
install -d -m 0700 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg (hostPath "oidc-client.secret")} "$secret"
fi
# `LoadCredential` copies the file once, at the responder's start.
refresh_consumer ${lib.escapeShellArg machine} swarm-nats-auth.service ${lib.escapeShellArg (hostPath "oidc-client.secret")}
'';
# `LoadCredential` copies the file once, at the responder's start.
refresh_consumer ${lib.escapeShellArg machine} swarm-nats-auth.service ${lib.escapeShellArg (hostPath "oidc-client.secret")}
'';
};
# The callout xkey's public half, from the store into the file
@ -1157,57 +1155,48 @@ in
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs authStoreActive {
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script =
if !authStoreActive then
''
echo "the swarm queue responder has no store identity, so the callout xkey can be neither created nor fetched and the queue does not start: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
exit 1
''
else
''
set -euo pipefail
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${atomicWriteSecret}
${refreshConsumer}
swarm-nats-auth ensure-xkey --store-cert-role ${lib.escapeShellArg authCertRole}
swarm-nats-auth ensure-xkey --store-cert-role ${lib.escapeShellArg authCertRole}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout xkey file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout xkey file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
if ! xkey="$(bao kv get -field=value ${lib.escapeShellArg xkeyPublicPath} 2>"$err")"; then
echo "swarm-bao did not return ${xkeyPublicPath}." >&2
cat "$err" >&2
exit 1
fi
if [ -z "$xkey" ]; then
echo "swarm-bao returned an empty ${xkeyPublicPath}." >&2
exit 1
fi
if ! xkey="$(bao kv get -field=value ${lib.escapeShellArg xkeyPublicPath} 2>"$err")"; then
echo "swarm-bao did not return ${xkeyPublicPath}." >&2
cat "$err" >&2
exit 1
fi
if [ -z "$xkey" ]; then
echo "swarm-bao returned an empty ${xkeyPublicPath}." >&2
exit 1
fi
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
if secret_differs ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"; then
atomic_write_secret 0444 "" ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"
fi
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
if secret_differs ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"; then
atomic_write_secret 0444 "" ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"
fi
# `renderConfig` reads the file as the server starts, and only then.
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"}
'';
# `renderConfig` reads the file as the server starts, and only then.
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"}
'';
};
# ⚠️ Minted on the HOST, not in the container, because the responder is