vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-agent: read the per-agent queue secret from bao in process

Browse source 
The harness now reads swarm/agents/<agent>/queue from the store itself,
under the agent's own store certificate, and holds it in memory only.
It reads once before the first connect and again on every reconnect
attempt (async-nats `ConnectOptions::with_auth_callback`), so an agent
whose secret was re-minted reconnects with the new value instead of
being refused until the container restarts.

hive-agent-queue-credential.service, the /run file it wrote, and
HIVE_AGENT_QUEUE_AGENT_SECRET_FILE are gone; queue-identity.nix now
hands hive-agent.service the store address, its certificate paths and
the agent name.

A failed or empty read before the first connect still falls back to
the hive's shared client. Each read is bounded by a 10s timeout, and
retries wait out the existing reconnect backoff (500ms doubling, capped
at 60s).

Closes #4783
This commit is contained in:
atlas 2026-09-29 09:40:48 +02:00 • committed by mara
commit ccb5bd3b38
10 changed files with 770 additions and 529 deletions
Download patch file Download diff file Expand all files Collapse all files

114
nix/module-eval/agent-queue-bao.nix
View file

@ -40,8 +40,15 @@ let
agentNoBao = agentWith { };
# Both at once, for the property that only exists when both are configured:
# the store certificate never reaches a queue variable.
agentQueueBao = agentWith {
hyperhive.queue.natsUrl = "nats://10.42.0.1:4222";
hyperhive.queue.tokenEndpoint = "https://auth.t.local/api/oidc/token";
services.hyperhive.agent.bao.addr = "https://bao.t.local:8200";
};
agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
agentQueueCredential = machine: machine.systemd.services.hive-agent-queue-credential;
cases = [
{
# Both ids or neither: the secret authenticates nobody without the id it
@ -166,91 +173,62 @@ let
ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity);
}
{
# The per-agent credential's fetch rides on the same store identity the
# check above proves, because there is one identity per agent. A fetch
# unit loading different ids would be a second certificate nothing
# mints.
name = "the queue-credential fetch presents the agent's own store identity";
# The harness reads its per-agent queue secret from the store itself,
# under the same store identity the check above proves, because there is
# one identity per agent.
name = "the harness presents the agent's own store identity";
ok =
let
u = agentQueueCredential agentBao;
u = agentHarness agentBao;
in
builtins.elem "hive-agent-bao-cert" u.serviceConfig.LoadCredential
&& builtins.elem "hive-agent-bao-key" u.serviceConfig.LoadCredential
&& u.environment.BAO_CLIENT_CERT == "%d/hive-agent-bao-cert"
&& u.environment.BAO_CLIENT_KEY == "%d/hive-agent-bao-key";
&& u.environment.BAO_CLIENT_KEY == "%d/hive-agent-bao-key"
&& u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
}
{
# Same 403-not-a-miss reason as every other reader here: the path
# `swarm_secret_client::queue::agent_queue_path` builds is the one this
# agent's own policy stanza covers. Built from the agent's own name,
# because the name is what makes it this agent's credential and not a
# neighbour's — which is the entire point of minting one per agent.
name = "the queue-credential fetch reads the agent's own per-agent path";
# `swarm_secret_client` builds both the store path and the cert-auth role
# from this name, so it has to be the name the credential was minted
# under.
name = "the harness is told the agent name its queue secret is minted under";
ok =
let
m = agentBao;
name = m.services.hyperhive.agent.user.name;
in
lib.hasInfix "secret/swarm/agents/${name}/queue" (agentQueueCredential m).script;
(agentHarness agentBao).environment.HIVE_AGENT_NAME == agentBao.services.hyperhive.agent.user.name;
}
{
# The per-agent queue secret lives in the harness's memory only: no unit
# fetches it to a file, and the harness is handed no file to read it
# from.
name = "no unit writes the per-agent queue secret to disk";
ok =
!(agentBao.systemd.services ? hive-agent-queue-credential)
&& !((agentHarness agentBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE);
}
{
# ⛔ The store certificate authenticates against the store and nothing
# else. It must never reach the queue, so nothing in this unit may hand
# a `BAO_CLIENT_*` path to anything queue-shaped.
name = "the queue-credential fetch never points the queue at the store certificate";
# else. With both the store and the queue configured, no queue variable
# may name one of its paths.
name = "the harness never points the queue at the store certificate";
ok =
let
u = agentQueueCredential agentBao;
harness = agentHarness agentBao;
e = (agentHarness agentQueueBao).environment;
queueVars = lib.filterAttrs (
n: _: lib.hasPrefix "HIVE_AGENT_OIDC_" n || n == "HIVE_AGENT_NATS_URL"
) e;
in
!(lib.hasInfix "nats" u.script)
&& !(lib.any (lib.hasPrefix "BAO_") (builtins.attrNames harness.environment));
}
{
# A secret fetched at boot has no business surviving one, and the
# directory has to be the unit's own so the file is owned by the agent
# rather than needing a mode change. `RemainAfterExit` is what keeps
# systemd from removing it out from under the harness.
name = "the fetched credential lands in a runtime directory the unit keeps alive";
ok =
let
u = agentQueueCredential agentBao;
c = u.serviceConfig;
in
c.RuntimeDirectory == "hive-agent-queue-credential"
&& c.RemainAfterExit
&& lib.hasInfix "/run/hive-agent-queue-credential/secret" u.script;
}
{
# 🩸 Degrades where the identity check fails, and the reason is that
# both reads are governed by one policy stanza: a refusal this unit
# sees and that check did not is an object not yet minted, not a policy
# that drifted. An agent created before its swarm minted one is a
# deployment doing nothing wrong.
name = "a per-agent credential that was never minted does not fail the unit";
ok = lib.hasInfix "exit 0" (agentQueueCredential agentBao).script;
}
{
# The harness reads a path and never a value — the same discipline
# ../agent-modules/queue.nix keeps for the hive-scoped secret. Not
# `%d`: this one is not a systemd credential, it is a file the
# container fetched for itself.
name = "the harness is handed the fetched credential as a path";
ok =
let
m = agentBao;
in
(agentHarness m).environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE
== m.services.hyperhive.agent.queue.agentSecretFile;
queueVars != { } && !(lib.any (lib.hasInfix "hive-agent-bao") (builtins.attrValues queueVars));
}
{
# The absence arm. An agent whose swarm gave it no store has nothing to
# log in with, so there is nothing to fetch with either — and the
# harness is then told no path rather than one that never fills.
name = "an agent told no store address fetches no queue credential";
# log in with, so the harness is handed no store coordinates at all.
name = "an agent told no store address hands the harness no store identity";
ok =
!(agentNoBao.systemd.services ? hive-agent-queue-credential)
&& !((agentHarness agentNoBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE);
let
u = agentHarness agentNoBao;
in
!(u.environment ? BAO_ADDR)
&& !(u.environment ? HIVE_AGENT_NAME)
&& !(builtins.elem "hive-agent-bao-cert" (u.serviceConfig.LoadCredential or [ ]));
}
];
in