diff --git a/hive-agent/src/swarm_queue.rs b/hive-agent/src/swarm_queue.rs index c5d6250a..b8d267e9 100644 --- a/hive-agent/src/swarm_queue.rs +++ b/hive-agent/src/swarm_queue.rs @@ -13,8 +13,21 @@ //! has already spawned threads, and this one has. So the rule is restated //! here over the inputs this consumer actually has, and [`decide`] is the //! single place it lives. +//! +//! Beside all four sits a fifth coordinate, resolved by +//! [`decide_agent_secret`] and not part of their group. The four are the +//! *hive's* — one OIDC client shared by every container on it — so at the +//! queue's auth callout they say which hive is connecting and never which +//! agent. The fifth is this agent's own, minted per agent at swarm level and +//! fetched by the container itself (`nix/agent-modules/queue-identity.nix`). +//! +//! It is reported here but not yet *presented*: the queue's auth-callout +//! responder (`swarm-nats-auth`) validates only the hive-scoped token, and an +//! agent offering a credential nothing on the other end reads back would be +//! refused. Until that responder learns the same path, the connect path below +//! is unchanged and this is the fetching half. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::OnceLock; use tokio::sync::OnceCell; @@ -49,6 +62,13 @@ struct QueueEnv { /// `hive_c0re::meta` embeds at build time — so it is here for a /// deployment that needs a different one, not for ours. ca_file: Option, + /// Where `nix/agent-modules/queue-identity.nix` fetched this agent's own + /// per-agent secret to. Outside the all-or-none group above because it is + /// governed by a different switch entirely — that unit is generated by + /// the agent having a *store* address, not by its hive having queue + /// coordinates — so an agent can legally have this and none of the four, + /// or the four and not this. + agent_secret_file: Option, } impl QueueEnv { @@ -60,6 +80,7 @@ impl QueueEnv { client_id_file: var("OIDC_CLIENT_ID_FILE"), client_secret_file: var("OIDC_CLIENT_SECRET_FILE"), ca_file: var("OIDC_CA_FILE"), + agent_secret_file: var("QUEUE_AGENT_SECRET_FILE"), } } } @@ -96,6 +117,36 @@ fn read_client_id(path: &Path) -> Option { } } +/// Decide whether this agent has its own per-agent queue secret, given the +/// path the fetch unit was told to write it to. +/// +/// Three states collapse to two answers. No variable means no store address +/// for this container, so no fetch unit was generated at all. A variable +/// naming a file that is missing or empty means the unit ran and found +/// nothing minted — the ordinary state of an agent created before its swarm +/// knew to mint one, which that unit reports and survives. Only a non-empty +/// file is a credential. +/// +/// The file is not read. Its *contents* are the secret and belong nowhere but +/// the moment of use; what a caller needs from here is whether there is one +/// and where, which `metadata` answers without opening it. +fn decide_agent_secret(path: Option<&str>) -> Option { + let path = PathBuf::from(path?); + match std::fs::metadata(&path) { + Ok(m) if m.len() > 0 => Some(path), + Ok(_) => None, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "checking for this agent's own queue credential failed" + ); + None + } + } +} + /// Decide what this agent's queue configuration is, given the environment and /// whatever the client-id file held. /// @@ -163,6 +214,28 @@ pub fn init() { } }; let _ = CONFIG.set(resolved); + + // Independent of everything above: this agent may hold its own secret on + // a hive with no queue coordinates, or hold the coordinates and no secret + // of its own yet. Reported either way, because "which credential is this + // agent able to present" is a question only this process can answer, and + // it is the one the next slice's rollout will be asked repeatedly. + // + // The answer is only logged here. Presenting it needs the queue's + // auth-callout responder to verify it, which is the next slice — see this + // module's header. + if let Some(path) = decide_agent_secret(env.agent_secret_file.as_deref()) { + // The path, never the bytes: the file holds the secret itself. + tracing::info!( + path = %path.display(), + "this agent has its own swarm queue credential" + ); + } else { + tracing::info!( + "no per-agent swarm queue credential; this agent is known to the queue \ + by its hive's shared client" + ); + } } /// What [`init`] resolved, or `None` when this agent has no queue. @@ -213,7 +286,7 @@ async fn connect_once() -> Option { #[cfg(test)] mod tests { - use super::{QueueEnv, Resolution, decide, read_client_id}; + use super::{QueueEnv, Resolution, decide, decide_agent_secret, read_client_id}; fn env(parts: [Option<&str>; 4]) -> QueueEnv { let [nats_url, token_endpoint, client_id_file, client_secret_file] = parts; @@ -223,6 +296,7 @@ mod tests { client_id_file: client_id_file.map(str::to_owned), client_secret_file: client_secret_file.map(str::to_owned), ca_file: None, + agent_secret_file: None, } } @@ -313,4 +387,52 @@ mod tests { ); } } + + /// The whole point of the fetch: a non-empty file is this agent's own + /// credential, and the answer is the path rather than what is in it. + #[test] + fn a_fetched_secret_resolves_to_its_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("secret"); + std::fs::write(&path, "s3cr3t").expect("write"); + assert_eq!( + decide_agent_secret(path.to_str()).as_deref(), + Some(path.as_path()) + ); + } + + /// The rollout state, and the one this must not confuse with a + /// credential: the fetch unit ran, found nothing minted for this agent, + /// and left no file. Treating that as a secret would have the harness + /// present zero bytes to the queue. + #[test] + fn a_missing_or_empty_fetched_secret_is_no_credential() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("secret"); + assert_eq!(decide_agent_secret(path.to_str()), None); + std::fs::write(&path, "").expect("write"); + assert_eq!(decide_agent_secret(path.to_str()), None); + } + + /// No variable at all: this container was given no store address, so no + /// fetch unit exists to have written anything. + #[test] + fn no_fetch_path_is_no_credential() { + assert_eq!(decide_agent_secret(None), None); + } + + /// The two credentials are resolved by separate switches, and this is the + /// asymmetry that makes keeping them apart worth it: an agent whose hive + /// has no queue can still hold its own swarm-minted secret, because that + /// one is minted with no hive in the chain. + #[test] + fn the_per_agent_secret_is_independent_of_the_hive_coordinates() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("secret"); + std::fs::write(&path, "s3cr3t").expect("write"); + + let e = env([None, None, None, None]); + assert!(matches!(decide(&e, None), Resolution::Absent(_))); + assert!(decide_agent_secret(path.to_str()).is_some()); + } } diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index bab54e47..a244ad02 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -55,6 +55,7 @@ in ./otel.nix ./packages.nix ./queue.nix + ./queue-identity.nix ./renamed-options.nix ./user.nix ./screen.nix diff --git a/nix/agent-modules/queue-identity.nix b/nix/agent-modules/queue-identity.nix new file mode 100644 index 00000000..7aad8897 --- /dev/null +++ b/nix/agent-modules/queue-identity.nix @@ -0,0 +1,203 @@ +# This agent's own credential for the swarm queue, fetched from the store by +# the agent itself. +# +# The credential beside it in ./queue.nix is keyed per **hive**: one OIDC +# client minted at deploy time and handed to every agent container on the +# hive, so at the queue's auth callout one agent is indistinguishable from its +# co-hived neighbours. This one is keyed per **agent** — `swarm-controller` +# mints it at agent creation into `swarm/agents//queue` +# (`swarm_secret_client::queue::agent_queue_path`), at swarm level, with no +# hive anywhere in the chain. +# +# That is also why this unit *fetches* rather than being handed a credential: +# a hive courier in the path would be the hive vouching for which agent this +# is, which is the property the per-agent credential exists to remove. The +# agent authenticates to the store as itself, with the certificate ./bao.nix +# already proves it can log in with, and reads its own path. +# +# ⛔ The store certificate is for reaching the store and nothing else. It is +# never presented to the queue: what goes to the queue is the secret read +# back from this path. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.agent.bao; + + # This container's agent name — the same string `swarm-controller` minted + # the credential under, because the agent's unix user is named for the + # agent (see ./user.nix). + agentName = config.services.hyperhive.agent.user.name; + + # The same three ids ./bao.nix loads. Both units present the same + # certificate because there is one store identity per agent; the ids are + # `hive_c0re::lifecycle::agent_identity`'s and a rename is a rename there + # too. + certCredential = "hive-agent-bao-cert"; + keyCredential = "hive-agent-bao-key"; + serverCaCredential = "hive-agent-bao-server-ca"; + + unitName = "hive-agent-queue-credential"; + + # The nix half of `swarm_secret_client::queue::agent_queue_path` plus + # `path::MOUNT`, spelled exactly as ./bao.nix spells its own sibling path. + queuePath = "secret/swarm/agents/${agentName}/queue"; + + # `RuntimeDirectory=` under the unit's own `User=`, so the file is owned by + # the agent and readable by the harness without a mode change. /run and not + # the state dir on purpose: a secret fetched at boot has no business + # surviving one. + runtimeDir = "${unitName}"; + secretFile = "/run/${runtimeDir}/secret"; + + # The store's address is the whole switch, exactly as in ./bao.nix — and + # deliberately *not* the queue coordinates in ./queue.nix. Those are the + # hive's, and gating a swarm-minted per-agent credential on a hive-level + # option would put the hive back in a path whose entire purpose is not + # having one. + configured = cfg.addr != null; +in +{ + options.services.hyperhive.agent.queue.agentSecretFile = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = secretFile; + description = '' + Path this agent's own swarm-queue secret is fetched to, for a consumer + outside the harness unit. Read-only for the same reason + {option}`services.hyperhive.agent.queue.clientSecretFile` is: it is a + fact about where the fetch writes, not a knob. + + 🩸 A PATH and never a value. The file is `0400` to the agent user and + is read at the moment it is needed; nothing in this tree puts its + contents in an environment variable, where `/proc//environ` would + publish them to every process in the container. + + The file exists only once the swarm has minted a credential for this + agent. An agent created before its swarm did so has none, and + `${unitName}.service` says so in the journal rather than failing — + see that unit. + ''; + }; + + config = lib.mkIf configured { + systemd.services.${unitName} = { + description = "fetch this agent's own swarm-queue credential from the secret store"; + after = [ + "network.target" + # Ordering only, not a requirement: that unit is the one that reports + # a store this agent cannot reach as itself, and it should get to say + # so before this one reports a path it could not read. + "hive-agent-bao-identity.service" + ]; + before = [ "hive-agent.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + pkgs.openbao + pkgs.coreutils + ]; + # Same sizing and the same `[Unit]`-not-`[Service]` placement as + # ./bao.nix's check: a few short attempts cover a store that comes up + # alongside this container, and a longer window only delays the report. + startLimitBurst = 4; + startLimitIntervalSec = 300; + serviceConfig = { + Type = "oneshot"; + # Keeps the unit active, which is what keeps `RuntimeDirectory=` + # from being removed out from under the harness. + RemainAfterExit = true; + TimeoutStartSec = 30; + Restart = "on-failure"; + RestartSec = 15; + User = agentName; + Group = agentName; + RuntimeDirectory = runtimeDir; + RuntimeDirectoryMode = "0700"; + UMask = "0377"; + # Bare ids, no paths: the terse `LoadCredential=` form that inherits + # a credential the service *manager* received, which is what the + # container manager passed in. ./bao.nix and ./queue.nix state the + # same shape. + LoadCredential = [ + certCredential + keyCredential + serverCaCredential + ]; + }; + environment = { + BAO_ADDR = cfg.addr; + # `%d` is `$CREDENTIALS_DIRECTORY`, per-unit and owned by `User=`. + BAO_CLIENT_CERT = "%d/${certCredential}"; + BAO_CLIENT_KEY = "%d/${keyCredential}"; + }; + script = '' + set -euo pipefail + + # No identity delivered at all. ./bao.nix's check reports this as the + # failure it is; there is nothing for this unit to add, and failing + # here too would be the same cause stated twice. + for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do + if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then + echo "this agent has no store identity, so it cannot fetch its own queue credential." >&2 + exit 0 + fi + done + + # Only when one was delivered — absent means verify the store's + # listener against the container's own trust store, which is what a + # deployment with a real CA wants. ./bao.nix says the same. + if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then + export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}" + fi + + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*` + # variables above only pick the certificate the handshake presents. + # `-token-only` answers on stdout and skips the token helper, which + # is a `sh` this unit's `path` does not carry. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2 + if [ -s "$err" ]; then cat "$err" >&2; fi + exit 1 + fi + export BAO_TOKEN + + # 🩸 Degrades where ./bao.nix's check fails, and the reason is that + # the two reads are governed by the *same* policy stanza: + # `swarm_secret_client::policy::render_agent` grants read on + # `secret/data/swarm/agents//*`, which covers this path and + # the `bao-mtls` one beside it alike. So a refusal this unit sees and + # that check did not cannot be a policy that drifted — it is an + # object that has not been minted, which is the ordinary state of + # every agent created before its swarm knew to mint one. A unit that + # failed at every boot over that would be loud about a deployment + # doing nothing wrong. + # + # ⚠️ Written by redirect into the runtime directory, never echoed: + # the field is the secret itself. + if ! bao kv get -field=value ${lib.escapeShellArg queuePath} > ${lib.escapeShellArg secretFile} 2>"$err"; then + rm -f ${lib.escapeShellArg secretFile} + echo "no per-agent queue credential at ${queuePath} yet; this agent falls back to its hive's shared one." >&2 + if [ -s "$err" ]; then cat "$err" >&2; fi + exit 0 + fi + + echo "fetched this agent's own queue credential from ${queuePath}." + ''; + }; + + # The harness reads a path and never a value, the same shape ./queue.nix + # hands it the hive-scoped secret in. Not `%d` here: this credential is + # not a systemd credential at all — it is a file this container fetched + # for itself, which is the whole point. + systemd.services.hive-agent = { + after = [ "${unitName}.service" ]; + environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE = secretFile; + }; + }; +} diff --git a/nix/module-eval/agent-queue-bao.nix b/nix/module-eval/agent-queue-bao.nix index 6ebd09dd..4768d71a 100644 --- a/nix/module-eval/agent-queue-bao.nix +++ b/nix/module-eval/agent-queue-bao.nix @@ -41,6 +41,7 @@ let agentNoBao = agentWith { }; agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity; + agentQueueCredential = machine: machine.systemd.services.hive-agent-queue-credential; cases = [ { # Both ids or neither: the secret authenticates nobody without the id it @@ -164,6 +165,93 @@ let name = "an agent told no store address runs no identity check"; ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity); } + { + # The per-agent credential's fetch rides on the same store identity the + # check above proves, because there is one identity per agent. A fetch + # unit loading different ids would be a second certificate nothing + # mints. + name = "the queue-credential fetch presents the agent's own store identity"; + ok = + let + u = agentQueueCredential agentBao; + in + builtins.elem "hive-agent-bao-cert" u.serviceConfig.LoadCredential + && u.environment.BAO_CLIENT_CERT == "%d/hive-agent-bao-cert" + && u.environment.BAO_CLIENT_KEY == "%d/hive-agent-bao-key"; + } + { + # Same 403-not-a-miss reason as every other reader here: the path + # `swarm_secret_client::queue::agent_queue_path` builds is the one this + # agent's own policy stanza covers. Built from the agent's own name, + # because the name is what makes it this agent's credential and not a + # neighbour's — which is the entire point of minting one per agent. + name = "the queue-credential fetch reads the agent's own per-agent path"; + ok = + let + m = agentBao; + name = m.services.hyperhive.agent.user.name; + in + lib.hasInfix "secret/swarm/agents/${name}/queue" (agentQueueCredential m).script; + } + { + # ⛔ The store certificate authenticates against the store and nothing + # else. It must never reach the queue, so nothing in this unit may hand + # a `BAO_CLIENT_*` path to anything queue-shaped. + name = "the queue-credential fetch never points the queue at the store certificate"; + ok = + let + u = agentQueueCredential agentBao; + harness = agentHarness agentBao; + in + !(lib.hasInfix "nats" u.script) + && !(lib.any (lib.hasPrefix "BAO_") (builtins.attrNames harness.environment)); + } + { + # A secret fetched at boot has no business surviving one, and the + # directory has to be the unit's own so the file is owned by the agent + # rather than needing a mode change. `RemainAfterExit` is what keeps + # systemd from removing it out from under the harness. + name = "the fetched credential lands in a runtime directory the unit keeps alive"; + ok = + let + u = agentQueueCredential agentBao; + c = u.serviceConfig; + in + c.RuntimeDirectory == "hive-agent-queue-credential" + && c.RemainAfterExit + && lib.hasInfix "/run/hive-agent-queue-credential/secret" u.script; + } + { + # 🩸 Degrades where the identity check fails, and the reason is that + # both reads are governed by one policy stanza: a refusal this unit + # sees and that check did not is an object not yet minted, not a policy + # that drifted. An agent created before its swarm minted one is a + # deployment doing nothing wrong. + name = "a per-agent credential that was never minted does not fail the unit"; + ok = lib.hasInfix "exit 0" (agentQueueCredential agentBao).script; + } + { + # The harness reads a path and never a value — the same discipline + # ../agent-modules/queue.nix keeps for the hive-scoped secret. Not + # `%d`: this one is not a systemd credential, it is a file the + # container fetched for itself. + name = "the harness is handed the fetched credential as a path"; + ok = + let + m = agentBao; + in + (agentHarness m).environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE + == m.services.hyperhive.agent.queue.agentSecretFile; + } + { + # The absence arm. An agent whose swarm gave it no store has nothing to + # log in with, so there is nothing to fetch with either — and the + # harness is then told no path rather than one that never fills. + name = "an agent told no store address fetches no queue credential"; + ok = + !(agentNoBao.systemd.services ? hive-agent-queue-credential) + && !((agentHarness agentNoBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE); + } ]; in runGroup "agent-queue-bao" cases