swarm: carry the agent queue credential from the host into the harness
hive-c0re stats the two files `swarm-bao-queue-agent` lands and forwards them into every agent container as systemd credentials, and the harness resolves a `QueueConfig` out of them at boot. Nothing connects yet. A credential and not a bind mount, and the mode is what forces it: the secret is root:0600 and the harness runs as the unprivileged agent user, so a bind would deliver a file that user cannot open. nspawn's `--load-credential` is read by the container manager as root and re-exposed under the consuming unit's own `User=`. hive-c0re never reads the bytes either way, which is just as well — it runs as `hive-core`. Absent files stay legal and become visible rather than silent: the publisher lives on the authelia host and mints on its first boot, so "nothing at that path" is the ordinary early state of a swarm. c0re forwards nothing and logs why; the harness logs that it has no queue. The client id comes out of the delivered file rather than being rebuilt from `hiveName` in nix, which is the agreement `swarm-secret-client` states. `QueueConfig::from_env` wants it as a value, so the harness reads the file itself — assigning the variable instead would need `std::env::set_var` in a process that has already spawned threads. Refs #3805
This commit is contained in:
parent
f8dd737456
commit
353cdd9264
6 changed files with 436 additions and 7 deletions
|
|
@ -130,6 +130,71 @@ fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Env var naming the host directory `swarm-bao-queue-agent.service` lands
|
||||
/// this hive's agent queue credential in. Set by the hive-c0re NixOS module
|
||||
/// from `deploy.hive-controller.queue.agentCredentialDir`; absent means this
|
||||
/// daemon runs outside its unit, which is the same "no queue" answer as an
|
||||
/// empty directory.
|
||||
const QUEUE_CREDENTIAL_DIR_ENV: &str = "HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR";
|
||||
|
||||
/// systemd credential ids the two files arrive under inside the container.
|
||||
/// `nix/agent-modules/queue.nix` spells the same two names in the harness
|
||||
/// unit's `LoadCredential=`; neither side can discover the other's, so a
|
||||
/// rename here is a rename there.
|
||||
const QUEUE_SECRET_CREDENTIAL: &str = "hive-queue-agent-secret";
|
||||
const QUEUE_CLIENT_ID_CREDENTIAL: &str = "hive-queue-agent-client-id";
|
||||
|
||||
/// Forward this hive's agent queue credential into `agent_name`'s container
|
||||
/// as two systemd credentials, or nothing when the publisher has not landed
|
||||
/// it yet.
|
||||
///
|
||||
/// **A credential and not a bind mount, because of the mode.** The secret is
|
||||
/// `root:0600` on the host and the harness runs as the unprivileged agent
|
||||
/// user, so a bind would deliver a file that user cannot open. nspawn's
|
||||
/// `--load-credential` is read by the container manager as root and
|
||||
/// re-exposed under the consuming unit's own `User=`, which is the whole
|
||||
/// difference. hive-c0re never reads the bytes either way — it runs as
|
||||
/// `hive-core` and only ever needs to know the file is *there*, which a
|
||||
/// `0755` directory permits.
|
||||
///
|
||||
/// **Absent files are legal.** Authelia mints the secret on its first boot
|
||||
/// and a publisher on that host puts it in the store, so "nothing at that
|
||||
/// path" is the ordinary early state of a swarm rather than a fault. Saying
|
||||
/// so at `info!` is what keeps it from being invisible: without a line here,
|
||||
/// an agent that never connects looks identical to one that was never
|
||||
/// configured.
|
||||
fn queue_agent_credentials(agent_name: &str, dir: Option<&Path>) -> Vec<CredentialMount> {
|
||||
let Some(dir) = dir else {
|
||||
tracing::info!(
|
||||
%agent_name,
|
||||
"no {QUEUE_CREDENTIAL_DIR_ENV} in this daemon's environment — agent gets no swarm queue"
|
||||
);
|
||||
return Vec::new();
|
||||
};
|
||||
let secret = dir.join("secret");
|
||||
let client_id = dir.join("client_id");
|
||||
// Both or neither, for the same reason `QueueConfig::from_env` refuses a
|
||||
// half-set environment: a client holding one of the two comes up fine and
|
||||
// never connects.
|
||||
if !secret.is_file() || !client_id.is_file() {
|
||||
tracing::info!(
|
||||
%agent_name, dir = %dir.display(),
|
||||
"swarm queue credential not published yet — agent gets no swarm queue"
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
vec![
|
||||
CredentialMount {
|
||||
name: QUEUE_SECRET_CREDENTIAL.to_owned(),
|
||||
host_path: secret.to_string_lossy().into_owned(),
|
||||
},
|
||||
CredentialMount {
|
||||
name: QUEUE_CLIENT_ID_CREDENTIAL.to_owned(),
|
||||
host_path: client_id.to_string_lossy().into_owned(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Idempotently rewrite the lines in `/etc/nixos-containers/<container>.conf`
|
||||
/// that hive-c0re owns: `PRIVATE_NETWORK` (always 1), `HOST_ADDRESS` (the
|
||||
/// bridge gateway IP) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind). What
|
||||
|
|
@ -185,12 +250,11 @@ async fn set_nspawn_flags(
|
|||
// is needed here — the bind alone is enough.
|
||||
let claude_mount = container_claude_mount(agent_name);
|
||||
|
||||
// No hive-wide secrets are forwarded into agent containers. hive-priv
|
||||
// still accepts a credential list (see `write_nspawn_flags`), but
|
||||
// nothing produces one: the only entry was the OTEL upstream token,
|
||||
// and an agent has no business holding the hive's credential for
|
||||
// anything outside it.
|
||||
let load_creds: Vec<CredentialMount> = Vec::new();
|
||||
// The agent's own swarm-queue credential — the one thing forwarded in
|
||||
// here, and forwarded *as a credential* rather than a bind for the
|
||||
// reason `queue_agent_credentials` records.
|
||||
let queue_credential_dir = std::env::var_os(QUEUE_CREDENTIAL_DIR_ENV).map(PathBuf::from);
|
||||
let load_creds = queue_agent_credentials(agent_name, queue_credential_dir.as_deref());
|
||||
|
||||
let mut binds: Vec<BindMount> = vec![
|
||||
BindMount {
|
||||
|
|
@ -324,7 +388,10 @@ async fn set_nspawn_flags(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{BindMount, bind_child_agent_dirs};
|
||||
use super::{
|
||||
BindMount, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs,
|
||||
queue_agent_credentials,
|
||||
};
|
||||
|
||||
fn child_binds() -> Vec<BindMount> {
|
||||
let mut binds = Vec::new();
|
||||
|
|
@ -393,4 +460,49 @@ mod tests {
|
|||
bind_child_agent_dirs("../escape", &mut binds);
|
||||
assert!(binds.is_empty());
|
||||
}
|
||||
|
||||
/// The ordinary state of a swarm before the publisher on the authelia
|
||||
/// host has run: the directory is named and empty. Forwarding a
|
||||
/// credential whose source does not exist would make every agent
|
||||
/// container refuse to start, so this has to be silence-with-a-log
|
||||
/// rather than a partial list.
|
||||
#[test]
|
||||
fn an_unpublished_queue_credential_forwards_nothing() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
assert!(queue_agent_credentials("iris", None).is_empty());
|
||||
assert!(queue_agent_credentials("iris", Some(dir.path())).is_empty());
|
||||
std::fs::write(dir.path().join("secret"), "s").expect("write secret");
|
||||
assert!(
|
||||
queue_agent_credentials("iris", Some(dir.path())).is_empty(),
|
||||
"a secret with no client id is not a usable credential"
|
||||
);
|
||||
}
|
||||
|
||||
/// Both files present: two credentials, named the same two ids the
|
||||
/// harness unit imports, pointing at the two files the reader unit
|
||||
/// wrote.
|
||||
#[test]
|
||||
fn a_published_queue_credential_forwards_both_files() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::write(dir.path().join("secret"), "s").expect("write secret");
|
||||
std::fs::write(dir.path().join("client_id"), "hive-h1-agent").expect("write client_id");
|
||||
let creds = queue_agent_credentials("iris", Some(dir.path()));
|
||||
let named: Vec<(String, String)> = creds
|
||||
.into_iter()
|
||||
.map(|c| (c.name, c.host_path))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
named,
|
||||
[
|
||||
(
|
||||
QUEUE_SECRET_CREDENTIAL.to_owned(),
|
||||
dir.path().join("secret").to_string_lossy().into_owned(),
|
||||
),
|
||||
(
|
||||
QUEUE_CLIENT_ID_CREDENTIAL.to_owned(),
|
||||
dir.path().join("client_id").to_string_lossy().into_owned(),
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -700,6 +700,12 @@ const CANONICAL_INPUTS: &[&str] = &["nixpkgs"];
|
|||
/// trivial to stub from tests (which build their own slice instead of
|
||||
/// touching process-wide env).
|
||||
const FORWARDED_VARS: &[&str] = &[
|
||||
// Where the swarm queue is and where its tokens are minted, as an agent
|
||||
// container reaches them. Both are addresses the *host* computes — the
|
||||
// queue's from the bridge IP an agent routes over, never a loopback
|
||||
// address, which inside a container is the agent itself.
|
||||
"HIVE_AGENT_NATS_URL",
|
||||
"HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
|
||||
"HIVE_FORGE_URL",
|
||||
"HIVE_FORGE_PUBLIC_URL",
|
||||
"HIVE_MATRIX_URL",
|
||||
|
|
@ -724,6 +730,16 @@ const FORWARDED_VARS: &[&str] = &[
|
|||
/// answer. Half-wiring this map is not a missing nicety; it is a value that
|
||||
/// evaluates fine and is silently wrong.
|
||||
const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[
|
||||
// Read at build time to decide whether the harness unit declares the
|
||||
// queue credential at all — see `nix/agent-modules/queue.nix`. An agent
|
||||
// whose option says no queue and whose env says otherwise logs the
|
||||
// partial-config error rather than half-connecting, which is the same
|
||||
// failure this map's doc describes and the reason both halves are wired.
|
||||
("HIVE_AGENT_NATS_URL", "hyperhive.queue.natsUrl"),
|
||||
(
|
||||
"HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
|
||||
"hyperhive.queue.tokenEndpoint",
|
||||
),
|
||||
("HIVE_FORGE_URL", "hyperhive.forge.url"),
|
||||
("HIVE_MATRIX_URL", "hyperhive.matrix.url"),
|
||||
("HYPERHIVE_HIVE_NAME", "hyperhive.hiveName"),
|
||||
|
|
@ -2027,6 +2043,37 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// The queue coordinates travel as a pair and are wired through both
|
||||
/// halves of the forwarded-var machinery: the env var the harness reads at
|
||||
/// runtime and the option the harness *unit* is built from. An agent that
|
||||
/// got only the env var would declare no credential and then report a
|
||||
/// partial config — visible, but a rebuild away from working.
|
||||
#[test]
|
||||
fn queue_coordinates_are_forwarded_as_both_env_and_option() {
|
||||
for var in ["HIVE_AGENT_NATS_URL", "HIVE_AGENT_OIDC_TOKEN_ENDPOINT"] {
|
||||
assert!(
|
||||
FORWARDED_VARS.contains(&var),
|
||||
"{var} must be forwarded into every agent's env"
|
||||
);
|
||||
}
|
||||
let mut out = String::new();
|
||||
push_forwarded_var_options(
|
||||
&mut out,
|
||||
&[
|
||||
("HIVE_AGENT_NATS_URL", "nats://10.42.0.1:4222".to_string()),
|
||||
(
|
||||
"HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
|
||||
"https://auth.t.local/api/oidc/token".to_string(),
|
||||
),
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
" hyperhive.queue.natsUrl = \"nats://10.42.0.1:4222\";\n\
|
||||
\x20 hyperhive.queue.tokenEndpoint = \"https://auth.t.local/api/oidc/token\";\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn require_service_urls_accepts_a_rendered_forge_url() {
|
||||
require_service_urls(&[
|
||||
|
|
|
|||
Loading…
Reference in a new issue