diff --git a/Cargo.lock b/Cargo.lock index 6b2e9f51..f14c6419 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1668,7 +1668,6 @@ dependencies = [ "schemars", "serde", "serde_json", - "swarm-queue-client", "tempfile", "tokio", "tokio-stream", diff --git a/docs/swarm/README.md b/docs/swarm/README.md index e67bd379..a012bb04 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -361,34 +361,6 @@ list, which would look like a silent swarm rather than a controller that can't see. The body says which. Status survives a controller restart: it's stored in the queue, not in the daemon. -### Giving the agents the queue too - -An agent authenticates as its own client, not as its hive, so it needs its -own coordinates. Two of them are options on the hive; the other two arrive -with the credential itself and aren't configurable. - -| option | what to set it to | -| ------------------------------------------- | -------------------------------------------------------------- | -| `deploy.hive-controller.queue.agentNatsUrl` | where the queue listens, as an agent **container** reaches it | -| `swarm.statusPublish.tokenEndpoint` | the swarm IdP's `/api/oidc/token` — the same one the hive uses | - -On a host that runs the queue, `agentNatsUrl` defaults to -`nats://:`, which is the only address that -works from inside a container: the port is opened on the bridge interface and -nowhere else. ⚠️ **Never a loopback address here** — the hive's own -`statusPublish.natsUrl` is loopback and correct, because `hive-c0re` shares the -host's network namespace. An agent doesn't, so `127.0.0.1` reaches the agent. - -The harness sees four variables, and treats them as all-or-none: -`HIVE_AGENT_NATS_URL` and `HIVE_AGENT_OIDC_TOKEN_ENDPOINT` from the two options -above, plus `HIVE_AGENT_OIDC_CLIENT_SECRET_FILE` and -`HIVE_AGENT_OIDC_CLIENT_ID_FILE`, which point into the unit's own credentials -directory. The last two come from the delivered credential rather than from -config — see [`secrets.md`](secrets.md#hive-level--one-of-each-per-hive) for -how it gets there. A hive that hasn't been given the queue's address for its -agents sets none of the four and each agent logs that it has none; a half-set -environment logs an error and the harness keeps serving. - ### Swarm-wide forge webhooks At startup the controller registers two Forgejo hooks pointing at diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index afce2a45..6a3dde95 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -168,21 +168,6 @@ reader runs before `hive-c0re.service` and is wanted (not required) by it, so an agent container never renders ahead of the credential; an unreachable store delays the daemon's start rather than failing it. -That credential still has one hop left, because the reader of it lives inside -an agent container. **It crosses as a systemd credential, not as a bind -mount**, and the mode decides that: the secret is `root:0600` and a harness -runs as its own unprivileged agent user, so a bind would deliver a file that -user can't open. `hive-c0re` stats the two files — the directory is `0755`, so -it needs no read access to either — and hands them to -`systemd-nspawn --load-credential` through `hive-priv`, which runs as root. -Inside, `hive-agent.service` names the same two ids in `LoadCredential=` and -reads them out of its own `$CREDENTIALS_DIRECTORY`, owned by the agent user -and by nobody else. `hive-c0re` never reads the bytes at any point: it runs as -`hive-core`, which is the reason a copy wasn't an option either. When the -files aren't there the daemon forwards nothing and says so in its journal, and -the harness logs that it has no queue coordinates — the same absent-and-legal state, twice, -rather than a container that refuses to start. - ⚠️ **Service↔store mTLS is its own trust domain.** A credential you must already hold to authenticate can't be fetched from the thing it authenticates you to, so the store's identity can't come from an authority the store diff --git a/hive-agent/Cargo.toml b/hive-agent/Cargo.toml index 205fba07..7e4c90d2 100644 --- a/hive-agent/Cargo.toml +++ b/hive-agent/Cargo.toml @@ -31,9 +31,6 @@ rusqlite.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true -# Bare: `kv`/`notices` name buckets and streams this harness opens neither -# end of. All it wants from the crate is `QueueConfig` and, next, a connect. -swarm-queue-client.workspace = true tokio.workspace = true tokio-stream.workspace = true tower-http.workspace = true diff --git a/hive-agent/src/main.rs b/hive-agent/src/main.rs index 28d371b6..92b59d97 100644 --- a/hive-agent/src/main.rs +++ b/hive-agent/src/main.rs @@ -28,7 +28,6 @@ mod serve_common; mod state_entry_watch; mod stats; mod stream_enrich; -mod swarm_queue; mod term_msg; mod todo_server; mod todos; @@ -496,9 +495,6 @@ async fn serve_main(socket: &Path, poll_ms: u64) -> Result<()> { let claude_dir = login::default_dir(); let initial = LoginState::from_dir(&claude_dir); tracing::info!(state = ?initial, claude_dir = %claude_dir.display(), "harness boot"); - // Resolve the swarm-queue credential while the boot log is still the - // thing an operator is reading. Nothing connects yet. - swarm_queue::init(); // Config fact, stamped once — see `harness_state::write_api_key_mode`'s // doc for why hive-c0re needs this to stop reporting `needs_login` for // an agent whose `~/.claude/` is empty by design. diff --git a/hive-agent/src/swarm_queue.rs b/hive-agent/src/swarm_queue.rs deleted file mode 100644 index ab93fac7..00000000 --- a/hive-agent/src/swarm_queue.rs +++ /dev/null @@ -1,263 +0,0 @@ -//! This agent's swarm-queue credentials, resolved once at boot. -//! -//! Three of the four coordinates arrive as environment variables the meta -//! flake renders into the harness unit. The fourth — the OIDC client id — -//! arrives as a *file*, delivered beside the secret as one systemd -//! credential pair (`nix/agent-modules/queue.nix`). That is the whole reason -//! this module exists rather than a bare [`QueueConfig::from_env`] call: the -//! id rides with the secret so a reader never has to spell `hive--agent` -//! a second time, and `from_env` wants it as a value. -//! -//! Reading the file and assigning the variable would put the same rule back -//! in `from_env`'s hands, but `std::env::set_var` is unsound in a process that -//! has already spawned threads, and this one has. So the rule is restated -//! here over the inputs this consumer actually has, and [`decide`] is the -//! single place it lives. - -use std::path::Path; -use std::sync::OnceLock; - -use swarm_queue_client::QueueConfig; - -/// Variable prefix for this agent's coordinates. Distinct from `HIVE_C0RE`'s -/// on purpose: an agent authenticates as its own client, not as its hive. -const ENV_PREFIX: &str = "HIVE_AGENT"; - -/// Resolved once at boot, so the publisher that connects to the queue reads -/// one answer rather than re-deriving it per call. -static CONFIG: OnceLock> = OnceLock::new(); - -/// The four variables the harness unit sets, before the client-id file is -/// read. Collected into a struct so [`decide`] is pure over them and the -/// process env is touched in exactly one place. -struct QueueEnv { - nats_url: Option, - token_endpoint: Option, - client_id_file: Option, - client_secret_file: Option, - /// Independent of the all-or-none group below, exactly as in - /// `QueueConfig::from_env`: a swarm behind a publicly-trusted - /// certificate needs no extra anchor. Nothing in this tree sets it for - /// an agent — a container already trusts the swarm root, which - /// `hive_c0re::meta` embeds at build time — so it is here for a - /// deployment that needs a different one, not for ours. - ca_file: Option, -} - -impl QueueEnv { - fn from_env() -> Self { - let var = |suffix: &str| std::env::var(format!("{ENV_PREFIX}_{suffix}")).ok(); - Self { - nats_url: var("NATS_URL"), - token_endpoint: var("OIDC_TOKEN_ENDPOINT"), - client_id_file: var("OIDC_CLIENT_ID_FILE"), - client_secret_file: var("OIDC_CLIENT_SECRET_FILE"), - ca_file: var("OIDC_CA_FILE"), - } - } -} - -/// What the environment plus the client-id file add up to. -enum Resolution { - /// Everything is here; the agent can reach the queue. - Configured(Box), - /// No queue coordinates for this agent, and that is a legal state — carries why. - Absent(&'static str), - /// Some of the environment, not all of it. A deployment bug rather than - /// an absent integration, so it is reported and then survived. - Partial, -} - -/// Read the client id out of the file the credential landed at. -/// -/// `None` for a missing or empty file, which is the ordinary state of a hive -/// whose secret store has nothing published yet — the credential simply is -/// not there, and nspawn forwards nothing. Trailing newline stripped: the -/// reader unit writes one and an id with a newline in it authenticates as -/// nobody. -fn read_client_id(path: &Path) -> Option { - match std::fs::read_to_string(path) { - Ok(raw) => { - let id = raw.trim(); - (!id.is_empty()).then(|| id.to_owned()) - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, - Err(e) => { - tracing::warn!(path = %path.display(), error = %e, "reading the queue client id failed"); - None - } - } -} - -/// Decide what this agent's queue configuration is, given the environment and -/// whatever the client-id file held. -/// -/// All four variables or none, for the reason `QueueConfig::from_env` gives: -/// a half-set environment produces a process that comes up fine and never -/// connects. The client id is graded separately from the four because its -/// file has a legal absence the variables do not — the unit names the path a -/// credential *would* arrive at whether or not one has been published yet. -fn decide(env: &QueueEnv, client_id: Option) -> Resolution { - match ( - env.nats_url.as_ref(), - env.token_endpoint.as_ref(), - env.client_id_file.as_ref(), - env.client_secret_file.as_ref(), - ) { - (None, None, None, None) => Resolution::Absent("this hive has no swarm queue configured"), - (Some(url), Some(token_endpoint), Some(_), Some(secret)) => match client_id { - Some(client_id) => Resolution::Configured(Box::new(QueueConfig { - url: url.clone(), - token_endpoint: token_endpoint.clone(), - client_id, - client_secret_file: secret.into(), - ca_file: env.ca_file.as_ref().map(Into::into), - })), - None => { - Resolution::Absent("the queue credential has not been published to this hive yet") - } - }, - _ => Resolution::Partial, - } -} - -/// Resolve this agent's queue configuration and record it for later use. -/// -/// Never fails: a harness that cannot reach the queue still serves its -/// operator, its web UI and its turn loop, so every outcome here is a log -/// line and not an exit. -pub fn init() { - let env = QueueEnv::from_env(); - let client_id = env - .client_id_file - .as_deref() - .map(Path::new) - .and_then(read_client_id); - let resolved = match decide(&env, client_id) { - Resolution::Configured(cfg) => { - // url + client id only. The secret is a path in this struct and - // stays one: neither it nor its contents belong in a log. - tracing::info!(url = %cfg.url, client_id = %cfg.client_id, "swarm queue configured"); - Some(*cfg) - } - Resolution::Absent(why) => { - tracing::info!(why, "no swarm queue coordinates for this agent"); - None - } - Resolution::Partial => { - tracing::error!( - prefix = ENV_PREFIX, - "swarm queue half-configured: {ENV_PREFIX}_NATS_URL, \ - {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, {ENV_PREFIX}_OIDC_CLIENT_ID_FILE and \ - {ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are set together or not at all — \ - this agent will not connect" - ); - None - } - }; - let _ = CONFIG.set(resolved); -} - -#[cfg(test)] -mod tests { - use super::{QueueEnv, Resolution, decide, read_client_id}; - - fn env(parts: [Option<&str>; 4]) -> QueueEnv { - let [nats_url, token_endpoint, client_id_file, client_secret_file] = parts; - QueueEnv { - nats_url: nats_url.map(str::to_owned), - token_endpoint: token_endpoint.map(str::to_owned), - client_id_file: client_id_file.map(str::to_owned), - client_secret_file: client_secret_file.map(str::to_owned), - ca_file: None, - } - } - - fn full() -> QueueEnv { - env([ - Some("nats://10.42.0.1:4222"), - Some("https://auth.t.local/api/oidc/token"), - Some("/run/credentials/hive-agent.service/hive-queue-agent-client-id"), - Some("/run/credentials/hive-agent.service/hive-queue-agent-secret"), - ]) - } - - /// The reader unit writes the id with a trailing newline; an id carrying - /// one is a client authelia has never heard of. - #[test] - fn a_client_id_file_is_read_without_its_newline() { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("client_id"); - std::fs::write(&path, "hive-h1-agent\n").expect("write"); - assert_eq!(read_client_id(&path).as_deref(), Some("hive-h1-agent")); - } - - /// Both shapes of "no credential here": never delivered, or delivered - /// empty. Neither is an id, and treating an empty string as one would - /// authenticate as the anonymous client rather than failing. - #[test] - fn a_missing_or_empty_client_id_file_reads_as_no_id() { - let dir = tempfile::tempdir().expect("tempdir"); - let missing = dir.path().join("client_id"); - assert_eq!(read_client_id(&missing), None); - std::fs::write(&missing, "\n").expect("write"); - assert_eq!(read_client_id(&missing), None); - } - - #[test] - fn a_complete_environment_with_a_published_id_configures_the_queue() { - let e = full(); - let Resolution::Configured(cfg) = decide(&e, Some("hive-h1-agent".to_owned())) else { - panic!("expected a configured queue"); - }; - assert_eq!(cfg.url, "nats://10.42.0.1:4222"); - assert_eq!(cfg.client_id, "hive-h1-agent"); - assert!( - cfg.client_secret_file.ends_with("hive-queue-agent-secret"), - "the secret stays a path: {}", - cfg.client_secret_file.display() - ); - } - - /// A hive that has not been given its queue's coordinates at all. Silence - /// here is correct, and it has to be distinguishable from the half-set - /// case below — that distinction is the only thing that makes the error - /// branch worth logging. - #[test] - fn an_empty_environment_is_no_queue_rather_than_an_error() { - let e = env([None, None, None, None]); - assert!(matches!(decide(&e, None), Resolution::Absent(_))); - } - - /// The credential's own absence. The unit names the path unconditionally - /// once the hive has a queue, so this is the state of every agent on a - /// swarm whose publisher has not run — legal, and not the error branch. - #[test] - fn a_complete_environment_with_no_published_id_is_no_queue() { - let e = full(); - assert!(matches!(decide(&e, None), Resolution::Absent(_))); - } - - /// Each single-variable omission, because the failure a partial set - /// produces is a harness that looks healthy and publishes nothing. - #[test] - fn any_missing_variable_is_a_partial_configuration() { - for drop in 0..4 { - let mut parts = [ - Some("nats://10.42.0.1:4222"), - Some("https://auth.t.local/api/oidc/token"), - Some("/run/credentials/hive-agent.service/hive-queue-agent-client-id"), - Some("/run/credentials/hive-agent.service/hive-queue-agent-secret"), - ]; - parts[drop] = None; - let e = env(parts); - assert!( - matches!( - decide(&e, Some("hive-h1-agent".to_owned())), - Resolution::Partial - ), - "dropping variable {drop} must report a partial configuration" - ); - } - } -} diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index 09b55c66..3eeddbe9 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -130,71 +130,6 @@ fn bind_child_agent_dirs(child: &str, binds: &mut Vec) { } } -/// Env var naming the host directory `swarm-bao-queue-agent.service` lands -/// this hive's agent queue credential in. Set by the hive-c0re NixOS module -/// from `deploy.hive-controller.queue.agentCredentialDir`; absent means this -/// daemon runs outside its unit, which is the same "no queue credential" -/// answer as an empty directory. -const QUEUE_CREDENTIAL_DIR_ENV: &str = "HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR"; - -/// systemd credential ids the two files arrive under inside the container. -/// `nix/agent-modules/queue.nix` spells the same two names in the harness -/// unit's `LoadCredential=`; neither side can discover the other's, so a -/// rename here is a rename there. -const QUEUE_SECRET_CREDENTIAL: &str = "hive-queue-agent-secret"; -const QUEUE_CLIENT_ID_CREDENTIAL: &str = "hive-queue-agent-client-id"; - -/// Forward this hive's agent queue credential into `agent_name`'s container -/// as two systemd credentials, or nothing when the publisher has not landed -/// it yet. -/// -/// **A credential and not a bind mount, because of the mode.** The secret is -/// `root:0600` on the host and the harness runs as the unprivileged agent -/// user, so a bind would deliver a file that user cannot open. nspawn's -/// `--load-credential` is read by the container manager as root and -/// re-exposed under the consuming unit's own `User=`, which is the whole -/// difference. hive-c0re never reads the bytes either way — it runs as -/// `hive-core` and only ever needs to know the file is *there*, which a -/// `0755` directory permits. -/// -/// **Absent files are legal.** Authelia mints the secret on its first boot -/// and a publisher on that host puts it in the store, so "nothing at that -/// path" is the ordinary early state of a swarm rather than a fault. Saying -/// so at `info!` is what keeps it from being invisible: without a line here, -/// an agent that never connects looks identical to one that was never -/// configured. -fn queue_agent_credentials(agent_name: &str, dir: Option<&Path>) -> Vec { - let Some(dir) = dir else { - tracing::info!( - %agent_name, - "no {QUEUE_CREDENTIAL_DIR_ENV} in this daemon's environment — agent gets no swarm queue credential" - ); - return Vec::new(); - }; - let secret = dir.join("secret"); - let client_id = dir.join("client_id"); - // Both or neither, for the same reason `QueueConfig::from_env` refuses a - // half-set environment: a client holding one of the two comes up fine and - // never connects. - if !secret.is_file() || !client_id.is_file() { - tracing::info!( - %agent_name, dir = %dir.display(), - "swarm queue credential not published yet — agent gets no swarm queue credential" - ); - return Vec::new(); - } - vec![ - CredentialMount { - name: QUEUE_SECRET_CREDENTIAL.to_owned(), - host_path: secret.to_string_lossy().into_owned(), - }, - CredentialMount { - name: QUEUE_CLIENT_ID_CREDENTIAL.to_owned(), - host_path: client_id.to_string_lossy().into_owned(), - }, - ] -} - /// Idempotently rewrite the lines in `/etc/nixos-containers/.conf` /// that hive-c0re owns: `PRIVATE_NETWORK` (always 1), `HOST_ADDRESS` (the /// bridge gateway IP) and `EXTRA_NSPAWN_FLAGS` (the runtime-dir bind). What @@ -250,11 +185,12 @@ async fn set_nspawn_flags( // is needed here — the bind alone is enough. let claude_mount = container_claude_mount(agent_name); - // The agent's own swarm-queue credential — the one thing forwarded in - // here, and forwarded *as a credential* rather than a bind for the - // reason `queue_agent_credentials` records. - let queue_credential_dir = std::env::var_os(QUEUE_CREDENTIAL_DIR_ENV).map(PathBuf::from); - let load_creds = queue_agent_credentials(agent_name, queue_credential_dir.as_deref()); + // No hive-wide secrets are forwarded into agent containers. hive-priv + // still accepts a credential list (see `write_nspawn_flags`), but + // nothing produces one: the only entry was the OTEL upstream token, + // and an agent has no business holding the hive's credential for + // anything outside it. + let load_creds: Vec = Vec::new(); let mut binds: Vec = vec![ BindMount { @@ -388,10 +324,7 @@ async fn set_nspawn_flags( #[cfg(test)] mod tests { - use super::{ - BindMount, QUEUE_CLIENT_ID_CREDENTIAL, QUEUE_SECRET_CREDENTIAL, bind_child_agent_dirs, - queue_agent_credentials, - }; + use super::{BindMount, bind_child_agent_dirs}; fn child_binds() -> Vec { let mut binds = Vec::new(); @@ -460,49 +393,4 @@ mod tests { bind_child_agent_dirs("../escape", &mut binds); assert!(binds.is_empty()); } - - /// The ordinary state of a swarm before the publisher on the authelia - /// host has run: the directory is named and empty. Forwarding a - /// credential whose source does not exist would make every agent - /// container refuse to start, so this has to be silence-with-a-log - /// rather than a partial list. - #[test] - fn an_unpublished_queue_credential_forwards_nothing() { - let dir = tempfile::tempdir().expect("tempdir"); - assert!(queue_agent_credentials("iris", None).is_empty()); - assert!(queue_agent_credentials("iris", Some(dir.path())).is_empty()); - std::fs::write(dir.path().join("secret"), "s").expect("write secret"); - assert!( - queue_agent_credentials("iris", Some(dir.path())).is_empty(), - "a secret with no client id is not a usable credential" - ); - } - - /// Both files present: two credentials, named the same two ids the - /// harness unit imports, pointing at the two files the reader unit - /// wrote. - #[test] - fn a_published_queue_credential_forwards_both_files() { - let dir = tempfile::tempdir().expect("tempdir"); - std::fs::write(dir.path().join("secret"), "s").expect("write secret"); - std::fs::write(dir.path().join("client_id"), "hive-h1-agent").expect("write client_id"); - let creds = queue_agent_credentials("iris", Some(dir.path())); - let named: Vec<(String, String)> = creds - .into_iter() - .map(|c| (c.name, c.host_path)) - .collect::>(); - assert_eq!( - named, - [ - ( - QUEUE_SECRET_CREDENTIAL.to_owned(), - dir.path().join("secret").to_string_lossy().into_owned(), - ), - ( - QUEUE_CLIENT_ID_CREDENTIAL.to_owned(), - dir.path().join("client_id").to_string_lossy().into_owned(), - ), - ] - ); - } } diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 8199a958..2f716f80 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -700,12 +700,6 @@ const CANONICAL_INPUTS: &[&str] = &["nixpkgs"]; /// trivial to stub from tests (which build their own slice instead of /// touching process-wide env). const FORWARDED_VARS: &[&str] = &[ - // Where the swarm queue is and where its tokens are minted, as an agent - // container reaches them. Both are addresses the *host* computes — the - // queue's from the bridge IP an agent routes over, never a loopback - // address, which inside a container is the agent itself. - "HIVE_AGENT_NATS_URL", - "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", "HIVE_FORGE_URL", "HIVE_FORGE_PUBLIC_URL", "HIVE_MATRIX_URL", @@ -730,16 +724,6 @@ const FORWARDED_VARS: &[&str] = &[ /// answer. Half-wiring this map is not a missing nicety; it is a value that /// evaluates fine and is silently wrong. const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ - // Read at build time to decide whether the harness unit declares the - // queue credential at all — see `nix/agent-modules/queue.nix`. An agent - // whose option says no queue coordinates and whose env says otherwise logs the - // partial-config error rather than half-connecting, which is the same - // failure this map's doc describes and the reason both halves are wired. - ("HIVE_AGENT_NATS_URL", "hyperhive.queue.natsUrl"), - ( - "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", - "hyperhive.queue.tokenEndpoint", - ), ("HIVE_FORGE_URL", "hyperhive.forge.url"), ("HIVE_MATRIX_URL", "hyperhive.matrix.url"), ("HYPERHIVE_HIVE_NAME", "hyperhive.hiveName"), @@ -2043,37 +2027,6 @@ mod tests { ); } - /// The queue coordinates travel as a pair and are wired through both - /// halves of the forwarded-var machinery: the env var the harness reads at - /// runtime and the option the harness *unit* is built from. An agent that - /// got only the env var would declare no credential and then report a - /// partial config — visible, but a rebuild away from working. - #[test] - fn queue_coordinates_are_forwarded_as_both_env_and_option() { - for var in ["HIVE_AGENT_NATS_URL", "HIVE_AGENT_OIDC_TOKEN_ENDPOINT"] { - assert!( - FORWARDED_VARS.contains(&var), - "{var} must be forwarded into every agent's env" - ); - } - let mut out = String::new(); - push_forwarded_var_options( - &mut out, - &[ - ("HIVE_AGENT_NATS_URL", "nats://10.42.0.1:4222".to_string()), - ( - "HIVE_AGENT_OIDC_TOKEN_ENDPOINT", - "https://auth.t.local/api/oidc/token".to_string(), - ), - ], - ); - assert_eq!( - out, - " hyperhive.queue.natsUrl = \"nats://10.42.0.1:4222\";\n\ - \x20 hyperhive.queue.tokenEndpoint = \"https://auth.t.local/api/oidc/token\";\n" - ); - } - #[test] fn require_service_urls_accepts_a_rendered_forge_url() { require_service_urls(&[ diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 8fb48656..94f5e680 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -33,7 +33,6 @@ ./network.nix ./otel.nix ./packages.nix - ./queue.nix ./user.nix ./screen.nix ./weston-vnc.nix diff --git a/nix/agent-modules/queue.nix b/nix/agent-modules/queue.nix deleted file mode 100644 index 430c1b7a..00000000 --- a/nix/agent-modules/queue.nix +++ /dev/null @@ -1,97 +0,0 @@ -# Swarm-queue coordinates for this agent's harness. -# -# Three of the four the harness needs are addresses (this file's two options -# plus the secret's path); the fourth, the client id, arrives as a file beside -# the secret so a reader never spells `hive--agent` a second time. -# -# ⚠️ The credential arrives as a systemd credential and NOT as a bind mount, -# and the mode is why: the host file is `root:0600` and this unit runs as the -# unprivileged agent user. nspawn's `--load-credential` (written by -# `hive_c0re::lifecycle::host_config`) is read by the container manager as -# root and re-exposed under this unit's own `User=`; a bind would deliver a -# file the harness cannot open. -{ - lib, - config, - ... -}: -let - cfg = config.hyperhive.queue; - configured = cfg.natsUrl != null && cfg.tokenEndpoint != null; - - # The two ids `hive_c0re::lifecycle::host_config` forwards under. Neither - # side can discover the other's spelling, so a rename is a rename there too. - secretCredential = "hive-queue-agent-secret"; - clientIdCredential = "hive-queue-agent-client-id"; -in -{ - options.hyperhive.queue = { - natsUrl = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "nats://10.42.0.1:4222"; - description = '' - Where the swarm queue listens, as this container reaches it. - - Set by the generated meta flake from the host's - {option}`services.hyperhive.deploy.hive-controller.queue.agentNatsUrl`, - which is the bridge address rather than a loopback one — inside this - container `127.0.0.1` is the agent itself. - - `null` means this hive has not been given the queue's address for its - agents, and the harness then declares no credential and logs that it - has none. It is deliberately not defaulted to anything: a guessed - address builds fine and talks to the wrong machine. - ''; - }; - - tokenEndpoint = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "https://auth.example.com/api/oidc/token"; - description = '' - The swarm IdP's OAuth2 token endpoint. The harness mints a - `client_credentials` token there and presents it to the queue, which - authenticates it as the client named in the delivered credential. - - Set together with {option}`hyperhive.queue.natsUrl` or not at all — - the harness treats a half-set pair as a deployment bug rather than as - "no queue coordinates". - ''; - }; - }; - - config = lib.mkIf configured { - systemd.services.hive-agent = { - # Bare ids, no paths: this is the terse `LoadCredential=` form that - # inherits a credential the service *manager* received, which is what - # the container manager passed in. `man systemd.exec` also makes that - # form non-fatal when the credential is absent, which is exactly the - # behaviour a hive whose publisher has not run yet needs — the unit - # starts, finds no id, and says so. - serviceConfig.LoadCredential = [ - secretCredential - clientIdCredential - ]; - environment = { - # `%d` is `$CREDENTIALS_DIRECTORY`, per-unit and owned by `User=`. - # Same shape hive-c0re's own queue client is handed its secret in - # (`nix/host-modules/hive-c0re/environment.nix`) — the harness reads - # a path and never a value. - HIVE_AGENT_OIDC_CLIENT_SECRET_FILE = "%d/${secretCredential}"; - # The id is public (it is sent to the token endpoint on every - # connection) but still arrives as a path, because it arrives *with* - # the secret. `QueueConfig::from_env` wants it as a value, so the - # harness reads this file itself — see `hive-agent`'s `swarm_queue`. - HIVE_AGENT_OIDC_CLIENT_ID_FILE = "%d/${clientIdCredential}"; - }; - # No `HIVE_AGENT_OIDC_CA_FILE`. The hive's own client needs one because - # the host does not trust the swarm's CA, but an agent does: the meta - # flake embeds the hive CA and the swarm root it is issued under into - # this container's `security.pki.certificateFiles` at build time, and - # reqwest's rustls backend verifies against the system store. A path - # here would need the bundle delivered as a third credential to say - # nothing new. - }; - }; -} diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 254fa978..8ff76562 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -275,33 +275,6 @@ in # ./default.nix. The daemon reads a path, never a value. HIVE_C0RE_OIDC_CLIENT_SECRET_FILE = "%d/swarm-status-client.secret"; } -// { - # Where ../glue-queue-agent-credential.nix lands the AGENTS' queue - # credential. Read by `hive_c0re::lifecycle::host_config`, which stats the - # two files and forwards them into each container as systemd credentials. - # Never read for its contents here: the secret is `0600` root-owned and - # this daemon is `hive-core`, which is exactly why the transport is a - # credential rather than a bind mount. - HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR = toString config.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; -} -// - # The agents' half of the same queue, forwarded by `hive_c0re::meta` into - # every container. A pair, gated together, because an agent that got one of - # them would report a half-configured queue instead of none. - # - # ⚠️ The url is the bridge address, NOT the loopback one beside it in - # `HIVE_C0RE_NATS_URL` above. Both are correct for their reader: this daemon - # shares the host netns, an agent does not, and inside a container - # `127.0.0.1` is the agent itself. - lib.optionalAttrs - ( - config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl != null - && config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null - ) - { - HIVE_AGENT_NATS_URL = config.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl; - HIVE_AGENT_OIDC_TOKEN_ENDPOINT = config.services.hyperhive.swarm.statusPublish.tokenEndpoint; - } // # Where the swarm's secret store is, and the identity this hive presents to # it (hive-c0re::workers::credential). `swarm_secret_client` reads these diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 123c2748..0267d1c6 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -587,36 +587,4 @@ in }; }; - # The same queue, reached from one layer further in. An agent container has - # its own network namespace, so it needs an address of its own rather than - # the one beside it in `statusPublish.natsUrl` — sharing that option would - # hand every agent a loopback address that resolves to the agent. - # - # Only the address lives here. The credential does not: it is published per - # hive and read out of the store by ./glue-queue-agent-credential.nix, which - # owns `queue.agentCredentialDir` in the same namespace. - options.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = - if queueLocal then "nats://${cfg.network.bridgeIp}:${toString swarmCfg.nats.port}" else null; - defaultText = lib.literalExpression ''"nats://''${network.bridgeIp}:''${swarm.nats.port}" when this host runs the queue and the IdP, else null''; - example = "nats://10.100.0.1:4222"; - description = '' - Where the swarm queue listens, as an agent *container* on this host - reaches it. - - Defaults to the bridge address when this host runs the queue, because - that is the only address it is reachable at from a container: - {option}`services.hyperhive.swarm.nats.port` is opened on the bridge - interface alone. ⚠️ Never a loopback address — inside an agent's network - namespace `127.0.0.1` is the agent, not this host. - - Null means this hive's agents have not been given the queue's address. - Together with - {option}`services.hyperhive.swarm.statusPublish.tokenEndpoint` it is - what decides whether the harness is handed queue coordinates at all; a - hive whose queue is elsewhere names the address its containers route to. - ''; - }; - } diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 4cc04261..a35580c0 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -464,16 +464,6 @@ let otel.protocol = "grpc"; }; agentNoOtel = agent { }; - - # The agent side of the swarm queue. Both coordinates set is the only state - # in which the harness unit declares a credential at all, so the pair and - # the empty fixture beside it are the two arms worth having. - agentQueue = agent { - queue.natsUrl = "nats://10.42.0.1:4222"; - queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; - }; - agentNoQueue = agent { }; - agentHarness = machine: machine.systemd.services.hive-agent; agentSettings = machine: machine.services.opentelemetry-collector.settings; # This hive's own collector, which is a HOST service — unlike the swarm @@ -1252,90 +1242,6 @@ let && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ])) && !(builtins.elem "hive-c0re.service" (u.requires or [ ])); } - { - # Where the reader puts the files and where the daemon looks for them is - # one agreement spanning two modules. Asserted against the option rather - # than the literal so moving the directory moves both ends. - name = "hive-c0re is told where the agents' queue credential lands"; - ok = - allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR - == toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; - } - { - # The one address in this file that must NOT be loopback. Both spellings - # sit in the same unit's environment and are correct for their own - # reader: hive-c0re shares the host netns, an agent container does not, - # so a copy-paste between them reaches the agent itself and the symptom - # is a connect that hangs. - name = "the agents' queue address is the bridge, not the loopback one the hive itself uses"; - ok = - let - e = allLocal.systemd.services.hive-c0re.environment; - in - e.HIVE_AGENT_NATS_URL == "nats://${allLocal.services.hyperhive.network.bridgeIp}:4222" - && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) - && e.HIVE_AGENT_NATS_URL != e.HIVE_C0RE_NATS_URL; - } - { - # The agents mint against the swarm's IdP, the same endpoint the hive's - # own client uses — a hive-local guess would produce a token the queue - # would not accept. - name = "the agents' token endpoint is the swarm IdP's"; - ok = - let - e = allLocal.systemd.services.hive-c0re.environment; - in - lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT - && e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT; - } - { - # The absence arm, and what makes the two above able to fail: a hive - # with no queue address must forward neither coordinate, because half a - # pair reaches the harness as a partial configuration rather than as - # none. - name = "a hive with no swarm queue forwards no agent queue coordinates"; - ok = - let - e = bare.systemd.services.hive-c0re.environment; - in - !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); - } - { - # Both ids or neither: the secret authenticates nobody without the id it - # belongs to, and the harness refuses to treat one of the two as a queue. - name = "an agent with queue coordinates imports both halves of its credential"; - ok = - let - c = (agentHarness agentQueue).serviceConfig.LoadCredential; - in - builtins.elem "hive-queue-agent-secret" c && builtins.elem "hive-queue-agent-client-id" c; - } - { - # `%d` and not a path under the state dir: the host file is `0600` - # root-owned, so the only copy this unprivileged unit can open is the - # one systemd puts in its own credentials directory. - name = "the harness reads its queue credential out of the credentials directory"; - ok = - let - e = (agentHarness agentQueue).environment; - in - e.HIVE_AGENT_OIDC_CLIENT_SECRET_FILE == "%d/hive-queue-agent-secret" - && e.HIVE_AGENT_OIDC_CLIENT_ID_FILE == "%d/hive-queue-agent-client-id"; - } - { - # An agent built before its hive was handed the queue's address. It - # must declare nothing rather than name a credential that never - # arrives — and the harness then reports "no queue coordinates" - # instead of a half-set environment. - name = "an agent with no queue coordinates declares no credential"; - ok = - let - u = agentHarness agentNoQueue; - in - !(u.serviceConfig ? LoadCredential) - && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) - && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); - } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts,