swarm: carry the agent queue credential from the host into the harness
hive-c0re stats the two files `swarm-bao-queue-agent` lands and forwards them into every agent container as systemd credentials, and the harness resolves a `QueueConfig` out of them at boot. Nothing connects yet. A credential and not a bind mount, and the mode is what forces it: the secret is root:0600 and the harness runs as the unprivileged agent user, so a bind would deliver a file that user cannot open. nspawn's `--load-credential` is read by the container manager as root and re-exposed under the consuming unit's own `User=`. hive-c0re never reads the bytes either way, which is just as well — it runs as `hive-core`. Absent files stay legal and become visible rather than silent: the publisher lives on the authelia host and mints on its first boot, so "nothing at that path" is the ordinary early state of a swarm. c0re forwards nothing and logs why; the harness logs that it has no queue. The client id comes out of the delivered file rather than being rebuilt from `hiveName` in nix, which is the agreement `swarm-secret-client` states. `QueueConfig::from_env` wants it as a value, so the harness reads the file itself — assigning the variable instead would need `std::env::set_var` in a process that has already spawned threads. Refs #3805
This commit is contained in:
parent
f8dd737456
commit
353cdd9264
6 changed files with 436 additions and 7 deletions
|
|
@ -28,6 +28,7 @@ mod serve_common;
|
|||
mod state_entry_watch;
|
||||
mod stats;
|
||||
mod stream_enrich;
|
||||
mod swarm_queue;
|
||||
mod term_msg;
|
||||
mod todo_server;
|
||||
mod todos;
|
||||
|
|
@ -495,6 +496,9 @@ async fn serve_main<S: Surface>(socket: &Path, poll_ms: u64) -> Result<()> {
|
|||
let claude_dir = login::default_dir();
|
||||
let initial = LoginState::from_dir(&claude_dir);
|
||||
tracing::info!(state = ?initial, claude_dir = %claude_dir.display(), "harness boot");
|
||||
// Resolve the swarm-queue credential while the boot log is still the
|
||||
// thing an operator is reading. Nothing connects yet.
|
||||
swarm_queue::init();
|
||||
// Config fact, stamped once — see `harness_state::write_api_key_mode`'s
|
||||
// doc for why hive-c0re needs this to stop reporting `needs_login` for
|
||||
// an agent whose `~/.claude/` is empty by design.
|
||||
|
|
|
|||
262
hive-agent/src/swarm_queue.rs
Normal file
262
hive-agent/src/swarm_queue.rs
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
//! This agent's swarm-queue credentials, resolved once at boot.
|
||||
//!
|
||||
//! Three of the four coordinates arrive as environment variables the meta
|
||||
//! flake renders into the harness unit. The fourth — the OIDC client id —
|
||||
//! arrives as a *file*, delivered beside the secret as one systemd
|
||||
//! credential pair (`nix/agent-modules/queue.nix`). That is the whole reason
|
||||
//! this module exists rather than a bare [`QueueConfig::from_env`] call: the
|
||||
//! id rides with the secret so a reader never has to spell `hive-<name>-agent`
|
||||
//! a second time, and `from_env` wants it as a value.
|
||||
//!
|
||||
//! Reading the file and assigning the variable would put the same rule back
|
||||
//! in `from_env`'s hands, but `std::env::set_var` is unsound in a process that
|
||||
//! has already spawned threads, and this one has. So the rule is restated
|
||||
//! here over the inputs this consumer actually has, and [`decide`] is the
|
||||
//! single place it lives.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use swarm_queue_client::QueueConfig;
|
||||
|
||||
/// Variable prefix for this agent's coordinates. Distinct from `HIVE_C0RE`'s
|
||||
/// on purpose: an agent authenticates as its own client, not as its hive.
|
||||
const ENV_PREFIX: &str = "HIVE_AGENT";
|
||||
|
||||
/// Resolved once at boot, so the publisher that connects to the queue reads
|
||||
/// one answer rather than re-deriving it per call.
|
||||
static CONFIG: OnceLock<Option<QueueConfig>> = OnceLock::new();
|
||||
|
||||
/// The four variables the harness unit sets, before the client-id file is
|
||||
/// read. Collected into a struct so [`decide`] is pure over them and the
|
||||
/// process env is touched in exactly one place.
|
||||
struct QueueEnv {
|
||||
nats_url: Option<String>,
|
||||
token_endpoint: Option<String>,
|
||||
client_id_file: Option<String>,
|
||||
client_secret_file: Option<String>,
|
||||
/// Independent of the all-or-none group below, exactly as in
|
||||
/// `QueueConfig::from_env`: a swarm behind a publicly-trusted
|
||||
/// certificate needs no extra anchor. Nothing in this tree sets it for
|
||||
/// an agent — a container already trusts the swarm root, which
|
||||
/// `hive_c0re::meta` embeds at build time — so it is here for a
|
||||
/// deployment that needs a different one, not for ours.
|
||||
ca_file: Option<String>,
|
||||
}
|
||||
|
||||
impl QueueEnv {
|
||||
fn from_env() -> Self {
|
||||
let var = |suffix: &str| std::env::var(format!("{ENV_PREFIX}_{suffix}")).ok();
|
||||
Self {
|
||||
nats_url: var("NATS_URL"),
|
||||
token_endpoint: var("OIDC_TOKEN_ENDPOINT"),
|
||||
client_id_file: var("OIDC_CLIENT_ID_FILE"),
|
||||
client_secret_file: var("OIDC_CLIENT_SECRET_FILE"),
|
||||
ca_file: var("OIDC_CA_FILE"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What the environment plus the client-id file add up to.
|
||||
enum Resolution {
|
||||
/// Everything is here; the agent can reach the queue.
|
||||
Configured(Box<QueueConfig>),
|
||||
/// No queue for this agent, and that is a legal state — carries why.
|
||||
Absent(&'static str),
|
||||
/// Some of the environment, not all of it. A deployment bug rather than
|
||||
/// an absent integration, so it is reported and then survived.
|
||||
Partial,
|
||||
}
|
||||
|
||||
/// Read the client id out of the file the credential landed at.
|
||||
///
|
||||
/// `None` for a missing or empty file, which is the ordinary state of a hive
|
||||
/// whose secret store has nothing published yet — the credential simply is
|
||||
/// not there, and nspawn forwards nothing. Trailing newline stripped: the
|
||||
/// reader unit writes one and an id with a newline in it authenticates as
|
||||
/// nobody.
|
||||
fn read_client_id(path: &Path) -> Option<String> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(raw) => {
|
||||
let id = raw.trim();
|
||||
(!id.is_empty()).then(|| id.to_owned())
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %path.display(), error = %e, "reading the queue client id failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide what this agent's queue configuration is, given the environment and
|
||||
/// whatever the client-id file held.
|
||||
///
|
||||
/// All four variables or none, for the reason `QueueConfig::from_env` gives:
|
||||
/// a half-set environment produces a process that comes up fine and never
|
||||
/// connects. The client id is graded separately from the four because its
|
||||
/// file has a legal absence the variables do not — the unit names the path a
|
||||
/// credential *would* arrive at whether or not one has been published yet.
|
||||
fn decide(env: &QueueEnv, client_id: Option<String>) -> Resolution {
|
||||
match (
|
||||
env.nats_url.as_ref(),
|
||||
env.token_endpoint.as_ref(),
|
||||
env.client_id_file.as_ref(),
|
||||
env.client_secret_file.as_ref(),
|
||||
) {
|
||||
(None, None, None, None) => Resolution::Absent("this hive has no swarm queue configured"),
|
||||
(Some(url), Some(token_endpoint), Some(_), Some(secret)) => match client_id {
|
||||
Some(client_id) => Resolution::Configured(Box::new(QueueConfig {
|
||||
url: url.clone(),
|
||||
token_endpoint: token_endpoint.clone(),
|
||||
client_id,
|
||||
client_secret_file: secret.into(),
|
||||
ca_file: env.ca_file.as_ref().map(Into::into),
|
||||
})),
|
||||
None => {
|
||||
Resolution::Absent("the queue credential has not been published to this hive yet")
|
||||
}
|
||||
},
|
||||
_ => Resolution::Partial,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve this agent's queue configuration and record it for later use.
|
||||
///
|
||||
/// Never fails: a harness that cannot reach the queue still serves its
|
||||
/// operator, its web UI and its turn loop, so every outcome here is a log
|
||||
/// line and not an exit.
|
||||
pub fn init() {
|
||||
let env = QueueEnv::from_env();
|
||||
let client_id = env
|
||||
.client_id_file
|
||||
.as_deref()
|
||||
.map(Path::new)
|
||||
.and_then(read_client_id);
|
||||
let resolved = match decide(&env, client_id) {
|
||||
Resolution::Configured(cfg) => {
|
||||
// url + client id only. The secret is a path in this struct and
|
||||
// stays one: neither it nor its contents belong in a log.
|
||||
tracing::info!(url = %cfg.url, client_id = %cfg.client_id, "swarm queue configured");
|
||||
Some(*cfg)
|
||||
}
|
||||
Resolution::Absent(why) => {
|
||||
tracing::info!(why, "no swarm queue for this agent");
|
||||
None
|
||||
}
|
||||
Resolution::Partial => {
|
||||
tracing::error!(
|
||||
prefix = ENV_PREFIX,
|
||||
"swarm queue half-configured: {ENV_PREFIX}_NATS_URL, \
|
||||
{ENV_PREFIX}_OIDC_TOKEN_ENDPOINT, {ENV_PREFIX}_OIDC_CLIENT_ID_FILE and \
|
||||
{ENV_PREFIX}_OIDC_CLIENT_SECRET_FILE are set together or not at all — \
|
||||
this agent will not connect"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let _ = CONFIG.set(resolved);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{QueueEnv, Resolution, decide, read_client_id};
|
||||
|
||||
fn env(parts: [Option<&str>; 4]) -> QueueEnv {
|
||||
let [nats_url, token_endpoint, client_id_file, client_secret_file] = parts;
|
||||
QueueEnv {
|
||||
nats_url: nats_url.map(str::to_owned),
|
||||
token_endpoint: token_endpoint.map(str::to_owned),
|
||||
client_id_file: client_id_file.map(str::to_owned),
|
||||
client_secret_file: client_secret_file.map(str::to_owned),
|
||||
ca_file: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn full() -> QueueEnv {
|
||||
env([
|
||||
Some("nats://10.42.0.1:4222"),
|
||||
Some("https://auth.t.local/api/oidc/token"),
|
||||
Some("/run/credentials/hive-agent.service/hive-queue-agent-client-id"),
|
||||
Some("/run/credentials/hive-agent.service/hive-queue-agent-secret"),
|
||||
])
|
||||
}
|
||||
|
||||
/// The reader unit writes the id with a trailing newline; an id carrying
|
||||
/// one is a client authelia has never heard of.
|
||||
#[test]
|
||||
fn a_client_id_file_is_read_without_its_newline() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("client_id");
|
||||
std::fs::write(&path, "hive-h1-agent\n").expect("write");
|
||||
assert_eq!(read_client_id(&path).as_deref(), Some("hive-h1-agent"));
|
||||
}
|
||||
|
||||
/// Both shapes of "no credential here": never delivered, or delivered
|
||||
/// empty. Neither is an id, and treating an empty string as one would
|
||||
/// authenticate as the anonymous client rather than failing.
|
||||
#[test]
|
||||
fn a_missing_or_empty_client_id_file_reads_as_no_id() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let missing = dir.path().join("client_id");
|
||||
assert_eq!(read_client_id(&missing), None);
|
||||
std::fs::write(&missing, "\n").expect("write");
|
||||
assert_eq!(read_client_id(&missing), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_complete_environment_with_a_published_id_configures_the_queue() {
|
||||
let e = full();
|
||||
let Resolution::Configured(cfg) = decide(&e, Some("hive-h1-agent".to_owned())) else {
|
||||
panic!("expected a configured queue");
|
||||
};
|
||||
assert_eq!(cfg.url, "nats://10.42.0.1:4222");
|
||||
assert_eq!(cfg.client_id, "hive-h1-agent");
|
||||
assert!(
|
||||
cfg.client_secret_file.ends_with("hive-queue-agent-secret"),
|
||||
"the secret stays a path: {}",
|
||||
cfg.client_secret_file.display()
|
||||
);
|
||||
}
|
||||
|
||||
/// A hive with no swarm queue at all. Silence here is correct, and it has
|
||||
/// to be distinguishable from the half-set case below — that distinction
|
||||
/// is the only thing that makes the error branch worth logging.
|
||||
#[test]
|
||||
fn an_empty_environment_is_no_queue_rather_than_an_error() {
|
||||
let e = env([None, None, None, None]);
|
||||
assert!(matches!(decide(&e, None), Resolution::Absent(_)));
|
||||
}
|
||||
|
||||
/// The credential's own absence. The unit names the path unconditionally
|
||||
/// once the hive has a queue, so this is the state of every agent on a
|
||||
/// swarm whose publisher has not run — legal, and not the error branch.
|
||||
#[test]
|
||||
fn a_complete_environment_with_no_published_id_is_no_queue() {
|
||||
let e = full();
|
||||
assert!(matches!(decide(&e, None), Resolution::Absent(_)));
|
||||
}
|
||||
|
||||
/// Each single-variable omission, because the failure a partial set
|
||||
/// produces is a harness that looks healthy and publishes nothing.
|
||||
#[test]
|
||||
fn any_missing_variable_is_a_partial_configuration() {
|
||||
for drop in 0..4 {
|
||||
let mut parts = [
|
||||
Some("nats://10.42.0.1:4222"),
|
||||
Some("https://auth.t.local/api/oidc/token"),
|
||||
Some("/run/credentials/hive-agent.service/hive-queue-agent-client-id"),
|
||||
Some("/run/credentials/hive-agent.service/hive-queue-agent-secret"),
|
||||
];
|
||||
parts[drop] = None;
|
||||
let e = env(parts);
|
||||
assert!(
|
||||
matches!(
|
||||
decide(&e, Some("hive-h1-agent".to_owned())),
|
||||
Resolution::Partial
|
||||
),
|
||||
"dropping variable {drop} must report a partial configuration"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue