diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index bb4af221..2ebf02c9 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -33,6 +33,8 @@ use utoipa::ToSchema; use crate::webhook::DeliveryKind; +pub mod agent_token; + /// An agent's open config-PR, as [`Client::list_open_config_prs`] reports it /// and `GET /api/agents/{name}/config-pr` serves it. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] @@ -135,6 +137,10 @@ const TOKEN_FILE_ENV: &str = "SWARM_CONTROLLER_FORGE_TOKEN_FILE"; /// Typed forgejo client, built once at startup from env. pub struct Client { api: Forgejo, + /// The forge's base URL, kept so a client authenticated as someone + /// else can be built against the same forge (see + /// [`agent_token`]'s read-back). + url: url::Url, } impl Client { @@ -168,8 +174,12 @@ impl Client { } let parsed_url = url::Url::parse(&url).with_context(|| format!("parse {URL_ENV} ({url}) as a URL"))?; - let api = Forgejo::new(Auth::Token(token), parsed_url).context("build forgejo client")?; - Ok(Some(Self { api })) + let api = + Forgejo::new(Auth::Token(token), parsed_url.clone()).context("build forgejo client")?; + Ok(Some(Self { + api, + url: parsed_url, + })) } /// Creation options for an empty repo defaulting to `main`. diff --git a/swarm-controller/src/forge/agent_token.rs b/swarm-controller/src/forge/agent_token.rs new file mode 100644 index 00000000..ee8faec9 --- /dev/null +++ b/swarm-controller/src/forge/agent_token.rs @@ -0,0 +1,539 @@ +//! Each agent's own forge access token: minted here with the admin API, stored +//! at `swarm/agents//forge-token`, and pulled from there by the agent +//! container itself (`nix/agent-modules/forge-token.nix`). +//! +//! One token per agent, under the fixed name [`AGENT_TOKEN_NAME`]. Forgejo +//! refuses a second token with a name the user already has, so a rotation is a +//! delete then a create, and there is never more than one of these per agent. +//! The `hyperhive-` tokens `hive-c0re` used to mint are never +//! touched here: the name cannot match them. +//! +//! Two callers insert the same `MintAgentForgeToken` job node: agent creation, +//! and [`spawn`]'s pass at start and every [`RECONCILE_INTERVAL`] over every +//! agent that holds a store identity. The decision is pure ([`classify`] and +//! [`plan`]), so the tests pin it; the IO on either side only reads or acts. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use forgejo_api::structs::{AccessToken, CreateAccessTokenOption}; +use forgejo_api::{ApiErrorKind, Auth, Forgejo, ForgejoError}; +use reqwest::StatusCode; +use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, forge, policy}; + +use super::Client; + +/// The forge's name for every agent token this module mints. +/// +/// Deliberately not `hyperhive-…`: that prefix is what `hive-c0re` named each +/// of its re-mints, and a later sweep of those must never match this one. +pub const AGENT_TOKEN_NAME: &str = "swarm-agent"; + +/// The scopes an agent token carries. Byte-identical to `hive-c0re`'s +/// `forge::users::TOKEN_SCOPES`, so the move from hive to swarm changes +/// nothing an agent can do. Narrowing it is a separate decision. +/// +/// ⚠️ Duplicated across the crate boundary until the last `hive-c0re` caller +/// of that constant goes. A test here pins the literal. +pub const AGENT_TOKEN_SCOPES: &str = "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; + +/// How often [`spawn`] re-checks every agent's token. +const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); + +/// Why a token has to be (re)minted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MintReason { + /// The store holds nothing for this agent. + NotStored, + /// The forge lists no [`AGENT_TOKEN_NAME`] token for this agent. + NotOnForge, + /// The forge's token is not the one the store holds. + LastEightMismatch, + /// The forge's token carries other scopes than [`AGENT_TOKEN_SCOPES`]. + ScopeMismatch, +} + +/// What to do about one agent's token. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Decision { + /// The stored token is the forge's token, with the right scopes. + Keep, + /// Delete any [`AGENT_TOKEN_NAME`] token, create a new one, store it. + Mint(MintReason), + /// The forge lists more than one [`AGENT_TOKEN_NAME`] token. Forgejo + /// refuses a duplicate name, so this is a forge in a state nothing here + /// can reason about; a delete by name would 422 on it anyway. + Conflict, +} + +/// A scope list reduced to what Forgejo would store for it. +/// +/// Forgejo normalises a new token's scopes before storing them +/// (`models/auth/access_token_scope.go`, `Normalize` → `toScope`): a `read:X` +/// beside `write:X` is dropped, because the write bit implies the read one. +/// The admin list then returns the stored form. Comparing +/// [`AGENT_TOKEN_SCOPES`] to that list as written would never match, and every +/// pass would rotate every agent's token. +fn normalized_scopes<'a>(scopes: impl IntoIterator) -> BTreeSet { + let all: BTreeSet<&str> = scopes.into_iter().map(str::trim).collect(); + all.iter() + .filter(|s| { + s.strip_prefix("read:") + .is_none_or(|area| !all.contains(format!("write:{area}").as_str())) + }) + .map(|s| (*s).to_owned()) + .collect() +} + +/// The last eight characters of `token`, as Forgejo reports them for a token +/// it holds (`token_last_eight`). `None` for a token shorter than that, which +/// no Forgejo token is. +fn last_eight(token: &str) -> Option<&str> { + token.len().checked_sub(8).and_then(|at| token.get(at..)) +} + +/// Decide what to do about an agent's token, from what the store holds and +/// what the forge lists for the agent. +/// +/// Every token not named [`AGENT_TOKEN_NAME`] is ignored, so the +/// `hyperhive-*` tokens `hive-c0re` left behind never affect the decision. +pub fn classify(stored: Option<&forge::Credential>, listed: &[AccessToken]) -> Decision { + let ours: Vec<&AccessToken> = listed + .iter() + .filter(|t| t.name.as_deref() == Some(AGENT_TOKEN_NAME)) + .collect(); + let token = match ours[..] { + [] => return Decision::Mint(MintReason::NotOnForge), + [token] => token, + _ => return Decision::Conflict, + }; + let Some(stored) = stored else { + return Decision::Mint(MintReason::NotStored); + }; + if token.token_last_eight.as_deref() != last_eight(&stored.value) { + return Decision::Mint(MintReason::LastEightMismatch); + } + let listed_scopes = normalized_scopes(token.scopes.iter().flatten().map(String::as_str)); + if listed_scopes != normalized_scopes(AGENT_TOKEN_SCOPES.split(',')) { + return Decision::Mint(MintReason::ScopeMismatch); + } + Decision::Keep +} + +/// What one pass found for one agent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Observed { + /// Both reads worked, and this is what [`classify`] made of them. + Decided(Decision), + /// The forge has no user by this agent's name. Creating one is agent + /// creation's job, not this module's. + NoForgeUser, + /// A read failed. Nothing is known, so nothing is done. + Unknown, +} + +/// The agents a pass mints for. +/// +/// Only [`Decision::Mint`]. [`Observed::Unknown`] in particular never mints: a +/// forge or store outage must not turn into a rotation of every agent's token. +pub fn plan(observed: &[(String, Observed)]) -> Vec { + observed + .iter() + .filter(|(_, o)| matches!(o, Observed::Decided(Decision::Mint(_)))) + .map(|(agent, _)| agent.clone()) + .collect() +} + +/// Whether a forge error is a 404, whichever of its two shapes it came in. +fn is_not_found(e: &ForgejoError) -> bool { + match e { + ForgejoError::ApiError(api) => { + matches!(api.error_kind(), ApiErrorKind::NotFound { .. }) + || matches!(api.error_kind(), ApiErrorKind::Other(s) if *s == StatusCode::NOT_FOUND) + } + ForgejoError::UnexpectedStatusCode(s) => *s == StatusCode::NOT_FOUND, + _ => false, + } +} + +impl Client { + /// Every access token the forge lists for `agent`, or `None` when the + /// forge has no such user. + async fn list_agent_tokens(&self, agent: &str) -> Result>> { + match self.api.admin_list_user_access_tokens(agent).all().await { + Ok(tokens) => Ok(Some(tokens)), + Err(e) if is_not_found(&e) => Ok(None), + Err(e) => Err(e).with_context(|| format!("listing {agent}'s forge tokens")), + } + } + + /// Make sure `agent`'s stored token is a live [`AGENT_TOKEN_NAME`] token + /// with [`AGENT_TOKEN_SCOPES`], and mint one if it is not. The whole job + /// of the `MintAgentForgeToken` node. + /// + /// A rotation deletes the old token first, then creates, stores, and uses + /// the new one: it logs in with it and asks the forge who it is, so the + /// node does not report success on a write nobody has read. A crash + /// between any two steps leaves a state the next run classifies as + /// [`Decision::Mint`] again. + /// + /// # Errors + /// When the store or the forge refuses a step, when the agent has no forge + /// user, or when the new token authenticates as someone else. + pub async fn ensure_agent_forge_token(&self, agent: &str) -> Result<()> { + let path = forge::agent_token_path(agent)?; + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + let stored: Option = store + .read_optional(&path) + .await + .with_context(|| format!("reading {path}"))?; + let Some(listed) = self.list_agent_tokens(agent).await? else { + bail!("the forge has no user {agent:?}, so there is nothing to mint a token for"); + }; + let reason = match classify(stored.as_ref(), &listed) { + Decision::Keep => { + tracing::debug!(agent, "agent forge token is current; left as it is"); + return Ok(()); + } + Decision::Conflict => bail!( + "the forge lists more than one {AGENT_TOKEN_NAME:?} token for {agent:?}; \ + delete them by id and re-run" + ), + Decision::Mint(reason) => reason, + }; + + if listed + .iter() + .any(|t| t.name.as_deref() == Some(AGENT_TOKEN_NAME)) + { + match self + .api + .admin_delete_user_access_token(agent, AGENT_TOKEN_NAME) + .await + { + Ok(()) => {} + Err(e) if is_not_found(&e) => {} + Err(e) => { + return Err(e) + .with_context(|| format!("deleting {agent}'s {AGENT_TOKEN_NAME} token")); + } + } + } + let created = self + .api + .admin_create_user_access_token( + agent, + CreateAccessTokenOption { + name: AGENT_TOKEN_NAME.to_owned(), + repositories: None, + scopes: Some(AGENT_TOKEN_SCOPES.split(',').map(str::to_owned).collect()), + }, + ) + .await + .with_context(|| format!("creating {agent}'s {AGENT_TOKEN_NAME} token"))?; + let Some(value) = created.sha1.filter(|v| !v.is_empty()) else { + bail!("the forge created {agent}'s token but returned no value for it"); + }; + let credential = forge::Credential { + value, + name: AGENT_TOKEN_NAME.to_owned(), + }; + store + .write(&path, &credential) + .await + .with_context(|| format!("storing {agent}'s forge token at {path}"))?; + + let as_agent = Forgejo::new(Auth::Token(&credential.value), self.url.clone()) + .context("building a forge client with the new token")?; + let me = as_agent + .user_get_current() + .await + .with_context(|| format!("using {agent}'s new forge token"))?; + if me.login.as_deref() != Some(agent) { + bail!("{agent}'s new forge token authenticates as someone else"); + } + tracing::info!(agent, ?reason, %path, "agent forge token minted and stored"); + Ok(()) + } + + /// What the forge and the store say about `agent`'s token. + async fn observe_agent( + &self, + store: &swarm_secret_client::SecretStore, + agent: &str, + ) -> Observed { + let stored = match forge::agent_token_path(agent) { + Ok(path) => store.read_optional::(&path).await, + Err(e) => Err(e), + }; + let stored = match stored { + Ok(stored) => stored, + Err(e) => { + tracing::warn!(agent, error = %e, "agent forge token: store read failed"); + return Observed::Unknown; + } + }; + match self.list_agent_tokens(agent).await { + Ok(Some(listed)) => Observed::Decided(classify(stored.as_ref(), &listed)), + Ok(None) => Observed::NoForgeUser, + Err(e) => { + tracing::warn!(agent, error = %format!("{e:#}"), "agent forge token: forge read failed"); + Observed::Unknown + } + } + } + + /// One pass: every agent holding a store identity, observed. + async fn observe_all(&self) -> Result> { + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + let roles = store + .list_cert_roles(DEFAULT_CERT_MOUNT) + .await + .context("listing the store's cert-auth roles")?; + let mut observed = Vec::new(); + for agent in policy::agents_from_role_names(&roles) { + let o = self.observe_agent(&store, &agent).await; + if o == Observed::NoForgeUser { + tracing::debug!(agent, "agent forge token: no forge user; skipped"); + } + observed.push((agent, o)); + } + Ok(observed) + } +} + +/// Check every agent's token now and every [`RECONCILE_INTERVAL`] after, and +/// hand the agents that need one to `enqueue`, which inserts a +/// `MintAgentForgeToken` node for each. +/// +/// The roster is the store's `hive-agent-*` cert-auth roles: exactly the +/// agents that can read what the node writes. The first tick fires +/// immediately (`tokio::time::interval`'s default). A pass that fails is +/// logged and retried on the next tick; it never stops the daemon. +pub fn spawn(client: Arc, enqueue: impl Fn(Vec) + Send + 'static) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + loop { + ticker.tick().await; + match client.observe_all().await { + Ok(observed) => { + let agents = plan(&observed); + if agents.is_empty() { + tracing::debug!( + checked = observed.len(), + "agent forge tokens: all current" + ); + } else { + tracing::info!( + checked = observed.len(), + minting = agents.len(), + "agent forge tokens: queueing mints" + ); + enqueue(agents); + } + } + Err(e) => tracing::warn!( + error = %format!("{e:#}"), + retry_in_s = RECONCILE_INTERVAL.as_secs(), + "agent forge tokens: pass failed; retrying next tick" + ), + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + const VALUE: &str = "0123456789abcdef0123456789abcdef01234567"; + + fn stored() -> forge::Credential { + forge::Credential { + value: VALUE.to_owned(), + name: AGENT_TOKEN_NAME.to_owned(), + } + } + + fn token(name: &str, last_eight: &str, scopes: &[&str]) -> AccessToken { + AccessToken { + created_at: None, + id: Some(1), + name: Some(name.to_owned()), + repositories: None, + scopes: Some(scopes.iter().map(|s| (*s).to_owned()).collect()), + sha1: None, + token_last_eight: Some(last_eight.to_owned()), + } + } + + /// What Forgejo v16 stores and lists for [`AGENT_TOKEN_SCOPES`]: the + /// `toScope` order, with each `read:X` folded into its `write:X`. + const LISTED_SCOPES: [&str; 6] = [ + "write:misc", + "write:notification", + "write:organization", + "write:issue", + "write:repository", + "write:user", + ]; + + fn ours() -> AccessToken { + token(AGENT_TOKEN_NAME, "01234567", &LISTED_SCOPES) + } + + #[test] + fn the_scopes_are_hive_c0res_byte_for_byte() { + assert_eq!( + AGENT_TOKEN_SCOPES, + "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc" + ); + } + + #[test] + fn a_current_token_is_kept() { + assert_eq!(classify(Some(&stored()), &[ours()]), Decision::Keep); + } + + #[test] + fn scopes_as_written_are_kept_too() { + // A forge that listed the scopes un-normalised must not rotate either. + let t = token( + AGENT_TOKEN_NAME, + "01234567", + &AGENT_TOKEN_SCOPES.split(',').collect::>(), + ); + assert_eq!(classify(Some(&stored()), &[t]), Decision::Keep); + } + + #[test] + fn nothing_stored_mints() { + assert_eq!( + classify(None, &[ours()]), + Decision::Mint(MintReason::NotStored) + ); + } + + #[test] + fn nothing_on_the_forge_mints() { + assert_eq!( + classify(Some(&stored()), &[]), + Decision::Mint(MintReason::NotOnForge) + ); + assert_eq!(classify(None, &[]), Decision::Mint(MintReason::NotOnForge)); + } + + #[test] + fn a_different_last_eight_mints() { + let t = token(AGENT_TOKEN_NAME, "ffffffff", &LISTED_SCOPES); + assert_eq!( + classify(Some(&stored()), &[t]), + Decision::Mint(MintReason::LastEightMismatch) + ); + } + + #[test] + fn a_missing_scope_mints() { + let t = token(AGENT_TOKEN_NAME, "01234567", &LISTED_SCOPES[1..]); + assert_eq!( + classify(Some(&stored()), &[t]), + Decision::Mint(MintReason::ScopeMismatch) + ); + } + + #[test] + fn an_extra_scope_mints() { + let mut scopes = LISTED_SCOPES.to_vec(); + scopes.push("write:admin"); + let t = token(AGENT_TOKEN_NAME, "01234567", &scopes); + assert_eq!( + classify(Some(&stored()), &[t]), + Decision::Mint(MintReason::ScopeMismatch) + ); + } + + #[test] + fn scope_order_does_not_matter() { + let mut scopes = LISTED_SCOPES.to_vec(); + scopes.reverse(); + let t = token(AGENT_TOKEN_NAME, "01234567", &scopes); + assert_eq!(classify(Some(&stored()), &[t]), Decision::Keep); + } + + #[test] + fn legacy_tokens_are_ignored() { + // A pile of `hyperhive-*` tokens, one of them even matching the + // stored value's last eight, is still "no swarm token". + let legacy = [ + token("hyperhive-1700000000", "01234567", &LISTED_SCOPES), + token("hyperhive-1700000001", "aaaaaaaa", &LISTED_SCOPES), + ]; + assert_eq!( + classify(Some(&stored()), &legacy), + Decision::Mint(MintReason::NotOnForge) + ); + let mut with_ours = legacy.to_vec(); + with_ours.push(ours()); + assert_eq!(classify(Some(&stored()), &with_ours), Decision::Keep); + } + + #[test] + fn two_swarm_tokens_are_a_conflict_not_a_mint() { + assert_eq!( + classify(Some(&stored()), &[ours(), ours()]), + Decision::Conflict + ); + } + + #[test] + fn last_eight_is_the_tail() { + assert_eq!(last_eight(VALUE), Some("01234567")); + assert_eq!(last_eight("short"), None); + } + + #[test] + fn only_a_mint_decision_is_planned() { + let observed = [ + ("a".to_owned(), Observed::Decided(Decision::Keep)), + ( + "b".to_owned(), + Observed::Decided(Decision::Mint(MintReason::NotStored)), + ), + ("c".to_owned(), Observed::Unknown), + ("d".to_owned(), Observed::NoForgeUser), + ("e".to_owned(), Observed::Decided(Decision::Conflict)), + ( + "f".to_owned(), + Observed::Decided(Decision::Mint(MintReason::ScopeMismatch)), + ), + ]; + assert_eq!(plan(&observed), ["b", "f"]); + } + + #[test] + fn an_outage_plans_nothing() { + let observed = [ + ("a".to_owned(), Observed::Unknown), + ("b".to_owned(), Observed::Unknown), + ]; + assert!(plan(&observed).is_empty()); + } + + #[test] + fn both_404_shapes_are_not_found() { + assert!(is_not_found(&ForgejoError::UnexpectedStatusCode( + StatusCode::NOT_FOUND + ))); + assert!(is_not_found(&ForgejoError::ApiError( + ApiErrorKind::NotFound { errors: None }.into() + ))); + assert!(!is_not_found(&ForgejoError::UnexpectedStatusCode( + StatusCode::FORBIDDEN + ))); + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 6ebb4c40..fe785c87 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -105,6 +105,14 @@ enum SwarmNodeKind { /// its hive's shared queue credential, so the document cannot be rendered /// without knowing which hive the agent belongs to. MintAgentIdentity { hive: String, agent: String }, + /// Make sure `agent` holds a live forge access token in the swarm secret + /// store, minting one with the forge's admin API when it does not. See + /// `forge::agent_token` — including why a rotation is a delete then a + /// create. + /// + /// Carries no hive: the token's store path has no hive segment, and the + /// agent pulls it from wherever it runs. + MintAgentForgeToken { agent: String }, /// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state /// store, so a freshly created agent does not start driving turns the /// moment it's deployed — the operator has to explicitly flip it to `Up`. @@ -130,6 +138,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(), SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(), SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(), + SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(), SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(), SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), } @@ -147,7 +156,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { | SwarmNodeKind::CreateRepo { agent } | SwarmNodeKind::CreateForgeUser { agent } | SwarmNodeKind::AddRepoMember { agent } - | SwarmNodeKind::InitAgentConfigRepo { agent } => { + | SwarmNodeKind::InitAgentConfigRepo { agent } + | SwarmNodeKind::MintAgentForgeToken { agent } => { serde_json::json!({ "agent": agent }) } SwarmNodeKind::TriggerDeploy { hive, agent } @@ -308,6 +318,7 @@ async fn run_swarm_node( } } }, + SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(deps.forge, &agent).await, SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted { None => Outcome::Failed( "no swarm queue is configured on this host, so no wanted-state \ @@ -327,6 +338,27 @@ async fn run_swarm_node( (builder, outcome) } +/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under +/// `clippy::too_many_lines`. +async fn mint_forge_token( + forge: Option>, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge configured on this host (SWARM_CONTROLLER_FORGE_URL / \ + SWARM_CONTROLLER_FORGE_TOKEN_FILE unset), so no agent forge token can be minted" + .to_owned(), + ); + }; + match client.ensure_agent_forge_token(agent).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + /// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns /// out not to be new: an agent that already has a declaration (other than /// `Destroyed`, which this treats as reusable) is left alone, so a retried @@ -1470,6 +1502,14 @@ fn declare_agent_job( agent: agent.to_owned(), }) .after_ok(create_identity); + // The agent's own forge token, in the store before the hive deploys + // the container that pulls it. After the forge user, because the token + // is minted for that user. + let mint_forge_token = b + .node(SwarmNodeKind::MintAgentForgeToken { + agent: agent.to_owned(), + }) + .after_ok(create_forge_user); // Declared before the deploy trigger so the pause is visible in the // wanted-state store before the hive brings the container up — see the // node's own doc comment for why "before", not just "eventually". Needs @@ -1491,6 +1531,11 @@ fn declare_agent_job( // create agents exactly as it does today. `after_ok` there would turn an // unconfigured option into an agent nobody runs. // + // The forge-token mint gets `after_any` for the same reason: a host with + // no forge or no store must still create agents; only that node fails, + // by name, and the agent's container fetches the token on its next + // timer tick once one is minted. + // // `after_ok` on the declaration, though: an agent deployed without its // pause landing first is the race this node exists to close, so a // declaration that did not land must not be deployed past. A host with no @@ -1504,6 +1549,7 @@ fn declare_agent_job( }) .after_ok(init_config) .after_any(mint_identity) + .after_any(mint_forge_token) .after_ok(set_wanted); vec![create_identity.guid()] } @@ -1703,6 +1749,73 @@ async fn mint_agent_identity( Ok(Json(MintAgentIdentityResponse { node_id: id.get() })) } +/// Success body of `POST /api/agents/{name}/forge-token`. +#[derive(Clone, Debug, Serialize, ToSchema)] +struct MintAgentForgeTokenResponse { + /// The queued node, so a caller can follow it in the job view. + node_id: u64, +} + +/// Insert one `MintAgentForgeToken` node per agent, each its own job. +/// +/// The one place that node is queued outside agent creation: both the manual +/// route below and `forge::agent_token::spawn`'s periodic pass come through +/// here, so a backfilled mint is the same node a new agent gets. +fn queue_forge_token_mints( + sched: &Mutex>, + agents: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(agents.len()); + for agent in agents { + let queued = sched + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintAgentForgeToken { agent }).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Check an agent's forge token now, and mint one if it is missing or stale. +/// +/// The periodic pass (`forge::agent_token::spawn`) does the same every five +/// minutes for every agent with a store identity; this is for an operator who +/// does not want to wait, or for an agent the pass skipped. Queues the same +/// node agent creation does. Idempotent: a current token is left alone. +#[utoipa::path( + post, + path = "/api/agents/{name}/forge-token", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 200, description = "mint queued", body = MintAgentForgeTokenResponse), + (status = 400, description = "`name` is not a valid identifier (problem+json)", body = String), + (status = 500, description = "the job could not be queued (problem+json)", body = String), + ), + tag = "agents" +)] +async fn mint_agent_forge_token( + State(state): State, + Path(name): Path, +) -> Result, problem_details::ProblemDetails> { + let agent = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + let ids = queue_forge_token_mints(&state.jobq, vec![agent]).map_err(|e| { + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &e.to_string(), + ) + })?; + let [id] = ids[..] else { + unreachable!("exactly one handle was asked for"); + }; + Ok(Json(MintAgentForgeTokenResponse { node_id: id.get() })) +} + /// Every agent with an open config PR, in one response — the bulk /// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs /// every agent's status to render, and fetching them one at a time doesn't @@ -1990,6 +2103,14 @@ async fn main() -> Result<()> { ); } + if let Some(client) = forge_client.clone() { + let sched = Arc::clone(&jobq); + forge::agent_token::spawn(client, move |agents| { + if let Err(e) = queue_forge_token_mints(&sched, agents) { + tracing::warn!(error = %format!("{e:#}"), "agent forge tokens: queueing failed"); + } + }); + } let config_prs = forge_client.clone().map(config_pr::spawn); let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone()); @@ -2041,6 +2162,7 @@ fn build_app(state: AppState) -> axum::Router { .routes(routes!(get_config_prs)) .routes(routes!(create_agent)) .routes(routes!(mint_agent_identity)) + .routes(routes!(mint_agent_forge_token)) .routes(routes!(get_agents)) .routes(routes!(get_agents_status)) .routes(routes!(set_agent_state)) @@ -2842,6 +2964,105 @@ mod tests { ); } + /// The forge-token mint sits between the forge user and the deploy: it + /// needs the user to exist, and the deploy must not overtake it — but a + /// host with no forge or no store must still deploy the agent. + #[tokio::test] + async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = super::create_agent( + axum::extract::State(state), + axum::Json(super::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let edge = |from: hive_jobq::NodeId, to: &str| { + graph + .node(id_of(to)) + .expect("just found") + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == from => Some(*when), + _ => None, + }) + .unwrap_or_else(|| panic!("{to} waits for node {}", from.get())) + }; + + let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token"); + assert!( + !user_to_mint.accepts(hive_jobq::TerminalState::Failed), + "a token for a user that was never created cannot be minted; this \ + edge has to be `after_ok`" + ); + let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy"); + assert!( + mint_to_deploy.accepts(hive_jobq::TerminalState::Failed), + "a host with no forge or store must not cancel the deploy; this edge \ + has to be `after_any`, not `after_ok`" + ); + } + + #[test] + fn a_forge_token_node_renders_the_agent() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::MintAgentForgeToken { + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "mint_agent_forge_token"); + assert_eq!(kind.data(1)["agent"], "atlas"); + } + + /// The manual route and the periodic pass both go through + /// `queue_forge_token_mints`, so this is the assertion that a backfilled + /// mint is the same node agent creation inserts. + #[test] + fn a_queued_mint_is_one_forge_token_node_per_agent() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()]) + .expect("two single-node jobs insert"); + assert_eq!(ids.len(), 2); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut agents: Vec = guard + .graph() + .nodes() + .map(|n| { + assert_eq!(n.payload.label(), "mint_agent_forge_token"); + n.payload.data(n.id.get())["agent"] + .as_str() + .expect("agent is a string") + .to_owned() + }) + .collect(); + agents.sort(); + assert_eq!(agents, ["a", "b"]); + } + /// The socket must not share a directory with anything else, because /// the socket is `0666` and the directory is therefore the only access /// control it has. `/run/hyperhive` in particular holds hive-c0re's diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index 0a8232f3..a0df7d36 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -260,6 +260,25 @@ impl SecretStore { .await?; Ok(()) } + + /// The names of every cert-auth role under `mount`, or none when the + /// mount has no roles at all. + /// + /// **Only a 404 is "none".** The store answers a `LIST` on an empty + /// directory with a 404; a denial is a 403 and stays an error, so a caller + /// iterating the result never mistakes "not allowed to look" for "nothing + /// there". + /// + /// # Errors + /// [`Error::Vault`] for anything that is not a 404: a denial, or an + /// unreachable store. + pub async fn list_cert_roles(&self, mount: &str) -> Result, Error> { + match vaultrs::auth::cert::ca_cert_role::list(&self.inner, mount).await { + Ok(listed) => Ok(listed.keys), + Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(Vec::new()), + Err(e) => Err(e.into()), + } + } } /// Write an ACL policy, spelled out here because [`vaultrs`] does not have it: diff --git a/swarm-secret-client/src/forge.rs b/swarm-secret-client/src/forge.rs new file mode 100644 index 00000000..8a0f68ce --- /dev/null +++ b/swarm-secret-client/src/forge.rs @@ -0,0 +1,137 @@ +//! The forge agreement: where an agent's own forge token lives in the store, +//! and what the object at that path holds. +//! +//! `swarm-controller` mints the token with the forge's admin API and writes it +//! here; the agent container reads it back under its own certificate +//! (`nix/agent-modules/forge-token.nix`). Neither end is senior, so both halves +//! are stated once, here, beside [`crate::matrix`]. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +/// The path holding `agent`'s own forge access token. +/// +/// A flat leaf under the agent's prefix, like its controller-minted siblings +/// [`crate::queue::agent_queue_path`] and [`crate::mtls::identity_path`], so +/// the agent's own read stanza ([`crate::policy::render_agent_with_queue`]) +/// already covers it. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another agent's secret. +pub fn agent_token_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/forge-token")) +} + +/// What an agent's forge-token path holds. +/// +/// No `Debug` derive: `value` is a live forge credential, and a derived +/// `Debug` is one `{:?}` away from a log line. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Credential { + /// The token itself. `forge-token.nix` reads the store with + /// `bao kv get -field=value`, so this name is load-bearing for a reader + /// this crate does not control, the same as [`crate::matrix::Credential`]'s. + pub value: String, + + /// The forge's name for the token — not secret. Stored beside the value + /// so whoever rotates it can name the token it replaces. + pub name: String, +} + +impl std::fmt::Debug for Credential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credential") + .field("value", &"") + .field("name", &self.name) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_token_lands_under_the_agent_prefix() { + // Spelled out rather than rebuilt from the pieces the code uses: the + // nix reader spells the same string, and a rename here that moved it + // would leave that reader fetching nothing. + assert_eq!( + agent_token_path("atlas").expect("a plain name is legal"), + "swarm/agents/atlas/forge-token" + ); + } + + #[test] + fn a_traversal_in_the_agent_name_is_refused() { + for bad in ["../argus", "a/b", "atlas/../argus", "a.b", ""] { + let e = agent_token_path(bad).expect_err("a traversal is not legal"); + assert!(matches!(e, Error::PathSegment { kind: "agent", .. }), "{e}"); + } + } + + #[test] + fn the_legal_charset_is_actually_reachable() { + // The control for the test above: a guard that refused everything + // would pass it for the wrong reason. + assert!(agent_token_path("a-b_C9").is_ok()); + } + + #[test] + fn the_agents_own_read_grant_covers_the_path() { + // The agent reads this path under `hive-agent-`, and that + // policy is rendered elsewhere. If the path ever moved out from under + // it, the agent's fetch would 403 at boot, naming neither. + let path = agent_token_path("atlas").expect("legal"); + let policy = crate::policy::render_agent_with_queue("atlas", "pr1ma").expect("legal"); + let covered = policy.lines().any(|line| { + line.strip_prefix("path \"") + .and_then(|rest| rest.split_once("\" {")) + .and_then(|(p, _)| p.strip_suffix('*')) + .is_some_and(|prefix| format!("secret/data/{path}").starts_with(prefix)) + }); + assert!(covered, "no stanza in\n{policy}\ncovers {path}"); + } + + #[test] + fn another_agents_read_grant_does_not_cover_the_path() { + // Control for the test above: the prefix match must actually be + // discriminating, or it proves nothing. + let path = agent_token_path("atlas").expect("legal"); + let policy = crate::policy::render_agent_with_queue("argus", "pr1ma").expect("legal"); + let covered = policy.lines().any(|line| { + line.strip_prefix("path \"") + .and_then(|rest| rest.split_once("\" {")) + .and_then(|(p, _)| p.strip_suffix('*')) + .is_some_and(|prefix| format!("secret/data/{path}").starts_with(prefix)) + }); + assert!(!covered, "argus's policy must not reach atlas's token"); + } + + #[test] + fn the_value_field_matches_what_the_nix_reader_asks_for() { + let json = serde_json::to_string(&Credential { + value: "t".to_owned(), + name: "swarm-agent".to_owned(), + }) + .expect("two Strings serialise"); + assert_eq!(json, r#"{"value":"t","name":"swarm-agent"}"#); + } + + #[test] + fn debug_never_prints_the_value() { + let c = Credential { + value: "0123456789abcdef".to_owned(), + name: "swarm-agent".to_owned(), + }; + let shown = format!("{c:?}"); + assert!(!shown.contains("0123456789abcdef"), "{shown}"); + assert!(shown.contains("swarm-agent"), "{shown}"); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 28fd566c..cb93e056 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -5,7 +5,7 @@ //! the rules every path obeys ([`path`]), the translation from this //! deployment's environment into a logged-in client ([`client`]), and, per kind //! of secret, the path it lives at together with the fields it holds -//! ([`matrix`], [`queue`], [`mtls`]). Each of those is a thing the controller +//! ([`matrix`], [`queue`], [`mtls`], [`forge`]). Each of those is a thing the controller //! and a hive must say identically, so it is said once here. //! //! [`policy`] is the same kind of agreement seen from the other side: which of @@ -23,6 +23,7 @@ //! value inside it, and its doc explains why that is not circular. pub mod client; +pub mod forge; pub mod matrix; pub mod mtls; pub mod path; diff --git a/swarm-secret-client/src/policy.rs b/swarm-secret-client/src/policy.rs index a04e4e57..72cdfa0c 100644 --- a/swarm-secret-client/src/policy.rs +++ b/swarm-secret-client/src/policy.rs @@ -96,6 +96,23 @@ pub fn agent_object_name(agent: &str) -> Result { Ok(format!("{AGENT_PREFIX}{agent}")) } +/// The agents among `names`, a listing of cert-auth roles: the inverse of +/// [`agent_object_name`], for a caller that needs the set of agents holding a +/// store identity. +/// +/// A name that carries the prefix but whose suffix is not a legal segment is +/// dropped rather than returned: [`agent_object_name`] could not have produced +/// it, so it is not an agent this crate minted. +#[must_use] +pub fn agents_from_role_names(names: &[String]) -> Vec { + names + .iter() + .filter_map(|name| name.strip_prefix(AGENT_PREFIX)) + .filter(|agent| checked_segment("agent", agent).is_ok()) + .map(str::to_owned) + .collect() +} + /// One read stanza. The only shape this module emits, so "read-only" is a /// property of the renderer rather than of each call site. fn read_stanza(path: &str) -> String { @@ -595,4 +612,33 @@ mod tests { let expected_queue_path = crate::queue::agent_client_path("pr1ma").expect("legal"); assert!(p.contains(&expected_queue_path)); } + + #[test] + fn only_agent_roles_come_back_and_without_their_prefix() { + let roles: Vec = [ + "hive-agent-atlas", + "hive-pr1ma", + "swarm-controller", + "hive-agent-argus", + ] + .map(str::to_owned) + .to_vec(); + assert_eq!(agents_from_role_names(&roles), ["atlas", "argus"]); + } + + #[test] + fn a_role_agent_object_name_could_not_have_produced_is_dropped() { + // An empty suffix and one carrying a dot are both names the prefix + // matches and no agent can have. + let roles: Vec = ["hive-agent-", "hive-agent-a.b", "hive-agent-ok"] + .map(str::to_owned) + .to_vec(); + assert_eq!(agents_from_role_names(&roles), ["ok"]); + } + + #[test] + fn the_listing_inverts_agent_object_name() { + let name = agent_object_name("atlas").expect("legal"); + assert_eq!(agents_from_role_names(&[name]), ["atlas"]); + } }