From 11fec06a506edc908ec0c85cff429093688d919b Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 16 Nov 2026 21:48:41 +0100 Subject: [PATCH] cut off hives that left the roster: forge token, team seat, token secret, login role --- nix/host-modules/swarm-bao.nix | 4 + nix/module-eval/bao-grants.nix | 22 + swarm-controller/src/forge.rs | 1 + swarm-controller/src/forge/hive_cutoff.rs | 642 ++++++++++++++++++++++ swarm-controller/src/forge/hive_token.rs | 17 +- swarm-controller/src/jobq.rs | 2 + swarm-controller/src/main.rs | 2 +- swarm-controller/src/read_policy.rs | 18 +- swarm-controller/src/startup.rs | 95 +++- swarm-secret-client/src/policy.rs | 48 ++ 10 files changed, 830 insertions(+), 21 deletions(-) create mode 100644 swarm-controller/src/forge/hive_cutoff.rs diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 269a9fdf0..02c8434b0 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -533,6 +533,10 @@ let capabilities = ["create", "read", "update"] } + path "${credentialMountPath}/metadata/swarm/hives/+/forge/token" { + capabilities = ["delete"] + } + path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" { capabilities = ["create", "read"] } diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 3cdfa80aa..0cbfd56a9 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -2570,6 +2570,28 @@ let && lib.hasInfix "path \"secret/metadata/swarm/agents/+/github-token\" {\n capabilities = [\"delete\"]\n}" s && lib.hasInfix "path \"secret/metadata/swarm/agents/+/backend-env\" {\n capabilities = [\"delete\"]\n}" s; } + { + # `forge::hive_cutoff` deletes the token secret of a hive that left the + # roster through `delete_all_versions`, which addresses the metadata + # path. `+` is one segment, so the stanza reaches the `forge/token` leaf + # of every hive and no other key under `hives/`: not the attic facts, the + # squid credential, the OIDC client or the archive keys, which carry + # their own stanzas. Pinned as the whole stanza, so a `read` or `list` + # that would expose version history fails. The control is the archive + # revocation's metadata stanza, which the same search must find. + name = "the controller may delete a hive's forge token secret, and no other key of a hive's by that path"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "path \"secret/metadata/swarm/hives/+/forge/token\" {\n capabilities = [\"delete\"]\n}" s + && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s + && !(lib.hasInfix "secret/metadata/swarm/hives/*" s) + && !(lib.hasInfix "secret/metadata/swarm/hives/+/forge/*" s) + && !(lib.hasInfix "secret/metadata/swarm/hives/+/attic" s) + && !(lib.hasInfix "secret/metadata/swarm/hives/+/squid-selfcheck" s) + && !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc" s); + } { # The swarm appservice token is a homeserver-admin credential. The # controller mints agents' accounts with it and has no business replacing diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index 4cc909920..2494faf37 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -52,6 +52,7 @@ pub mod builder_account; pub mod ci_metrics_account; pub mod classifier_account; pub mod docs_grant; +pub mod hive_cutoff; pub mod hive_token; pub mod legacy_tokens; mod meta_repo; diff --git a/swarm-controller/src/forge/hive_cutoff.rs b/swarm-controller/src/forge/hive_cutoff.rs new file mode 100644 index 000000000..b7bb014ef --- /dev/null +++ b/swarm-controller/src/forge/hive_cutoff.rs @@ -0,0 +1,642 @@ +//! Cutting a removed hive's credentials off at the forge and the store. +//! +//! A hive dropped from `swarm.hives` keeps everything the controller gave it: +//! the forge token (read on every config repo), the seat in the +//! `agent-configs/hives` team, the token secret and the cert-auth role it +//! logs in to the store under. [`cut_off_removed`] ends each, for every hive +//! that holds one of them and is not in the roster, in this order: +//! +//! 1. the hive's `swarm-hive` forge token is deleted, +//! 2. its account leaves the `hives` team, +//! 3. the token secret at `swarm/hives//forge/token` is deleted, +//! 4. the cert-auth role `hive-` is deleted. +//! +//! The role goes last because it is the marker that work is left: a failure +//! anywhere above it leaves the role, the next pass finds the hive again and +//! runs every step again. Each step finds nothing to do when it already ran. +//! The forge account stays, so a hive put back in the roster gets its token +//! and seat from [`super::hive_token`]'s pass and its role from the store +//! access written at boot. +//! +//! Nothing is cut off from a roster that did not load or that holds no hive +//! (`startup::HiveRoster::absence_means_removal`): an unreadable roster would +//! otherwise read as every hive removed. A forge account counts as a hive's +//! only with the hive email (`is_hive_account`), and a role only with the +//! `hive-` prefix and not the agents' `hive-agent-` one. +//! +//! The grants are the `auth/cert/certs/hive-*` stanza and the +//! `metadata/swarm/hives/+/forge/token` delete of `swarm-bao.nix`'s +//! `controllerPolicyText`. + +use std::collections::BTreeSet; +use std::future::Future; + +use anyhow::{Context, Result}; +use forgejo_api::structs::AdminSearchUsersQuery; +use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy}; + +use super::agent_token::is_not_found; +use super::hive_token::{HIVE_TOKEN_NAME, HIVES_TEAM, TeamSeen, is_hive_account}; +use super::{CONFIG_ORG, Client}; + +/// Accounts read per page of the forge's user listing. +const USER_PAGE: u32 = 50; + +/// A forge account as the user listing shows it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Account { + pub login: String, + pub email: Option, +} + +/// What one cut-off removed from one hive, by name, in the order of the steps. +pub type Removed = Vec<&'static str>; + +/// The hives to cut off: those with a hive account on the forge or a +/// cert-auth role in `roles`, and not in `roster`. +/// +/// Empty for an empty `roster`, whatever the forge and the store hold. +pub fn removed_hives( + roster: &[String], + accounts: &[Account], + roles: &[String], +) -> BTreeSet { + if roster.is_empty() { + return BTreeSet::new(); + } + let from_accounts = accounts + .iter() + .filter_map(|a| { + let hive = policy::hive_from_object_name(&a.login)?; + is_hive_account(hive, a.email.as_deref()).then(|| hive.to_owned()) + }) + .collect::>(); + from_accounts + .into_iter() + .chain(policy::hives_from_role_names(roles)) + .filter(|hive| !roster.contains(hive)) + .collect() +} + +/// What a cut-off reads and removes. [`Live`] is the forge and the store; +/// tests stand in with a record of the calls. +pub trait Revoke { + /// Every forge account. + fn accounts(&self) -> impl Future>> + Send; + /// Every cert-auth role name. + fn roles(&self) -> impl Future>> + Send; + /// Delete `hive`'s [`HIVE_TOKEN_NAME`] forge token; whether there was one. + fn delete_token(&self, hive: &str) -> impl Future> + Send; + /// Take `hive`'s account out of the `hives` team; whether it was in. + fn remove_team_seat(&self, hive: &str) -> impl Future> + Send; + /// Delete the secret holding `hive`'s forge token; whether there was one. + fn delete_token_secret(&self, hive: &str) -> impl Future> + Send; + /// Delete the cert-auth role `hive-`; safe when there is none. + fn delete_role(&self, hive: &str) -> impl Future> + Send; +} + +/// Run the four steps for `hive`, in order, stopping at the first that fails. +/// +/// `has_role` is whether the role listing named it, which is what makes the +/// role count as removed. +/// +/// # Errors +/// The first failing step, with its name. Steps after it did not run, and the +/// role is still there. +pub async fn cut_off(revoke: &impl Revoke, hive: &str, has_role: bool) -> Result { + let mut removed = Removed::new(); + if revoke + .delete_token(hive) + .await + .context("deleting the forge token")? + { + removed.push("forge token"); + } + if revoke + .remove_team_seat(hive) + .await + .context("removing the team seat")? + { + removed.push("team seat"); + } + if revoke + .delete_token_secret(hive) + .await + .context("deleting the token secret")? + { + removed.push("token secret"); + } + revoke + .delete_role(hive) + .await + .context("deleting the cert-auth role")?; + if has_role { + removed.push("login role"); + } + Ok(removed) +} + +/// [`cut_off`] every hive [`removed_hives`] names for `roster`, logging one +/// line per hive. A hive that fails is retried by the next pass. +pub async fn cut_off_removed(revoke: &impl Revoke, roster: &[String]) { + if roster.is_empty() { + tracing::error!("hive cut-off: the roster is empty; nothing is cut off"); + return; + } + let accounts = revoke.accounts().await.unwrap_or_else(|e| { + tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the forge accounts failed"); + Vec::new() + }); + let roles = revoke.roles().await.unwrap_or_else(|e| { + tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the cert-auth roles failed"); + Vec::new() + }); + let with_role = policy::hives_from_role_names(&roles); + for hive in removed_hives(roster, &accounts, &roles) { + match cut_off(revoke, &hive, with_role.contains(&hive)).await { + Ok(removed) if !removed.is_empty() => tracing::info!( + hive, + removed = %removed.join(", "), + "hive is not in the roster: its credentials are revoked" + ), + Ok(_) => tracing::debug!( + hive, + "hive is not in the roster and holds nothing to revoke" + ), + Err(e) => tracing::warn!( + hive, + error = %format!("{e:#}"), + "hive is not in the roster: revoking its credentials failed; retried on the next pass" + ), + } + } +} + +/// The forge and the store as one [`Revoke`]. +pub struct Live<'a> { + pub client: &'a Client, + pub store: &'a SecretStore, +} + +impl Client { + /// Whether the forge account `hive-` exists and is the hive's. + /// An account by that name that is not (another email) is left alone. + async fn has_hive_account(&self, hive: &str) -> Result { + let login = forge::hive_login(hive); + match self + .api + .call(|api| api.user_get(&login).into_future()) + .await + { + Ok(user) if is_hive_account(hive, user.email.as_deref()) => Ok(true), + Ok(user) => { + tracing::warn!( + hive, + email = ?user.email, + "forge account {login} is not the hive's; left alone" + ); + Ok(false) + } + Err(e) if is_not_found(&e) => Ok(false), + Err(e) => Err(e).with_context(|| format!("reading forge user {login}")), + } + } + + /// Every forge account, a page at a time. + async fn all_accounts(&self) -> Result> { + let mut accounts = Vec::new(); + let mut page = 1; + loop { + let (_, users) = self + .api + .call(|api| { + api.admin_search_users(AdminSearchUsersQuery::default()) + .page(page) + .page_size(USER_PAGE) + .into_future() + }) + .await + .context("listing the forge's users")?; + let last = users.len() < USER_PAGE as usize; + accounts.extend(users.into_iter().filter_map(|u| { + Some(Account { + login: u.login?, + email: u.email, + }) + })); + if last { + return Ok(accounts); + } + page += 1; + } + } + + /// Delete the [`HIVE_TOKEN_NAME`] token of the hive's account, if the + /// account is the hive's and holds one. + async fn delete_hive_forge_token(&self, hive: &str) -> Result { + if !self.has_hive_account(hive).await? { + return Ok(false); + } + let login = forge::hive_login(hive); + let held = self.list_agent_tokens(&login).await?.is_some_and(|tokens| { + tokens + .iter() + .any(|t| t.name.as_deref() == Some(HIVE_TOKEN_NAME)) + }); + if !held { + return Ok(false); + } + match self + .api + .call(|api| { + api.admin_delete_user_access_token(&login, HIVE_TOKEN_NAME) + .into_future() + }) + .await + { + Ok(()) => Ok(true), + Err(e) if is_not_found(&e) => Ok(false), + Err(e) => Err(e).with_context(|| format!("deleting {login}'s {HIVE_TOKEN_NAME} token")), + } + } + + /// Take the hive's account out of [`HIVES_TEAM`], if the account is the + /// hive's and is a member. + async fn remove_hive_team_seat(&self, hive: &str) -> Result { + if !self.has_hive_account(hive).await? { + return Ok(false); + } + let login = forge::hive_login(hive); + let team = match self.observe_hives_team().await { + TeamSeen::Matches(id) | TeamSeen::Drifted(id) => id, + TeamSeen::Absent => return Ok(false), + TeamSeen::Unknown => { + anyhow::bail!("could not read the teams of {CONFIG_ORG}") + } + }; + match self + .api + .call(|api| api.org_list_team_member(team, &login).into_future()) + .await + { + Ok(_) => {} + Err(e) if is_not_found(&e) => return Ok(false), + Err(e) => { + return Err(e).with_context(|| format!("reading {login}'s team membership")); + } + } + match self + .api + .call(|api| api.org_remove_team_member(team, &login).into_future()) + .await + { + Ok(()) => Ok(true), + Err(e) if is_not_found(&e) => Ok(false), + Err(e) => { + Err(e).with_context(|| format!("removing {login} from {CONFIG_ORG}/{HIVES_TEAM}")) + } + } + } +} + +impl Revoke for Live<'_> { + async fn accounts(&self) -> Result> { + self.client.all_accounts().await + } + + async fn roles(&self) -> Result> { + Ok(self.store.list_cert_roles(DEFAULT_CERT_MOUNT).await?) + } + + async fn delete_token(&self, hive: &str) -> Result { + self.client.delete_hive_forge_token(hive).await + } + + async fn remove_team_seat(&self, hive: &str) -> Result { + self.client.remove_hive_team_seat(hive).await + } + + async fn delete_token_secret(&self, hive: &str) -> Result { + let path = forge::hive_token_path(hive)?; + let held = self + .store + .read_optional::(&path) + .await + .with_context(|| format!("reading {path}"))? + .is_some(); + if held { + self.store + .delete_all_versions(&path) + .await + .with_context(|| format!("deleting {path}"))?; + } + Ok(held) + } + + async fn delete_role(&self, hive: &str) -> Result<()> { + let role = policy::hive_object_name(hive)?; + self.store + .delete_cert_role(DEFAULT_CERT_MOUNT, &role) + .await + .with_context(|| format!("deleting {role}")) + } +} + +/// One pass: log in to the store and [`cut_off_removed`] against `roster`. +pub async fn run(client: &Client, roster: &[String]) { + let store = match crate::store::connect().await { + Ok(store) => store, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "hive cut-off: logging in to the store failed"); + return; + } + }; + cut_off_removed( + &Live { + client, + store: &store, + }, + roster, + ) + .await; +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::super::agent_token::{Decision, MintReason, Observed, classify_named}; + use super::super::hive_token::{HIVE_TOKEN_SCOPES, HiveSeen, plan}; + use super::*; + + fn roster(names: &[&str]) -> Vec { + names.iter().map(|n| (*n).to_owned()).collect() + } + + fn account(hive: &str) -> Account { + Account { + login: format!("hive-{hive}"), + email: Some(format!("hive-{hive}@hives.hyperhive.local")), + } + } + + /// What the forge and the store hold for the hives it was built with, and a + /// record of every call. + #[derive(Default)] + struct Fake { + state: Mutex, + /// The step that fails, as its [`Revoke`] method name. + failing: Option<&'static str>, + accounts: Vec, + } + + #[derive(Default)] + struct State { + token: BTreeSet, + seat: BTreeSet, + secret: BTreeSet, + role: BTreeSet, + calls: Vec, + } + + impl Fake { + /// A hive holding all four credentials, and a forge account. + fn holding(hives: &[&str]) -> Self { + let fake = Self { + accounts: hives.iter().map(|h| account(h)).collect(), + ..Self::default() + }; + { + let mut s = fake.state.lock().unwrap(); + for h in hives { + s.token.insert((*h).to_owned()); + s.seat.insert((*h).to_owned()); + s.secret.insert((*h).to_owned()); + s.role.insert(format!("hive-{h}")); + } + } + fake + } + + fn failing(mut self, step: &'static str) -> Self { + self.failing = Some(step); + self + } + + fn calls(&self) -> Vec { + self.state.lock().unwrap().calls.clone() + } + + fn holds(&self, hive: &str) -> [bool; 4] { + let s = self.state.lock().unwrap(); + [ + s.token.contains(hive), + s.seat.contains(hive), + s.secret.contains(hive), + s.role.contains(&format!("hive-{hive}")), + ] + } + + fn step(&self, name: &'static str, hive: &str) -> Result<()> { + self.state + .lock() + .unwrap() + .calls + .push(format!("{name} {hive}")); + if self.failing == Some(name) { + anyhow::bail!("{name} refused"); + } + Ok(()) + } + } + + impl Revoke for Fake { + async fn accounts(&self) -> Result> { + Ok(self.accounts.clone()) + } + + async fn roles(&self) -> Result> { + Ok(self.state.lock().unwrap().role.iter().cloned().collect()) + } + + async fn delete_token(&self, hive: &str) -> Result { + self.step("delete_token", hive)?; + Ok(self.state.lock().unwrap().token.remove(hive)) + } + + async fn remove_team_seat(&self, hive: &str) -> Result { + self.step("remove_team_seat", hive)?; + Ok(self.state.lock().unwrap().seat.remove(hive)) + } + + async fn delete_token_secret(&self, hive: &str) -> Result { + self.step("delete_token_secret", hive)?; + Ok(self.state.lock().unwrap().secret.remove(hive)) + } + + async fn delete_role(&self, hive: &str) -> Result<()> { + self.step("delete_role", hive)?; + self.state + .lock() + .unwrap() + .role + .remove(&format!("hive-{hive}")); + Ok(()) + } + } + + #[test] + fn a_hive_outside_the_roster_with_an_account_or_a_role_is_a_target() { + let accounts = [account("gone"), account("kept")]; + let roles = ["hive-kept".to_owned(), "hive-orphan".to_owned()]; + assert_eq!( + removed_hives(&roster(&["kept"]), &accounts, &roles), + BTreeSet::from(["gone".to_owned(), "orphan".to_owned()]) + ); + // The control: with `gone` and `orphan` in the roster too, nothing is. + assert!(removed_hives(&roster(&["kept", "gone", "orphan"]), &accounts, &roles).is_empty()); + } + + #[test] + fn an_agents_role_or_account_is_never_a_hive_target() { + let agent_account = Account { + login: "hive-x".to_owned(), + email: Some("hive-x@hyperhive.local".to_owned()), + }; + let no_email = Account { + login: "hive-y".to_owned(), + email: None, + }; + let roles = [ + "hive-agent-atlas".to_owned(), + "swarm-controller".to_owned(), + "hive-agent-".to_owned(), + ]; + assert!( + removed_hives( + &roster(&["kept"]), + &[agent_account, no_email, account("ok")], + &roles + ) + .iter() + .eq(["ok"]), + "only the account with the hive email counts" + ); + } + + #[tokio::test] + async fn a_removed_hive_loses_its_token_seat_secret_and_role_in_that_order() { + let fake = Fake::holding(&["gone", "kept"]); + cut_off_removed(&fake, &roster(&["kept"])).await; + assert_eq!(fake.holds("gone"), [false; 4], "{:?}", fake.calls()); + assert_eq!( + fake.calls(), + [ + "delete_token gone", + "remove_team_seat gone", + "delete_token_secret gone", + "delete_role gone", + ] + ); + } + + #[tokio::test] + async fn a_hive_in_the_roster_is_untouched() { + let fake = Fake::holding(&["gone", "kept"]); + cut_off_removed(&fake, &roster(&["kept"])).await; + assert_eq!(fake.holds("kept"), [true; 4], "{:?}", fake.calls()); + assert!(fake.calls().iter().all(|c| !c.ends_with("kept"))); + // The control: the same pass did cut the removed one. + assert_eq!(fake.holds("gone"), [false; 4]); + } + + #[tokio::test] + async fn an_empty_roster_cuts_nothing_off() { + let fake = Fake::holding(&["a", "b"]); + cut_off_removed(&fake, &[]).await; + assert_eq!(fake.holds("a"), [true; 4]); + assert_eq!(fake.holds("b"), [true; 4]); + assert!(fake.calls().is_empty(), "{:?}", fake.calls()); + // The control: a roster that names another hive cuts both. + cut_off_removed(&fake, &roster(&["c"])).await; + assert_eq!(fake.holds("a"), [false; 4]); + assert_eq!(fake.holds("b"), [false; 4]); + } + + #[tokio::test] + async fn an_agents_role_is_not_deleted() { + let fake = Fake::holding(&["gone"]); + fake.state + .lock() + .unwrap() + .role + .insert("hive-agent-atlas".to_owned()); + cut_off_removed(&fake, &roster(&["kept"])).await; + assert!(fake.state.lock().unwrap().role.contains("hive-agent-atlas")); + assert!( + fake.calls().iter().all(|c| !c.contains("atlas")), + "{:?}", + fake.calls() + ); + // The control: the hive's own role went. + assert!(!fake.state.lock().unwrap().role.contains("hive-gone")); + } + + #[tokio::test] + async fn a_failed_step_stops_the_rest_and_keeps_the_role_for_the_next_pass() { + for (step, left) in [ + ("delete_token", [true, true, true, true]), + ("remove_team_seat", [false, true, true, true]), + ("delete_token_secret", [false, false, true, true]), + ("delete_role", [false, false, false, true]), + ] { + let broken = Fake::holding(&["gone"]).failing(step); + cut_off_removed(&broken, &roster(&["kept"])).await; + assert_eq!(broken.holds("gone"), left, "{step}: {:?}", broken.calls()); + assert!( + broken.holds("gone")[3], + "{step}: the role is the marker that work is left" + ); + + // The next pass, against a store and a forge that now accept it. + let healthy = Fake { + state: Mutex::new(std::mem::take(&mut *broken.state.lock().unwrap())), + accounts: broken.accounts.clone(), + failing: None, + }; + cut_off_removed(&healthy, &roster(&["kept"])).await; + assert_eq!(healthy.holds("gone"), [false; 4], "{step}"); + } + } + + #[tokio::test] + async fn a_second_pass_finds_nothing_left_to_remove() { + let fake = Fake::holding(&["gone"]); + let first = cut_off(&fake, "gone", true).await.expect("first"); + assert_eq!( + first, + ["forge token", "team seat", "token secret", "login role"] + ); + let again = cut_off(&fake, "gone", false).await.expect("second"); + assert!(again.is_empty(), "{again:?}"); + } + + #[tokio::test] + async fn a_hive_put_back_in_the_roster_is_minted_again() { + let fake = Fake::holding(&["back"]); + cut_off_removed(&fake, &roster(&["kept"])).await; + + // What hive_token's pass reads of the cut-off hive: no stored token, + // none listed, not in the team. The account is still there. + let token = classify_named(HIVE_TOKEN_NAME, HIVE_TOKEN_SCOPES, None, &[]); + assert_eq!(token, Decision::Mint(MintReason::NotOnForge)); + let seen = HiveSeen { + token: Observed::Decided(token), + member: Some(fake.holds("back")[1]), + }; + assert_eq!( + plan(TeamSeen::Matches(7), &[("back".to_owned(), seen)]), + ["back"] + ); + } +} diff --git a/swarm-controller/src/forge/hive_token.rs b/swarm-controller/src/forge/hive_token.rs index c44284cf9..bacfa0407 100644 --- a/swarm-controller/src/forge/hive_token.rs +++ b/swarm-controller/src/forge/hive_token.rs @@ -37,7 +37,7 @@ pub const HIVE_TOKEN_NAME: &str = "swarm-hive"; pub const HIVE_TOKEN_SCOPES: &str = "read:repository"; /// The team in `agent-configs` every hive account is a member of. -const HIVES_TEAM: &str = "hives"; +pub(super) const HIVES_TEAM: &str = "hives"; const HIVES_TEAM_DESCRIPTION: &str = "hyperhive hives — read on every agent config repo"; @@ -60,7 +60,7 @@ fn hive_email(hive: &str) -> String { /// Whether the account named [`forge::hive_login`]`(hive)`, carrying `email`, /// is the one this module creates. -fn is_hive_account(hive: &str, email: Option<&str>) -> bool { +pub(super) fn is_hive_account(hive: &str, email: Option<&str>) -> bool { email == Some(hive_email(hive).as_str()) } @@ -393,16 +393,27 @@ impl Client { /// node for each. /// /// The roster is the controller's hive directory. A pass that fails is -/// logged and retried on the next tick; it never stops the daemon. +/// logged and retried on the next tick; it never stops the daemon. With +/// `cut_off_removed`, each tick also revokes the credentials of the hives +/// that are not in `hives` ([`super::hive_cutoff`]); the caller passes `false` +/// for a roster that did not load. pub fn spawn( client: Arc, hives: Vec, + cut_off_removed: bool, enqueue: impl Fn(Vec) + Send + 'static, ) { tokio::spawn(async move { let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); loop { ticker.tick().await; + if cut_off_removed { + super::within_forge_deadline( + "hive credential cut-off", + super::hive_cutoff::run(&client, &hives), + ) + .await; + } let Some(outcome) = super::within_forge_deadline("hive forge tokens", client.observe_hives(&hives)) .await diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs index 6a4861b5c..df447328c 100644 --- a/swarm-controller/src/jobq.rs +++ b/swarm-controller/src/jobq.rs @@ -487,6 +487,7 @@ pub(crate) async fn mint_identity( let declarations = match wanted { Some(writer) => { let hives: Vec = crate::startup::load_hives() + .into_entries() .into_iter() .map(|h| h.name) .collect(); @@ -1954,6 +1955,7 @@ fn read_meta_deps() -> Option { Some(MetaDeps { settings, hives: crate::startup::load_hives() + .into_entries() .into_iter() .map(|h| h.name) .collect(), diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 3074451d6..2ec3e8bcb 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -177,7 +177,7 @@ async fn main() -> Result<()> { hive_jobq::Graph::new(), hive_jobq::resources::ResourceTable::new(), ))); - let hives = load_hives(); + let hives = load_hives().into_entries(); spawn_matrix_account_backfill(&jobq, deps.matrix_homeserver.clone(), auth.clone()); spawn_jobq_worker(Arc::clone(&jobq), deps); // Bound to a named variable, not `_` — dropping the provider stops its diff --git a/swarm-controller/src/read_policy.rs b/swarm-controller/src/read_policy.rs index 5723b8b05..67510b770 100644 --- a/swarm-controller/src/read_policy.rs +++ b/swarm-controller/src/read_policy.rs @@ -358,11 +358,11 @@ async fn write_agent_policy_and_entities( mod tests { use std::{cell::Cell, time::Duration}; - use crate::store::fake::FakeAccess; + use crate::store::{AccessStore, fake::FakeAccess}; use super::{ RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, attached_policies, policy, - provision_with, retry_until_ok, write_agent_policy_and_entities, + provision_with, retry_until_ok, write_agent_policy_and_entities, write_role, }; #[test] @@ -472,6 +472,20 @@ mod tests { ); } + #[tokio::test] + async fn a_hive_whose_role_was_deleted_gets_it_back_when_it_is_in_the_roster_again() { + let store = FakeAccess::default(); + write_role(&store, "pr1ma", "CA").await.expect("accepted"); + assert!(store.role("hive-pr1ma").is_some()); + store + .delete_cert_role("auth/cert", "hive-pr1ma") + .await + .expect("accepted"); + assert_eq!(store.role("hive-pr1ma"), None); + write_role(&store, "pr1ma", "CA").await.expect("accepted"); + assert!(store.role("hive-pr1ma").is_some()); + } + #[tokio::test] async fn without_an_authority_the_policy_is_written_and_no_role_is() { let store = FakeAccess::default(); diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index b1896cd54..00a89226b 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -91,19 +91,51 @@ pub(crate) struct ApiDoc; /// (this daemon has no "self" to exclude) rather than peers-minus-self. const HIVES_ENV: &str = "SWARM_CONTROLLER_HIVES"; +/// How [`load_hives`] read [`HIVES_ENV`]. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum HiveRoster { + Loaded(Vec), + /// [`HIVES_ENV`] is not set. + Unset, + /// [`HIVES_ENV`] is set and is not a hive directory. + Unparseable, +} + +impl HiveRoster { + pub(crate) fn into_entries(self) -> Vec { + match self { + Self::Loaded(hives) => hives, + Self::Unset | Self::Unparseable => Vec::new(), + } + } + + /// Whether an absence from the roster can be read as a removal: the roster + /// loaded and holds a hive. An unset, unparseable or empty roster reads + /// the same as "every hive removed", so it is refused with the reason. + pub(crate) fn absence_means_removal(&self) -> Result<(), &'static str> { + match self { + Self::Loaded(hives) if !hives.is_empty() => Ok(()), + Self::Loaded(_) => Err("the hive roster is empty"), + Self::Unset => Err("the hive roster env var is not set"), + Self::Unparseable => Err("the hive roster env var does not parse"), + } + } +} + /// Parses [`HIVES_ENV`] into the swarm's hive directory. Unset or -/// unparseable both fall back to an empty list with a warning rather than -/// failing startup — a swarm-controller that can't yet see its own config -/// (dev run, a module not wired up yet) should still serve `/health`. -pub(crate) fn load_hives() -> Vec { +/// unparseable both serve an empty list rather than failing startup — a +/// swarm-controller that can't yet see its own config (dev run, a module not +/// wired up yet) should still serve `/health` — and the [`HiveRoster`] says +/// which one it was. +pub(crate) fn load_hives() -> HiveRoster { let Some(raw) = std::env::var_os(HIVES_ENV) else { - return Vec::new(); + return HiveRoster::Unset; }; match serde_json::from_str(&raw.to_string_lossy()) { - Ok(hives) => hives, + Ok(hives) => HiveRoster::Loaded(hives), Err(e) => { tracing::warn!(error = %e, env = HIVES_ENV, "failed to parse hive directory, serving an empty list"); - Vec::new() + HiveRoster::Unparseable } } } @@ -288,11 +320,26 @@ fn spawn_forge_workers( } let sched = Arc::clone(jobq); let hive_names = hives.iter().map(|h| h.name.clone()).collect(); - forge::hive_token::spawn(Arc::clone(&client), hive_names, move |hives| { - if let Err(e) = queue_hive_forge_token_mints(&sched, hives) { - tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed"); + let cut_off_removed = match load_hives().absence_means_removal() { + Ok(()) => true, + Err(reason) => { + tracing::error!( + reason, + "removed hives are not cut off from the forge and the store until the roster loads" + ); + false } - }); + }; + forge::hive_token::spawn( + Arc::clone(&client), + hive_names, + cut_off_removed, + move |hives| { + if let Err(e) = queue_hive_forge_token_mints(&sched, hives) { + tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed"); + } + }, + ); let sched = Arc::clone(jobq); forge::classifier_account::spawn(Arc::clone(&client), move || { if let Err(e) = queue_classifier_forge_token_mint(&sched) { @@ -773,21 +820,37 @@ mod tests { unsafe { std::env::remove_var(HIVES_ENV); } + let unset = load_hives(); + assert_eq!(unset, HiveRoster::Unset); assert_eq!( - load_hives(), + unset.clone().into_entries(), Vec::::new(), "unset env var is an empty directory, not a startup failure" ); + assert!(unset.absence_means_removal().is_err()); unsafe { std::env::set_var(HIVES_ENV, "not json"); } + let garbled = load_hives(); + assert_eq!(garbled, HiveRoster::Unparseable); assert_eq!( - load_hives(), + garbled.clone().into_entries(), Vec::::new(), "unparseable env var falls back to empty rather than panicking — \ /health must still answer even if this daemon's own config is wrong" ); + assert!(garbled.absence_means_removal().is_err()); + + unsafe { + std::env::set_var(HIVES_ENV, "[]"); + } + let empty = load_hives(); + assert_eq!(empty, HiveRoster::Loaded(Vec::new())); + assert!( + empty.absence_means_removal().is_err(), + "a roster that loaded empty reads as every hive removed" + ); unsafe { std::env::set_var( @@ -795,9 +858,10 @@ mod tests { r#"[{"name":"pr1ma","domain":"pr1ma.example.com","dashboard":"pr1ma.hives.example"},{"name":"umbra","domain":"umbra.example.com","dashboard":"umbra.hives.example"}]"#, ); } + let loaded = load_hives(); assert_eq!( - load_hives(), - vec![ + loaded.clone().into_entries(), + [ HiveEntry { name: "pr1ma".to_string(), domain: "pr1ma.example.com".to_string(), @@ -810,6 +874,7 @@ mod tests { }, ] ); + assert_eq!(loaded.absence_means_removal(), Ok(())); unsafe { std::env::remove_var(HIVES_ENV); diff --git a/swarm-secret-client/src/policy.rs b/swarm-secret-client/src/policy.rs index 755f8e2d5..c0ed31adc 100644 --- a/swarm-secret-client/src/policy.rs +++ b/swarm-secret-client/src/policy.rs @@ -111,6 +111,32 @@ pub fn agents_from_role_names(names: &[String]) -> Vec { .collect() } +/// The hive that holds the cert-auth role or forge login `name`: the inverse +/// of [`hive_object_name`], for a caller that needs the hives with a role. +/// +/// `None` for an agent's object ([`AGENT_PREFIX`]), which shares the +/// [`HIVE_PREFIX`] and is never a hive's, and for a name whose suffix is not a +/// legal segment. +#[must_use] +pub fn hive_from_object_name(name: &str) -> Option<&str> { + if name.starts_with(AGENT_PREFIX) { + return None; + } + name.strip_prefix(HIVE_PREFIX) + .filter(|hive| checked_segment("hive", hive).is_ok()) +} + +/// The hives among `names`, a listing of cert-auth roles +/// ([`hive_from_object_name`] for each). +#[must_use] +pub fn hives_from_role_names(names: &[String]) -> Vec { + names + .iter() + .filter_map(|name| hive_from_object_name(name)) + .map(str::to_owned) + .collect() +} + /// One read stanza. With [`list_stanza`], [`create_stanza`] and [`sign_stanza`], /// the only shapes this module emits, so what an agent may write is a property /// of the renderers rather than of each call site. @@ -753,4 +779,26 @@ mod tests { let name = agent_object_name("atlas").expect("legal"); assert_eq!(agents_from_role_names(&[name]), ["atlas"]); } + + #[test] + fn only_hive_roles_come_back_and_without_their_prefix() { + let roles: Vec = [ + "hive-pr1ma", + "hive-agent-atlas", + "swarm-controller", + "hive-umbra", + "hive-", + "hive-a.b", + ] + .map(str::to_owned) + .to_vec(); + assert_eq!(hives_from_role_names(&roles), ["pr1ma", "umbra"]); + } + + #[test] + fn the_hive_listing_inverts_hive_object_name() { + let name = hive_object_name("pr1ma").expect("legal"); + assert_eq!(hive_from_object_name(&name), Some("pr1ma")); + assert_eq!(hive_from_object_name("hive-agent-pr1ma"), None); + } }