Watch
0
0
Fork
You've already forked hyperhive
0

cut off hives that left the roster: forge token, team seat, token secret, login role

This commit is contained in:
atlas 2026-11-16 21:48:41 +01:00 • committed by mara
commit 11fec06a50
10 changed files with 830 additions and 21 deletions

View file

@ -533,6 +533,10 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/metadata/swarm/hives/+/forge/token" {
capabilities = ["delete"]
}
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
capabilities = ["create", "read"]
}

View file

@ -2570,6 +2570,28 @@ let
&& lib.hasInfix "path \"secret/metadata/swarm/agents/+/github-token\" {\n capabilities = [\"delete\"]\n}" s
&& lib.hasInfix "path \"secret/metadata/swarm/agents/+/backend-env\" {\n capabilities = [\"delete\"]\n}" s;
}
{
# `forge::hive_cutoff` deletes the token secret of a hive that left the
# roster through `delete_all_versions`, which addresses the metadata
# path. `+` is one segment, so the stanza reaches the `forge/token` leaf
# of every hive and no other key under `hives/`: not the attic facts, the
# squid credential, the OIDC client or the archive keys, which carry
# their own stanzas. Pinned as the whole stanza, so a `read` or `list`
# that would expose version history fails. The control is the archive
# revocation's metadata stanza, which the same search must find.
name = "the controller may delete a hive's forge token secret, and no other key of a hive's by that path";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/metadata/swarm/hives/+/forge/token\" {\n capabilities = [\"delete\"]\n}" s
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s
&& !(lib.hasInfix "secret/metadata/swarm/hives/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/forge/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/attic" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/squid-selfcheck" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc" s);
}
{
# The swarm appservice token is a homeserver-admin credential. The
# controller mints agents' accounts with it and has no business replacing

View file

@ -52,6 +52,7 @@ pub mod builder_account;
pub mod ci_metrics_account;
pub mod classifier_account;
pub mod docs_grant;
pub mod hive_cutoff;
pub mod hive_token;
pub mod legacy_tokens;
mod meta_repo;

View file

@ -0,0 +1,642 @@
//! Cutting a removed hive's credentials off at the forge and the store.
//!
//! A hive dropped from `swarm.hives` keeps everything the controller gave it:
//! the forge token (read on every config repo), the seat in the
//! `agent-configs/hives` team, the token secret and the cert-auth role it
//! logs in to the store under. [`cut_off_removed`] ends each, for every hive
//! that holds one of them and is not in the roster, in this order:
//!
//! 1. the hive's `swarm-hive` forge token is deleted,
//! 2. its account leaves the `hives` team,
//! 3. the token secret at `swarm/hives/<hive>/forge/token` is deleted,
//! 4. the cert-auth role `hive-<hive>` is deleted.
//!
//! The role goes last because it is the marker that work is left: a failure
//! anywhere above it leaves the role, the next pass finds the hive again and
//! runs every step again. Each step finds nothing to do when it already ran.
//! The forge account stays, so a hive put back in the roster gets its token
//! and seat from [`super::hive_token`]'s pass and its role from the store
//! access written at boot.
//!
//! Nothing is cut off from a roster that did not load or that holds no hive
//! (`startup::HiveRoster::absence_means_removal`): an unreadable roster would
//! otherwise read as every hive removed. A forge account counts as a hive's
//! only with the hive email (`is_hive_account`), and a role only with the
//! `hive-` prefix and not the agents' `hive-agent-` one.
//!
//! The grants are the `auth/cert/certs/hive-*` stanza and the
//! `metadata/swarm/hives/+/forge/token` delete of `swarm-bao.nix`'s
//! `controllerPolicyText`.
use std::collections::BTreeSet;
use std::future::Future;
use anyhow::{Context, Result};
use forgejo_api::structs::AdminSearchUsersQuery;
use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy};
use super::agent_token::is_not_found;
use super::hive_token::{HIVE_TOKEN_NAME, HIVES_TEAM, TeamSeen, is_hive_account};
use super::{CONFIG_ORG, Client};
/// Accounts read per page of the forge's user listing.
const USER_PAGE: u32 = 50;
/// A forge account as the user listing shows it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Account {
pub login: String,
pub email: Option<String>,
}
/// What one cut-off removed from one hive, by name, in the order of the steps.
pub type Removed = Vec<&'static str>;
/// The hives to cut off: those with a hive account on the forge or a
/// cert-auth role in `roles`, and not in `roster`.
///
/// Empty for an empty `roster`, whatever the forge and the store hold.
pub fn removed_hives(
roster: &[String],
accounts: &[Account],
roles: &[String],
) -> BTreeSet<String> {
if roster.is_empty() {
return BTreeSet::new();
}
let from_accounts = accounts
.iter()
.filter_map(|a| {
let hive = policy::hive_from_object_name(&a.login)?;
is_hive_account(hive, a.email.as_deref()).then(|| hive.to_owned())
})
.collect::<Vec<_>>();
from_accounts
.into_iter()
.chain(policy::hives_from_role_names(roles))
.filter(|hive| !roster.contains(hive))
.collect()
}
/// What a cut-off reads and removes. [`Live`] is the forge and the store;
/// tests stand in with a record of the calls.
pub trait Revoke {
/// Every forge account.
fn accounts(&self) -> impl Future<Output = Result<Vec<Account>>> + Send;
/// Every cert-auth role name.
fn roles(&self) -> impl Future<Output = Result<Vec<String>>> + Send;
/// Delete `hive`'s [`HIVE_TOKEN_NAME`] forge token; whether there was one.
fn delete_token(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
/// Take `hive`'s account out of the `hives` team; whether it was in.
fn remove_team_seat(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
/// Delete the secret holding `hive`'s forge token; whether there was one.
fn delete_token_secret(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
/// Delete the cert-auth role `hive-<hive>`; safe when there is none.
fn delete_role(&self, hive: &str) -> impl Future<Output = Result<()>> + Send;
}
/// Run the four steps for `hive`, in order, stopping at the first that fails.
///
/// `has_role` is whether the role listing named it, which is what makes the
/// role count as removed.
///
/// # Errors
/// The first failing step, with its name. Steps after it did not run, and the
/// role is still there.
pub async fn cut_off(revoke: &impl Revoke, hive: &str, has_role: bool) -> Result<Removed> {
let mut removed = Removed::new();
if revoke
.delete_token(hive)
.await
.context("deleting the forge token")?
{
removed.push("forge token");
}
if revoke
.remove_team_seat(hive)
.await
.context("removing the team seat")?
{
removed.push("team seat");
}
if revoke
.delete_token_secret(hive)
.await
.context("deleting the token secret")?
{
removed.push("token secret");
}
revoke
.delete_role(hive)
.await
.context("deleting the cert-auth role")?;
if has_role {
removed.push("login role");
}
Ok(removed)
}
/// [`cut_off`] every hive [`removed_hives`] names for `roster`, logging one
/// line per hive. A hive that fails is retried by the next pass.
pub async fn cut_off_removed(revoke: &impl Revoke, roster: &[String]) {
if roster.is_empty() {
tracing::error!("hive cut-off: the roster is empty; nothing is cut off");
return;
}
let accounts = revoke.accounts().await.unwrap_or_else(|e| {
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the forge accounts failed");
Vec::new()
});
let roles = revoke.roles().await.unwrap_or_else(|e| {
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the cert-auth roles failed");
Vec::new()
});
let with_role = policy::hives_from_role_names(&roles);
for hive in removed_hives(roster, &accounts, &roles) {
match cut_off(revoke, &hive, with_role.contains(&hive)).await {
Ok(removed) if !removed.is_empty() => tracing::info!(
hive,
removed = %removed.join(", "),
"hive is not in the roster: its credentials are revoked"
),
Ok(_) => tracing::debug!(
hive,
"hive is not in the roster and holds nothing to revoke"
),
Err(e) => tracing::warn!(
hive,
error = %format!("{e:#}"),
"hive is not in the roster: revoking its credentials failed; retried on the next pass"
),
}
}
}
/// The forge and the store as one [`Revoke`].
pub struct Live<'a> {
pub client: &'a Client,
pub store: &'a SecretStore,
}
impl Client {
/// Whether the forge account `hive-<hive>` exists and is the hive's.
/// An account by that name that is not (another email) is left alone.
async fn has_hive_account(&self, hive: &str) -> Result<bool> {
let login = forge::hive_login(hive);
match self
.api
.call(|api| api.user_get(&login).into_future())
.await
{
Ok(user) if is_hive_account(hive, user.email.as_deref()) => Ok(true),
Ok(user) => {
tracing::warn!(
hive,
email = ?user.email,
"forge account {login} is not the hive's; left alone"
);
Ok(false)
}
Err(e) if is_not_found(&e) => Ok(false),
Err(e) => Err(e).with_context(|| format!("reading forge user {login}")),
}
}
/// Every forge account, a page at a time.
async fn all_accounts(&self) -> Result<Vec<Account>> {
let mut accounts = Vec::new();
let mut page = 1;
loop {
let (_, users) = self
.api
.call(|api| {
api.admin_search_users(AdminSearchUsersQuery::default())
.page(page)
.page_size(USER_PAGE)
.into_future()
})
.await
.context("listing the forge's users")?;
let last = users.len() < USER_PAGE as usize;
accounts.extend(users.into_iter().filter_map(|u| {
Some(Account {
login: u.login?,
email: u.email,
})
}));
if last {
return Ok(accounts);
}
page += 1;
}
}
/// Delete the [`HIVE_TOKEN_NAME`] token of the hive's account, if the
/// account is the hive's and holds one.
async fn delete_hive_forge_token(&self, hive: &str) -> Result<bool> {
if !self.has_hive_account(hive).await? {
return Ok(false);
}
let login = forge::hive_login(hive);
let held = self.list_agent_tokens(&login).await?.is_some_and(|tokens| {
tokens
.iter()
.any(|t| t.name.as_deref() == Some(HIVE_TOKEN_NAME))
});
if !held {
return Ok(false);
}
match self
.api
.call(|api| {
api.admin_delete_user_access_token(&login, HIVE_TOKEN_NAME)
.into_future()
})
.await
{
Ok(()) => Ok(true),
Err(e) if is_not_found(&e) => Ok(false),
Err(e) => Err(e).with_context(|| format!("deleting {login}'s {HIVE_TOKEN_NAME} token")),
}
}
/// Take the hive's account out of [`HIVES_TEAM`], if the account is the
/// hive's and is a member.
async fn remove_hive_team_seat(&self, hive: &str) -> Result<bool> {
if !self.has_hive_account(hive).await? {
return Ok(false);
}
let login = forge::hive_login(hive);
let team = match self.observe_hives_team().await {
TeamSeen::Matches(id) | TeamSeen::Drifted(id) => id,
TeamSeen::Absent => return Ok(false),
TeamSeen::Unknown => {
anyhow::bail!("could not read the teams of {CONFIG_ORG}")
}
};
match self
.api
.call(|api| api.org_list_team_member(team, &login).into_future())
.await
{
Ok(_) => {}
Err(e) if is_not_found(&e) => return Ok(false),
Err(e) => {
return Err(e).with_context(|| format!("reading {login}'s team membership"));
}
}
match self
.api
.call(|api| api.org_remove_team_member(team, &login).into_future())
.await
{
Ok(()) => Ok(true),
Err(e) if is_not_found(&e) => Ok(false),
Err(e) => {
Err(e).with_context(|| format!("removing {login} from {CONFIG_ORG}/{HIVES_TEAM}"))
}
}
}
}
impl Revoke for Live<'_> {
async fn accounts(&self) -> Result<Vec<Account>> {
self.client.all_accounts().await
}
async fn roles(&self) -> Result<Vec<String>> {
Ok(self.store.list_cert_roles(DEFAULT_CERT_MOUNT).await?)
}
async fn delete_token(&self, hive: &str) -> Result<bool> {
self.client.delete_hive_forge_token(hive).await
}
async fn remove_team_seat(&self, hive: &str) -> Result<bool> {
self.client.remove_hive_team_seat(hive).await
}
async fn delete_token_secret(&self, hive: &str) -> Result<bool> {
let path = forge::hive_token_path(hive)?;
let held = self
.store
.read_optional::<serde_json::Value>(&path)
.await
.with_context(|| format!("reading {path}"))?
.is_some();
if held {
self.store
.delete_all_versions(&path)
.await
.with_context(|| format!("deleting {path}"))?;
}
Ok(held)
}
async fn delete_role(&self, hive: &str) -> Result<()> {
let role = policy::hive_object_name(hive)?;
self.store
.delete_cert_role(DEFAULT_CERT_MOUNT, &role)
.await
.with_context(|| format!("deleting {role}"))
}
}
/// One pass: log in to the store and [`cut_off_removed`] against `roster`.
pub async fn run(client: &Client, roster: &[String]) {
let store = match crate::store::connect().await {
Ok(store) => store,
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: logging in to the store failed");
return;
}
};
cut_off_removed(
&Live {
client,
store: &store,
},
roster,
)
.await;
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::super::agent_token::{Decision, MintReason, Observed, classify_named};
use super::super::hive_token::{HIVE_TOKEN_SCOPES, HiveSeen, plan};
use super::*;
fn roster(names: &[&str]) -> Vec<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
fn account(hive: &str) -> Account {
Account {
login: format!("hive-{hive}"),
email: Some(format!("hive-{hive}@hives.hyperhive.local")),
}
}
/// What the forge and the store hold for the hives it was built with, and a
/// record of every call.
#[derive(Default)]
struct Fake {
state: Mutex<State>,
/// The step that fails, as its [`Revoke`] method name.
failing: Option<&'static str>,
accounts: Vec<Account>,
}
#[derive(Default)]
struct State {
token: BTreeSet<String>,
seat: BTreeSet<String>,
secret: BTreeSet<String>,
role: BTreeSet<String>,
calls: Vec<String>,
}
impl Fake {
/// A hive holding all four credentials, and a forge account.
fn holding(hives: &[&str]) -> Self {
let fake = Self {
accounts: hives.iter().map(|h| account(h)).collect(),
..Self::default()
};
{
let mut s = fake.state.lock().unwrap();
for h in hives {
s.token.insert((*h).to_owned());
s.seat.insert((*h).to_owned());
s.secret.insert((*h).to_owned());
s.role.insert(format!("hive-{h}"));
}
}
fake
}
fn failing(mut self, step: &'static str) -> Self {
self.failing = Some(step);
self
}
fn calls(&self) -> Vec<String> {
self.state.lock().unwrap().calls.clone()
}
fn holds(&self, hive: &str) -> [bool; 4] {
let s = self.state.lock().unwrap();
[
s.token.contains(hive),
s.seat.contains(hive),
s.secret.contains(hive),
s.role.contains(&format!("hive-{hive}")),
]
}
fn step(&self, name: &'static str, hive: &str) -> Result<()> {
self.state
.lock()
.unwrap()
.calls
.push(format!("{name} {hive}"));
if self.failing == Some(name) {
anyhow::bail!("{name} refused");
}
Ok(())
}
}
impl Revoke for Fake {
async fn accounts(&self) -> Result<Vec<Account>> {
Ok(self.accounts.clone())
}
async fn roles(&self) -> Result<Vec<String>> {
Ok(self.state.lock().unwrap().role.iter().cloned().collect())
}
async fn delete_token(&self, hive: &str) -> Result<bool> {
self.step("delete_token", hive)?;
Ok(self.state.lock().unwrap().token.remove(hive))
}
async fn remove_team_seat(&self, hive: &str) -> Result<bool> {
self.step("remove_team_seat", hive)?;
Ok(self.state.lock().unwrap().seat.remove(hive))
}
async fn delete_token_secret(&self, hive: &str) -> Result<bool> {
self.step("delete_token_secret", hive)?;
Ok(self.state.lock().unwrap().secret.remove(hive))
}
async fn delete_role(&self, hive: &str) -> Result<()> {
self.step("delete_role", hive)?;
self.state
.lock()
.unwrap()
.role
.remove(&format!("hive-{hive}"));
Ok(())
}
}
#[test]
fn a_hive_outside_the_roster_with_an_account_or_a_role_is_a_target() {
let accounts = [account("gone"), account("kept")];
let roles = ["hive-kept".to_owned(), "hive-orphan".to_owned()];
assert_eq!(
removed_hives(&roster(&["kept"]), &accounts, &roles),
BTreeSet::from(["gone".to_owned(), "orphan".to_owned()])
);
// The control: with `gone` and `orphan` in the roster too, nothing is.
assert!(removed_hives(&roster(&["kept", "gone", "orphan"]), &accounts, &roles).is_empty());
}
#[test]
fn an_agents_role_or_account_is_never_a_hive_target() {
let agent_account = Account {
login: "hive-x".to_owned(),
email: Some("hive-x@hyperhive.local".to_owned()),
};
let no_email = Account {
login: "hive-y".to_owned(),
email: None,
};
let roles = [
"hive-agent-atlas".to_owned(),
"swarm-controller".to_owned(),
"hive-agent-".to_owned(),
];
assert!(
removed_hives(
&roster(&["kept"]),
&[agent_account, no_email, account("ok")],
&roles
)
.iter()
.eq(["ok"]),
"only the account with the hive email counts"
);
}
#[tokio::test]
async fn a_removed_hive_loses_its_token_seat_secret_and_role_in_that_order() {
let fake = Fake::holding(&["gone", "kept"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
assert_eq!(fake.holds("gone"), [false; 4], "{:?}", fake.calls());
assert_eq!(
fake.calls(),
[
"delete_token gone",
"remove_team_seat gone",
"delete_token_secret gone",
"delete_role gone",
]
);
}
#[tokio::test]
async fn a_hive_in_the_roster_is_untouched() {
let fake = Fake::holding(&["gone", "kept"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
assert_eq!(fake.holds("kept"), [true; 4], "{:?}", fake.calls());
assert!(fake.calls().iter().all(|c| !c.ends_with("kept")));
// The control: the same pass did cut the removed one.
assert_eq!(fake.holds("gone"), [false; 4]);
}
#[tokio::test]
async fn an_empty_roster_cuts_nothing_off() {
let fake = Fake::holding(&["a", "b"]);
cut_off_removed(&fake, &[]).await;
assert_eq!(fake.holds("a"), [true; 4]);
assert_eq!(fake.holds("b"), [true; 4]);
assert!(fake.calls().is_empty(), "{:?}", fake.calls());
// The control: a roster that names another hive cuts both.
cut_off_removed(&fake, &roster(&["c"])).await;
assert_eq!(fake.holds("a"), [false; 4]);
assert_eq!(fake.holds("b"), [false; 4]);
}
#[tokio::test]
async fn an_agents_role_is_not_deleted() {
let fake = Fake::holding(&["gone"]);
fake.state
.lock()
.unwrap()
.role
.insert("hive-agent-atlas".to_owned());
cut_off_removed(&fake, &roster(&["kept"])).await;
assert!(fake.state.lock().unwrap().role.contains("hive-agent-atlas"));
assert!(
fake.calls().iter().all(|c| !c.contains("atlas")),
"{:?}",
fake.calls()
);
// The control: the hive's own role went.
assert!(!fake.state.lock().unwrap().role.contains("hive-gone"));
}
#[tokio::test]
async fn a_failed_step_stops_the_rest_and_keeps_the_role_for_the_next_pass() {
for (step, left) in [
("delete_token", [true, true, true, true]),
("remove_team_seat", [false, true, true, true]),
("delete_token_secret", [false, false, true, true]),
("delete_role", [false, false, false, true]),
] {
let broken = Fake::holding(&["gone"]).failing(step);
cut_off_removed(&broken, &roster(&["kept"])).await;
assert_eq!(broken.holds("gone"), left, "{step}: {:?}", broken.calls());
assert!(
broken.holds("gone")[3],
"{step}: the role is the marker that work is left"
);
// The next pass, against a store and a forge that now accept it.
let healthy = Fake {
state: Mutex::new(std::mem::take(&mut *broken.state.lock().unwrap())),
accounts: broken.accounts.clone(),
failing: None,
};
cut_off_removed(&healthy, &roster(&["kept"])).await;
assert_eq!(healthy.holds("gone"), [false; 4], "{step}");
}
}
#[tokio::test]
async fn a_second_pass_finds_nothing_left_to_remove() {
let fake = Fake::holding(&["gone"]);
let first = cut_off(&fake, "gone", true).await.expect("first");
assert_eq!(
first,
["forge token", "team seat", "token secret", "login role"]
);
let again = cut_off(&fake, "gone", false).await.expect("second");
assert!(again.is_empty(), "{again:?}");
}
#[tokio::test]
async fn a_hive_put_back_in_the_roster_is_minted_again() {
let fake = Fake::holding(&["back"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
// What hive_token's pass reads of the cut-off hive: no stored token,
// none listed, not in the team. The account is still there.
let token = classify_named(HIVE_TOKEN_NAME, HIVE_TOKEN_SCOPES, None, &[]);
assert_eq!(token, Decision::Mint(MintReason::NotOnForge));
let seen = HiveSeen {
token: Observed::Decided(token),
member: Some(fake.holds("back")[1]),
};
assert_eq!(
plan(TeamSeen::Matches(7), &[("back".to_owned(), seen)]),
["back"]
);
}
}

View file

@ -37,7 +37,7 @@ pub const HIVE_TOKEN_NAME: &str = "swarm-hive";
pub const HIVE_TOKEN_SCOPES: &str = "read:repository";
/// The team in `agent-configs` every hive account is a member of.
const HIVES_TEAM: &str = "hives";
pub(super) const HIVES_TEAM: &str = "hives";
const HIVES_TEAM_DESCRIPTION: &str = "hyperhive hives — read on every agent config repo";
@ -60,7 +60,7 @@ fn hive_email(hive: &str) -> String {
/// Whether the account named [`forge::hive_login`]`(hive)`, carrying `email`,
/// is the one this module creates.
fn is_hive_account(hive: &str, email: Option<&str>) -> bool {
pub(super) fn is_hive_account(hive: &str, email: Option<&str>) -> bool {
email == Some(hive_email(hive).as_str())
}
@ -393,16 +393,27 @@ impl Client {
/// node for each.
///
/// The roster is the controller's hive directory. A pass that fails is
/// logged and retried on the next tick; it never stops the daemon.
/// logged and retried on the next tick; it never stops the daemon. With
/// `cut_off_removed`, each tick also revokes the credentials of the hives
/// that are not in `hives` ([`super::hive_cutoff`]); the caller passes `false`
/// for a roster that did not load.
pub fn spawn(
client: Arc<Client>,
hives: Vec<String>,
cut_off_removed: bool,
enqueue: impl Fn(Vec<String>) + Send + 'static,
) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
if cut_off_removed {
super::within_forge_deadline(
"hive credential cut-off",
super::hive_cutoff::run(&client, &hives),
)
.await;
}
let Some(outcome) =
super::within_forge_deadline("hive forge tokens", client.observe_hives(&hives))
.await

View file

@ -487,6 +487,7 @@ pub(crate) async fn mint_identity(
let declarations = match wanted {
Some(writer) => {
let hives: Vec<String> = crate::startup::load_hives()
.into_entries()
.into_iter()
.map(|h| h.name)
.collect();
@ -1954,6 +1955,7 @@ fn read_meta_deps() -> Option<MetaDeps> {
Some(MetaDeps {
settings,
hives: crate::startup::load_hives()
.into_entries()
.into_iter()
.map(|h| h.name)
.collect(),

View file

@ -177,7 +177,7 @@ async fn main() -> Result<()> {
hive_jobq::Graph::new(),
hive_jobq::resources::ResourceTable::new(),
)));
let hives = load_hives();
let hives = load_hives().into_entries();
spawn_matrix_account_backfill(&jobq, deps.matrix_homeserver.clone(), auth.clone());
spawn_jobq_worker(Arc::clone(&jobq), deps);
// Bound to a named variable, not `_` — dropping the provider stops its

View file

@ -358,11 +358,11 @@ async fn write_agent_policy_and_entities(
mod tests {
use std::{cell::Cell, time::Duration};
use crate::store::fake::FakeAccess;
use crate::store::{AccessStore, fake::FakeAccess};
use super::{
RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, attached_policies, policy,
provision_with, retry_until_ok, write_agent_policy_and_entities,
provision_with, retry_until_ok, write_agent_policy_and_entities, write_role,
};
#[test]
@ -472,6 +472,20 @@ mod tests {
);
}
#[tokio::test]
async fn a_hive_whose_role_was_deleted_gets_it_back_when_it_is_in_the_roster_again() {
let store = FakeAccess::default();
write_role(&store, "pr1ma", "CA").await.expect("accepted");
assert!(store.role("hive-pr1ma").is_some());
store
.delete_cert_role("auth/cert", "hive-pr1ma")
.await
.expect("accepted");
assert_eq!(store.role("hive-pr1ma"), None);
write_role(&store, "pr1ma", "CA").await.expect("accepted");
assert!(store.role("hive-pr1ma").is_some());
}
#[tokio::test]
async fn without_an_authority_the_policy_is_written_and_no_role_is() {
let store = FakeAccess::default();

View file

@ -91,19 +91,51 @@ pub(crate) struct ApiDoc;
/// (this daemon has no "self" to exclude) rather than peers-minus-self.
const HIVES_ENV: &str = "SWARM_CONTROLLER_HIVES";
/// How [`load_hives`] read [`HIVES_ENV`].
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum HiveRoster {
Loaded(Vec<HiveEntry>),
/// [`HIVES_ENV`] is not set.
Unset,
/// [`HIVES_ENV`] is set and is not a hive directory.
Unparseable,
}
impl HiveRoster {
pub(crate) fn into_entries(self) -> Vec<HiveEntry> {
match self {
Self::Loaded(hives) => hives,
Self::Unset | Self::Unparseable => Vec::new(),
}
}
/// Whether an absence from the roster can be read as a removal: the roster
/// loaded and holds a hive. An unset, unparseable or empty roster reads
/// the same as "every hive removed", so it is refused with the reason.
pub(crate) fn absence_means_removal(&self) -> Result<(), &'static str> {
match self {
Self::Loaded(hives) if !hives.is_empty() => Ok(()),
Self::Loaded(_) => Err("the hive roster is empty"),
Self::Unset => Err("the hive roster env var is not set"),
Self::Unparseable => Err("the hive roster env var does not parse"),
}
}
}
/// Parses [`HIVES_ENV`] into the swarm's hive directory. Unset or
/// unparseable both fall back to an empty list with a warning rather than
/// failing startup — a swarm-controller that can't yet see its own config
/// (dev run, a module not wired up yet) should still serve `/health`.
pub(crate) fn load_hives() -> Vec<HiveEntry> {
/// unparseable both serve an empty list rather than failing startup — a
/// swarm-controller that can't yet see its own config (dev run, a module not
/// wired up yet) should still serve `/health` — and the [`HiveRoster`] says
/// which one it was.
pub(crate) fn load_hives() -> HiveRoster {
let Some(raw) = std::env::var_os(HIVES_ENV) else {
return Vec::new();
return HiveRoster::Unset;
};
match serde_json::from_str(&raw.to_string_lossy()) {
Ok(hives) => hives,
Ok(hives) => HiveRoster::Loaded(hives),
Err(e) => {
tracing::warn!(error = %e, env = HIVES_ENV, "failed to parse hive directory, serving an empty list");
Vec::new()
HiveRoster::Unparseable
}
}
}
@ -288,11 +320,26 @@ fn spawn_forge_workers(
}
let sched = Arc::clone(jobq);
let hive_names = hives.iter().map(|h| h.name.clone()).collect();
forge::hive_token::spawn(Arc::clone(&client), hive_names, move |hives| {
if let Err(e) = queue_hive_forge_token_mints(&sched, hives) {
tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed");
let cut_off_removed = match load_hives().absence_means_removal() {
Ok(()) => true,
Err(reason) => {
tracing::error!(
reason,
"removed hives are not cut off from the forge and the store until the roster loads"
);
false
}
});
};
forge::hive_token::spawn(
Arc::clone(&client),
hive_names,
cut_off_removed,
move |hives| {
if let Err(e) = queue_hive_forge_token_mints(&sched, hives) {
tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed");
}
},
);
let sched = Arc::clone(jobq);
forge::classifier_account::spawn(Arc::clone(&client), move || {
if let Err(e) = queue_classifier_forge_token_mint(&sched) {
@ -773,21 +820,37 @@ mod tests {
unsafe {
std::env::remove_var(HIVES_ENV);
}
let unset = load_hives();
assert_eq!(unset, HiveRoster::Unset);
assert_eq!(
load_hives(),
unset.clone().into_entries(),
Vec::<HiveEntry>::new(),
"unset env var is an empty directory, not a startup failure"
);
assert!(unset.absence_means_removal().is_err());
unsafe {
std::env::set_var(HIVES_ENV, "not json");
}
let garbled = load_hives();
assert_eq!(garbled, HiveRoster::Unparseable);
assert_eq!(
load_hives(),
garbled.clone().into_entries(),
Vec::<HiveEntry>::new(),
"unparseable env var falls back to empty rather than panicking — \
/health must still answer even if this daemon's own config is wrong"
);
assert!(garbled.absence_means_removal().is_err());
unsafe {
std::env::set_var(HIVES_ENV, "[]");
}
let empty = load_hives();
assert_eq!(empty, HiveRoster::Loaded(Vec::new()));
assert!(
empty.absence_means_removal().is_err(),
"a roster that loaded empty reads as every hive removed"
);
unsafe {
std::env::set_var(
@ -795,9 +858,10 @@ mod tests {
r#"[{"name":"pr1ma","domain":"pr1ma.example.com","dashboard":"pr1ma.hives.example"},{"name":"umbra","domain":"umbra.example.com","dashboard":"umbra.hives.example"}]"#,
);
}
let loaded = load_hives();
assert_eq!(
load_hives(),
vec![
loaded.clone().into_entries(),
[
HiveEntry {
name: "pr1ma".to_string(),
domain: "pr1ma.example.com".to_string(),
@ -810,6 +874,7 @@ mod tests {
},
]
);
assert_eq!(loaded.absence_means_removal(), Ok(()));
unsafe {
std::env::remove_var(HIVES_ENV);

View file

@ -111,6 +111,32 @@ pub fn agents_from_role_names(names: &[String]) -> Vec<String> {
.collect()
}
/// The hive that holds the cert-auth role or forge login `name`: the inverse
/// of [`hive_object_name`], for a caller that needs the hives with a role.
///
/// `None` for an agent's object ([`AGENT_PREFIX`]), which shares the
/// [`HIVE_PREFIX`] and is never a hive's, and for a name whose suffix is not a
/// legal segment.
#[must_use]
pub fn hive_from_object_name(name: &str) -> Option<&str> {
if name.starts_with(AGENT_PREFIX) {
return None;
}
name.strip_prefix(HIVE_PREFIX)
.filter(|hive| checked_segment("hive", hive).is_ok())
}
/// The hives among `names`, a listing of cert-auth roles
/// ([`hive_from_object_name`] for each).
#[must_use]
pub fn hives_from_role_names(names: &[String]) -> Vec<String> {
names
.iter()
.filter_map(|name| hive_from_object_name(name))
.map(str::to_owned)
.collect()
}
/// One read stanza. With [`list_stanza`], [`create_stanza`] and [`sign_stanza`],
/// the only shapes this module emits, so what an agent may write is a property
/// of the renderers rather than of each call site.
@ -753,4 +779,26 @@ mod tests {
let name = agent_object_name("atlas").expect("legal");
assert_eq!(agents_from_role_names(&[name]), ["atlas"]);
}
#[test]
fn only_hive_roles_come_back_and_without_their_prefix() {
let roles: Vec<String> = [
"hive-pr1ma",
"hive-agent-atlas",
"swarm-controller",
"hive-umbra",
"hive-",
"hive-a.b",
]
.map(str::to_owned)
.to_vec();
assert_eq!(hives_from_role_names(&roles), ["pr1ma", "umbra"]);
}
#[test]
fn the_hive_listing_inverts_hive_object_name() {
let name = hive_object_name("pr1ma").expect("legal");
assert_eq!(hive_from_object_name(&name), Some("pr1ma"));
assert_eq!(hive_from_object_name("hive-agent-pr1ma"), None);
}
}