cut off hives that left the roster: forge token, team seat, token secret, login role
This commit is contained in:
parent
f8a0388457
commit
11fec06a50
10 changed files with 830 additions and 21 deletions
|
|
@ -533,6 +533,10 @@ let
|
|||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/metadata/swarm/hives/+/forge/token" {
|
||||
capabilities = ["delete"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
|
||||
capabilities = ["create", "read"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2570,6 +2570,28 @@ let
|
|||
&& lib.hasInfix "path \"secret/metadata/swarm/agents/+/github-token\" {\n capabilities = [\"delete\"]\n}" s
|
||||
&& lib.hasInfix "path \"secret/metadata/swarm/agents/+/backend-env\" {\n capabilities = [\"delete\"]\n}" s;
|
||||
}
|
||||
{
|
||||
# `forge::hive_cutoff` deletes the token secret of a hive that left the
|
||||
# roster through `delete_all_versions`, which addresses the metadata
|
||||
# path. `+` is one segment, so the stanza reaches the `forge/token` leaf
|
||||
# of every hive and no other key under `hives/`: not the attic facts, the
|
||||
# squid credential, the OIDC client or the archive keys, which carry
|
||||
# their own stanzas. Pinned as the whole stanza, so a `read` or `list`
|
||||
# that would expose version history fails. The control is the archive
|
||||
# revocation's metadata stanza, which the same search must find.
|
||||
name = "the controller may delete a hive's forge token secret, and no other key of a hive's by that path";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/metadata/swarm/hives/+/forge/token\" {\n capabilities = [\"delete\"]\n}" s
|
||||
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives/*" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/forge/*" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/attic" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/squid-selfcheck" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc" s);
|
||||
}
|
||||
{
|
||||
# The swarm appservice token is a homeserver-admin credential. The
|
||||
# controller mints agents' accounts with it and has no business replacing
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ pub mod builder_account;
|
|||
pub mod ci_metrics_account;
|
||||
pub mod classifier_account;
|
||||
pub mod docs_grant;
|
||||
pub mod hive_cutoff;
|
||||
pub mod hive_token;
|
||||
pub mod legacy_tokens;
|
||||
mod meta_repo;
|
||||
|
|
|
|||
642
swarm-controller/src/forge/hive_cutoff.rs
Normal file
642
swarm-controller/src/forge/hive_cutoff.rs
Normal file
|
|
@ -0,0 +1,642 @@
|
|||
//! Cutting a removed hive's credentials off at the forge and the store.
|
||||
//!
|
||||
//! A hive dropped from `swarm.hives` keeps everything the controller gave it:
|
||||
//! the forge token (read on every config repo), the seat in the
|
||||
//! `agent-configs/hives` team, the token secret and the cert-auth role it
|
||||
//! logs in to the store under. [`cut_off_removed`] ends each, for every hive
|
||||
//! that holds one of them and is not in the roster, in this order:
|
||||
//!
|
||||
//! 1. the hive's `swarm-hive` forge token is deleted,
|
||||
//! 2. its account leaves the `hives` team,
|
||||
//! 3. the token secret at `swarm/hives/<hive>/forge/token` is deleted,
|
||||
//! 4. the cert-auth role `hive-<hive>` is deleted.
|
||||
//!
|
||||
//! The role goes last because it is the marker that work is left: a failure
|
||||
//! anywhere above it leaves the role, the next pass finds the hive again and
|
||||
//! runs every step again. Each step finds nothing to do when it already ran.
|
||||
//! The forge account stays, so a hive put back in the roster gets its token
|
||||
//! and seat from [`super::hive_token`]'s pass and its role from the store
|
||||
//! access written at boot.
|
||||
//!
|
||||
//! Nothing is cut off from a roster that did not load or that holds no hive
|
||||
//! (`startup::HiveRoster::absence_means_removal`): an unreadable roster would
|
||||
//! otherwise read as every hive removed. A forge account counts as a hive's
|
||||
//! only with the hive email (`is_hive_account`), and a role only with the
|
||||
//! `hive-` prefix and not the agents' `hive-agent-` one.
|
||||
//!
|
||||
//! The grants are the `auth/cert/certs/hive-*` stanza and the
|
||||
//! `metadata/swarm/hives/+/forge/token` delete of `swarm-bao.nix`'s
|
||||
//! `controllerPolicyText`.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use forgejo_api::structs::AdminSearchUsersQuery;
|
||||
use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy};
|
||||
|
||||
use super::agent_token::is_not_found;
|
||||
use super::hive_token::{HIVE_TOKEN_NAME, HIVES_TEAM, TeamSeen, is_hive_account};
|
||||
use super::{CONFIG_ORG, Client};
|
||||
|
||||
/// Accounts read per page of the forge's user listing.
|
||||
const USER_PAGE: u32 = 50;
|
||||
|
||||
/// A forge account as the user listing shows it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Account {
|
||||
pub login: String,
|
||||
pub email: Option<String>,
|
||||
}
|
||||
|
||||
/// What one cut-off removed from one hive, by name, in the order of the steps.
|
||||
pub type Removed = Vec<&'static str>;
|
||||
|
||||
/// The hives to cut off: those with a hive account on the forge or a
|
||||
/// cert-auth role in `roles`, and not in `roster`.
|
||||
///
|
||||
/// Empty for an empty `roster`, whatever the forge and the store hold.
|
||||
pub fn removed_hives(
|
||||
roster: &[String],
|
||||
accounts: &[Account],
|
||||
roles: &[String],
|
||||
) -> BTreeSet<String> {
|
||||
if roster.is_empty() {
|
||||
return BTreeSet::new();
|
||||
}
|
||||
let from_accounts = accounts
|
||||
.iter()
|
||||
.filter_map(|a| {
|
||||
let hive = policy::hive_from_object_name(&a.login)?;
|
||||
is_hive_account(hive, a.email.as_deref()).then(|| hive.to_owned())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
from_accounts
|
||||
.into_iter()
|
||||
.chain(policy::hives_from_role_names(roles))
|
||||
.filter(|hive| !roster.contains(hive))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// What a cut-off reads and removes. [`Live`] is the forge and the store;
|
||||
/// tests stand in with a record of the calls.
|
||||
pub trait Revoke {
|
||||
/// Every forge account.
|
||||
fn accounts(&self) -> impl Future<Output = Result<Vec<Account>>> + Send;
|
||||
/// Every cert-auth role name.
|
||||
fn roles(&self) -> impl Future<Output = Result<Vec<String>>> + Send;
|
||||
/// Delete `hive`'s [`HIVE_TOKEN_NAME`] forge token; whether there was one.
|
||||
fn delete_token(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
|
||||
/// Take `hive`'s account out of the `hives` team; whether it was in.
|
||||
fn remove_team_seat(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
|
||||
/// Delete the secret holding `hive`'s forge token; whether there was one.
|
||||
fn delete_token_secret(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
|
||||
/// Delete the cert-auth role `hive-<hive>`; safe when there is none.
|
||||
fn delete_role(&self, hive: &str) -> impl Future<Output = Result<()>> + Send;
|
||||
}
|
||||
|
||||
/// Run the four steps for `hive`, in order, stopping at the first that fails.
|
||||
///
|
||||
/// `has_role` is whether the role listing named it, which is what makes the
|
||||
/// role count as removed.
|
||||
///
|
||||
/// # Errors
|
||||
/// The first failing step, with its name. Steps after it did not run, and the
|
||||
/// role is still there.
|
||||
pub async fn cut_off(revoke: &impl Revoke, hive: &str, has_role: bool) -> Result<Removed> {
|
||||
let mut removed = Removed::new();
|
||||
if revoke
|
||||
.delete_token(hive)
|
||||
.await
|
||||
.context("deleting the forge token")?
|
||||
{
|
||||
removed.push("forge token");
|
||||
}
|
||||
if revoke
|
||||
.remove_team_seat(hive)
|
||||
.await
|
||||
.context("removing the team seat")?
|
||||
{
|
||||
removed.push("team seat");
|
||||
}
|
||||
if revoke
|
||||
.delete_token_secret(hive)
|
||||
.await
|
||||
.context("deleting the token secret")?
|
||||
{
|
||||
removed.push("token secret");
|
||||
}
|
||||
revoke
|
||||
.delete_role(hive)
|
||||
.await
|
||||
.context("deleting the cert-auth role")?;
|
||||
if has_role {
|
||||
removed.push("login role");
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// [`cut_off`] every hive [`removed_hives`] names for `roster`, logging one
|
||||
/// line per hive. A hive that fails is retried by the next pass.
|
||||
pub async fn cut_off_removed(revoke: &impl Revoke, roster: &[String]) {
|
||||
if roster.is_empty() {
|
||||
tracing::error!("hive cut-off: the roster is empty; nothing is cut off");
|
||||
return;
|
||||
}
|
||||
let accounts = revoke.accounts().await.unwrap_or_else(|e| {
|
||||
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the forge accounts failed");
|
||||
Vec::new()
|
||||
});
|
||||
let roles = revoke.roles().await.unwrap_or_else(|e| {
|
||||
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: listing the cert-auth roles failed");
|
||||
Vec::new()
|
||||
});
|
||||
let with_role = policy::hives_from_role_names(&roles);
|
||||
for hive in removed_hives(roster, &accounts, &roles) {
|
||||
match cut_off(revoke, &hive, with_role.contains(&hive)).await {
|
||||
Ok(removed) if !removed.is_empty() => tracing::info!(
|
||||
hive,
|
||||
removed = %removed.join(", "),
|
||||
"hive is not in the roster: its credentials are revoked"
|
||||
),
|
||||
Ok(_) => tracing::debug!(
|
||||
hive,
|
||||
"hive is not in the roster and holds nothing to revoke"
|
||||
),
|
||||
Err(e) => tracing::warn!(
|
||||
hive,
|
||||
error = %format!("{e:#}"),
|
||||
"hive is not in the roster: revoking its credentials failed; retried on the next pass"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The forge and the store as one [`Revoke`].
|
||||
pub struct Live<'a> {
|
||||
pub client: &'a Client,
|
||||
pub store: &'a SecretStore,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
/// Whether the forge account `hive-<hive>` exists and is the hive's.
|
||||
/// An account by that name that is not (another email) is left alone.
|
||||
async fn has_hive_account(&self, hive: &str) -> Result<bool> {
|
||||
let login = forge::hive_login(hive);
|
||||
match self
|
||||
.api
|
||||
.call(|api| api.user_get(&login).into_future())
|
||||
.await
|
||||
{
|
||||
Ok(user) if is_hive_account(hive, user.email.as_deref()) => Ok(true),
|
||||
Ok(user) => {
|
||||
tracing::warn!(
|
||||
hive,
|
||||
email = ?user.email,
|
||||
"forge account {login} is not the hive's; left alone"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) if is_not_found(&e) => Ok(false),
|
||||
Err(e) => Err(e).with_context(|| format!("reading forge user {login}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every forge account, a page at a time.
|
||||
async fn all_accounts(&self) -> Result<Vec<Account>> {
|
||||
let mut accounts = Vec::new();
|
||||
let mut page = 1;
|
||||
loop {
|
||||
let (_, users) = self
|
||||
.api
|
||||
.call(|api| {
|
||||
api.admin_search_users(AdminSearchUsersQuery::default())
|
||||
.page(page)
|
||||
.page_size(USER_PAGE)
|
||||
.into_future()
|
||||
})
|
||||
.await
|
||||
.context("listing the forge's users")?;
|
||||
let last = users.len() < USER_PAGE as usize;
|
||||
accounts.extend(users.into_iter().filter_map(|u| {
|
||||
Some(Account {
|
||||
login: u.login?,
|
||||
email: u.email,
|
||||
})
|
||||
}));
|
||||
if last {
|
||||
return Ok(accounts);
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Delete the [`HIVE_TOKEN_NAME`] token of the hive's account, if the
|
||||
/// account is the hive's and holds one.
|
||||
async fn delete_hive_forge_token(&self, hive: &str) -> Result<bool> {
|
||||
if !self.has_hive_account(hive).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
let login = forge::hive_login(hive);
|
||||
let held = self.list_agent_tokens(&login).await?.is_some_and(|tokens| {
|
||||
tokens
|
||||
.iter()
|
||||
.any(|t| t.name.as_deref() == Some(HIVE_TOKEN_NAME))
|
||||
});
|
||||
if !held {
|
||||
return Ok(false);
|
||||
}
|
||||
match self
|
||||
.api
|
||||
.call(|api| {
|
||||
api.admin_delete_user_access_token(&login, HIVE_TOKEN_NAME)
|
||||
.into_future()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(true),
|
||||
Err(e) if is_not_found(&e) => Ok(false),
|
||||
Err(e) => Err(e).with_context(|| format!("deleting {login}'s {HIVE_TOKEN_NAME} token")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take the hive's account out of [`HIVES_TEAM`], if the account is the
|
||||
/// hive's and is a member.
|
||||
async fn remove_hive_team_seat(&self, hive: &str) -> Result<bool> {
|
||||
if !self.has_hive_account(hive).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
let login = forge::hive_login(hive);
|
||||
let team = match self.observe_hives_team().await {
|
||||
TeamSeen::Matches(id) | TeamSeen::Drifted(id) => id,
|
||||
TeamSeen::Absent => return Ok(false),
|
||||
TeamSeen::Unknown => {
|
||||
anyhow::bail!("could not read the teams of {CONFIG_ORG}")
|
||||
}
|
||||
};
|
||||
match self
|
||||
.api
|
||||
.call(|api| api.org_list_team_member(team, &login).into_future())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(e) if is_not_found(&e) => return Ok(false),
|
||||
Err(e) => {
|
||||
return Err(e).with_context(|| format!("reading {login}'s team membership"));
|
||||
}
|
||||
}
|
||||
match self
|
||||
.api
|
||||
.call(|api| api.org_remove_team_member(team, &login).into_future())
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(true),
|
||||
Err(e) if is_not_found(&e) => Ok(false),
|
||||
Err(e) => {
|
||||
Err(e).with_context(|| format!("removing {login} from {CONFIG_ORG}/{HIVES_TEAM}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Revoke for Live<'_> {
|
||||
async fn accounts(&self) -> Result<Vec<Account>> {
|
||||
self.client.all_accounts().await
|
||||
}
|
||||
|
||||
async fn roles(&self) -> Result<Vec<String>> {
|
||||
Ok(self.store.list_cert_roles(DEFAULT_CERT_MOUNT).await?)
|
||||
}
|
||||
|
||||
async fn delete_token(&self, hive: &str) -> Result<bool> {
|
||||
self.client.delete_hive_forge_token(hive).await
|
||||
}
|
||||
|
||||
async fn remove_team_seat(&self, hive: &str) -> Result<bool> {
|
||||
self.client.remove_hive_team_seat(hive).await
|
||||
}
|
||||
|
||||
async fn delete_token_secret(&self, hive: &str) -> Result<bool> {
|
||||
let path = forge::hive_token_path(hive)?;
|
||||
let held = self
|
||||
.store
|
||||
.read_optional::<serde_json::Value>(&path)
|
||||
.await
|
||||
.with_context(|| format!("reading {path}"))?
|
||||
.is_some();
|
||||
if held {
|
||||
self.store
|
||||
.delete_all_versions(&path)
|
||||
.await
|
||||
.with_context(|| format!("deleting {path}"))?;
|
||||
}
|
||||
Ok(held)
|
||||
}
|
||||
|
||||
async fn delete_role(&self, hive: &str) -> Result<()> {
|
||||
let role = policy::hive_object_name(hive)?;
|
||||
self.store
|
||||
.delete_cert_role(DEFAULT_CERT_MOUNT, &role)
|
||||
.await
|
||||
.with_context(|| format!("deleting {role}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass: log in to the store and [`cut_off_removed`] against `roster`.
|
||||
pub async fn run(client: &Client, roster: &[String]) {
|
||||
let store = match crate::store::connect().await {
|
||||
Ok(store) => store,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "hive cut-off: logging in to the store failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
cut_off_removed(
|
||||
&Live {
|
||||
client,
|
||||
store: &store,
|
||||
},
|
||||
roster,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::super::agent_token::{Decision, MintReason, Observed, classify_named};
|
||||
use super::super::hive_token::{HIVE_TOKEN_SCOPES, HiveSeen, plan};
|
||||
use super::*;
|
||||
|
||||
fn roster(names: &[&str]) -> Vec<String> {
|
||||
names.iter().map(|n| (*n).to_owned()).collect()
|
||||
}
|
||||
|
||||
fn account(hive: &str) -> Account {
|
||||
Account {
|
||||
login: format!("hive-{hive}"),
|
||||
email: Some(format!("hive-{hive}@hives.hyperhive.local")),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the forge and the store hold for the hives it was built with, and a
|
||||
/// record of every call.
|
||||
#[derive(Default)]
|
||||
struct Fake {
|
||||
state: Mutex<State>,
|
||||
/// The step that fails, as its [`Revoke`] method name.
|
||||
failing: Option<&'static str>,
|
||||
accounts: Vec<Account>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
token: BTreeSet<String>,
|
||||
seat: BTreeSet<String>,
|
||||
secret: BTreeSet<String>,
|
||||
role: BTreeSet<String>,
|
||||
calls: Vec<String>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
/// A hive holding all four credentials, and a forge account.
|
||||
fn holding(hives: &[&str]) -> Self {
|
||||
let fake = Self {
|
||||
accounts: hives.iter().map(|h| account(h)).collect(),
|
||||
..Self::default()
|
||||
};
|
||||
{
|
||||
let mut s = fake.state.lock().unwrap();
|
||||
for h in hives {
|
||||
s.token.insert((*h).to_owned());
|
||||
s.seat.insert((*h).to_owned());
|
||||
s.secret.insert((*h).to_owned());
|
||||
s.role.insert(format!("hive-{h}"));
|
||||
}
|
||||
}
|
||||
fake
|
||||
}
|
||||
|
||||
fn failing(mut self, step: &'static str) -> Self {
|
||||
self.failing = Some(step);
|
||||
self
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<String> {
|
||||
self.state.lock().unwrap().calls.clone()
|
||||
}
|
||||
|
||||
fn holds(&self, hive: &str) -> [bool; 4] {
|
||||
let s = self.state.lock().unwrap();
|
||||
[
|
||||
s.token.contains(hive),
|
||||
s.seat.contains(hive),
|
||||
s.secret.contains(hive),
|
||||
s.role.contains(&format!("hive-{hive}")),
|
||||
]
|
||||
}
|
||||
|
||||
fn step(&self, name: &'static str, hive: &str) -> Result<()> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.calls
|
||||
.push(format!("{name} {hive}"));
|
||||
if self.failing == Some(name) {
|
||||
anyhow::bail!("{name} refused");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Revoke for Fake {
|
||||
async fn accounts(&self) -> Result<Vec<Account>> {
|
||||
Ok(self.accounts.clone())
|
||||
}
|
||||
|
||||
async fn roles(&self) -> Result<Vec<String>> {
|
||||
Ok(self.state.lock().unwrap().role.iter().cloned().collect())
|
||||
}
|
||||
|
||||
async fn delete_token(&self, hive: &str) -> Result<bool> {
|
||||
self.step("delete_token", hive)?;
|
||||
Ok(self.state.lock().unwrap().token.remove(hive))
|
||||
}
|
||||
|
||||
async fn remove_team_seat(&self, hive: &str) -> Result<bool> {
|
||||
self.step("remove_team_seat", hive)?;
|
||||
Ok(self.state.lock().unwrap().seat.remove(hive))
|
||||
}
|
||||
|
||||
async fn delete_token_secret(&self, hive: &str) -> Result<bool> {
|
||||
self.step("delete_token_secret", hive)?;
|
||||
Ok(self.state.lock().unwrap().secret.remove(hive))
|
||||
}
|
||||
|
||||
async fn delete_role(&self, hive: &str) -> Result<()> {
|
||||
self.step("delete_role", hive)?;
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.role
|
||||
.remove(&format!("hive-{hive}"));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hive_outside_the_roster_with_an_account_or_a_role_is_a_target() {
|
||||
let accounts = [account("gone"), account("kept")];
|
||||
let roles = ["hive-kept".to_owned(), "hive-orphan".to_owned()];
|
||||
assert_eq!(
|
||||
removed_hives(&roster(&["kept"]), &accounts, &roles),
|
||||
BTreeSet::from(["gone".to_owned(), "orphan".to_owned()])
|
||||
);
|
||||
// The control: with `gone` and `orphan` in the roster too, nothing is.
|
||||
assert!(removed_hives(&roster(&["kept", "gone", "orphan"]), &accounts, &roles).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_agents_role_or_account_is_never_a_hive_target() {
|
||||
let agent_account = Account {
|
||||
login: "hive-x".to_owned(),
|
||||
email: Some("hive-x@hyperhive.local".to_owned()),
|
||||
};
|
||||
let no_email = Account {
|
||||
login: "hive-y".to_owned(),
|
||||
email: None,
|
||||
};
|
||||
let roles = [
|
||||
"hive-agent-atlas".to_owned(),
|
||||
"swarm-controller".to_owned(),
|
||||
"hive-agent-".to_owned(),
|
||||
];
|
||||
assert!(
|
||||
removed_hives(
|
||||
&roster(&["kept"]),
|
||||
&[agent_account, no_email, account("ok")],
|
||||
&roles
|
||||
)
|
||||
.iter()
|
||||
.eq(["ok"]),
|
||||
"only the account with the hive email counts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_removed_hive_loses_its_token_seat_secret_and_role_in_that_order() {
|
||||
let fake = Fake::holding(&["gone", "kept"]);
|
||||
cut_off_removed(&fake, &roster(&["kept"])).await;
|
||||
assert_eq!(fake.holds("gone"), [false; 4], "{:?}", fake.calls());
|
||||
assert_eq!(
|
||||
fake.calls(),
|
||||
[
|
||||
"delete_token gone",
|
||||
"remove_team_seat gone",
|
||||
"delete_token_secret gone",
|
||||
"delete_role gone",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hive_in_the_roster_is_untouched() {
|
||||
let fake = Fake::holding(&["gone", "kept"]);
|
||||
cut_off_removed(&fake, &roster(&["kept"])).await;
|
||||
assert_eq!(fake.holds("kept"), [true; 4], "{:?}", fake.calls());
|
||||
assert!(fake.calls().iter().all(|c| !c.ends_with("kept")));
|
||||
// The control: the same pass did cut the removed one.
|
||||
assert_eq!(fake.holds("gone"), [false; 4]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_empty_roster_cuts_nothing_off() {
|
||||
let fake = Fake::holding(&["a", "b"]);
|
||||
cut_off_removed(&fake, &[]).await;
|
||||
assert_eq!(fake.holds("a"), [true; 4]);
|
||||
assert_eq!(fake.holds("b"), [true; 4]);
|
||||
assert!(fake.calls().is_empty(), "{:?}", fake.calls());
|
||||
// The control: a roster that names another hive cuts both.
|
||||
cut_off_removed(&fake, &roster(&["c"])).await;
|
||||
assert_eq!(fake.holds("a"), [false; 4]);
|
||||
assert_eq!(fake.holds("b"), [false; 4]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agents_role_is_not_deleted() {
|
||||
let fake = Fake::holding(&["gone"]);
|
||||
fake.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.role
|
||||
.insert("hive-agent-atlas".to_owned());
|
||||
cut_off_removed(&fake, &roster(&["kept"])).await;
|
||||
assert!(fake.state.lock().unwrap().role.contains("hive-agent-atlas"));
|
||||
assert!(
|
||||
fake.calls().iter().all(|c| !c.contains("atlas")),
|
||||
"{:?}",
|
||||
fake.calls()
|
||||
);
|
||||
// The control: the hive's own role went.
|
||||
assert!(!fake.state.lock().unwrap().role.contains("hive-gone"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_step_stops_the_rest_and_keeps_the_role_for_the_next_pass() {
|
||||
for (step, left) in [
|
||||
("delete_token", [true, true, true, true]),
|
||||
("remove_team_seat", [false, true, true, true]),
|
||||
("delete_token_secret", [false, false, true, true]),
|
||||
("delete_role", [false, false, false, true]),
|
||||
] {
|
||||
let broken = Fake::holding(&["gone"]).failing(step);
|
||||
cut_off_removed(&broken, &roster(&["kept"])).await;
|
||||
assert_eq!(broken.holds("gone"), left, "{step}: {:?}", broken.calls());
|
||||
assert!(
|
||||
broken.holds("gone")[3],
|
||||
"{step}: the role is the marker that work is left"
|
||||
);
|
||||
|
||||
// The next pass, against a store and a forge that now accept it.
|
||||
let healthy = Fake {
|
||||
state: Mutex::new(std::mem::take(&mut *broken.state.lock().unwrap())),
|
||||
accounts: broken.accounts.clone(),
|
||||
failing: None,
|
||||
};
|
||||
cut_off_removed(&healthy, &roster(&["kept"])).await;
|
||||
assert_eq!(healthy.holds("gone"), [false; 4], "{step}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_second_pass_finds_nothing_left_to_remove() {
|
||||
let fake = Fake::holding(&["gone"]);
|
||||
let first = cut_off(&fake, "gone", true).await.expect("first");
|
||||
assert_eq!(
|
||||
first,
|
||||
["forge token", "team seat", "token secret", "login role"]
|
||||
);
|
||||
let again = cut_off(&fake, "gone", false).await.expect("second");
|
||||
assert!(again.is_empty(), "{again:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hive_put_back_in_the_roster_is_minted_again() {
|
||||
let fake = Fake::holding(&["back"]);
|
||||
cut_off_removed(&fake, &roster(&["kept"])).await;
|
||||
|
||||
// What hive_token's pass reads of the cut-off hive: no stored token,
|
||||
// none listed, not in the team. The account is still there.
|
||||
let token = classify_named(HIVE_TOKEN_NAME, HIVE_TOKEN_SCOPES, None, &[]);
|
||||
assert_eq!(token, Decision::Mint(MintReason::NotOnForge));
|
||||
let seen = HiveSeen {
|
||||
token: Observed::Decided(token),
|
||||
member: Some(fake.holds("back")[1]),
|
||||
};
|
||||
assert_eq!(
|
||||
plan(TeamSeen::Matches(7), &[("back".to_owned(), seen)]),
|
||||
["back"]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -37,7 +37,7 @@ pub const HIVE_TOKEN_NAME: &str = "swarm-hive";
|
|||
pub const HIVE_TOKEN_SCOPES: &str = "read:repository";
|
||||
|
||||
/// The team in `agent-configs` every hive account is a member of.
|
||||
const HIVES_TEAM: &str = "hives";
|
||||
pub(super) const HIVES_TEAM: &str = "hives";
|
||||
|
||||
const HIVES_TEAM_DESCRIPTION: &str = "hyperhive hives — read on every agent config repo";
|
||||
|
||||
|
|
@ -60,7 +60,7 @@ fn hive_email(hive: &str) -> String {
|
|||
|
||||
/// Whether the account named [`forge::hive_login`]`(hive)`, carrying `email`,
|
||||
/// is the one this module creates.
|
||||
fn is_hive_account(hive: &str, email: Option<&str>) -> bool {
|
||||
pub(super) fn is_hive_account(hive: &str, email: Option<&str>) -> bool {
|
||||
email == Some(hive_email(hive).as_str())
|
||||
}
|
||||
|
||||
|
|
@ -393,16 +393,27 @@ impl Client {
|
|||
/// node for each.
|
||||
///
|
||||
/// The roster is the controller's hive directory. A pass that fails is
|
||||
/// logged and retried on the next tick; it never stops the daemon.
|
||||
/// logged and retried on the next tick; it never stops the daemon. With
|
||||
/// `cut_off_removed`, each tick also revokes the credentials of the hives
|
||||
/// that are not in `hives` ([`super::hive_cutoff`]); the caller passes `false`
|
||||
/// for a roster that did not load.
|
||||
pub fn spawn(
|
||||
client: Arc<Client>,
|
||||
hives: Vec<String>,
|
||||
cut_off_removed: bool,
|
||||
enqueue: impl Fn(Vec<String>) + Send + 'static,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
if cut_off_removed {
|
||||
super::within_forge_deadline(
|
||||
"hive credential cut-off",
|
||||
super::hive_cutoff::run(&client, &hives),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let Some(outcome) =
|
||||
super::within_forge_deadline("hive forge tokens", client.observe_hives(&hives))
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -487,6 +487,7 @@ pub(crate) async fn mint_identity(
|
|||
let declarations = match wanted {
|
||||
Some(writer) => {
|
||||
let hives: Vec<String> = crate::startup::load_hives()
|
||||
.into_entries()
|
||||
.into_iter()
|
||||
.map(|h| h.name)
|
||||
.collect();
|
||||
|
|
@ -1954,6 +1955,7 @@ fn read_meta_deps() -> Option<MetaDeps> {
|
|||
Some(MetaDeps {
|
||||
settings,
|
||||
hives: crate::startup::load_hives()
|
||||
.into_entries()
|
||||
.into_iter()
|
||||
.map(|h| h.name)
|
||||
.collect(),
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ async fn main() -> Result<()> {
|
|||
hive_jobq::Graph::new(),
|
||||
hive_jobq::resources::ResourceTable::new(),
|
||||
)));
|
||||
let hives = load_hives();
|
||||
let hives = load_hives().into_entries();
|
||||
spawn_matrix_account_backfill(&jobq, deps.matrix_homeserver.clone(), auth.clone());
|
||||
spawn_jobq_worker(Arc::clone(&jobq), deps);
|
||||
// Bound to a named variable, not `_` — dropping the provider stops its
|
||||
|
|
|
|||
|
|
@ -358,11 +358,11 @@ async fn write_agent_policy_and_entities(
|
|||
mod tests {
|
||||
use std::{cell::Cell, time::Duration};
|
||||
|
||||
use crate::store::fake::FakeAccess;
|
||||
use crate::store::{AccessStore, fake::FakeAccess};
|
||||
|
||||
use super::{
|
||||
RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, attached_policies, policy,
|
||||
provision_with, retry_until_ok, write_agent_policy_and_entities,
|
||||
provision_with, retry_until_ok, write_agent_policy_and_entities, write_role,
|
||||
};
|
||||
|
||||
#[test]
|
||||
|
|
@ -472,6 +472,20 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hive_whose_role_was_deleted_gets_it_back_when_it_is_in_the_roster_again() {
|
||||
let store = FakeAccess::default();
|
||||
write_role(&store, "pr1ma", "CA").await.expect("accepted");
|
||||
assert!(store.role("hive-pr1ma").is_some());
|
||||
store
|
||||
.delete_cert_role("auth/cert", "hive-pr1ma")
|
||||
.await
|
||||
.expect("accepted");
|
||||
assert_eq!(store.role("hive-pr1ma"), None);
|
||||
write_role(&store, "pr1ma", "CA").await.expect("accepted");
|
||||
assert!(store.role("hive-pr1ma").is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_an_authority_the_policy_is_written_and_no_role_is() {
|
||||
let store = FakeAccess::default();
|
||||
|
|
|
|||
|
|
@ -91,19 +91,51 @@ pub(crate) struct ApiDoc;
|
|||
/// (this daemon has no "self" to exclude) rather than peers-minus-self.
|
||||
const HIVES_ENV: &str = "SWARM_CONTROLLER_HIVES";
|
||||
|
||||
/// How [`load_hives`] read [`HIVES_ENV`].
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum HiveRoster {
|
||||
Loaded(Vec<HiveEntry>),
|
||||
/// [`HIVES_ENV`] is not set.
|
||||
Unset,
|
||||
/// [`HIVES_ENV`] is set and is not a hive directory.
|
||||
Unparseable,
|
||||
}
|
||||
|
||||
impl HiveRoster {
|
||||
pub(crate) fn into_entries(self) -> Vec<HiveEntry> {
|
||||
match self {
|
||||
Self::Loaded(hives) => hives,
|
||||
Self::Unset | Self::Unparseable => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an absence from the roster can be read as a removal: the roster
|
||||
/// loaded and holds a hive. An unset, unparseable or empty roster reads
|
||||
/// the same as "every hive removed", so it is refused with the reason.
|
||||
pub(crate) fn absence_means_removal(&self) -> Result<(), &'static str> {
|
||||
match self {
|
||||
Self::Loaded(hives) if !hives.is_empty() => Ok(()),
|
||||
Self::Loaded(_) => Err("the hive roster is empty"),
|
||||
Self::Unset => Err("the hive roster env var is not set"),
|
||||
Self::Unparseable => Err("the hive roster env var does not parse"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses [`HIVES_ENV`] into the swarm's hive directory. Unset or
|
||||
/// unparseable both fall back to an empty list with a warning rather than
|
||||
/// failing startup — a swarm-controller that can't yet see its own config
|
||||
/// (dev run, a module not wired up yet) should still serve `/health`.
|
||||
pub(crate) fn load_hives() -> Vec<HiveEntry> {
|
||||
/// unparseable both serve an empty list rather than failing startup — a
|
||||
/// swarm-controller that can't yet see its own config (dev run, a module not
|
||||
/// wired up yet) should still serve `/health` — and the [`HiveRoster`] says
|
||||
/// which one it was.
|
||||
pub(crate) fn load_hives() -> HiveRoster {
|
||||
let Some(raw) = std::env::var_os(HIVES_ENV) else {
|
||||
return Vec::new();
|
||||
return HiveRoster::Unset;
|
||||
};
|
||||
match serde_json::from_str(&raw.to_string_lossy()) {
|
||||
Ok(hives) => hives,
|
||||
Ok(hives) => HiveRoster::Loaded(hives),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, env = HIVES_ENV, "failed to parse hive directory, serving an empty list");
|
||||
Vec::new()
|
||||
HiveRoster::Unparseable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -288,11 +320,26 @@ fn spawn_forge_workers(
|
|||
}
|
||||
let sched = Arc::clone(jobq);
|
||||
let hive_names = hives.iter().map(|h| h.name.clone()).collect();
|
||||
forge::hive_token::spawn(Arc::clone(&client), hive_names, move |hives| {
|
||||
if let Err(e) = queue_hive_forge_token_mints(&sched, hives) {
|
||||
tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed");
|
||||
let cut_off_removed = match load_hives().absence_means_removal() {
|
||||
Ok(()) => true,
|
||||
Err(reason) => {
|
||||
tracing::error!(
|
||||
reason,
|
||||
"removed hives are not cut off from the forge and the store until the roster loads"
|
||||
);
|
||||
false
|
||||
}
|
||||
});
|
||||
};
|
||||
forge::hive_token::spawn(
|
||||
Arc::clone(&client),
|
||||
hive_names,
|
||||
cut_off_removed,
|
||||
move |hives| {
|
||||
if let Err(e) = queue_hive_forge_token_mints(&sched, hives) {
|
||||
tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed");
|
||||
}
|
||||
},
|
||||
);
|
||||
let sched = Arc::clone(jobq);
|
||||
forge::classifier_account::spawn(Arc::clone(&client), move || {
|
||||
if let Err(e) = queue_classifier_forge_token_mint(&sched) {
|
||||
|
|
@ -773,21 +820,37 @@ mod tests {
|
|||
unsafe {
|
||||
std::env::remove_var(HIVES_ENV);
|
||||
}
|
||||
let unset = load_hives();
|
||||
assert_eq!(unset, HiveRoster::Unset);
|
||||
assert_eq!(
|
||||
load_hives(),
|
||||
unset.clone().into_entries(),
|
||||
Vec::<HiveEntry>::new(),
|
||||
"unset env var is an empty directory, not a startup failure"
|
||||
);
|
||||
assert!(unset.absence_means_removal().is_err());
|
||||
|
||||
unsafe {
|
||||
std::env::set_var(HIVES_ENV, "not json");
|
||||
}
|
||||
let garbled = load_hives();
|
||||
assert_eq!(garbled, HiveRoster::Unparseable);
|
||||
assert_eq!(
|
||||
load_hives(),
|
||||
garbled.clone().into_entries(),
|
||||
Vec::<HiveEntry>::new(),
|
||||
"unparseable env var falls back to empty rather than panicking — \
|
||||
/health must still answer even if this daemon's own config is wrong"
|
||||
);
|
||||
assert!(garbled.absence_means_removal().is_err());
|
||||
|
||||
unsafe {
|
||||
std::env::set_var(HIVES_ENV, "[]");
|
||||
}
|
||||
let empty = load_hives();
|
||||
assert_eq!(empty, HiveRoster::Loaded(Vec::new()));
|
||||
assert!(
|
||||
empty.absence_means_removal().is_err(),
|
||||
"a roster that loaded empty reads as every hive removed"
|
||||
);
|
||||
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
|
|
@ -795,9 +858,10 @@ mod tests {
|
|||
r#"[{"name":"pr1ma","domain":"pr1ma.example.com","dashboard":"pr1ma.hives.example"},{"name":"umbra","domain":"umbra.example.com","dashboard":"umbra.hives.example"}]"#,
|
||||
);
|
||||
}
|
||||
let loaded = load_hives();
|
||||
assert_eq!(
|
||||
load_hives(),
|
||||
vec![
|
||||
loaded.clone().into_entries(),
|
||||
[
|
||||
HiveEntry {
|
||||
name: "pr1ma".to_string(),
|
||||
domain: "pr1ma.example.com".to_string(),
|
||||
|
|
@ -810,6 +874,7 @@ mod tests {
|
|||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(loaded.absence_means_removal(), Ok(()));
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var(HIVES_ENV);
|
||||
|
|
|
|||
|
|
@ -111,6 +111,32 @@ pub fn agents_from_role_names(names: &[String]) -> Vec<String> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// The hive that holds the cert-auth role or forge login `name`: the inverse
|
||||
/// of [`hive_object_name`], for a caller that needs the hives with a role.
|
||||
///
|
||||
/// `None` for an agent's object ([`AGENT_PREFIX`]), which shares the
|
||||
/// [`HIVE_PREFIX`] and is never a hive's, and for a name whose suffix is not a
|
||||
/// legal segment.
|
||||
#[must_use]
|
||||
pub fn hive_from_object_name(name: &str) -> Option<&str> {
|
||||
if name.starts_with(AGENT_PREFIX) {
|
||||
return None;
|
||||
}
|
||||
name.strip_prefix(HIVE_PREFIX)
|
||||
.filter(|hive| checked_segment("hive", hive).is_ok())
|
||||
}
|
||||
|
||||
/// The hives among `names`, a listing of cert-auth roles
|
||||
/// ([`hive_from_object_name`] for each).
|
||||
#[must_use]
|
||||
pub fn hives_from_role_names(names: &[String]) -> Vec<String> {
|
||||
names
|
||||
.iter()
|
||||
.filter_map(|name| hive_from_object_name(name))
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// One read stanza. With [`list_stanza`], [`create_stanza`] and [`sign_stanza`],
|
||||
/// the only shapes this module emits, so what an agent may write is a property
|
||||
/// of the renderers rather than of each call site.
|
||||
|
|
@ -753,4 +779,26 @@ mod tests {
|
|||
let name = agent_object_name("atlas").expect("legal");
|
||||
assert_eq!(agents_from_role_names(&[name]), ["atlas"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_hive_roles_come_back_and_without_their_prefix() {
|
||||
let roles: Vec<String> = [
|
||||
"hive-pr1ma",
|
||||
"hive-agent-atlas",
|
||||
"swarm-controller",
|
||||
"hive-umbra",
|
||||
"hive-",
|
||||
"hive-a.b",
|
||||
]
|
||||
.map(str::to_owned)
|
||||
.to_vec();
|
||||
assert_eq!(hives_from_role_names(&roles), ["pr1ma", "umbra"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_hive_listing_inverts_hive_object_name() {
|
||||
let name = hive_object_name("pr1ma").expect("legal");
|
||||
assert_eq!(hive_from_object_name(&name), Some("pr1ma"));
|
||||
assert_eq!(hive_from_object_name("hive-agent-pr1ma"), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue