Watch
0
0
Fork
You've already forked hyperhive
0

disable a destroyed agent's identity entity and delete its login role

This commit is contained in:
atlas 2026-11-16 16:50:53 +01:00
commit 299b922039
5 changed files with 528 additions and 9 deletions

View file

@ -0,0 +1,291 @@
//! Cutting a destroyed agent's identity off at the store.
//!
//! Destroying an agent stops its renewal and nothing else: its client
//! certificate stays valid for days, and the cert-auth role it logs in under
//! stays in the store. [`cut_off`] ends both. It disables the agent's identity
//! entity, which the store enforces at once — tokens already issued stop
//! working and a new login is refused — and then deletes the role, so the
//! certificate cannot start a login.
//!
//! The renewal pass ([`crate::agent_renewal`]) calls [`cut_off_destroyed`] on
//! every agent that still has a role and is declared `Destroyed` on some hive
//! and live on none ([`cut_off_targets`]). The role is the marker that work is
//! left, so the entity is disabled before the role is deleted: a failure
//! between the two leaves the role, and the next pass runs both again. An
//! agent declared nowhere is not cut off, since an unreadable or emptied
//! declaration would otherwise read as every agent destroyed.
//!
//! A new agent with the same name is brought back by [`restore`], which the
//! mint runs ([`crate::agent_identity::mint_and_verify`]) before it writes the
//! role. The entity keeps its name and alias throughout, so the shared policy
//! reads the same entity afterwards.
//!
//! The grants are the `identity/entity/name/+` and `auth/cert/certs/hive-*`
//! stanzas of `swarm-bao.nix`'s `controllerPolicyText`.
use std::collections::BTreeSet;
use anyhow::{Context, Result};
use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, policy};
use swarm_wire::wanted::{AgentState, HiveWanted};
use crate::store::AccessStore;
/// Agents that are declared `Destroyed` on at least one hive and live on none.
fn destroyed_agents(declarations: &[HiveWanted]) -> BTreeSet<String> {
let live = crate::agent_renewal::live_agents(declarations);
declarations
.iter()
.flat_map(|d| &d.agents)
.filter(|(agent, wanted)| wanted.state == AgentState::Destroyed && !live.contains(*agent))
.map(|(agent, _)| agent.clone())
.collect()
}
/// The agents of `with_role`, those that have a cert-auth role, that are
/// destroyed in `declarations`.
pub fn cut_off_targets(
with_role: &BTreeSet<String>,
declarations: &[HiveWanted],
) -> BTreeSet<String> {
destroyed_agents(declarations)
.intersection(with_role)
.cloned()
.collect()
}
/// Disable `agent`'s identity entity, then delete its cert-auth role.
///
/// A missing entity is not an error: an agent that never logged in has none,
/// and disabling would make one. Safe to repeat.
///
/// # Errors
/// When a store call fails. The entity is disabled before the role is deleted,
/// so a failed role delete leaves the entity disabled and the role in place.
pub async fn cut_off(store: &impl AccessStore, agent: &str) -> Result<()> {
let role = policy::agent_object_name(agent)?;
let entity = store
.entity_id_by_name(agent)
.await
.with_context(|| format!("looking up the entity named {agent}"))?;
if entity.is_some() {
store
.set_entity_disabled(agent, true)
.await
.with_context(|| format!("disabling the entity {agent}"))?;
}
store
.delete_cert_role(DEFAULT_CERT_MOUNT, &role)
.await
.with_context(|| format!("deleting the cert-auth role {role}"))?;
tracing::info!(
agent,
entity_disabled = entity.is_some(),
%role,
"destroyed agent cut off from the store: identity entity disabled, cert-auth role deleted"
);
Ok(())
}
/// Undo [`cut_off`]'s disable for `agent`, whose entity exists.
///
/// Called for every mint, a live agent's renewal included, where it rewrites a
/// flag that is already clear.
///
/// # Errors
/// When the store refuses the write.
pub async fn restore(store: &impl AccessStore, agent: &str) -> Result<()> {
store
.set_entity_disabled(agent, false)
.await
.with_context(|| format!("enabling the entity {agent}"))
}
/// [`cut_off`] every agent of `with_role` that `declarations` leave destroyed
/// ([`cut_off_targets`]), logging the ones that fail; the next pass retries
/// them, since their role is still there.
pub async fn cut_off_destroyed(
store: &impl AccessStore,
with_role: &BTreeSet<String>,
declarations: &[HiveWanted],
) {
for agent in cut_off_targets(with_role, declarations) {
if let Err(e) = cut_off(store, &agent).await {
tracing::warn!(
agent,
error = %format!("{e:#}"),
"cutting this destroyed agent off from the store failed; its certificate still logs in until the next pass succeeds"
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::store::fake::{FakeAccess, FakeRole};
use std::collections::BTreeMap;
use swarm_secret_client::client::{CERT_ROLE_TOKEN_TTL_SECS, CertRoleTokenTtls};
use swarm_wire::wanted::AgentWanted;
fn declared(agents: &[(&str, AgentState)]) -> HiveWanted {
HiveWanted {
agents: agents
.iter()
.map(|(name, state)| ((*name).to_owned(), AgentWanted { state: *state }))
.collect::<BTreeMap<_, _>>(),
}
}
fn role() -> FakeRole {
FakeRole {
certificate: "CA".to_owned(),
common_name: "hive-agent-x".to_owned(),
policies: vec!["hive-agent".to_owned()],
ttls: CertRoleTokenTtls {
token_ttl: CERT_ROLE_TOKEN_TTL_SECS,
token_max_ttl: CERT_ROLE_TOKEN_TTL_SECS,
},
}
}
fn with_roles(names: &[&str]) -> BTreeSet<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
/// A store that holds `agents`' roles and entities.
fn store_with(agents: &[&str]) -> FakeAccess {
agents.iter().fold(FakeAccess::default(), |store, a| {
store
.with_role(&format!("hive-agent-{a}"), role())
.with_entity(a)
})
}
#[test]
fn only_an_agent_destroyed_everywhere_and_holding_a_role_is_a_target() {
let declarations = [
declared(&[
("gone", AgentState::Destroyed),
("moved", AgentState::Destroyed),
("alive", AgentState::Up),
("asleep", AgentState::Offline),
("held", AgentState::Paused),
("no-role", AgentState::Destroyed),
]),
declared(&[("moved", AgentState::Up)]),
];
let roles = with_roles(&[
"gone",
"moved",
"alive",
"asleep",
"held",
"undeclared",
"unnamed",
]);
assert_eq!(
cut_off_targets(&roles, &declarations),
with_roles(&["gone"]),
"destroyed on one hive and live on another is live; declared nowhere is left alone"
);
// The control: without the live declaration on `b`, `moved` is a target
// too, so the line above can see a destroyed agent.
assert_eq!(
cut_off_targets(&roles, &declarations[..1]),
with_roles(&["gone", "moved"])
);
}
#[tokio::test]
async fn a_destroyed_agent_loses_its_entity_and_its_role() {
let store = store_with(&["gone"]);
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
cut_off_destroyed(&store, &with_roles(&["gone"]), &declarations).await;
assert!(store.is_disabled("gone"), "{:?}", store.calls());
assert_eq!(store.role("hive-agent-gone"), None, "{:?}", store.calls());
}
#[tokio::test]
async fn a_live_agent_is_never_touched() {
let store = store_with(&["alive", "gone"]);
let declarations = [declared(&[
("alive", AgentState::Up),
("gone", AgentState::Destroyed),
])];
cut_off_destroyed(&store, &with_roles(&["alive", "gone"]), &declarations).await;
assert!(!store.is_disabled("alive"), "{:?}", store.calls());
assert_eq!(store.role("hive-agent-alive"), Some(role()));
// The control: the same pass did cut the destroyed one off.
assert!(store.is_disabled("gone"));
assert_eq!(store.role("hive-agent-gone"), None);
}
#[tokio::test]
async fn an_agent_without_an_entity_has_its_role_deleted_and_no_entity_made() {
let store = FakeAccess::default().with_role("hive-agent-gone", role());
cut_off(&store, "gone").await.expect("accepted");
assert_eq!(store.role("hive-agent-gone"), None);
assert!(
!store
.calls()
.iter()
.any(|c| c.starts_with("set_entity_disabled")),
"{:?}",
store.calls()
);
}
#[tokio::test]
async fn a_failed_disable_keeps_the_role_so_the_next_pass_retries() {
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
let refused = store_with(&["gone"]).refusing_disable();
cut_off_destroyed(&refused, &with_roles(&["gone"]), &declarations).await;
assert_eq!(
refused.role("hive-agent-gone"),
Some(role()),
"the role is the marker that work is left"
);
assert!(
!refused
.calls()
.iter()
.any(|c| c.starts_with("delete_cert_role")),
"{:?}",
refused.calls()
);
}
#[tokio::test]
async fn a_failed_role_delete_leaves_the_entity_disabled_and_the_next_pass_finishes() {
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
let refused = store_with(&["gone"]).refusing_role_delete();
cut_off_destroyed(&refused, &with_roles(&["gone"]), &declarations).await;
assert!(refused.is_disabled("gone"));
assert_eq!(refused.role("hive-agent-gone"), Some(role()));
// The next pass, against a store that now accepts the delete.
let healthy = store_with(&["gone"]);
cut_off_destroyed(&healthy, &with_roles(&["gone"]), &declarations).await;
assert!(healthy.is_disabled("gone"));
assert_eq!(healthy.role("hive-agent-gone"), None);
}
#[tokio::test]
async fn a_cut_off_is_repeatable() {
let store = store_with(&["gone"]);
cut_off(&store, "gone").await.expect("first");
cut_off(&store, "gone").await.expect("second");
assert!(store.is_disabled("gone"));
assert_eq!(store.role("hive-agent-gone"), None);
}
#[tokio::test]
async fn restore_clears_what_a_cut_off_set() {
let store = store_with(&["gone"]);
cut_off(&store, "gone").await.expect("accepted");
assert!(store.is_disabled("gone"));
restore(&store, "gone").await.expect("accepted");
assert!(!store.is_disabled("gone"));
}
}

View file

@ -8,8 +8,8 @@
//! and its grant covers exactly the calls made here (`swarm-bao.nix`'s
//! `controllerPolicyText`: `update` on the agent PKI role's `issue` path, and
//! `create/update` on `secret/data/swarm/agents/*`, on
//! `sys/policies/acl/hive-*`, and on `auth/cert/certs/hive-*`, plus the
//! `identity/` and `sys/auth/cert` stanzas [`crate::agent_entity`] uses).
//! `sys/policies/acl/hive-*`, and on `auth/cert/certs/hive-*` (`delete` too), plus the
//! `identity/` and `sys/auth/cert` stanzas [`crate::agent_entity`] and [`crate::agent_cutoff`] use).
//!
//! **Two credentials, deliberately unrelated.** The certificate reaches the
//! store; the queue secret identifies the agent to the swarm queue. Both sit
@ -80,8 +80,11 @@ fn role_inputs<'a>(agent: &str, credential: &'a mtls::Credential) -> Result<Role
})
}
/// Write the shared policy, name the agent's entity, then write the cert-auth
/// role that attaches the policy.
/// Write the shared policy, name the agent's entity and enable it, then write
/// the cert-auth role that attaches the policy.
///
/// Enabling clears the flag [`crate::agent_cutoff::cut_off`] set when an agent
/// of this name was destroyed, so a new agent of that name can log in.
///
/// The entity comes before the role and is a precondition for it. The role
/// attaches the shared policy alone, so an agent whose entity is not named after
@ -108,6 +111,10 @@ async fn publish_access(
.await
.with_context(|| format!("naming {agent}'s identity entity; the role is not changed"))
.at(MintStep::Entity)?;
crate::agent_cutoff::restore(store, agent)
.await
.with_context(|| format!("enabling {agent}'s identity entity; the role is not changed"))
.at(MintStep::Entity)?;
store
.write_cert_role(
DEFAULT_CERT_MOUNT,
@ -235,7 +242,7 @@ impl<T, E: Into<anyhow::Error>> AtStep<T> for Result<T, E> {
/// presents names *it* rather than its hive;
/// 4. write the shared ACL document [`policy::render_shared_agent`] renders;
/// 5. name the agent's identity entity after the agent ([`crate::agent_entity`]),
/// which the policy reads;
/// which the policy reads, and enable it ([`crate::agent_cutoff::restore`]);
/// 6. write the cert-auth role that ties the three together, attaching that
/// policy and pinning the CA the store named as this leaf's issuer.
///
@ -610,6 +617,45 @@ mod tests {
);
}
/// An agent created under the name of a destroyed one gets its entity
/// enabled and its role back, the enable first.
#[tokio::test]
async fn re_creating_a_cut_off_agent_enables_its_entity_and_writes_its_role() {
let store = FakeAccess::default().with_entity("atlas");
crate::agent_cutoff::cut_off(&store, "atlas")
.await
.expect("accepted");
assert!(store.is_disabled("atlas"), "the control: it was cut off");
assert_eq!(store.role("hive-agent-atlas"), None);
let credential = credential();
let inputs = role_inputs("atlas", &credential).expect("legal");
publish_access(&store, "atlas", &inputs)
.await
.expect("a store that accepts everything");
assert!(!store.is_disabled("atlas"), "{:?}", store.calls());
assert!(store.role("hive-agent-atlas").is_some());
let calls = store.calls();
let at = |call: &str| calls.iter().position(|c| c == call).expect(call);
assert!(
at("set_entity_disabled atlas false")
< at("write_cert_role hive-agent-atlas hive-agent"),
"the entity is enabled before the role is written: {calls:?}"
);
}
#[tokio::test]
async fn a_mint_whose_enable_is_refused_leaves_the_role_alone() {
let store = FakeAccess::default().refusing_disable();
let credential = credential();
let inputs = role_inputs("atlas", &credential).expect("legal");
let err = publish_access(&store, "atlas", &inputs)
.await
.expect_err("the enable is refused");
assert_eq!(failed_step(&err), Some(MintStep::Entity), "{err:#}");
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
}
#[tokio::test]
async fn a_renewal_whose_entity_step_fails_leaves_the_role_alone() {
let store = FakeAccess::without_accessor();

View file

@ -348,6 +348,11 @@ async fn observe_all(
None
}
};
// Destroyed agents that still have a role are cut off here, each pass until
// it succeeds.
if let Some(with_role) = &roles {
crate::agent_cutoff::cut_off_destroyed(&store, with_role, &declarations).await;
}
let mut observed = Vec::new();
for agent in live_agents(&declarations) {
let o = AgentObserved {

View file

@ -26,6 +26,7 @@ mod agent_commands;
mod agent_config_repo;
mod agent_control;
mod agent_custom;
mod agent_cutoff;
mod agent_entity;
mod agent_icon;
mod agent_identity;

View file

@ -91,6 +91,20 @@ pub(crate) trait AccessStore {
fn rename_entity(&self, id: &str, name: &str)
-> impl Future<Output = Result<(), Error>> + Send;
/// As [`SecretStore::set_entity_disabled`].
fn set_entity_disabled(
&self,
name: &str,
disabled: bool,
) -> impl Future<Output = Result<(), Error>> + Send;
/// As [`SecretStore::delete_cert_role`].
fn delete_cert_role(
&self,
mount: &str,
name: &str,
) -> impl Future<Output = Result<(), Error>> + Send;
/// As [`SecretStore::create_entity_alias`].
fn create_entity_alias(
&self,
@ -160,6 +174,14 @@ impl AccessStore for SecretStore {
) -> Result<(), Error> {
SecretStore::create_entity_alias(self, alias_name, mount_accessor, entity_id).await
}
async fn set_entity_disabled(&self, name: &str, disabled: bool) -> Result<(), Error> {
SecretStore::set_entity_disabled(self, name, disabled).await
}
async fn delete_cert_role(&self, mount: &str, name: &str) -> Result<(), Error> {
SecretStore::delete_cert_role(self, mount, name).await
}
}
/// A store that records what it is asked to do, for tests of the order and the
@ -204,6 +226,10 @@ pub(crate) mod fake {
/// The entity that owns each alias, as `(id, name)`.
alias_owner: Option<(String, String)>,
entities: Mutex<Vec<String>>,
/// The entities, by name, whose `disabled` flag is set.
disabled: Mutex<std::collections::BTreeSet<String>>,
/// The calls, by verb, that are refused.
refused: Vec<&'static str>,
calls: Mutex<Vec<String>>,
}
@ -261,6 +287,32 @@ pub(crate) mod fake {
self
}
/// A store that already holds the entity `name`.
pub(crate) fn with_entity(self, name: &str) -> Self {
self.entities
.lock()
.expect("not poisoned")
.push(name.to_owned());
self
}
/// A store that refuses to set an entity's `disabled` flag.
pub(crate) fn refusing_disable(mut self) -> Self {
self.refused.push("set_entity_disabled");
self
}
/// A store that refuses to delete a cert-auth role.
pub(crate) fn refusing_role_delete(mut self) -> Self {
self.refused.push("delete_cert_role");
self
}
/// Whether the entity `name` is disabled now.
pub(crate) fn is_disabled(&self, name: &str) -> bool {
self.disabled.lock().expect("not poisoned").contains(name)
}
/// The role `name` as the store holds it now.
pub(crate) fn role(&self, name: &str) -> Option<FakeRole> {
self.roles.lock().expect("not poisoned").get(name).cloned()
@ -403,5 +455,28 @@ pub(crate) mod fake {
self.log(format!("create_entity_alias {alias_name}"));
Ok(())
}
async fn set_entity_disabled(&self, name: &str, disabled: bool) -> Result<(), Error> {
self.log(format!("set_entity_disabled {name} {disabled}"));
if self.refused.contains(&"set_entity_disabled") {
return Err(Self::denied());
}
let mut flagged = self.disabled.lock().expect("not poisoned");
if disabled {
flagged.insert(name.to_owned());
} else {
flagged.remove(name);
}
Ok(())
}
async fn delete_cert_role(&self, _mount: &str, name: &str) -> Result<(), Error> {
self.log(format!("delete_cert_role {name}"));
if self.refused.contains(&"delete_cert_role") {
return Err(Self::denied());
}
self.roles.lock().expect("not poisoned").remove(name);
Ok(())
}
}
}