disable a destroyed agent's identity entity and delete its login role
This commit is contained in:
parent
db924d9e77
commit
299b922039
5 changed files with 528 additions and 9 deletions
291
swarm-controller/src/agent_cutoff.rs
Normal file
291
swarm-controller/src/agent_cutoff.rs
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
//! Cutting a destroyed agent's identity off at the store.
|
||||
//!
|
||||
//! Destroying an agent stops its renewal and nothing else: its client
|
||||
//! certificate stays valid for days, and the cert-auth role it logs in under
|
||||
//! stays in the store. [`cut_off`] ends both. It disables the agent's identity
|
||||
//! entity, which the store enforces at once — tokens already issued stop
|
||||
//! working and a new login is refused — and then deletes the role, so the
|
||||
//! certificate cannot start a login.
|
||||
//!
|
||||
//! The renewal pass ([`crate::agent_renewal`]) calls [`cut_off_destroyed`] on
|
||||
//! every agent that still has a role and is declared `Destroyed` on some hive
|
||||
//! and live on none ([`cut_off_targets`]). The role is the marker that work is
|
||||
//! left, so the entity is disabled before the role is deleted: a failure
|
||||
//! between the two leaves the role, and the next pass runs both again. An
|
||||
//! agent declared nowhere is not cut off, since an unreadable or emptied
|
||||
//! declaration would otherwise read as every agent destroyed.
|
||||
//!
|
||||
//! A new agent with the same name is brought back by [`restore`], which the
|
||||
//! mint runs ([`crate::agent_identity::mint_and_verify`]) before it writes the
|
||||
//! role. The entity keeps its name and alias throughout, so the shared policy
|
||||
//! reads the same entity afterwards.
|
||||
//!
|
||||
//! The grants are the `identity/entity/name/+` and `auth/cert/certs/hive-*`
|
||||
//! stanzas of `swarm-bao.nix`'s `controllerPolicyText`.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, policy};
|
||||
use swarm_wire::wanted::{AgentState, HiveWanted};
|
||||
|
||||
use crate::store::AccessStore;
|
||||
|
||||
/// Agents that are declared `Destroyed` on at least one hive and live on none.
|
||||
fn destroyed_agents(declarations: &[HiveWanted]) -> BTreeSet<String> {
|
||||
let live = crate::agent_renewal::live_agents(declarations);
|
||||
declarations
|
||||
.iter()
|
||||
.flat_map(|d| &d.agents)
|
||||
.filter(|(agent, wanted)| wanted.state == AgentState::Destroyed && !live.contains(*agent))
|
||||
.map(|(agent, _)| agent.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The agents of `with_role`, those that have a cert-auth role, that are
|
||||
/// destroyed in `declarations`.
|
||||
pub fn cut_off_targets(
|
||||
with_role: &BTreeSet<String>,
|
||||
declarations: &[HiveWanted],
|
||||
) -> BTreeSet<String> {
|
||||
destroyed_agents(declarations)
|
||||
.intersection(with_role)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Disable `agent`'s identity entity, then delete its cert-auth role.
|
||||
///
|
||||
/// A missing entity is not an error: an agent that never logged in has none,
|
||||
/// and disabling would make one. Safe to repeat.
|
||||
///
|
||||
/// # Errors
|
||||
/// When a store call fails. The entity is disabled before the role is deleted,
|
||||
/// so a failed role delete leaves the entity disabled and the role in place.
|
||||
pub async fn cut_off(store: &impl AccessStore, agent: &str) -> Result<()> {
|
||||
let role = policy::agent_object_name(agent)?;
|
||||
let entity = store
|
||||
.entity_id_by_name(agent)
|
||||
.await
|
||||
.with_context(|| format!("looking up the entity named {agent}"))?;
|
||||
if entity.is_some() {
|
||||
store
|
||||
.set_entity_disabled(agent, true)
|
||||
.await
|
||||
.with_context(|| format!("disabling the entity {agent}"))?;
|
||||
}
|
||||
store
|
||||
.delete_cert_role(DEFAULT_CERT_MOUNT, &role)
|
||||
.await
|
||||
.with_context(|| format!("deleting the cert-auth role {role}"))?;
|
||||
tracing::info!(
|
||||
agent,
|
||||
entity_disabled = entity.is_some(),
|
||||
%role,
|
||||
"destroyed agent cut off from the store: identity entity disabled, cert-auth role deleted"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Undo [`cut_off`]'s disable for `agent`, whose entity exists.
|
||||
///
|
||||
/// Called for every mint, a live agent's renewal included, where it rewrites a
|
||||
/// flag that is already clear.
|
||||
///
|
||||
/// # Errors
|
||||
/// When the store refuses the write.
|
||||
pub async fn restore(store: &impl AccessStore, agent: &str) -> Result<()> {
|
||||
store
|
||||
.set_entity_disabled(agent, false)
|
||||
.await
|
||||
.with_context(|| format!("enabling the entity {agent}"))
|
||||
}
|
||||
|
||||
/// [`cut_off`] every agent of `with_role` that `declarations` leave destroyed
|
||||
/// ([`cut_off_targets`]), logging the ones that fail; the next pass retries
|
||||
/// them, since their role is still there.
|
||||
pub async fn cut_off_destroyed(
|
||||
store: &impl AccessStore,
|
||||
with_role: &BTreeSet<String>,
|
||||
declarations: &[HiveWanted],
|
||||
) {
|
||||
for agent in cut_off_targets(with_role, declarations) {
|
||||
if let Err(e) = cut_off(store, &agent).await {
|
||||
tracing::warn!(
|
||||
agent,
|
||||
error = %format!("{e:#}"),
|
||||
"cutting this destroyed agent off from the store failed; its certificate still logs in until the next pass succeeds"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::fake::{FakeAccess, FakeRole};
|
||||
use std::collections::BTreeMap;
|
||||
use swarm_secret_client::client::{CERT_ROLE_TOKEN_TTL_SECS, CertRoleTokenTtls};
|
||||
use swarm_wire::wanted::AgentWanted;
|
||||
|
||||
fn declared(agents: &[(&str, AgentState)]) -> HiveWanted {
|
||||
HiveWanted {
|
||||
agents: agents
|
||||
.iter()
|
||||
.map(|(name, state)| ((*name).to_owned(), AgentWanted { state: *state }))
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
}
|
||||
}
|
||||
|
||||
fn role() -> FakeRole {
|
||||
FakeRole {
|
||||
certificate: "CA".to_owned(),
|
||||
common_name: "hive-agent-x".to_owned(),
|
||||
policies: vec!["hive-agent".to_owned()],
|
||||
ttls: CertRoleTokenTtls {
|
||||
token_ttl: CERT_ROLE_TOKEN_TTL_SECS,
|
||||
token_max_ttl: CERT_ROLE_TOKEN_TTL_SECS,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn with_roles(names: &[&str]) -> BTreeSet<String> {
|
||||
names.iter().map(|n| (*n).to_owned()).collect()
|
||||
}
|
||||
|
||||
/// A store that holds `agents`' roles and entities.
|
||||
fn store_with(agents: &[&str]) -> FakeAccess {
|
||||
agents.iter().fold(FakeAccess::default(), |store, a| {
|
||||
store
|
||||
.with_role(&format!("hive-agent-{a}"), role())
|
||||
.with_entity(a)
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_an_agent_destroyed_everywhere_and_holding_a_role_is_a_target() {
|
||||
let declarations = [
|
||||
declared(&[
|
||||
("gone", AgentState::Destroyed),
|
||||
("moved", AgentState::Destroyed),
|
||||
("alive", AgentState::Up),
|
||||
("asleep", AgentState::Offline),
|
||||
("held", AgentState::Paused),
|
||||
("no-role", AgentState::Destroyed),
|
||||
]),
|
||||
declared(&[("moved", AgentState::Up)]),
|
||||
];
|
||||
let roles = with_roles(&[
|
||||
"gone",
|
||||
"moved",
|
||||
"alive",
|
||||
"asleep",
|
||||
"held",
|
||||
"undeclared",
|
||||
"unnamed",
|
||||
]);
|
||||
assert_eq!(
|
||||
cut_off_targets(&roles, &declarations),
|
||||
with_roles(&["gone"]),
|
||||
"destroyed on one hive and live on another is live; declared nowhere is left alone"
|
||||
);
|
||||
// The control: without the live declaration on `b`, `moved` is a target
|
||||
// too, so the line above can see a destroyed agent.
|
||||
assert_eq!(
|
||||
cut_off_targets(&roles, &declarations[..1]),
|
||||
with_roles(&["gone", "moved"])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_destroyed_agent_loses_its_entity_and_its_role() {
|
||||
let store = store_with(&["gone"]);
|
||||
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
|
||||
cut_off_destroyed(&store, &with_roles(&["gone"]), &declarations).await;
|
||||
assert!(store.is_disabled("gone"), "{:?}", store.calls());
|
||||
assert_eq!(store.role("hive-agent-gone"), None, "{:?}", store.calls());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_live_agent_is_never_touched() {
|
||||
let store = store_with(&["alive", "gone"]);
|
||||
let declarations = [declared(&[
|
||||
("alive", AgentState::Up),
|
||||
("gone", AgentState::Destroyed),
|
||||
])];
|
||||
cut_off_destroyed(&store, &with_roles(&["alive", "gone"]), &declarations).await;
|
||||
assert!(!store.is_disabled("alive"), "{:?}", store.calls());
|
||||
assert_eq!(store.role("hive-agent-alive"), Some(role()));
|
||||
// The control: the same pass did cut the destroyed one off.
|
||||
assert!(store.is_disabled("gone"));
|
||||
assert_eq!(store.role("hive-agent-gone"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_without_an_entity_has_its_role_deleted_and_no_entity_made() {
|
||||
let store = FakeAccess::default().with_role("hive-agent-gone", role());
|
||||
cut_off(&store, "gone").await.expect("accepted");
|
||||
assert_eq!(store.role("hive-agent-gone"), None);
|
||||
assert!(
|
||||
!store
|
||||
.calls()
|
||||
.iter()
|
||||
.any(|c| c.starts_with("set_entity_disabled")),
|
||||
"{:?}",
|
||||
store.calls()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_disable_keeps_the_role_so_the_next_pass_retries() {
|
||||
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
|
||||
let refused = store_with(&["gone"]).refusing_disable();
|
||||
cut_off_destroyed(&refused, &with_roles(&["gone"]), &declarations).await;
|
||||
assert_eq!(
|
||||
refused.role("hive-agent-gone"),
|
||||
Some(role()),
|
||||
"the role is the marker that work is left"
|
||||
);
|
||||
assert!(
|
||||
!refused
|
||||
.calls()
|
||||
.iter()
|
||||
.any(|c| c.starts_with("delete_cert_role")),
|
||||
"{:?}",
|
||||
refused.calls()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_role_delete_leaves_the_entity_disabled_and_the_next_pass_finishes() {
|
||||
let declarations = [declared(&[("gone", AgentState::Destroyed)])];
|
||||
let refused = store_with(&["gone"]).refusing_role_delete();
|
||||
cut_off_destroyed(&refused, &with_roles(&["gone"]), &declarations).await;
|
||||
assert!(refused.is_disabled("gone"));
|
||||
assert_eq!(refused.role("hive-agent-gone"), Some(role()));
|
||||
|
||||
// The next pass, against a store that now accepts the delete.
|
||||
let healthy = store_with(&["gone"]);
|
||||
cut_off_destroyed(&healthy, &with_roles(&["gone"]), &declarations).await;
|
||||
assert!(healthy.is_disabled("gone"));
|
||||
assert_eq!(healthy.role("hive-agent-gone"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_cut_off_is_repeatable() {
|
||||
let store = store_with(&["gone"]);
|
||||
cut_off(&store, "gone").await.expect("first");
|
||||
cut_off(&store, "gone").await.expect("second");
|
||||
assert!(store.is_disabled("gone"));
|
||||
assert_eq!(store.role("hive-agent-gone"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_clears_what_a_cut_off_set() {
|
||||
let store = store_with(&["gone"]);
|
||||
cut_off(&store, "gone").await.expect("accepted");
|
||||
assert!(store.is_disabled("gone"));
|
||||
restore(&store, "gone").await.expect("accepted");
|
||||
assert!(!store.is_disabled("gone"));
|
||||
}
|
||||
}
|
||||
|
|
@ -8,8 +8,8 @@
|
|||
//! and its grant covers exactly the calls made here (`swarm-bao.nix`'s
|
||||
//! `controllerPolicyText`: `update` on the agent PKI role's `issue` path, and
|
||||
//! `create/update` on `secret/data/swarm/agents/*`, on
|
||||
//! `sys/policies/acl/hive-*`, and on `auth/cert/certs/hive-*`, plus the
|
||||
//! `identity/` and `sys/auth/cert` stanzas [`crate::agent_entity`] uses).
|
||||
//! `sys/policies/acl/hive-*`, and on `auth/cert/certs/hive-*` (`delete` too), plus the
|
||||
//! `identity/` and `sys/auth/cert` stanzas [`crate::agent_entity`] and [`crate::agent_cutoff`] use).
|
||||
//!
|
||||
//! **Two credentials, deliberately unrelated.** The certificate reaches the
|
||||
//! store; the queue secret identifies the agent to the swarm queue. Both sit
|
||||
|
|
@ -80,8 +80,11 @@ fn role_inputs<'a>(agent: &str, credential: &'a mtls::Credential) -> Result<Role
|
|||
})
|
||||
}
|
||||
|
||||
/// Write the shared policy, name the agent's entity, then write the cert-auth
|
||||
/// role that attaches the policy.
|
||||
/// Write the shared policy, name the agent's entity and enable it, then write
|
||||
/// the cert-auth role that attaches the policy.
|
||||
///
|
||||
/// Enabling clears the flag [`crate::agent_cutoff::cut_off`] set when an agent
|
||||
/// of this name was destroyed, so a new agent of that name can log in.
|
||||
///
|
||||
/// The entity comes before the role and is a precondition for it. The role
|
||||
/// attaches the shared policy alone, so an agent whose entity is not named after
|
||||
|
|
@ -108,6 +111,10 @@ async fn publish_access(
|
|||
.await
|
||||
.with_context(|| format!("naming {agent}'s identity entity; the role is not changed"))
|
||||
.at(MintStep::Entity)?;
|
||||
crate::agent_cutoff::restore(store, agent)
|
||||
.await
|
||||
.with_context(|| format!("enabling {agent}'s identity entity; the role is not changed"))
|
||||
.at(MintStep::Entity)?;
|
||||
store
|
||||
.write_cert_role(
|
||||
DEFAULT_CERT_MOUNT,
|
||||
|
|
@ -235,7 +242,7 @@ impl<T, E: Into<anyhow::Error>> AtStep<T> for Result<T, E> {
|
|||
/// presents names *it* rather than its hive;
|
||||
/// 4. write the shared ACL document [`policy::render_shared_agent`] renders;
|
||||
/// 5. name the agent's identity entity after the agent ([`crate::agent_entity`]),
|
||||
/// which the policy reads;
|
||||
/// which the policy reads, and enable it ([`crate::agent_cutoff::restore`]);
|
||||
/// 6. write the cert-auth role that ties the three together, attaching that
|
||||
/// policy and pinning the CA the store named as this leaf's issuer.
|
||||
///
|
||||
|
|
@ -610,6 +617,45 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// An agent created under the name of a destroyed one gets its entity
|
||||
/// enabled and its role back, the enable first.
|
||||
#[tokio::test]
|
||||
async fn re_creating_a_cut_off_agent_enables_its_entity_and_writes_its_role() {
|
||||
let store = FakeAccess::default().with_entity("atlas");
|
||||
crate::agent_cutoff::cut_off(&store, "atlas")
|
||||
.await
|
||||
.expect("accepted");
|
||||
assert!(store.is_disabled("atlas"), "the control: it was cut off");
|
||||
assert_eq!(store.role("hive-agent-atlas"), None);
|
||||
|
||||
let credential = credential();
|
||||
let inputs = role_inputs("atlas", &credential).expect("legal");
|
||||
publish_access(&store, "atlas", &inputs)
|
||||
.await
|
||||
.expect("a store that accepts everything");
|
||||
assert!(!store.is_disabled("atlas"), "{:?}", store.calls());
|
||||
assert!(store.role("hive-agent-atlas").is_some());
|
||||
let calls = store.calls();
|
||||
let at = |call: &str| calls.iter().position(|c| c == call).expect(call);
|
||||
assert!(
|
||||
at("set_entity_disabled atlas false")
|
||||
< at("write_cert_role hive-agent-atlas hive-agent"),
|
||||
"the entity is enabled before the role is written: {calls:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mint_whose_enable_is_refused_leaves_the_role_alone() {
|
||||
let store = FakeAccess::default().refusing_disable();
|
||||
let credential = credential();
|
||||
let inputs = role_inputs("atlas", &credential).expect("legal");
|
||||
let err = publish_access(&store, "atlas", &inputs)
|
||||
.await
|
||||
.expect_err("the enable is refused");
|
||||
assert_eq!(failed_step(&err), Some(MintStep::Entity), "{err:#}");
|
||||
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_renewal_whose_entity_step_fails_leaves_the_role_alone() {
|
||||
let store = FakeAccess::without_accessor();
|
||||
|
|
|
|||
|
|
@ -348,6 +348,11 @@ async fn observe_all(
|
|||
None
|
||||
}
|
||||
};
|
||||
// Destroyed agents that still have a role are cut off here, each pass until
|
||||
// it succeeds.
|
||||
if let Some(with_role) = &roles {
|
||||
crate::agent_cutoff::cut_off_destroyed(&store, with_role, &declarations).await;
|
||||
}
|
||||
let mut observed = Vec::new();
|
||||
for agent in live_agents(&declarations) {
|
||||
let o = AgentObserved {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ mod agent_commands;
|
|||
mod agent_config_repo;
|
||||
mod agent_control;
|
||||
mod agent_custom;
|
||||
mod agent_cutoff;
|
||||
mod agent_entity;
|
||||
mod agent_icon;
|
||||
mod agent_identity;
|
||||
|
|
|
|||
|
|
@ -91,6 +91,20 @@ pub(crate) trait AccessStore {
|
|||
fn rename_entity(&self, id: &str, name: &str)
|
||||
-> impl Future<Output = Result<(), Error>> + Send;
|
||||
|
||||
/// As [`SecretStore::set_entity_disabled`].
|
||||
fn set_entity_disabled(
|
||||
&self,
|
||||
name: &str,
|
||||
disabled: bool,
|
||||
) -> impl Future<Output = Result<(), Error>> + Send;
|
||||
|
||||
/// As [`SecretStore::delete_cert_role`].
|
||||
fn delete_cert_role(
|
||||
&self,
|
||||
mount: &str,
|
||||
name: &str,
|
||||
) -> impl Future<Output = Result<(), Error>> + Send;
|
||||
|
||||
/// As [`SecretStore::create_entity_alias`].
|
||||
fn create_entity_alias(
|
||||
&self,
|
||||
|
|
@ -160,6 +174,14 @@ impl AccessStore for SecretStore {
|
|||
) -> Result<(), Error> {
|
||||
SecretStore::create_entity_alias(self, alias_name, mount_accessor, entity_id).await
|
||||
}
|
||||
|
||||
async fn set_entity_disabled(&self, name: &str, disabled: bool) -> Result<(), Error> {
|
||||
SecretStore::set_entity_disabled(self, name, disabled).await
|
||||
}
|
||||
|
||||
async fn delete_cert_role(&self, mount: &str, name: &str) -> Result<(), Error> {
|
||||
SecretStore::delete_cert_role(self, mount, name).await
|
||||
}
|
||||
}
|
||||
|
||||
/// A store that records what it is asked to do, for tests of the order and the
|
||||
|
|
@ -204,6 +226,10 @@ pub(crate) mod fake {
|
|||
/// The entity that owns each alias, as `(id, name)`.
|
||||
alias_owner: Option<(String, String)>,
|
||||
entities: Mutex<Vec<String>>,
|
||||
/// The entities, by name, whose `disabled` flag is set.
|
||||
disabled: Mutex<std::collections::BTreeSet<String>>,
|
||||
/// The calls, by verb, that are refused.
|
||||
refused: Vec<&'static str>,
|
||||
calls: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
|
|
@ -261,6 +287,32 @@ pub(crate) mod fake {
|
|||
self
|
||||
}
|
||||
|
||||
/// A store that already holds the entity `name`.
|
||||
pub(crate) fn with_entity(self, name: &str) -> Self {
|
||||
self.entities
|
||||
.lock()
|
||||
.expect("not poisoned")
|
||||
.push(name.to_owned());
|
||||
self
|
||||
}
|
||||
|
||||
/// A store that refuses to set an entity's `disabled` flag.
|
||||
pub(crate) fn refusing_disable(mut self) -> Self {
|
||||
self.refused.push("set_entity_disabled");
|
||||
self
|
||||
}
|
||||
|
||||
/// A store that refuses to delete a cert-auth role.
|
||||
pub(crate) fn refusing_role_delete(mut self) -> Self {
|
||||
self.refused.push("delete_cert_role");
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether the entity `name` is disabled now.
|
||||
pub(crate) fn is_disabled(&self, name: &str) -> bool {
|
||||
self.disabled.lock().expect("not poisoned").contains(name)
|
||||
}
|
||||
|
||||
/// The role `name` as the store holds it now.
|
||||
pub(crate) fn role(&self, name: &str) -> Option<FakeRole> {
|
||||
self.roles.lock().expect("not poisoned").get(name).cloned()
|
||||
|
|
@ -403,5 +455,28 @@ pub(crate) mod fake {
|
|||
self.log(format!("create_entity_alias {alias_name}"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set_entity_disabled(&self, name: &str, disabled: bool) -> Result<(), Error> {
|
||||
self.log(format!("set_entity_disabled {name} {disabled}"));
|
||||
if self.refused.contains(&"set_entity_disabled") {
|
||||
return Err(Self::denied());
|
||||
}
|
||||
let mut flagged = self.disabled.lock().expect("not poisoned");
|
||||
if disabled {
|
||||
flagged.insert(name.to_owned());
|
||||
} else {
|
||||
flagged.remove(name);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_cert_role(&self, _mount: &str, name: &str) -> Result<(), Error> {
|
||||
self.log(format!("delete_cert_role {name}"));
|
||||
if self.refused.contains(&"delete_cert_role") {
|
||||
return Err(Self::denied());
|
||||
}
|
||||
self.roles.lock().expect("not poisoned").remove(name);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue