vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

re-read the declaration before a mint, and mint for a current certificate whose role is missing

Browse source
This commit is contained in:
atlas 2026-10-29 09:06:53 +01:00 • committed by mara
commit 37d5fad0b3
2 changed files with 252 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

94
swarm-controller/src/agent_renewal.rs
View file

@ -26,14 +26,14 @@
//! **Only agents some hive is declared to run are renewed** ([`live_agents`]):
//! a destroy is recorded in the wanted-state declarations, and an agent declared
//! nowhere, or only as `Destroyed`, is skipped. A live agent with no stored
//! certificate gets one minted, as a failed mint at creation is retried nowhere
//! else, unless its mint is in flight. A missing queue secret stays missing.
//! certificate, or a current one with no listed cert-auth role, gets one minted
//! unless its mint is in flight. A missing queue secret stays missing.
use std::collections::BTreeSet;
use std::sync::Arc;
use anyhow::{Context, Result, bail};
use swarm_secret_client::{SecretStore, cert, mtls, queue};
use swarm_secret_client::{SecretStore, cert, client::DEFAULT_CERT_MOUNT, mtls, policy, queue};
use swarm_wire::wanted::{AgentState, HiveWanted};
use crate::wanted::WantedWriter;
@ -141,8 +141,9 @@ pub enum Observed {
Unstamped,
/// Stored and not yet due.
Current,
/// Nothing stored. A certificate is minted for a live agent; a queue secret
/// is left alone (see the module doc).
/// Nothing stored (a certificate: or stored with no cert-auth role, see
/// [`cert_needing_role`]). A certificate is minted for a live agent; a queue
/// secret is left alone (see the module doc).
Absent,
/// The read or the decode failed. Nothing is known, so nothing is done.
Unknown,
@ -290,13 +291,13 @@ async fn observe_secret(store: &SecretStore, agent: &str, now: i64) -> Observed
}
}
/// One pass: every live agent, observed. Fails as a whole when any hive's
/// declaration cannot be read, since that hive may be the one declaring an
/// agent destroyed.
async fn observe_all(
/// Every readable wanted-state declaration of `hives`. Fails as a whole when
/// one cannot be read, since that hive may be the one declaring an agent
/// destroyed.
pub(crate) async fn read_declarations(
wanted: &WantedWriter,
hives: &[String],
) -> Result<Vec<(String, AgentObserved)>> {
) -> Result<Vec<HiveWanted>> {
let mut declarations = Vec::with_capacity(hives.len());
for hive in hives {
if let Some(d) = wanted
@ -307,14 +308,54 @@ async fn observe_all(
declarations.push(d);
}
}
Ok(declarations)
}
/// A stored, current certificate whose cert-auth role is not in `roles` is as
/// good as absent: the agent cannot log in with it, and the mint is the only
/// thing that writes the role. `None` (the role listing failed) and every
/// other observation come back unchanged.
pub fn cert_needing_role(
cert: Observed,
roles: Option<&BTreeSet<String>>,
agent: &str,
) -> Observed {
match (cert, roles) {
(Observed::Current, Some(roles)) if !roles.contains(agent) => Observed::Absent,
_ => cert,
}
}
/// One pass: every live agent, observed. Fails as a whole when any hive's
/// declaration cannot be read.
async fn observe_all(
wanted: &WantedWriter,
hives: &[String],
) -> Result<Vec<(String, AgentObserved)>> {
let declarations = read_declarations(wanted, hives).await?;
let store = crate::store::connect()
.await
.context("logging in to the swarm secret store")?;
let now = unix_now();
let roles = match store.list_cert_roles(DEFAULT_CERT_MOUNT).await {
Ok(names) => Some(
policy::agents_from_role_names(&names)
.into_iter()
.collect::<BTreeSet<_>>(),
),
Err(e) => {
tracing::warn!(error = %e, "agent certificate: listing the cert-auth roles failed");
None
}
};
let mut observed = Vec::new();
for agent in live_agents(&declarations) {
let o = AgentObserved {
cert: observe_cert(&store, &agent, now).await,
cert: cert_needing_role(
observe_cert(&store, &agent, now).await,
roles.as_ref(),
&agent,
),
secret: observe_secret(&store, &agent, now).await,
};
if let Observed::Due(age) = o.cert {
@ -323,7 +364,7 @@ async fn observe_all(
if o.cert == Observed::Absent {
tracing::info!(
agent,
"agent has no stored certificate; minting its store identity"
"agent has no usable certificate (none stored, or no cert-auth role); minting its store identity"
);
}
if let Observed::Due(age) = o.secret {
@ -596,6 +637,35 @@ mod tests {
);
}
/// A current certificate with no cert-auth role listed is minted for; the
/// controls are the same certificate with its role listed, a failed role
/// listing, and a certificate that is not `Current` anyway.
#[test]
fn a_current_certificate_without_its_role_is_minted_for() {
let roles: BTreeSet<String> = ["sock".to_owned()].into();
let planned = |cert: Observed, roles: Option<&BTreeSet<String>>, agent: &str| {
let observed = [(
agent.to_owned(),
AgentObserved {
cert: cert_needing_role(cert, roles, agent),
secret: Observed::Current,
},
)];
plan(&observed).iter().map(|r| r.cert).collect::<Vec<_>>()
};
assert_eq!(planned(Observed::Current, Some(&roles), "paused"), [true]);
assert_eq!(
planned(Observed::Current, Some(&roles), "sock"),
[] as [bool; 0]
);
assert_eq!(planned(Observed::Current, None, "paused"), [] as [bool; 0]);
assert_eq!(
planned(Observed::Unknown, Some(&roles), "paused"),
[] as [bool; 0]
);
assert_eq!(planned(Observed::Due(1), Some(&roles), "sock"), [true]);
}
#[test]
fn the_age_logged_is_whole_days() {
assert_eq!(Age(46 * DAY + 5).to_string(), "46d");

182
swarm-controller/src/jobq.rs
View file

@ -404,7 +404,9 @@ async fn run_node_work(
},
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await,
SwarmNodeKind::MintAgentIdentity { agent } => {
mint_identity(deps.wanted.as_deref(), &agent).await
}
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
SwarmNodeKind::MintAgentMatrixAccount { agent } => {
mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
@ -462,10 +464,52 @@ const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
/// `clippy::too_many_lines`.
pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
///
/// Re-reads the wanted-state declarations first (see [`mint_if_live`]); with
/// no wanted-state writer there is nothing to read and the mint runs.
pub(crate) async fn mint_identity(
wanted: Option<&wanted::WantedWriter>,
agent: &str,
) -> hive_jobq::scheduler::Outcome {
let declarations = match wanted {
Some(writer) => {
let hives: Vec<String> = crate::startup::load_hives()
.into_iter()
.map(|h| h.name)
.collect();
Some(agent_renewal::read_declarations(writer, &hives).await)
}
None => None,
};
mint_if_live(agent, declarations, agent_identity::mint_and_verify(agent)).await
}
/// Run `mint` unless `declarations` (every hive's, read when the node runs)
/// leave `agent` not live: declared `Destroyed` everywhere, or nowhere. A mint
/// queued before a destroy then writes no credential for the destroyed name,
/// and the node is `Done`. Creation orders its mint after the declaration, so a
/// created agent is always declared. A failed read fails the node rather than
/// minting blind.
async fn mint_if_live(
agent: &str,
declarations: Option<anyhow::Result<Vec<swarm_wire::wanted::HiveWanted>>>,
mint: impl std::future::Future<Output = anyhow::Result<()>>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
match agent_identity::mint_and_verify(agent).await {
match declarations {
Some(Ok(declarations)) if !agent_renewal::live_agents(&declarations).contains(agent) => {
tracing::info!(agent, "identity mint skipped: {agent} is destroyed");
return Outcome::Done;
}
Some(Err(e)) => {
let error = format!("{e:#}");
tracing::warn!(%agent, step = "declaration", %error, "{MINT_IDENTITY_FAILED}");
return Outcome::Failed(error);
}
Some(Ok(_)) | None => {}
}
match mint.await {
Ok(()) => Outcome::Done,
Err(e) => {
let error = format!("{e:#}");
@ -1292,16 +1336,27 @@ pub(crate) fn declare_agent_job(
agent: agent.to_owned(),
})
.after_ok(create_forge_user);
// Declared before the deploy trigger so the pause is visible in the
// wanted-state store before the hive brings the container up — see the
// node's own doc comment for why "before", not just "eventually". Needs
// no parent of its own: the declaration is written from the hive/agent
// names known at request time, same as `create_forge_user`.
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The agent's own identity at the swarm secret store, minted and
// published at swarm level so no hive ever needs the capability to mint
// one. After `create_identity` because the certificate names a principal
// the swarm has agreed exists — not because anything in the store reads
// authelia.
// authelia. After `set_wanted` because the node re-reads the declaration
// and writes nothing for an agent declared nowhere or `Destroyed`.
let mint_identity = b
.node(SwarmNodeKind::MintAgentIdentity {
agent: agent.to_owned(),
})
.after_ok(create_identity);
.after_ok(create_identity)
.after_ok(set_wanted);
// The agent's own forge token, in the store before the hive deploys
// the container that pulls it. After the forge user, because the token
// is minted for that user.
@ -1317,15 +1372,6 @@ pub(crate) fn declare_agent_job(
let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount {
agent: agent.to_owned(),
});
// Declared before the deploy trigger so the pause is visible in the
// wanted-state store before the hive brings the container up — see the
// node's own doc comment for why "before", not just "eventually". Needs
// no parent of its own: the declaration is written from the hive/agent
// names known at request time, same as `create_forge_user`.
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The hive builds the agent from `internal/meta` only, so the deploy needs
// the update and the swarm's build from it to succeed (`after_ok`): after a
// failed update (no builder, an unreadable config repo) or a red build, the
@ -3884,6 +3930,29 @@ mod tests {
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
}
/// The mint waits for the declaration its node re-reads: with the
/// declaration failed it is skipped, with it written it runs.
#[tokio::test]
async fn creation_mints_the_identity_only_after_the_declaration() {
use hive_jobq::State;
let mint_states = |sched: &SharedSched| -> Vec<State> {
let guard = sched.lock().unwrap();
guard
.graph()
.nodes()
.filter(|n| matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { .. }))
.map(|n| n.state)
.collect()
};
let sched = fresh_sched();
create_and_run(&sched, "atlas", false, true).await;
assert_eq!(mint_states(&sched), [State::Skipped]);
let sched = fresh_sched();
create_and_run(&sched, "atlas", false, false).await;
assert_eq!(mint_states(&sched), [State::Done]);
}
/// Run every runnable node; the build of `agent` fails with `error` and
/// everything else succeeds.
async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
@ -4028,7 +4097,7 @@ mod mint_identity_tests {
use hive_jobq::scheduler::Outcome;
use tracing_subscriber::layer::SubscriberExt as _;
use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node};
use super::{SwarmNodeKind, WorkerDeps, mint_identity, mint_if_live, run_swarm_node};
/// One event's level and its fields, each by name.
type Event = (tracing::Level, Vec<(String, String)>);
@ -4067,7 +4136,7 @@ mod mint_identity_tests {
let _guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
let Outcome::Failed(error) = mint_identity("a/b").await else {
let Outcome::Failed(error) = mint_identity(None, "a/b").await else {
panic!("an illegal agent name cannot be minted");
};
@ -4088,6 +4157,87 @@ mod mint_identity_tests {
assert_eq!(field("error"), error);
}
fn declared_as(
agents: &[(&str, swarm_wire::wanted::AgentState)],
) -> swarm_wire::wanted::HiveWanted {
let mut d = swarm_wire::wanted::HiveWanted::default();
for (agent, state) in agents {
d.agents.insert(
(*agent).to_owned(),
swarm_wire::wanted::AgentWanted { state: *state },
);
}
d
}
/// The mint runs for a declared agent and not for one declared `Destroyed`
/// or not at all: nothing is written for the second, and the node is done.
#[tokio::test]
async fn a_mint_for_an_agent_destroyed_since_it_was_queued_writes_nothing() {
use swarm_wire::wanted::AgentState;
let events = Events::default();
let _guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
let ran = std::sync::atomic::AtomicBool::new(false);
let mint = async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
};
let declarations =
declared_as(&[("live", AgentState::Up), ("gone", AgentState::Destroyed)]);
for agent in ["gone", "unknown"] {
let outcome = mint_if_live(agent, Some(Ok(vec![declarations.clone()])), async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
assert!(matches!(outcome, Outcome::Done), "{agent}");
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst), "{agent}");
}
let got = events.0.lock().expect("unpoisoned").clone();
assert!(
got.iter()
.any(|(level, fields)| *level == tracing::Level::INFO
&& fields
.iter()
.any(|(n, v)| n == "message"
&& v == "identity mint skipped: gone is destroyed")),
"{got:?}"
);
// Presence controls: a live agent mints, and so does one with no
// wanted-state writer to ask.
let outcome = mint_if_live("live", Some(Ok(vec![declarations])), mint).await;
assert!(matches!(outcome, Outcome::Done));
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
ran.store(false, std::sync::atomic::Ordering::SeqCst);
let outcome = mint_if_live("anyone", None, async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
assert!(matches!(outcome, Outcome::Done));
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
}
/// A declaration that cannot be read fails the node, naming the cause,
/// and mints nothing.
#[tokio::test]
async fn an_unreadable_declaration_fails_the_mint_without_minting() {
let ran = std::sync::atomic::AtomicBool::new(false);
let outcome = mint_if_live("atlas", Some(Err(anyhow::anyhow!("hive down"))), async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
let Outcome::Failed(error) = outcome else {
panic!("a failed read must fail the node");
};
assert!(error.contains("hive down"), "{error}");
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
}
/// A node of any kind whose work returns `Failed` logs one warn naming the
/// node's id, its kind and the error, from the one place that runs nodes.
#[tokio::test]