vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

re-read the declaration before a mint, and mint for a current certificate whose role is missing

Browse source
This commit is contained in:
atlas 2026-10-29 09:06:53 +01:00 • committed by mara
commit 37d5fad0b3
1 changed files with 252 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

182
swarm-controller/src/jobq.rs
View file

@ -404,7 +404,9 @@ async fn run_node_work(
},
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await,
SwarmNodeKind::MintAgentIdentity { agent } => {
mint_identity(deps.wanted.as_deref(), &agent).await
}
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
SwarmNodeKind::MintAgentMatrixAccount { agent } => {
mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
@ -462,10 +464,52 @@ const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
/// `clippy::too_many_lines`.
pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
///
/// Re-reads the wanted-state declarations first (see [`mint_if_live`]); with
/// no wanted-state writer there is nothing to read and the mint runs.
pub(crate) async fn mint_identity(
wanted: Option<&wanted::WantedWriter>,
agent: &str,
) -> hive_jobq::scheduler::Outcome {
let declarations = match wanted {
Some(writer) => {
let hives: Vec<String> = crate::startup::load_hives()
.into_iter()
.map(|h| h.name)
.collect();
Some(agent_renewal::read_declarations(writer, &hives).await)
}
None => None,
};
mint_if_live(agent, declarations, agent_identity::mint_and_verify(agent)).await
}
/// Run `mint` unless `declarations` (every hive's, read when the node runs)
/// leave `agent` not live: declared `Destroyed` everywhere, or nowhere. A mint
/// queued before a destroy then writes no credential for the destroyed name,
/// and the node is `Done`. Creation orders its mint after the declaration, so a
/// created agent is always declared. A failed read fails the node rather than
/// minting blind.
async fn mint_if_live(
agent: &str,
declarations: Option<anyhow::Result<Vec<swarm_wire::wanted::HiveWanted>>>,
mint: impl std::future::Future<Output = anyhow::Result<()>>,
) -> hive_jobq::scheduler::Outcome {
use hive_jobq::scheduler::Outcome;
match agent_identity::mint_and_verify(agent).await {
match declarations {
Some(Ok(declarations)) if !agent_renewal::live_agents(&declarations).contains(agent) => {
tracing::info!(agent, "identity mint skipped: {agent} is destroyed");
return Outcome::Done;
}
Some(Err(e)) => {
let error = format!("{e:#}");
tracing::warn!(%agent, step = "declaration", %error, "{MINT_IDENTITY_FAILED}");
return Outcome::Failed(error);
}
Some(Ok(_)) | None => {}
}
match mint.await {
Ok(()) => Outcome::Done,
Err(e) => {
let error = format!("{e:#}");
@ -1292,16 +1336,27 @@ pub(crate) fn declare_agent_job(
agent: agent.to_owned(),
})
.after_ok(create_forge_user);
// Declared before the deploy trigger so the pause is visible in the
// wanted-state store before the hive brings the container up — see the
// node's own doc comment for why "before", not just "eventually". Needs
// no parent of its own: the declaration is written from the hive/agent
// names known at request time, same as `create_forge_user`.
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The agent's own identity at the swarm secret store, minted and
// published at swarm level so no hive ever needs the capability to mint
// one. After `create_identity` because the certificate names a principal
// the swarm has agreed exists — not because anything in the store reads
// authelia.
// authelia. After `set_wanted` because the node re-reads the declaration
// and writes nothing for an agent declared nowhere or `Destroyed`.
let mint_identity = b
.node(SwarmNodeKind::MintAgentIdentity {
agent: agent.to_owned(),
})
.after_ok(create_identity);
.after_ok(create_identity)
.after_ok(set_wanted);
// The agent's own forge token, in the store before the hive deploys
// the container that pulls it. After the forge user, because the token
// is minted for that user.
@ -1317,15 +1372,6 @@ pub(crate) fn declare_agent_job(
let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount {
agent: agent.to_owned(),
});
// Declared before the deploy trigger so the pause is visible in the
// wanted-state store before the hive brings the container up — see the
// node's own doc comment for why "before", not just "eventually". Needs
// no parent of its own: the declaration is written from the hive/agent
// names known at request time, same as `create_forge_user`.
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
hive: hive.to_owned(),
agent: agent.to_owned(),
});
// The hive builds the agent from `internal/meta` only, so the deploy needs
// the update and the swarm's build from it to succeed (`after_ok`): after a
// failed update (no builder, an unreadable config repo) or a red build, the
@ -3884,6 +3930,29 @@ mod tests {
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
}
/// The mint waits for the declaration its node re-reads: with the
/// declaration failed it is skipped, with it written it runs.
#[tokio::test]
async fn creation_mints_the_identity_only_after_the_declaration() {
use hive_jobq::State;
let mint_states = |sched: &SharedSched| -> Vec<State> {
let guard = sched.lock().unwrap();
guard
.graph()
.nodes()
.filter(|n| matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { .. }))
.map(|n| n.state)
.collect()
};
let sched = fresh_sched();
create_and_run(&sched, "atlas", false, true).await;
assert_eq!(mint_states(&sched), [State::Skipped]);
let sched = fresh_sched();
create_and_run(&sched, "atlas", false, false).await;
assert_eq!(mint_states(&sched), [State::Done]);
}
/// Run every runnable node; the build of `agent` fails with `error` and
/// everything else succeeds.
async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
@ -4028,7 +4097,7 @@ mod mint_identity_tests {
use hive_jobq::scheduler::Outcome;
use tracing_subscriber::layer::SubscriberExt as _;
use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node};
use super::{SwarmNodeKind, WorkerDeps, mint_identity, mint_if_live, run_swarm_node};
/// One event's level and its fields, each by name.
type Event = (tracing::Level, Vec<(String, String)>);
@ -4067,7 +4136,7 @@ mod mint_identity_tests {
let _guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
let Outcome::Failed(error) = mint_identity("a/b").await else {
let Outcome::Failed(error) = mint_identity(None, "a/b").await else {
panic!("an illegal agent name cannot be minted");
};
@ -4088,6 +4157,87 @@ mod mint_identity_tests {
assert_eq!(field("error"), error);
}
fn declared_as(
agents: &[(&str, swarm_wire::wanted::AgentState)],
) -> swarm_wire::wanted::HiveWanted {
let mut d = swarm_wire::wanted::HiveWanted::default();
for (agent, state) in agents {
d.agents.insert(
(*agent).to_owned(),
swarm_wire::wanted::AgentWanted { state: *state },
);
}
d
}
/// The mint runs for a declared agent and not for one declared `Destroyed`
/// or not at all: nothing is written for the second, and the node is done.
#[tokio::test]
async fn a_mint_for_an_agent_destroyed_since_it_was_queued_writes_nothing() {
use swarm_wire::wanted::AgentState;
let events = Events::default();
let _guard =
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
let ran = std::sync::atomic::AtomicBool::new(false);
let mint = async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
};
let declarations =
declared_as(&[("live", AgentState::Up), ("gone", AgentState::Destroyed)]);
for agent in ["gone", "unknown"] {
let outcome = mint_if_live(agent, Some(Ok(vec![declarations.clone()])), async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
assert!(matches!(outcome, Outcome::Done), "{agent}");
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst), "{agent}");
}
let got = events.0.lock().expect("unpoisoned").clone();
assert!(
got.iter()
.any(|(level, fields)| *level == tracing::Level::INFO
&& fields
.iter()
.any(|(n, v)| n == "message"
&& v == "identity mint skipped: gone is destroyed")),
"{got:?}"
);
// Presence controls: a live agent mints, and so does one with no
// wanted-state writer to ask.
let outcome = mint_if_live("live", Some(Ok(vec![declarations])), mint).await;
assert!(matches!(outcome, Outcome::Done));
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
ran.store(false, std::sync::atomic::Ordering::SeqCst);
let outcome = mint_if_live("anyone", None, async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
assert!(matches!(outcome, Outcome::Done));
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
}
/// A declaration that cannot be read fails the node, naming the cause,
/// and mints nothing.
#[tokio::test]
async fn an_unreadable_declaration_fails_the_mint_without_minting() {
let ran = std::sync::atomic::AtomicBool::new(false);
let outcome = mint_if_live("atlas", Some(Err(anyhow::anyhow!("hive down"))), async {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
.await;
let Outcome::Failed(error) = outcome else {
panic!("a failed read must fail the node");
};
assert!(error.contains("hive down"), "{error}");
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
}
/// A node of any kind whose work returns `Failed` logs one warn naming the
/// node's id, its kind and the error, from the one place that runs nodes.
#[tokio::test]