From 8126e0fffafa94f260b80fce273081e209772d22 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Thu, 29 Oct 2026 03:17:16 +0100
Subject: [PATCH] mint a live agent's missing store identity from the renewal
 pass, and log the step a mint stopped at

---
 docs/swarm/README.md                   |   8 +-
 docs/swarm/credentials-reference.md    |   2 +-
 swarm-controller/src/agent_identity.rs | 146 +++++++++++++++++++++----
 swarm-controller/src/agent_renewal.rs  |  64 +++++++++--
 swarm-controller/src/jobq.rs           | 105 +++++++++++++++++-
 5 files changed, 286 insertions(+), 39 deletions(-)

diff --git a/docs/swarm/README.md b/docs/swarm/README.md
index c7c6f044..19e93411 100644
--- a/docs/swarm/README.md
+++ b/docs/swarm/README.md
@@ -59,9 +59,11 @@ the agent roster and their wanted state, and the job graph. Creating an agent 
 from the swarm UI or `swarmctl agent create --hive <h>` — queues its SSO
 identity, forge user, config repo, store identity and matrix account, then
 sends the hive a deploy message. A new agent starts `paused`. A store identity
-that fails to mint logs `swarm jobq: mint_agent_identity failed` with the agent
-and the error, which the Grafana alert rule `hh-agent-identity-mint-failed`
-fires on. The agent is still created, without a store identity.
+that fails to mint logs `swarm jobq: mint_agent_identity failed` with the agent,
+the step it stopped at and the error, which the Grafana alert rule
+`hh-agent-identity-mint-failed` fires on. The agent is still created, without a
+store identity, and the controller's five-minute credential pass mints one for
+it as long as the agent isn't `Destroyed`.
 
 `services.hyperhive.deploy.swarm-controller.enable` runs it on this host;
 `singleHostSwarm` turns it on. **Otherwise off: set it on the one host that
diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md
index 57d3134e..c0024fdc 100644
--- a/docs/swarm/credentials-reference.md
+++ b/docs/swarm/credentials-reference.md
@@ -27,7 +27,7 @@ the rest of the cell says how.
 | `swarm/controller/swarm-controller/forge/token`                                                                        | the forge, once                                                                                                                                                                                                                                                | `swarm-controller`                                                     | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one                                               | ✅ re-read on a 401                                                                                                                                                                                                                                          |
 | `swarm/controller/swarm-controller/grafana/token`                                                                      | `swarm-bao-grafana-controller-token` on Grafana's host                                                                                                                                                                                                         | `swarm-controller`                                                     | ✅ every 10 minutes, when Grafana refuses it or none is stored                                                                                               | ✅ re-read on a 401                                                                                                                                                                                                                                          |
 | `swarm/controller/swarm-controller/forge/webhook-secret`                                                               | `swarm-controller`, at start, from its old `webhook-secret` file; a new one only when there is no file                                                                                                                                                         | `swarm-controller`                                                     | ❌ minted once; a rotated secret fails every delivery until the forge hooks are re-registered                                                                | ❌ read at start                                                                                                                                                                                                                                             |
-| `swarm/agents/<agent>/bao-mtls`                                                                                        | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation                                                                                                                                                                                | `hive-c0re`, then the agent                                            | ✅ re-issued every 5 minutes once past half its 7-day validity                                                                                               | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
+| `swarm/agents/<agent>/bao-mtls`                                                                                        | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation, and by the renewal pass when none is stored                                                                                                                                   | `hive-c0re`, then the agent                                            | ✅ re-issued every 5 minutes once past half its 7-day validity; minted when none is stored, for agents not `Destroyed`                                       | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
 | `swarm/agents/<agent>/queue`                                                                                           | `swarm-controller`, at agent creation                                                                                                                                                                                                                          | the agent; the queue, log and metrics auth services                    | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted                                                                               | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) |
 | `swarm/agents/<agent>/forge-token`                                                                                     | `swarm-controller`, at agent creation and every 5 minutes                                                                                                                                                                                                      | the agent                                                              | ✅ when missing or no longer matching the forge                                                                                                              | ✅ every 10 minutes                                                                                                                                                                                                                                          |
 | `swarm/agents/<agent>/forge/<label>`                                                                                   | `swarm-controller`, when you link an external forge account in the swarm UI                                                                                                                                                                                    | the agent                                                              | ❌ your token; delete it and link the label again                                                                                                            | ✅ every 2 minutes                                                                                                                                                                                                                                           |
diff --git a/swarm-controller/src/agent_identity.rs b/swarm-controller/src/agent_identity.rs
index 13eec673..34883282 100644
--- a/swarm-controller/src/agent_identity.rs
+++ b/swarm-controller/src/agent_identity.rs
@@ -27,7 +27,7 @@
 //! mount; its key never leaves the store. It signs no host leaf, and no host
 //! role pins it, so an agent's certificate satisfies only that agent's role.
 //! [`crate::agent_renewal`] re-issues a live agent's leaf once it is past half
-//! its validity (the role's `ttl`), by queueing the same node as creation.
+//! its validity (the role's `ttl`) or missing, by queueing the same node as creation.
 
 use crate::store::AccessStore;
 use anyhow::{Context, Result, bail};
@@ -102,10 +102,12 @@ async fn publish_access(
                 "writing the shared agent policy {}",
                 policy::SHARED_AGENT_POLICY
             )
-        })?;
+        })
+        .at(MintStep::Policy)?;
     crate::agent_entity::ensure_agent(store, agent)
         .await
-        .with_context(|| format!("naming {agent}'s identity entity; the role is not changed"))?;
+        .with_context(|| format!("naming {agent}'s identity entity; the role is not changed"))
+        .at(MintStep::Entity)?;
     store
         .write_cert_role(
             DEFAULT_CERT_MOUNT,
@@ -120,6 +122,7 @@ async fn publish_access(
         )
         .await
         .with_context(|| format!("writing the cert-auth role {}", inputs.name))
+        .at(MintStep::Role)
 }
 
 /// How many bytes of kernel randomness a queue secret is before encoding.
@@ -152,6 +155,73 @@ pub(crate) fn generate_queue_secret() -> Result<String> {
     ))
 }
 
+/// The part of [`mint_and_verify`] a failure came from.
+///
+/// Attached to the error as its outermost context, so the
+/// message leads with it and [`failed_step`] reads it back as a value.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MintStep {
+    /// Reading the environment and the agent's name, before any store call.
+    Setup,
+    /// Logging in to the store as the controller.
+    StoreLogin,
+    /// Issuing the leaf from the agent PKI role.
+    IssueLeaf,
+    /// Publishing the leaf at [`mtls::identity_path`].
+    WriteBaoMtls,
+    /// Reading or publishing the queue secret at [`queue::agent_queue_path`].
+    Queue,
+    /// Writing the shared ACL document.
+    Policy,
+    /// Naming the agent's identity entity.
+    Entity,
+    /// Deriving or writing the cert-auth role.
+    Role,
+    /// The login as the agent and the reads after it.
+    ReadBack,
+}
+
+impl MintStep {
+    /// The step as it reads in a log line.
+    #[must_use]
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            MintStep::Setup => "setup",
+            MintStep::StoreLogin => "store login",
+            MintStep::IssueLeaf => "issue leaf",
+            MintStep::WriteBaoMtls => "write bao-mtls",
+            MintStep::Queue => "queue",
+            MintStep::Policy => "policy",
+            MintStep::Entity => "entity",
+            MintStep::Role => "role",
+            MintStep::ReadBack => "read back",
+        }
+    }
+}
+
+impl std::fmt::Display for MintStep {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+/// The [`MintStep`] `error` was attached to, if [`mint_and_verify`] raised it.
+#[must_use]
+pub fn failed_step(error: &anyhow::Error) -> Option<MintStep> {
+    error.downcast_ref::<MintStep>().copied()
+}
+
+/// Names the [`MintStep`] a failing result belongs to.
+trait AtStep<T> {
+    fn at(self, step: MintStep) -> Result<T>;
+}
+
+impl<T, E: Into<anyhow::Error>> AtStep<T> for Result<T, E> {
+    fn at(self, step: MintStep) -> Result<T> {
+        self.map_err(|e| e.into().context(step))
+    }
+}
+
 /// Give `agent` an identity at the store, and prove it works.
 ///
 /// Six store calls' worth of agreement, then the login that checks it:
@@ -178,27 +248,30 @@ pub(crate) fn generate_queue_secret() -> Result<String> {
 /// replacing one by age is [`crate::agent_renewal`]'s.
 ///
 /// # Errors
-/// Anything that stops one of those six steps, with the step named, or the
-/// read-back login. A failure here fails the job node and nothing else — a new
-/// agent is still created, without a store identity, and a live agent keeps the
-/// role it had.
+/// Anything that stops one of those six steps, or the read-back login, with the
+/// [`MintStep`] ([`failed_step`]) leading the message. A failure here fails the
+/// job node and nothing else — a new agent is still created, without a store
+/// identity, and a live agent keeps the role it had.
 pub async fn mint_and_verify(agent: &str) -> Result<()> {
-    let (mount, pki_role) = agent_pki(|k| std::env::var(k).ok())?;
-    let name = policy::agent_object_name(agent)?;
-    let path = mtls::identity_path(agent)?;
-    let queue_path = queue::agent_queue_path(agent)?;
+    let (mount, pki_role) = agent_pki(|k| std::env::var(k).ok()).at(MintStep::Setup)?;
+    let name = policy::agent_object_name(agent).at(MintStep::Setup)?;
+    let path = mtls::identity_path(agent).at(MintStep::Setup)?;
+    let queue_path = queue::agent_queue_path(agent).at(MintStep::Setup)?;
 
     let store = crate::store::connect()
         .await
-        .context("logging in to the swarm secret store")?;
+        .context("logging in to the swarm secret store")
+        .at(MintStep::StoreLogin)?;
     let credential = store
         .issue_client_certificate(&mount, &pki_role, &name)
         .await
-        .with_context(|| format!("issuing {name}'s certificate from {mount}/issue/{pki_role}"))?;
+        .with_context(|| format!("issuing {name}'s certificate from {mount}/issue/{pki_role}"))
+        .at(MintStep::IssueLeaf)?;
     store
         .write(&path, &credential)
         .await
-        .with_context(|| format!("publishing the agent identity at {path}"))?;
+        .with_context(|| format!("publishing the agent identity at {path}"))
+        .at(MintStep::WriteBaoMtls)?;
 
     // Read before write, and `read_optional` rather than `read`, so that only
     // a genuine 404 leads to a new secret — see that method for why every
@@ -206,7 +279,8 @@ pub async fn mint_and_verify(agent: &str) -> Result<()> {
     let existing: Option<queue::AgentCredential> = store
         .read_optional(&queue_path)
         .await
-        .with_context(|| format!("checking whether {queue_path} already holds a credential"))?;
+        .with_context(|| format!("checking whether {queue_path} already holds a credential"))
+        .at(MintStep::Queue)?;
     // The secret and its mint time survive a re-run. An object naming a
     // different agent is corrected by rewriting the name around the *same*
     // `value`, which no live connection notices.
@@ -215,7 +289,8 @@ pub async fn mint_and_verify(agent: &str) -> Result<()> {
             agent: agent.to_owned(),
             ..existing.clone()
         },
-        None => crate::agent_renewal::fresh(agent, crate::agent_renewal::unix_now())?,
+        None => crate::agent_renewal::fresh(agent, crate::agent_renewal::unix_now())
+            .at(MintStep::Queue)?,
     };
     if existing.as_ref() == Some(&wanted) {
         tracing::info!(
@@ -227,7 +302,8 @@ pub async fn mint_and_verify(agent: &str) -> Result<()> {
         store
             .write(&queue_path, &wanted)
             .await
-            .with_context(|| format!("publishing the agent queue credential at {queue_path}"))?;
+            .with_context(|| format!("publishing the agent queue credential at {queue_path}"))
+            .at(MintStep::Queue)?;
         tracing::info!(
             agent,
             %queue_path,
@@ -239,11 +315,13 @@ pub async fn mint_and_verify(agent: &str) -> Result<()> {
     }
     let queue_credential = wanted;
 
-    let inputs = role_inputs(agent, &credential)?;
+    let inputs = role_inputs(agent, &credential).at(MintStep::Role)?;
     publish_access(&store, agent, &inputs).await?;
     tracing::info!(agent, %path, role = %name, "agent store identity published");
 
-    read_back_as_agent(&credential, &name, &path, &queue_path, &queue_credential).await?;
+    read_back_as_agent(&credential, &name, &path, &queue_path, &queue_credential)
+        .await
+        .at(MintStep::ReadBack)?;
     tracing::info!(
         agent,
         role = %name,
@@ -369,8 +447,8 @@ async fn read_back_as_agent(
 #[cfg(test)]
 mod tests {
     use super::{
-        ENV_AGENT_PKI_MOUNT, ENV_AGENT_PKI_ROLE, QUEUE_SECRET_BYTES, agent_pki,
-        generate_queue_secret, publish_access, role_inputs,
+        ENV_AGENT_PKI_MOUNT, ENV_AGENT_PKI_ROLE, MintStep, QUEUE_SECRET_BYTES, agent_pki,
+        failed_step, generate_queue_secret, publish_access, role_inputs,
     };
     use crate::store::fake::FakeAccess;
     use swarm_secret_client::{mtls, policy};
@@ -569,4 +647,30 @@ mod tests {
                 .contains(&"entity_by_alias hive-agent-atlas".to_owned())
         );
     }
+
+    /// Each refusal names the step it came from, in the message and as a value
+    /// the job runner logs. Two different steps, so a tag that named every
+    /// failure alike would fail one of them.
+    #[tokio::test]
+    async fn a_refused_step_is_named_in_the_message_and_by_failed_step() {
+        let credential = credential();
+        let inputs = role_inputs("atlas", &credential).expect("legal");
+
+        let err = publish_access(&FakeAccess::refusing_policies(), "atlas", &inputs)
+            .await
+            .expect_err("the policy write is refused");
+        assert_eq!(failed_step(&err), Some(MintStep::Policy), "{err:#}");
+        assert!(format!("{err:#}").starts_with("policy: "), "{err:#}");
+
+        let err = publish_access(&FakeAccess::without_accessor(), "atlas", &inputs)
+            .await
+            .expect_err("the entity step fails");
+        assert_eq!(failed_step(&err), Some(MintStep::Entity), "{err:#}");
+        assert!(format!("{err:#}").starts_with("entity: "), "{err:#}");
+    }
+
+    #[test]
+    fn an_error_from_outside_the_mint_has_no_step() {
+        assert_eq!(failed_step(&anyhow::anyhow!("elsewhere")), None);
+    }
 }
diff --git a/swarm-controller/src/agent_renewal.rs b/swarm-controller/src/agent_renewal.rs
index fadb1afc..dbf86a77 100644
--- a/swarm-controller/src/agent_renewal.rs
+++ b/swarm-controller/src/agent_renewal.rs
@@ -24,10 +24,10 @@
 //! tests pin them; the IO on either side only reads or acts.
 //!
 //! **Only agents some hive is declared to run are renewed** ([`live_agents`]):
-//! the wanted-state declarations are where a destroy is recorded, and an agent
-//! declared nowhere, or only as `Destroyed`, is skipped. A credential that is
-//! not stored is skipped too — creating one is agent creation's job, and a
-//! destroy deletes it.
+//! a destroy is recorded in the wanted-state declarations, and an agent declared
+//! nowhere, or only as `Destroyed`, is skipped. A live agent with no stored
+//! certificate gets one minted, as a failed mint at creation is retried nowhere
+//! else, unless its mint is in flight. A missing queue secret stays missing.
 
 use std::collections::BTreeSet;
 use std::sync::Arc;
@@ -141,7 +141,8 @@ pub enum Observed {
     Unstamped,
     /// Stored and not yet due.
     Current,
-    /// Nothing stored. Never renewed: see the module doc.
+    /// Nothing stored. A certificate is minted for a live agent; a queue secret
+    /// is left alone (see the module doc).
     Absent,
     /// The read or the decode failed. Nothing is known, so nothing is done.
     Unknown,
@@ -163,14 +164,14 @@ pub struct Renewal {
 }
 
 /// The renewals a pass queues: one per agent with a certificate that is
-/// [`Observed::Due`], or a queue secret that is [`Observed::Due`] or
-/// [`Observed::Unstamped`].
+/// [`Observed::Due`] or [`Observed::Absent`], or a queue secret that is
+/// [`Observed::Due`] or [`Observed::Unstamped`].
 pub fn plan(observed: &[(String, AgentObserved)]) -> Vec<Renewal> {
     observed
         .iter()
         .map(|(agent, o)| Renewal {
             agent: agent.clone(),
-            cert: matches!(o.cert, Observed::Due(_)),
+            cert: matches!(o.cert, Observed::Due(_) | Observed::Absent),
             secret: matches!(o.secret, Observed::Due(_) | Observed::Unstamped),
         })
         .filter(|r| r.cert || r.secret)
@@ -319,6 +320,12 @@ async fn observe_all(
         if let Observed::Due(age) = o.cert {
             tracing::info!(agent, age = %Age(age), "agent certificate past half its life; re-issuing");
         }
+        if o.cert == Observed::Absent {
+            tracing::info!(
+                agent,
+                "agent has no stored certificate; minting its store identity"
+            );
+        }
         if let Observed::Due(age) = o.secret {
             tracing::info!(agent, age = %Age(age), "agent queue credential due; re-minting");
         }
@@ -525,16 +532,19 @@ mod tests {
     }
 
     #[test]
-    fn only_due_credentials_are_planned() {
+    fn due_and_missing_certificates_and_due_secrets_are_planned() {
         use Observed::{Absent, Current, Due, Unknown, Unstamped};
         let o = |cert, secret| AgentObserved { cert, secret };
         let observed = [
             ("both".to_owned(), o(Due(46 * DAY), Due(45 * DAY))),
             ("cert".to_owned(), o(Due(46 * DAY), Current)),
-            ("secret".to_owned(), o(Absent, Due(45 * DAY))),
+            ("secret".to_owned(), o(Current, Due(45 * DAY))),
             ("unstamped".to_owned(), o(Current, Unstamped)),
             ("neither".to_owned(), o(Current, Absent)),
             ("unknown".to_owned(), o(Unknown, Unknown)),
+            ("no identity".to_owned(), o(Absent, Absent)),
+            ("no cert, old secret".to_owned(), o(Absent, Due(45 * DAY))),
+            ("unreadable cert".to_owned(), o(Unknown, Absent)),
         ];
         let r = |agent: &str, cert, secret| Renewal {
             agent: agent.to_owned(),
@@ -548,6 +558,40 @@ mod tests {
                 r("cert", true, false),
                 r("secret", false, true),
                 r("unstamped", false, true),
+                r("no identity", true, false),
+                r("no cert, old secret", true, true),
+            ]
+        );
+    }
+
+    /// A pass over declarations: the agent with no stored identity is minted
+    /// for when it is live and not when it is destroyed. The destroyed agent is
+    /// never observed, which is what [`live_agents`] does before the store is
+    /// read.
+    #[test]
+    fn a_missing_identity_is_minted_for_a_live_agent_and_not_a_destroyed_one() {
+        let declarations = [declared(&[
+            ("sock", AgentState::Up),
+            ("paused", AgentState::Paused),
+            ("gone", AgentState::Destroyed),
+        ])];
+        let nothing_stored = AgentObserved {
+            cert: Observed::Absent,
+            secret: Observed::Absent,
+        };
+        let observed: Vec<_> = live_agents(&declarations)
+            .into_iter()
+            .map(|agent| (agent, nothing_stored))
+            .collect();
+        let planned: Vec<_> = plan(&observed)
+            .into_iter()
+            .map(|r| (r.agent, r.cert, r.secret))
+            .collect();
+        assert_eq!(
+            planned,
+            [
+                ("paused".to_owned(), true, false),
+                ("sock".to_owned(), true, false)
             ]
         );
     }
diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs
index ee9d2317..a95d35a4 100644
--- a/swarm-controller/src/jobq.rs
+++ b/swarm-controller/src/jobq.rs
@@ -454,8 +454,8 @@ pub(crate) async fn forge_node<T>(
         })
 }
 
-/// Logged at `warn`, with the agent and the error, when [`mint_identity`]
-/// fails. The Grafana rule `hh-agent-identity-mint-failed`
+/// Logged at `warn`, with the agent, the step and the error, when
+/// [`mint_identity`] fails. The Grafana rule `hh-agent-identity-mint-failed`
 /// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both
 /// together.
 const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
@@ -469,7 +469,9 @@ pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome
         Ok(()) => Outcome::Done,
         Err(e) => {
             let error = format!("{e:#}");
-            tracing::warn!(%agent, %error, "{MINT_IDENTITY_FAILED}");
+            let step =
+                agent_identity::failed_step(&e).map_or("unknown", agent_identity::MintStep::as_str);
+            tracing::warn!(%agent, %step, %error, "{MINT_IDENTITY_FAILED}");
             Outcome::Failed(error)
         }
     }
@@ -1738,11 +1740,28 @@ pub(crate) fn queue_ci_metrics_forge_token_mint(
         .map_err(|e| anyhow::anyhow!("{e}"))
 }
 
+/// Whether a `MintAgentIdentity` for `agent` is queued or running.
+fn identity_mint_in_flight(
+    graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
+    agent: &str,
+) -> bool {
+    graph.nodes().any(|n| {
+        !n.state.is_terminal()
+            && matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
+    })
+}
+
 /// Insert one job per renewal and return the ids of its nodes:
 /// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a
 /// queue secret, the second `after_any` the first when an agent needs both.
 /// `agent_renewal::spawn`'s periodic pass comes through here.
 ///
+/// An agent whose `MintAgentIdentity` has not finished (agent creation's, or an
+/// earlier pass's) is skipped whole until the next pass: a mint issues a new
+/// leaf each run, and a pass comes every five minutes whether or not the last
+/// mint has run yet. Its secret renewal waits too, since the chain below
+/// cannot reach a mint it did not queue.
+///
 /// Chained rather than parallel because `mint_and_verify` reads the queue
 /// secret back and compares it with the one it read first; a re-mint landing
 /// in between fails that node. `after_any`, so a certificate that could not be
@@ -1756,6 +1775,13 @@ pub(crate) fn queue_agent_renewals(
         .unwrap_or_else(std::sync::PoisonError::into_inner);
     let mut ids = Vec::with_capacity(renewals.len());
     for renewal in renewals {
+        if renewal.cert && identity_mint_in_flight(sched.graph(), &renewal.agent) {
+            tracing::info!(
+                agent = %renewal.agent,
+                "agent identity mint already in flight; not queueing another"
+            );
+            continue;
+        }
         let queued = sched
             .insert_job(None, |b| {
                 let mut asked = Vec::new();
@@ -3347,6 +3373,75 @@ mod tests {
         );
     }
 
+    fn mint_nodes_for(
+        sched: &std::sync::Mutex<
+            hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
+        >,
+        agent: &str,
+    ) -> Vec<hive_jobq::NodeId> {
+        let guard = sched
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        guard
+            .graph()
+            .nodes()
+            .filter(|n| {
+                matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
+            })
+            .map(|n| n.id)
+            .collect()
+    }
+
+    /// A pass that finds a mint for the agent still queued or running (agent
+    /// creation's or its own earlier one) queues none; one that finds only
+    /// finished mints queues the next. The other agent is the control: the
+    /// check is per agent.
+    #[test]
+    fn a_mint_in_flight_blocks_another_for_that_agent_only() {
+        use crate::agent_renewal::Renewal;
+
+        let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
+            hive_jobq::Graph::new(),
+            hive_jobq::resources::ResourceTable::new(),
+        ));
+        let renewal = |agent: &str| Renewal {
+            agent: agent.to_owned(),
+            cert: true,
+            secret: false,
+        };
+
+        let first = super::queue_agent_renewals(&sched, vec![renewal("sock"), renewal("other")])
+            .expect("two jobs insert");
+        assert_eq!(first.len(), 2, "no mint in flight: both are queued");
+
+        let second = super::queue_agent_renewals(&sched, vec![renewal("sock")])
+            .expect("a skipped agent is not an error");
+        assert!(second.is_empty(), "the first pass's mint is still pending");
+        assert_eq!(mint_nodes_for(&sched, "sock").len(), 1);
+
+        // The next pass after that mint finished.
+        let pending = mint_nodes_for(&sched, "sock")[0];
+        assert!(
+            sched
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .cancel_node(pending)
+        );
+        let third = super::queue_agent_renewals(&sched, vec![renewal("sock")])
+            .expect("a finished mint does not block");
+        assert_eq!(
+            third.len(),
+            1,
+            "a finished mint is retried by the next pass"
+        );
+        assert_eq!(mint_nodes_for(&sched, "sock").len(), 2);
+        assert_eq!(
+            mint_nodes_for(&sched, "other").len(),
+            1,
+            "the other agent's pending mint was never touched"
+        );
+    }
+
     /// Agent creation mints the matrix account as a root of its own, and the
     /// deploy waits for it without being cancelled by it: a host with no
     /// homeserver configured must still deploy the agent.
@@ -3962,7 +4057,8 @@ mod mint_identity_tests {
     }
 
     /// An agent name the store would refuse fails the mint before any store
-    /// call, whatever this process's environment holds. The literal is the
+    /// call, whatever this process's environment holds. The line names the
+    /// step it stopped at. The literal is the
     /// `_msg` that `hh-agent-identity-mint-failed` in
     /// `nix/host-modules/swarm-grafana.nix` matches.
     #[tokio::test]
@@ -3988,6 +4084,7 @@ mod mint_identity_tests {
         };
         assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed");
         assert_eq!(field("agent"), "a/b");
+        assert_eq!(field("step"), "setup");
         assert_eq!(field("error"), error);
     }