mint a live agent's missing store identity from the renewal pass, and log the step a mint stopped at

swarm-controller/src/jobq.rs

@@ -454,8 +454,8 @@ pub(crate) async fn forge_node<T>(
         })
 }
 
-/// Logged at `warn`, with the agent and the error, when [`mint_identity`]
-/// fails. The Grafana rule `hh-agent-identity-mint-failed`
+/// Logged at `warn`, with the agent, the step and the error, when
+/// [`mint_identity`] fails. The Grafana rule `hh-agent-identity-mint-failed`
 /// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both
 /// together.
 const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
@@ -469,7 +469,9 @@ pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome
         Ok(()) => Outcome::Done,
         Err(e) => {
             let error = format!("{e:#}");
-            tracing::warn!(%agent, %error, "{MINT_IDENTITY_FAILED}");
+            let step =
+                agent_identity::failed_step(&e).map_or("unknown", agent_identity::MintStep::as_str);
+            tracing::warn!(%agent, %step, %error, "{MINT_IDENTITY_FAILED}");
             Outcome::Failed(error)
         }
     }
@@ -1738,11 +1740,28 @@ pub(crate) fn queue_ci_metrics_forge_token_mint(
         .map_err(|e| anyhow::anyhow!("{e}"))
 }
 
+/// Whether a `MintAgentIdentity` for `agent` is queued or running.
+fn identity_mint_in_flight(
+    graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
+    agent: &str,
+) -> bool {
+    graph.nodes().any(|n| {
+        !n.state.is_terminal()
+            && matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
+    })
+}
+
 /// Insert one job per renewal and return the ids of its nodes:
 /// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a
 /// queue secret, the second `after_any` the first when an agent needs both.
 /// `agent_renewal::spawn`'s periodic pass comes through here.
 ///
+/// An agent whose `MintAgentIdentity` has not finished (agent creation's, or an
+/// earlier pass's) is skipped whole until the next pass: a mint issues a new
+/// leaf each run, and a pass comes every five minutes whether or not the last
+/// mint has run yet. Its secret renewal waits too, since the chain below
+/// cannot reach a mint it did not queue.
+///
 /// Chained rather than parallel because `mint_and_verify` reads the queue
 /// secret back and compares it with the one it read first; a re-mint landing
 /// in between fails that node. `after_any`, so a certificate that could not be
@@ -1756,6 +1775,13 @@ pub(crate) fn queue_agent_renewals(
         .unwrap_or_else(std::sync::PoisonError::into_inner);
     let mut ids = Vec::with_capacity(renewals.len());
     for renewal in renewals {
+        if renewal.cert && identity_mint_in_flight(sched.graph(), &renewal.agent) {
+            tracing::info!(
+                agent = %renewal.agent,
+                "agent identity mint already in flight; not queueing another"
+            );
+            continue;
+        }
         let queued = sched
             .insert_job(None, |b| {
                 let mut asked = Vec::new();
@@ -3347,6 +3373,75 @@ mod tests {
         );
     }
 
+    fn mint_nodes_for(
+        sched: &std::sync::Mutex<
+            hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
+        >,
+        agent: &str,
+    ) -> Vec<hive_jobq::NodeId> {
+        let guard = sched
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        guard
+            .graph()
+            .nodes()
+            .filter(|n| {
+                matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
+            })
+            .map(|n| n.id)
+            .collect()
+    }
+
+    /// A pass that finds a mint for the agent still queued or running (agent
+    /// creation's or its own earlier one) queues none; one that finds only
+    /// finished mints queues the next. The other agent is the control: the
+    /// check is per agent.
+    #[test]
+    fn a_mint_in_flight_blocks_another_for_that_agent_only() {
+        use crate::agent_renewal::Renewal;
+
+        let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
+            hive_jobq::Graph::new(),
+            hive_jobq::resources::ResourceTable::new(),
+        ));
+        let renewal = |agent: &str| Renewal {
+            agent: agent.to_owned(),
+            cert: true,
+            secret: false,
+        };
+
+        let first = super::queue_agent_renewals(&sched, vec![renewal("sock"), renewal("other")])
+            .expect("two jobs insert");
+        assert_eq!(first.len(), 2, "no mint in flight: both are queued");
+
+        let second = super::queue_agent_renewals(&sched, vec![renewal("sock")])
+            .expect("a skipped agent is not an error");
+        assert!(second.is_empty(), "the first pass's mint is still pending");
+        assert_eq!(mint_nodes_for(&sched, "sock").len(), 1);
+
+        // The next pass after that mint finished.
+        let pending = mint_nodes_for(&sched, "sock")[0];
+        assert!(
+            sched
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .cancel_node(pending)
+        );
+        let third = super::queue_agent_renewals(&sched, vec![renewal("sock")])
+            .expect("a finished mint does not block");
+        assert_eq!(
+            third.len(),
+            1,
+            "a finished mint is retried by the next pass"
+        );
+        assert_eq!(mint_nodes_for(&sched, "sock").len(), 2);
+        assert_eq!(
+            mint_nodes_for(&sched, "other").len(),
+            1,
+            "the other agent's pending mint was never touched"
+        );
+    }
+
     /// Agent creation mints the matrix account as a root of its own, and the
     /// deploy waits for it without being cancelled by it: a host with no
     /// homeserver configured must still deploy the agent.
@@ -3962,7 +4057,8 @@ mod mint_identity_tests {
     }
 
     /// An agent name the store would refuse fails the mint before any store
-    /// call, whatever this process's environment holds. The literal is the
+    /// call, whatever this process's environment holds. The line names the
+    /// step it stopped at. The literal is the
     /// `_msg` that `hh-agent-identity-mint-failed` in
     /// `nix/host-modules/swarm-grafana.nix` matches.
     #[tokio::test]
@@ -3988,6 +4084,7 @@ mod mint_identity_tests {
         };
         assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed");
         assert_eq!(field("agent"), "a/b");
+        assert_eq!(field("step"), "setup");
         assert_eq!(field("error"), error);
     }