log the queue-connect and identity-mint failures the alert rules match

hh-swarm-queue-connect-failed matched "swarm queue connect failed; this
agent publishes nothing upward", which nothing has logged since the
harness started retrying its first connect and exiting (235bc481b). The
rule now matches the error the harness logs when every attempt has
failed, CONNECT_EXHAUSTED in swarm_queue.rs, and its comment and summary
describe the retry-then-exit behaviour.

A failed MintAgentIdentity node logged nothing: the error reached only
the job graph. mint_identity now warns with the agent and the error, and
hh-agent-identity-mint-failed fires on that line.

Refs #4651

swarm-controller/src/main.rs

@@ -460,6 +460,12 @@ async fn forge_node<T>(
         })
 }
 
+/// Logged at `warn`, with the agent and the error, when [`mint_identity`]
+/// fails. The Grafana rule `hh-agent-identity-mint-failed`
+/// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both
+/// together.
+const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
+
 /// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under
 /// `clippy::too_many_lines`.
 async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
@@ -467,7 +473,11 @@ async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
 
     match agent_identity::mint_and_verify(agent).await {
         Ok(()) => Outcome::Done,
-        Err(e) => Outcome::Failed(format!("{e:#}")),
+        Err(e) => {
+            let error = format!("{e:#}");
+            tracing::warn!(%agent, %error, "{MINT_IDENTITY_FAILED}");
+            Outcome::Failed(error)
+        }
     }
 }
 
@@ -6364,3 +6374,69 @@ mod create_agent_warning_tests {
         assert_eq!(json, r#"{"node_id":7,"warnings":["name is reserved"]}"#);
     }
 }
+
+#[cfg(test)]
+mod mint_identity_tests {
+    use std::sync::{Arc, Mutex};
+
+    use hive_jobq::scheduler::Outcome;
+    use tracing_subscriber::layer::SubscriberExt as _;
+
+    use super::mint_identity;
+
+    /// One event's level and its fields, each by name.
+    type Event = (tracing::Level, Vec<(String, String)>);
+
+    /// Every event.
+    #[derive(Clone, Default)]
+    struct Events(Arc<Mutex<Vec<Event>>>);
+
+    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Events {
+        fn on_event(
+            &self,
+            event: &tracing::Event<'_>,
+            _: tracing_subscriber::layer::Context<'_, S>,
+        ) {
+            let mut fields = Vec::new();
+            event.record(
+                &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
+                    fields.push((field.name().to_owned(), format!("{value:?}")));
+                },
+            );
+            self.0
+                .lock()
+                .expect("unpoisoned")
+                .push((*event.metadata().level(), fields));
+        }
+    }
+
+    /// An agent name the store would refuse fails the mint before any store
+    /// call, whatever this process's environment holds. The literal is the
+    /// `_msg` that `hh-agent-identity-mint-failed` in
+    /// `nix/host-modules/swarm-grafana.nix` matches.
+    #[tokio::test]
+    async fn a_failed_mint_logs_the_agent_and_the_error_on_the_line_the_alert_rule_matches() {
+        let events = Events::default();
+        let _guard =
+            tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
+
+        let Outcome::Failed(error) = mint_identity("a/b").await else {
+            panic!("an illegal agent name cannot be minted");
+        };
+
+        let got = events.0.lock().expect("unpoisoned").clone();
+        let [(level, fields)] = got.as_slice() else {
+            panic!("one event, got {got:?}");
+        };
+        assert_eq!(*level, tracing::Level::WARN);
+        let field = |name: &str| {
+            fields
+                .iter()
+                .find(|(n, _)| n == name)
+                .map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
+        };
+        assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed");
+        assert_eq!(field("agent"), "a/b");
+        assert_eq!(field("error"), error);
+    }
+}