hive-agent: present this agent's own queue credential, then fall back

When the per-agent secret `queue-identity.nix` fetched is present, the
harness connects with `swarm-agent.<agent>.<secret>` as a static token
and publishes on `$SWARM.term.<agent>` and `$SWARM.agent-state.<agent>`.
When it is absent, or that first connect fails for any reason, a refusal
from a responder that does not verify agent tokens included, it connects
with the hive's shared OIDC client and publishes on the hive-scoped
subjects as before. Which one it took is logged once per connect.

`swarm_queue_client::connect_with_token` is the static-token connect: no
retry on the initial attempt, so the caller sees the refusal and can
fall back. Reconnects share the existing backoff, now a named function.

Closes #4630

hive-agent/src/swarm_queue.rs

@@ -21,15 +21,16 @@
 //! agent. The fifth is this agent's own, minted per agent at swarm level and
 //! fetched by the container itself (`nix/agent-modules/queue-identity.nix`).
 //!
-//! It is reported here but not yet *presented*: the queue's auth-callout
-//! responder (`swarm-nats-auth`) validates only the hive-scoped token, and an
-//! agent offering a credential nothing on the other end reads back would be
-//! refused. Until that responder learns the same path, the connect path below
-//! is unchanged and this is the fetching half.
+//! When it is present, [`client`] connects with it first, and the agent
+//! publishes on its own hive-free subjects. When it is absent, or the queue
+//! refuses it, the connect falls back to the hive's shared client and the
+//! hive-scoped subjects. [`Connection::presented`] says which, so a publisher
+//! builds the subject that credential is granted.
 
 use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 
+use anyhow::Context as _;
 use tokio::sync::OnceCell;
 
 use swarm_queue_client::QueueConfig;
@@ -42,10 +43,39 @@ const ENV_PREFIX: &str = "HIVE_AGENT";
 /// one answer rather than re-deriving it per call.
 static CONFIG: OnceLock<Option<QueueConfig>> = OnceLock::new();
 
+/// Where to present this agent's own credential, resolved once at boot.
+static AGENT: OnceLock<Option<AgentPath>> = OnceLock::new();
+
 /// The one connection every publisher in this process shares. Separate from
 /// [`CONFIG`] because resolving the coordinates is synchronous boot work and
 /// connecting is not — see [`client`].
-static CLIENT: OnceCell<Option<async_nats::Client>> = OnceCell::const_new();
+static CLIENT: OnceCell<Option<Connection>> = OnceCell::const_new();
+
+/// What connecting with this agent's own credential needs.
+#[derive(Debug, PartialEq, Eq)]
+struct AgentPath {
+    url: String,
+    ca_file: Option<PathBuf>,
+    /// The fetched secret. A path; the bytes are read at connect.
+    secret_file: PathBuf,
+}
+
+/// Which credential the shared connection presented.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Presented {
+    /// This agent's own, granted `<prefix>.<agent>`.
+    Agent,
+    /// The hive's shared client, granted `<prefix>.<hive>.>` for the hive the
+    /// client id names.
+    Hive { client_id: String },
+}
+
+/// The shared queue connection and the credential it was made with.
+#[derive(Clone)]
+pub struct Connection {
+    pub client: async_nats::Client,
+    pub presented: Presented,
+}
 
 /// The four variables the harness unit sets, before the client-id file is
 /// read. Collected into a struct so [`decide`] is pure over them and the
@@ -147,6 +177,16 @@ fn decide_agent_secret(path: Option<&str>) -> Option<PathBuf> {
     }
 }
 
+/// Decide whether this agent can connect with its own credential: it needs the
+/// queue's address and a fetched secret, and nothing of the hive's client.
+fn decide_agent_path(env: &QueueEnv, secret_file: Option<PathBuf>) -> Option<AgentPath> {
+    Some(AgentPath {
+        url: env.nats_url.clone()?,
+        ca_file: env.ca_file.as_ref().map(Into::into),
+        secret_file: secret_file?,
+    })
+}
+
 /// Decide what this agent's queue configuration is, given the environment and
 /// whatever the client-id file held.
 ///
@@ -216,15 +256,10 @@ pub fn init() {
     let _ = CONFIG.set(resolved);
 
     // Independent of everything above: this agent may hold its own secret on
-    // a hive with no queue coordinates, or hold the coordinates and no secret
-    // of its own yet. Reported either way, because "which credential is this
-    // agent able to present" is a question only this process can answer, and
-    // it is the one the next slice's rollout will be asked repeatedly.
-    //
-    // The answer is only logged here. Presenting it needs the queue's
-    // auth-callout responder to verify it, which is the next slice — see this
-    // module's header.
-    if let Some(path) = decide_agent_secret(env.agent_secret_file.as_deref()) {
+    // a hive whose shared client is not published yet, or the shared client
+    // and no secret of its own.
+    let secret = decide_agent_secret(env.agent_secret_file.as_deref());
+    if let Some(path) = &secret {
         // The path, never the bytes: the file holds the secret itself.
         tracing::info!(
             path = %path.display(),
@@ -236,6 +271,13 @@ pub fn init() {
              by its hive's shared client"
         );
     }
+    let _ = AGENT.set(decide_agent_path(&env, secret));
+}
+
+/// Whether this agent has any credential to reach the queue with. `false`
+/// before [`init`] has run.
+pub fn configured() -> bool {
+    config().is_some() || AGENT.get().is_some_and(Option::is_some)
 }
 
 /// What [`init`] resolved, or `None` when this agent has no queue.
@@ -263,14 +305,48 @@ pub fn config() -> Option<&'static QueueConfig> {
 /// failed" — a caller does nothing differently between them, since either way
 /// there is nothing to publish onto. Connecting is lazy so that an agent on a
 /// hive with no queue pays nothing at boot.
-pub async fn client() -> Option<async_nats::Client> {
+pub async fn client() -> Option<Connection> {
     CLIENT.get_or_init(connect_once).await.clone()
 }
 
-async fn connect_once() -> Option<async_nats::Client> {
+/// This agent's own credential first, then the hive's. Each path taken is
+/// logged once, here.
+async fn connect_once() -> Option<Connection> {
+    if let Some(agent) = AGENT.get().and_then(Option::as_ref) {
+        match connect_as_agent(agent).await {
+            Ok(client) => {
+                tracing::info!(
+                    url = %agent.url,
+                    "connected to the swarm queue with this agent's own credential"
+                );
+                return Some(Connection {
+                    client,
+                    presented: Presented::Agent,
+                });
+            }
+            Err(e) => tracing::warn!(
+                error = format!("{e:#}"),
+                "connecting with this agent's own credential failed; falling back to \
+                 its hive's shared client"
+            ),
+        }
+    }
     let cfg = config()?;
     match swarm_queue_client::connect(cfg.clone()).await {
-        Ok(client) => Some(client),
+        Ok(client) => {
+            tracing::info!(
+                url = %cfg.url,
+                client_id = %cfg.client_id,
+                "connecting to the swarm queue with the hive's shared client; the \
+                 client retries in the background until the queue accepts it"
+            );
+            Some(Connection {
+                client,
+                presented: Presented::Hive {
+                    client_id: cfg.client_id.clone(),
+                },
+            })
+        }
         Err(e) => {
             // `chain`, not `{:#}`: this is `swarm_queue_client::Error`, whose
             // `Display` ignores the alternate flag, so `{:#}` renders the
@@ -284,9 +360,26 @@ async fn connect_once() -> Option<async_nats::Client> {
     }
 }
 
+/// Connect presenting this agent's own token. Any failure, a refusal
+/// included, is returned for [`connect_once`] to fall back on.
+async fn connect_as_agent(agent: &AgentPath) -> anyhow::Result<async_nats::Client> {
+    let name = crate::identity::label();
+    let secret = std::fs::read_to_string(&agent.secret_file)
+        .with_context(|| format!("reading {}", agent.secret_file.display()))?;
+    let token = swarm_queue_client::agent_token::format_agent_token(&name, secret.trim())?;
+    swarm_queue_client::connect_with_token(&agent.url, agent.ca_file.as_deref(), token)
+        .await
+        .map_err(|e| anyhow::anyhow!(swarm_queue_client::chain(&e)))
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{QueueEnv, Resolution, decide, decide_agent_secret, read_client_id};
+    use std::path::PathBuf;
+
+    use super::{
+        AgentPath, QueueEnv, Resolution, decide, decide_agent_path, decide_agent_secret,
+        read_client_id,
+    };
 
     fn env(parts: [Option<&str>; 4]) -> QueueEnv {
         let [nats_url, token_endpoint, client_id_file, client_secret_file] = parts;
@@ -435,4 +528,34 @@ mod tests {
         assert!(matches!(decide(&e, None), Resolution::Absent(_)));
         assert!(decide_agent_secret(path.to_str()).is_some());
     }
+
+    /// The agent's own path needs the queue's address and its own secret, and
+    /// not the hive's client id: that is what lets it connect on a hive whose
+    /// shared client is not published.
+    #[test]
+    fn an_agent_with_its_own_secret_and_the_queue_address_connects_as_itself() {
+        let secret = PathBuf::from("/run/queue-identity/secret");
+        assert_eq!(
+            decide_agent_path(&full(), Some(secret.clone())),
+            Some(AgentPath {
+                url: "nats://10.42.0.1:4222".to_owned(),
+                ca_file: None,
+                secret_file: secret.clone(),
+            })
+        );
+        let url_only = env([Some("nats://10.42.0.1:4222"), None, None, None]);
+        assert!(decide_agent_path(&url_only, Some(secret)).is_some());
+    }
+
+    #[test]
+    fn without_its_own_secret_or_the_queue_address_an_agent_does_not_connect_as_itself() {
+        assert_eq!(decide_agent_path(&full(), None), None);
+        assert_eq!(
+            decide_agent_path(
+                &env([None, None, None, None]),
+                Some(PathBuf::from("/run/queue-identity/secret"))
+            ),
+            None
+        );
+    }
 }