swarm-controller: re-issue agent certificates and re-mint queue secrets at half-life

A five-minute pass over every agent some hive's wanted state declares as
anything but destroyed queues, per agent:

- `MintAgentIdentity` (the node agent creation uses) when the stored
  certificate at swarm/agents/<agent>/bao-mtls is past half its validity,
  read from its own notBefore/notAfter: day 45 of the role's 90;
- the new `RenewAgentQueueCredential` node when the queue secret at
  swarm/agents/<agent>/queue is 45 days old or has no mint time. The node
  re-decides, writes a fresh value with `minted_at`, reads it back, and logs
  the agent and the old age.

When both are due the secret node runs after_any the certificate node,
because mint_and_verify compares the queue secret it read with the one it
reads back. A credential that is not stored is never created here.

`queue::AgentCredential` gains an optional `minted_at` (unix seconds);
agent creation now sets it. Stored objects without it decode unchanged and
count as due, so every existing queue secret is re-minted on the first pass.

Both replacements reach the agent at its next start. The old certificate
stays valid until it expires; the old queue secret does not, so a queue
reconnect before that restart is denied.

Adds x509-cert 0.2 (with der_derive and flagset) to read the validity.

docs/swarm/credentials.md: the renewal column splits into automatic re-mint
and automatic re-pull, filled from the code as it stands.

swarm-controller/src/main.rs

@@ -42,6 +42,7 @@ use utoipa_axum::{router::OpenApiRouter, routes};
 
 mod agent_icon;
 mod agent_identity;
+mod agent_renewal;
 mod agent_state_stream;
 mod agent_status;
 mod auth;
@@ -122,6 +123,11 @@ enum SwarmNodeKind {
     /// Carries no hive: the path it writes has no hive segment, so an agent
     /// that moves between hives keeps one matrix identity.
     MintAgentMatrixAccount { agent: String },
+    /// Re-mint `agent`'s queue secret if it is still stored and old enough.
+    /// See `agent_renewal`.
+    ///
+    /// Carries no hive: the secret's store path has none.
+    RenewAgentQueueCredential { agent: String },
     /// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state
     /// store, so a freshly created agent does not start driving turns the
     /// moment it's deployed — the operator has to explicitly flip it to `Up`.
@@ -148,6 +154,9 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
             SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(),
             SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(),
+            SwarmNodeKind::RenewAgentQueueCredential { .. } => {
+                "renew_agent_queue_credential".to_owned()
+            }
             SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
             SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
         }
@@ -168,7 +177,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
             | SwarmNodeKind::InitAgentConfigRepo { agent }
             | SwarmNodeKind::MintAgentIdentity { agent }
             | SwarmNodeKind::MintAgentForgeToken { agent }
-            | SwarmNodeKind::MintAgentMatrixAccount { agent } => {
+            | SwarmNodeKind::MintAgentMatrixAccount { agent }
+            | SwarmNodeKind::RenewAgentQueueCredential { agent } => {
                 serde_json::json!({ "agent": agent })
             }
             SwarmNodeKind::TriggerDeploy { hive, agent }
@@ -319,6 +329,12 @@ async fn run_swarm_node(
         SwarmNodeKind::MintAgentMatrixAccount { agent } => {
             mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
         }
+        SwarmNodeKind::RenewAgentQueueCredential { agent } => {
+            match agent_renewal::renew(&agent).await {
+                Ok(()) => Outcome::Done,
+                Err(e) => Outcome::Failed(format!("{e:#}")),
+            }
+        }
         SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted {
             None => Outcome::Failed(
                 "no swarm queue is configured on this host, so no wanted-state \
@@ -1875,6 +1891,73 @@ fn spawn_forge_workers(
     });
 }
 
+/// Insert one job per renewal and return the ids of its nodes:
+/// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a
+/// queue secret, the second `after_any` the first when an agent needs both.
+/// `agent_renewal::spawn`'s periodic pass comes through here.
+///
+/// Chained rather than parallel because `mint_and_verify` reads the queue
+/// secret back and compares it with the one it read first; a re-mint landing
+/// in between fails that node. `after_any`, so a certificate that could not be
+/// re-issued does not hold up the secret.
+fn queue_agent_renewals(
+    sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
+    renewals: Vec<agent_renewal::Renewal>,
+) -> Result<Vec<hive_jobq::NodeId>> {
+    let mut sched = sched
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let mut ids = Vec::with_capacity(renewals.len());
+    for renewal in renewals {
+        let queued = sched
+            .insert_job(None, |b| {
+                let mut asked = Vec::new();
+                let cert = renewal.cert.then(|| {
+                    b.node(SwarmNodeKind::MintAgentIdentity {
+                        agent: renewal.agent.clone(),
+                    })
+                    .guid()
+                });
+                asked.extend(cert);
+                if renewal.secret {
+                    let secret = b.node(SwarmNodeKind::RenewAgentQueueCredential {
+                        agent: renewal.agent.clone(),
+                    });
+                    let secret = match cert {
+                        Some(cert) => secret.after_any(cert),
+                        None => secret,
+                    };
+                    asked.push(secret.guid());
+                }
+                asked
+            })
+            .map_err(|e| anyhow::anyhow!("{e}"))?;
+        ids.extend(queued);
+    }
+    Ok(ids)
+}
+
+/// Start the agent credential renewal pass when a swarm queue is wired up:
+/// the pass reads the wanted-state declarations to tell live agents from
+/// destroyed ones, and without a queue it has none. Lifted out of `main` for
+/// `clippy::too_many_lines`.
+fn spawn_agent_renewal(
+    jobq: &Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
+    wanted: Option<Arc<wanted::WantedWriter>>,
+    hives: &[HiveEntry],
+) {
+    let Some(wanted) = wanted else {
+        return;
+    };
+    let hives = hives.iter().map(|h| h.name.clone()).collect();
+    let sched = Arc::clone(jobq);
+    agent_renewal::spawn(wanted, hives, move |renewals| {
+        if let Err(e) = queue_agent_renewals(&sched, renewals) {
+            tracing::warn!(error = %format!("{e:#}"), "agent credential renewal: queueing failed");
+        }
+    });
+}
+
 /// Check an agent's forge token now, and mint one if it is missing or stale.
 ///
 /// The periodic pass (`forge::agent_token::spawn`) does the same every five
@@ -2402,6 +2485,7 @@ async fn main() -> Result<()> {
     let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone());
 
     let hives = load_hives();
+    spawn_agent_renewal(&jobq, wanted_writer(status.as_ref()), &hives);
     // Before serving, because a hive whose role does not exist cannot log in,
     // and one whose policy does not exist logs in able to read nothing —
     // either way it cannot collect what this daemon writes for it. A store
@@ -3387,6 +3471,88 @@ mod tests {
         assert_eq!(kind.data(1)["agent"], "atlas");
     }
 
+    /// The periodic pass's queueing: a certificate re-issue is the node agent
+    /// creation mints with, a secret re-mint its own node, and an agent due
+    /// both gets the secret `after_any` the certificate.
+    #[test]
+    fn a_queued_renewal_chains_the_secret_after_the_certificate() {
+        use crate::agent_renewal::Renewal;
+        use hive_jobq_wire::WireNode as _;
+
+        let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
+            hive_jobq::Graph::new(),
+            hive_jobq::resources::ResourceTable::new(),
+        ));
+        let r = |agent: &str, cert, secret| Renewal {
+            agent: agent.to_owned(),
+            cert,
+            secret,
+        };
+        let ids = super::queue_agent_renewals(
+            &sched,
+            vec![
+                r("both", true, true),
+                r("cert", true, false),
+                r("secret", false, true),
+            ],
+        )
+        .expect("three jobs insert");
+        assert_eq!(ids.len(), 4, "one returned id per node");
+        let guard = sched
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let graph = guard.graph();
+        let agent_of = |n: &hive_jobq::Node<SwarmNodeKind, super::SwarmResourceKind>| {
+            n.payload.data(n.id.get())["agent"]
+                .as_str()
+                .expect("agent is a string")
+                .to_owned()
+        };
+        let mut nodes: Vec<(String, String)> = graph
+            .nodes()
+            .map(|n| (agent_of(n), n.payload.label()))
+            .collect();
+        nodes.sort();
+        assert_eq!(
+            nodes,
+            [
+                ("both".to_owned(), "mint_agent_identity".to_owned()),
+                ("both".to_owned(), "renew_agent_queue_credential".to_owned()),
+                ("cert".to_owned(), "mint_agent_identity".to_owned()),
+                (
+                    "secret".to_owned(),
+                    "renew_agent_queue_credential".to_owned()
+                ),
+            ]
+        );
+        let find = |agent: &str, label: &str| {
+            graph
+                .nodes()
+                .find(|n| n.payload.label() == label && agent_of(n) == agent)
+                .unwrap_or_else(|| panic!("{agent} has a {label} node"))
+        };
+        let cert = find("both", "mint_agent_identity").id;
+        let secret = find("both", "renew_agent_queue_credential");
+        let when = secret
+            .deps
+            .iter()
+            .find_map(|d| match d {
+                hive_jobq::Dep::Node { id, when } if *id == cert => Some(*when),
+                _ => None,
+            })
+            .expect("the secret waits for the certificate");
+        assert!(
+            when.accepts(hive_jobq::TerminalState::Failed),
+            "a certificate that could not be re-issued must not hold up the secret"
+        );
+        assert!(
+            find("secret", "renew_agent_queue_credential")
+                .deps
+                .is_empty(),
+            "a secret on its own waits for nothing"
+        );
+    }
+
     /// Agent creation mints the matrix account as a root of its own, and the
     /// deploy waits for it without being cancelled by it: a host with no
     /// homeserver configured must still deploy the agent.