re-read the declaration before a mint, and mint for a current certificate whose role is missing

docs/swarm/README.md

@@ -63,7 +63,9 @@ that fails to mint logs `swarm jobq: mint_agent_identity failed` with the agent,
 the step it stopped at and the error, which the Grafana alert rule
 `hh-agent-identity-mint-failed` fires on. The agent is still created, without a
 store identity, and the controller's five-minute credential pass mints one for
-it as long as the agent isn't `Destroyed`.
+it (also when only the certificate exists and its cert-auth role is missing) as
+long as the agent isn't `Destroyed`. A mint that runs after a hive declares the
+agent `Destroyed` logs `identity mint skipped` and writes nothing.
 
 `services.hyperhive.deploy.swarm-controller.enable` runs it on this host;
 `singleHostSwarm` turns it on. **Otherwise off: set it on the one host that

docs/swarm/credentials-reference.md

@@ -27,7 +27,7 @@ the rest of the cell says how.
 | `swarm/controller/swarm-controller/forge/token`                                                                        | the forge, once                                                                                                                                                                                                                                                | `swarm-controller`                                                     | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one                                               | ✅ re-read on a 401                                                                                                                                                                                                                                          |
 | `swarm/controller/swarm-controller/grafana/token`                                                                      | `swarm-bao-grafana-controller-token` on Grafana's host                                                                                                                                                                                                         | `swarm-controller`                                                     | ✅ every 10 minutes, when Grafana refuses it or none is stored                                                                                               | ✅ re-read on a 401                                                                                                                                                                                                                                          |
 | `swarm/controller/swarm-controller/forge/webhook-secret`                                                               | `swarm-controller`, at start, from its old `webhook-secret` file; a new one only when there is no file                                                                                                                                                         | `swarm-controller`                                                     | ❌ minted once; a rotated secret fails every delivery until the forge hooks are re-registered                                                                | ❌ read at start                                                                                                                                                                                                                                             |
-| `swarm/agents/<agent>/bao-mtls`                                                                                        | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation, and by the renewal pass when none is stored                                                                                                                                   | `hive-c0re`, then the agent                                            | ✅ re-issued every 5 minutes once past half its 7-day validity; minted when none is stored, for agents not `Destroyed`                                       | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
+| `swarm/agents/<agent>/bao-mtls`                                                                                        | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation, and by the renewal pass when none is stored or its cert-auth role is missing                                                                                                  | `hive-c0re`, then the agent                                            | ✅ re-issued every 5 minutes once past half its 7-day validity; minted when none is stored or its cert-auth role is missing, for agents not `Destroyed`      | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
 | `swarm/agents/<agent>/queue`                                                                                           | `swarm-controller`, at agent creation                                                                                                                                                                                                                          | the agent; the queue, log and metrics auth services                    | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted                                                                               | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) |
 | `swarm/agents/<agent>/forge-token`                                                                                     | `swarm-controller`, at agent creation and every 5 minutes                                                                                                                                                                                                      | the agent                                                              | ✅ when missing or no longer matching the forge                                                                                                              | ✅ every 10 minutes                                                                                                                                                                                                                                          |
 | `swarm/agents/<agent>/forge/<label>`                                                                                   | `swarm-controller`, when you link an external forge account in the swarm UI                                                                                                                                                                                    | the agent                                                              | ❌ your token; delete it and link the label again                                                                                                            | ✅ every 2 minutes                                                                                                                                                                                                                                           |

swarm-controller/src/agent_renewal.rs

@@ -26,14 +26,14 @@
 //! **Only agents some hive is declared to run are renewed** ([`live_agents`]):
 //! a destroy is recorded in the wanted-state declarations, and an agent declared
 //! nowhere, or only as `Destroyed`, is skipped. A live agent with no stored
-//! certificate gets one minted, as a failed mint at creation is retried nowhere
+//! certificate, or a current one with no listed cert-auth role, gets one minted
-//! else, unless its mint is in flight. A missing queue secret stays missing.
+//! unless its mint is in flight. A missing queue secret stays missing.
 
 use std::collections::BTreeSet;
 use std::sync::Arc;
 
 use anyhow::{Context, Result, bail};
-use swarm_secret_client::{SecretStore, cert, mtls, queue};
+use swarm_secret_client::{SecretStore, cert, client::DEFAULT_CERT_MOUNT, mtls, policy, queue};
 use swarm_wire::wanted::{AgentState, HiveWanted};
 
 use crate::wanted::WantedWriter;
@@ -141,8 +141,9 @@ pub enum Observed {
     Unstamped,
     /// Stored and not yet due.
     Current,
-    /// Nothing stored. A certificate is minted for a live agent; a queue secret
+    /// Nothing stored (a certificate: or stored with no cert-auth role, see
-    /// is left alone (see the module doc).
+    /// [`cert_needing_role`]). A certificate is minted for a live agent; a queue
+    /// secret is left alone (see the module doc).
     Absent,
     /// The read or the decode failed. Nothing is known, so nothing is done.
     Unknown,
@@ -290,13 +291,13 @@ async fn observe_secret(store: &SecretStore, agent: &str, now: i64) -> Observed
     }
 }
 
-/// One pass: every live agent, observed. Fails as a whole when any hive's
+/// Every readable wanted-state declaration of `hives`. Fails as a whole when
-/// declaration cannot be read, since that hive may be the one declaring an
+/// one cannot be read, since that hive may be the one declaring an agent
-/// agent destroyed.
+/// destroyed.
-async fn observe_all(
+pub(crate) async fn read_declarations(
     wanted: &WantedWriter,
     hives: &[String],
-) -> Result<Vec<(String, AgentObserved)>> {
+) -> Result<Vec<HiveWanted>> {
     let mut declarations = Vec::with_capacity(hives.len());
     for hive in hives {
         if let Some(d) = wanted
@@ -307,14 +308,54 @@ async fn observe_all(
             declarations.push(d);
         }
     }
+    Ok(declarations)
+}
+
+/// A stored, current certificate whose cert-auth role is not in `roles` is as
+/// good as absent: the agent cannot log in with it, and the mint is the only
+/// thing that writes the role. `None` (the role listing failed) and every
+/// other observation come back unchanged.
+pub fn cert_needing_role(
+    cert: Observed,
+    roles: Option<&BTreeSet<String>>,
+    agent: &str,
+) -> Observed {
+    match (cert, roles) {
+        (Observed::Current, Some(roles)) if !roles.contains(agent) => Observed::Absent,
+        _ => cert,
+    }
+}
+
+/// One pass: every live agent, observed. Fails as a whole when any hive's
+/// declaration cannot be read.
+async fn observe_all(
+    wanted: &WantedWriter,
+    hives: &[String],
+) -> Result<Vec<(String, AgentObserved)>> {
+    let declarations = read_declarations(wanted, hives).await?;
     let store = crate::store::connect()
         .await
         .context("logging in to the swarm secret store")?;
     let now = unix_now();
+    let roles = match store.list_cert_roles(DEFAULT_CERT_MOUNT).await {
+        Ok(names) => Some(
+            policy::agents_from_role_names(&names)
+                .into_iter()
+                .collect::<BTreeSet<_>>(),
+        ),
+        Err(e) => {
+            tracing::warn!(error = %e, "agent certificate: listing the cert-auth roles failed");
+            None
+        }
+    };
     let mut observed = Vec::new();
     for agent in live_agents(&declarations) {
         let o = AgentObserved {
-            cert: observe_cert(&store, &agent, now).await,
+            cert: cert_needing_role(
+                observe_cert(&store, &agent, now).await,
+                roles.as_ref(),
+                &agent,
+            ),
             secret: observe_secret(&store, &agent, now).await,
         };
         if let Observed::Due(age) = o.cert {
@@ -323,7 +364,7 @@ async fn observe_all(
         if o.cert == Observed::Absent {
             tracing::info!(
                 agent,
-                "agent has no stored certificate; minting its store identity"
+                "agent has no usable certificate (none stored, or no cert-auth role); minting its store identity"
             );
         }
         if let Observed::Due(age) = o.secret {
@@ -596,6 +637,35 @@ mod tests {
         );
     }
 
+    /// A current certificate with no cert-auth role listed is minted for; the
+    /// controls are the same certificate with its role listed, a failed role
+    /// listing, and a certificate that is not `Current` anyway.
+    #[test]
+    fn a_current_certificate_without_its_role_is_minted_for() {
+        let roles: BTreeSet<String> = ["sock".to_owned()].into();
+        let planned = |cert: Observed, roles: Option<&BTreeSet<String>>, agent: &str| {
+            let observed = [(
+                agent.to_owned(),
+                AgentObserved {
+                    cert: cert_needing_role(cert, roles, agent),
+                    secret: Observed::Current,
+                },
+            )];
+            plan(&observed).iter().map(|r| r.cert).collect::<Vec<_>>()
+        };
+        assert_eq!(planned(Observed::Current, Some(&roles), "paused"), [true]);
+        assert_eq!(
+            planned(Observed::Current, Some(&roles), "sock"),
+            [] as [bool; 0]
+        );
+        assert_eq!(planned(Observed::Current, None, "paused"), [] as [bool; 0]);
+        assert_eq!(
+            planned(Observed::Unknown, Some(&roles), "paused"),
+            [] as [bool; 0]
+        );
+        assert_eq!(planned(Observed::Due(1), Some(&roles), "sock"), [true]);
+    }
+
     #[test]
     fn the_age_logged_is_whole_days() {
         assert_eq!(Age(46 * DAY + 5).to_string(), "46d");

swarm-controller/src/jobq.rs

@@ -404,7 +404,9 @@ async fn run_node_work(
         },
         SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
         SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
-        SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await,
+        SwarmNodeKind::MintAgentIdentity { agent } => {
+            mint_identity(deps.wanted.as_deref(), &agent).await
+        }
         SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
         SwarmNodeKind::MintAgentMatrixAccount { agent } => {
             mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
@@ -462,10 +464,52 @@ const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
 
 /// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
 /// `clippy::too_many_lines`.
-pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome {
+///
+/// Re-reads the wanted-state declarations first (see [`mint_if_live`]); with
+/// no wanted-state writer there is nothing to read and the mint runs.
+pub(crate) async fn mint_identity(
+    wanted: Option<&wanted::WantedWriter>,
+    agent: &str,
+) -> hive_jobq::scheduler::Outcome {
+    let declarations = match wanted {
+        Some(writer) => {
+            let hives: Vec<String> = crate::startup::load_hives()
+                .into_iter()
+                .map(|h| h.name)
+                .collect();
+            Some(agent_renewal::read_declarations(writer, &hives).await)
+        }
+        None => None,
+    };
+    mint_if_live(agent, declarations, agent_identity::mint_and_verify(agent)).await
+}
+
+/// Run `mint` unless `declarations` (every hive's, read when the node runs)
+/// leave `agent` not live: declared `Destroyed` everywhere, or nowhere. A mint
+/// queued before a destroy then writes no credential for the destroyed name,
+/// and the node is `Done`. Creation orders its mint after the declaration, so a
+/// created agent is always declared. A failed read fails the node rather than
+/// minting blind.
+async fn mint_if_live(
+    agent: &str,
+    declarations: Option<anyhow::Result<Vec<swarm_wire::wanted::HiveWanted>>>,
+    mint: impl std::future::Future<Output = anyhow::Result<()>>,
+) -> hive_jobq::scheduler::Outcome {
     use hive_jobq::scheduler::Outcome;
 
-    match agent_identity::mint_and_verify(agent).await {
+    match declarations {
+        Some(Ok(declarations)) if !agent_renewal::live_agents(&declarations).contains(agent) => {
+            tracing::info!(agent, "identity mint skipped: {agent} is destroyed");
+            return Outcome::Done;
+        }
+        Some(Err(e)) => {
+            let error = format!("{e:#}");
+            tracing::warn!(%agent, step = "declaration", %error, "{MINT_IDENTITY_FAILED}");
+            return Outcome::Failed(error);
+        }
+        Some(Ok(_)) | None => {}
+    }
+    match mint.await {
         Ok(()) => Outcome::Done,
         Err(e) => {
             let error = format!("{e:#}");
@@ -1292,16 +1336,27 @@ pub(crate) fn declare_agent_job(
             agent: agent.to_owned(),
         })
         .after_ok(create_forge_user);
+    // Declared before the deploy trigger so the pause is visible in the
+    // wanted-state store before the hive brings the container up — see the
+    // node's own doc comment for why "before", not just "eventually". Needs
+    // no parent of its own: the declaration is written from the hive/agent
+    // names known at request time, same as `create_forge_user`.
+    let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
+        hive: hive.to_owned(),
+        agent: agent.to_owned(),
+    });
     // The agent's own identity at the swarm secret store, minted and
     // published at swarm level so no hive ever needs the capability to mint
     // one. After `create_identity` because the certificate names a principal
     // the swarm has agreed exists — not because anything in the store reads
-    // authelia.
+    // authelia. After `set_wanted` because the node re-reads the declaration
+    // and writes nothing for an agent declared nowhere or `Destroyed`.
     let mint_identity = b
         .node(SwarmNodeKind::MintAgentIdentity {
             agent: agent.to_owned(),
         })
-        .after_ok(create_identity);
+        .after_ok(create_identity)
+        .after_ok(set_wanted);
     // The agent's own forge token, in the store before the hive deploys
     // the container that pulls it. After the forge user, because the token
     // is minted for that user.
@@ -1317,15 +1372,6 @@ pub(crate) fn declare_agent_job(
     let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount {
         agent: agent.to_owned(),
     });
-    // Declared before the deploy trigger so the pause is visible in the
-    // wanted-state store before the hive brings the container up — see the
-    // node's own doc comment for why "before", not just "eventually". Needs
-    // no parent of its own: the declaration is written from the hive/agent
-    // names known at request time, same as `create_forge_user`.
-    let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
-        hive: hive.to_owned(),
-        agent: agent.to_owned(),
-    });
     // The hive builds the agent from `internal/meta` only, so the deploy needs
     // the update and the swarm's build from it to succeed (`after_ok`): after a
     // failed update (no builder, an unreadable config repo) or a red build, the
@@ -3884,6 +3930,29 @@ mod tests {
         assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
     }
 
+    /// The mint waits for the declaration its node re-reads: with the
+    /// declaration failed it is skipped, with it written it runs.
+    #[tokio::test]
+    async fn creation_mints_the_identity_only_after_the_declaration() {
+        use hive_jobq::State;
+
+        let mint_states = |sched: &SharedSched| -> Vec<State> {
+            let guard = sched.lock().unwrap();
+            guard
+                .graph()
+                .nodes()
+                .filter(|n| matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { .. }))
+                .map(|n| n.state)
+                .collect()
+        };
+        let sched = fresh_sched();
+        create_and_run(&sched, "atlas", false, true).await;
+        assert_eq!(mint_states(&sched), [State::Skipped]);
+        let sched = fresh_sched();
+        create_and_run(&sched, "atlas", false, false).await;
+        assert_eq!(mint_states(&sched), [State::Done]);
+    }
+
     /// Run every runnable node; the build of `agent` fails with `error` and
     /// everything else succeeds.
     async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
@@ -4028,7 +4097,7 @@ mod mint_identity_tests {
     use hive_jobq::scheduler::Outcome;
     use tracing_subscriber::layer::SubscriberExt as _;
 
-    use super::{SwarmNodeKind, WorkerDeps, mint_identity, run_swarm_node};
+    use super::{SwarmNodeKind, WorkerDeps, mint_identity, mint_if_live, run_swarm_node};
 
     /// One event's level and its fields, each by name.
     type Event = (tracing::Level, Vec<(String, String)>);
@@ -4067,7 +4136,7 @@ mod mint_identity_tests {
         let _guard =
             tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
 
-        let Outcome::Failed(error) = mint_identity("a/b").await else {
+        let Outcome::Failed(error) = mint_identity(None, "a/b").await else {
             panic!("an illegal agent name cannot be minted");
         };
 
@@ -4088,6 +4157,87 @@ mod mint_identity_tests {
         assert_eq!(field("error"), error);
     }
 
+    fn declared_as(
+        agents: &[(&str, swarm_wire::wanted::AgentState)],
+    ) -> swarm_wire::wanted::HiveWanted {
+        let mut d = swarm_wire::wanted::HiveWanted::default();
+        for (agent, state) in agents {
+            d.agents.insert(
+                (*agent).to_owned(),
+                swarm_wire::wanted::AgentWanted { state: *state },
+            );
+        }
+        d
+    }
+
+    /// The mint runs for a declared agent and not for one declared `Destroyed`
+    /// or not at all: nothing is written for the second, and the node is done.
+    #[tokio::test]
+    async fn a_mint_for_an_agent_destroyed_since_it_was_queued_writes_nothing() {
+        use swarm_wire::wanted::AgentState;
+
+        let events = Events::default();
+        let _guard =
+            tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
+        let ran = std::sync::atomic::AtomicBool::new(false);
+        let mint = async {
+            ran.store(true, std::sync::atomic::Ordering::SeqCst);
+            Ok(())
+        };
+        let declarations =
+            declared_as(&[("live", AgentState::Up), ("gone", AgentState::Destroyed)]);
+
+        for agent in ["gone", "unknown"] {
+            let outcome = mint_if_live(agent, Some(Ok(vec![declarations.clone()])), async {
+                ran.store(true, std::sync::atomic::Ordering::SeqCst);
+                Ok(())
+            })
+            .await;
+            assert!(matches!(outcome, Outcome::Done), "{agent}");
+            assert!(!ran.load(std::sync::atomic::Ordering::SeqCst), "{agent}");
+        }
+        let got = events.0.lock().expect("unpoisoned").clone();
+        assert!(
+            got.iter()
+                .any(|(level, fields)| *level == tracing::Level::INFO
+                    && fields
+                        .iter()
+                        .any(|(n, v)| n == "message"
+                            && v == "identity mint skipped: gone is destroyed")),
+            "{got:?}"
+        );
+
+        // Presence controls: a live agent mints, and so does one with no
+        // wanted-state writer to ask.
+        let outcome = mint_if_live("live", Some(Ok(vec![declarations])), mint).await;
+        assert!(matches!(outcome, Outcome::Done));
+        assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
+        ran.store(false, std::sync::atomic::Ordering::SeqCst);
+        let outcome = mint_if_live("anyone", None, async {
+            ran.store(true, std::sync::atomic::Ordering::SeqCst);
+            Ok(())
+        })
+        .await;
+        assert!(matches!(outcome, Outcome::Done));
+        assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
+    }
+
+    /// A declaration that cannot be read fails the node, naming the cause,
+    /// and mints nothing.
+    #[tokio::test]
+    async fn an_unreadable_declaration_fails_the_mint_without_minting() {
+        let ran = std::sync::atomic::AtomicBool::new(false);
+        let outcome = mint_if_live("atlas", Some(Err(anyhow::anyhow!("hive down"))), async {
+            ran.store(true, std::sync::atomic::Ordering::SeqCst);
+            Ok(())
+        })
+        .await;
+        let Outcome::Failed(error) = outcome else {
+            panic!("a failed read must fail the node");
+        };
+        assert!(error.contains("hive down"), "{error}");
+        assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
+    }
     /// A node of any kind whose work returns `Failed` logs one warn naming the
     /// node's id, its kind and the error, from the one place that runs nodes.
     #[tokio::test]