swarm-controller: retry the legacy forge token sweep from the mint pass

The sweep ran once at startup and was never retried: a boot where
store::connect or the roster read failed (e.g. the controller up
before bao) left the tokens live until the next restart. It now runs
off forge::agent_token::spawn's five-minute mint pass, reusing that
pass's roster observation instead of a second store/roster read, so a
failed first tick retries on the next one. Idempotent, so a re-run
after a partial sweep deletes nothing extra. Drops the one-shot
startup spawn; one call path.

Also updates the forge.md and agent_token.rs docs that still said the
legacy hyperhive-<seconds> tokens stay until manually removed.

Refs #4644

docs/integrations/forge.md

@@ -59,8 +59,9 @@ most one. The agent fetches the token under its own store certificate into
 and `hive-forge`, the git credential helper, the `forge_notify` poller and
 the avatar sync read it from there, falling back to `<state>/forge-token`,
 the file hive-c0re wrote before. hive-c0re no longer creates agent users or
-mints agent tokens; the `hyperhive-<unix-seconds>` tokens it minted stay on
+mints agent tokens. swarm-controller deletes the `hyperhive-<unix-seconds>`
-the forge until removed.
+tokens it left behind, for each agent whose current `swarm-agent` token is
+live; `core`'s `hyperhive`-named admin token is never touched.
 
 Two things live in the `agent-configs` Forgejo organization:
 

swarm-controller/src/forge/agent_token.rs

@@ -6,7 +6,8 @@
 //! refuses a second token with a name the user already has, so a rotation is a
 //! delete then a create, and there is never more than one of these per agent.
 //! The `hyperhive-<unix-seconds>` tokens `hive-c0re` used to mint are never
-//! touched here: the name cannot match them.
+//! touched here: the name cannot match them. [`spawn`]'s pass deletes them
+//! separately, via [`super::legacy_tokens`].
 //!
 //! Two callers insert the same `MintAgentForgeToken` job node: agent creation,
 //! and [`spawn`]'s pass at start and every [`RECONCILE_INTERVAL`] over every
@@ -322,7 +323,11 @@ impl Client {
 
 /// Check every agent's token now and every [`RECONCILE_INTERVAL`] after, and
 /// hand the agents that need one to `enqueue`, which inserts a
-/// `MintAgentForgeToken` node for each.
+/// `MintAgentForgeToken` node for each. Also sweeps each `Keep`-decided
+/// agent's legacy `hyperhive-<seconds>` tokens off the same observation
+/// (`super::legacy_tokens::sweepable`), so a boot where the first tick's
+/// store or forge read fails retries the sweep on the next tick instead of
+/// leaving those tokens live until a restart.
 ///
 /// The roster is the store's `hive-agent-*` cert-auth roles: exactly the
 /// agents that can read what the node writes. The first tick fires
@@ -335,6 +340,9 @@ pub fn spawn(client: Arc<Client>, enqueue: impl Fn(Vec<String>) + Send + 'static
             ticker.tick().await;
             match client.observe_all().await {
                 Ok(observed) => {
+                    let legacy = super::legacy_tokens::sweepable(&observed);
+                    client.sweep_legacy_tokens(&legacy).await;
+
                     let agents = plan(&observed);
                     if agents.is_empty() {
                         tracing::debug!(

swarm-controller/src/forge/legacy_tokens.rs

@@ -1,31 +1,31 @@
-//! A startup sweep of the `hyperhive-<unix-seconds>` access tokens `hive-c0re`
+//! Deletes the `hyperhive-<unix-seconds>` access tokens `hive-c0re` used to
-//! minted for agents on every spawn and rebuild, each one live on the forge
+//! mint for agents on every spawn and rebuild, each one live on the forge
 //! with write scopes.
 //!
 //! Two filters decide what goes, and both must pass:
 //!
 //! - **Whose tokens.** Only agents on the store's `hive-agent-*` roster, the
 //!   one [`super::agent_token`]'s pass walks, and only those whose
-//!   [`AGENT_TOKEN_NAME`] token that pass would [`Decision::Keep`]: until the
+//!   [`super::agent_token::AGENT_TOKEN_NAME`] token that pass would
-//!   replacement is stored and live, the agent may still be reading its last
+//!   [`Decision::Keep`]: until the replacement is stored and live, the agent
-//!   `hyperhive-*` token from `<state>/forge-token`. [`CORE_USER`] is refused
+//!   may still be reading its last `hyperhive-*` token from
-//!   by name at every step. `hive-c0re` still names `core`'s live admin token
+//!   `<state>/forge-token`. [`CORE_USER`] is refused by name at every step.
+//!   `hive-c0re` still names `core`'s live admin token
 //!   `hyperhive-<unix-seconds>`, so nothing about the token tells them apart.
 //! - **Which tokens.** A name that is exactly `hyperhive-` followed by one or
-//!   more ASCII digits ([`is_legacy_token_name`]). [`AGENT_TOKEN_NAME`] cannot
+//!   more ASCII digits ([`is_legacy_token_name`]).
-//!   match.
+//!   [`super::agent_token::AGENT_TOKEN_NAME`] cannot match.
 //!
-//! One pass per daemon start. A failed read or delete is logged and skipped;
+//! Runs from [`super::agent_token::spawn`]'s periodic pass, off the same
-//! the next start picks up whatever is left, and a start with nothing left
+//! roster observation, so a boot where that pass's first tick fails (the
-//! deletes nothing.
+//! store or forge unreachable) retries the sweep on the next tick instead of
+//! leaving the tokens live until a restart. A failed read or delete is
+//! logged and skipped; a re-run with nothing left to remove deletes nothing.
 
-use std::sync::Arc;
+use anyhow::Result;
-
-use anyhow::{Context, Result};
-use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, policy};
 
 use super::Client;
-use super::agent_token::{AGENT_TOKEN_NAME, Decision, Observed};
+use super::agent_token::{Decision, Observed};
 
 /// The forge user `hive-c0re` runs as. Its admin token carries a legacy-shaped
 /// name and is in use.
@@ -83,7 +83,7 @@ impl Client {
 
     /// Sweep each of `agents` and log what went, per agent and in total.
     /// Returns the total.
-    async fn sweep_legacy_tokens(&self, agents: &[String]) -> usize {
+    pub(super) async fn sweep_legacy_tokens(&self, agents: &[String]) -> usize {
         let mut total = 0;
         for agent in agents {
             match self.sweep_user(agent).await {
@@ -105,56 +105,16 @@ impl Client {
         );
         total
     }
-
-    /// The roster agents [`sweepable`] allows, observed the way the mint pass
-    /// observes them.
-    async fn sweep_roster(&self) -> Result<Vec<String>> {
-        let store = crate::store::connect()
-            .await
-            .context("logging in to the swarm secret store")?;
-        let roles = store
-            .list_cert_roles(DEFAULT_CERT_MOUNT)
-            .await
-            .context("listing the store's cert-auth roles")?;
-        let mut observed = Vec::new();
-        for agent in policy::agents_from_role_names(&roles) {
-            let o = self.observe_agent(&store, &agent).await;
-            if o != Observed::Decided(Decision::Keep) {
-                tracing::info!(
-                    agent,
-                    ?o,
-                    "legacy forge token sweep: {AGENT_TOKEN_NAME} token not current; agent skipped"
-                );
-            }
-            observed.push((agent, o));
-        }
-        Ok(sweepable(&observed))
-    }
-}
-
-/// Run one sweep in the background. A failure is logged and not retried.
-pub fn spawn(client: Arc<Client>) {
-    tokio::spawn(async move {
-        match client.sweep_roster().await {
-            Ok(agents) => {
-                client.sweep_legacy_tokens(&agents).await;
-            }
-            Err(e) => tracing::warn!(
-                error = %format!("{e:#}"),
-                "legacy forge token sweep: roster unavailable; not run"
-            ),
-        }
-    });
 }
 
 #[cfg(test)]
 mod tests {
     use std::collections::BTreeMap;
-    use std::sync::Mutex;
+    use std::sync::{Arc, Mutex};
 
     use forgejo_api::{Auth, Forgejo};
 
-    use super::super::agent_token::MintReason;
+    use super::super::agent_token::{AGENT_TOKEN_NAME, MintReason};
     use super::*;
 
     #[test]

swarm-controller/src/main.rs

@@ -2158,9 +2158,10 @@ fn spawn_matrix_account_backfill(
     });
 }
 
-/// Start the forge's periodic passes — the swarm-wide objects and agents'
+/// Start the forge's periodic passes — the swarm-wide objects, and agents'
-/// tokens — and the one-shot legacy token sweep, when a forge is configured.
+/// tokens (which also sweeps legacy tokens; see `forge::legacy_tokens`) —
-/// Lifted out of `main` for `clippy::too_many_lines`.
+/// when a forge is configured. Lifted out of `main` for
+/// `clippy::too_many_lines`.
 fn spawn_forge_workers(
     jobq: &Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
     forge_client: Option<Arc<forge::Client>>,
@@ -2169,7 +2170,6 @@ fn spawn_forge_workers(
         return;
     };
     forge::objects::spawn(Arc::clone(&client));
-    forge::legacy_tokens::spawn(Arc::clone(&client));
     let sched = Arc::clone(jobq);
     forge::agent_token::spawn(client, move |agents| {
         if let Err(e) = queue_forge_token_mints(&sched, agents) {