swarm-controller: KV-store + serve per-agent status (#3341 item 2)

swarm-controller/src/main.rs

@@ -40,6 +40,7 @@ use swarm_authelia_bridge_sock::BridgeResponse;
 use utoipa::{OpenApi, ToSchema};
 use utoipa_axum::{router::OpenApiRouter, routes};
 
+mod agent_status;
 mod auth;
 mod config_pr;
 mod forge;
@@ -436,6 +437,10 @@ struct AppState {
     /// up, so there is nowhere to publish a declaration to. Shares that
     /// reader's connection rather than opening a second one.
     wanted: Option<Arc<wanted::WantedWriter>>,
+    /// Per-agent status, sharing `status`'s connection — same "one queue
+    /// connection, several consumers" rationale as `wanted` above. `None`
+    /// in exactly the state `status` is.
+    agent_status: Option<Arc<agent_status::AgentStatusReader>>,
     /// The swarm-level job graph, wrapped in its
     /// [`hive_jobq::scheduler::Scheduler`] now that something drives it
     /// (`spawn_jobq_worker`) — the graph alone was enough for the
@@ -726,6 +731,22 @@ fn wanted_writer(status: Option<&Arc<status::StatusReader>>) -> Option<Arc<wante
     status.map(|s| Arc::new(wanted::WantedWriter::new(s.queue_client())))
 }
 
+/// The per-agent status reader, sharing the status reader's connection and
+/// staleness threshold — same rationale as [`wanted_writer`], and the same
+/// threshold on purpose: both buckets are fed by the same publisher cadence
+/// (`hive-c0re`'s `PUBLISH_INTERVAL`), so two separately-configured
+/// thresholds would just be two ways to get out of sync with one cadence.
+fn agent_status_reader(
+    status: Option<&Arc<status::StatusReader>>,
+) -> Option<Arc<agent_status::AgentStatusReader>> {
+    status.map(|s| {
+        Arc::new(agent_status::AgentStatusReader::new(
+            s.queue_client(),
+            status::StatusReader::stale_after_from_env(),
+        ))
+    })
+}
+
 /// Both handlers below take the same hive name and reject it the same way.
 ///
 /// Reports the status and the detail rather than a rendered
@@ -868,6 +889,49 @@ async fn get_hives_status(
     }
 }
 
+/// What each agent last said about itself, read from the swarm queue at
+/// request time.
+///
+/// Every agent in the roster gets a row whether or not it has ever
+/// reported — see the `agent_status` module for why, and for why its hive
+/// comes from its own bucket key rather than a second lookup.
+#[utoipa::path(
+    get,
+    path = "/api/agents/status",
+    responses(
+        (status = 200, description = "a row per agent, freshness derived now", body = Vec<agent_status::AgentStatusRow>),
+        (status = 503, description = "no swarm queue is configured here, no identity bridge is configured, or either store could not be read", body = String),
+    ),
+    tag = "agents"
+)]
+async fn get_agents_status(
+    State(state): State<AppState>,
+) -> Result<Json<Vec<agent_status::AgentStatusRow>>, StatusUnavailable> {
+    let Some(reader) = state.agent_status.as_ref() else {
+        return Err(StatusUnavailable(
+            "no swarm queue is configured on this host".to_owned(),
+        ));
+    };
+    let Some(bridge) = state.auth.as_ref() else {
+        return Err(StatusUnavailable(
+            "no identity bridge is configured on this host".to_owned(),
+        ));
+    };
+    let roster = bridge.list_agent_identities().await.map_err(|e| {
+        let detail = format!("{e:#}");
+        tracing::warn!(error = %detail, "reading the agent roster failed");
+        StatusUnavailable(detail)
+    })?;
+    match reader.view(&roster, std::time::SystemTime::now()).await {
+        Ok(rows) => Ok(Json(rows)),
+        Err(e) => {
+            let detail = format!("{e:#}");
+            tracing::warn!(error = %detail, "reading the agent-status bucket failed");
+            Err(StatusUnavailable(detail))
+        }
+    }
+}
+
 /// Body of `POST /api/agents` — the agent name to create, and the hive the
 /// creation is aimed at. The repo name inside `forge::CONFIG_ORG` is the
 /// same string as `name`: one repo per agent, named after it, same
@@ -1480,6 +1544,7 @@ async fn main() -> Result<()> {
         hives: Arc::new(load_hives()),
         links: Arc::new(load_links()),
         wanted: wanted_writer(status.as_ref()),
+        agent_status: agent_status_reader(status.as_ref()),
         status,
         jobq,
         webhook_secret,
@@ -1514,6 +1579,7 @@ fn build_app(state: AppState) -> axum::Router {
         .routes(routes!(get_config_prs))
         .routes(routes!(create_agent))
         .routes(routes!(get_agents))
+        .routes(routes!(get_agents_status))
         .routes(routes!(set_agent_state))
         .routes(routes!(get_hive_wanted))
         .routes(routes!(issue_report::get_repos))
@@ -1622,6 +1688,7 @@ mod tests {
             // No queue, for the same reason as `status`: these tests drive
             // agent creation, which publishes no declaration.
             wanted: None,
+            agent_status: None,
             jobq: std::sync::Arc::clone(&sched),
             webhook_secret: None,
             config_prs: None,