diff --git a/docs/agent-lifecycle/agent-roster.md b/docs/agent-lifecycle/agent-roster.md index 51541476..dbeccd7b 100644 --- a/docs/agent-lifecycle/agent-roster.md +++ b/docs/agent-lifecycle/agent-roster.md @@ -18,7 +18,7 @@ holds the hive directory, the agent roster and the job graph") is the **swarm-wide** one: `GET /api/agents` on swarm-controller reads straight from the swarm's authelia identity store over `swarm-authelia-bridge` (`swarm-controller/src/auth.rs::list_agent_identities`) -— "every agent the swarm holds an identity for" (`swarm-controller/src/main.rs`'s +— "every agent the swarm holds an identity for" (`swarm-controller/src/routes/agents.rs`'s `get_agents` doc comment). It's authoritative: an agent exists in the swarm if and only if it has an identity there, and `swarmctl agent create` (or the swarm UI, via `POST /api/agents`) writes that identity diff --git a/frontend/packages/swarm-ui/src/pages/JobsPage.tsx b/frontend/packages/swarm-ui/src/pages/JobsPage.tsx index 3b32f3c8..e0f2b672 100644 --- a/frontend/packages/swarm-ui/src/pages/JobsPage.tsx +++ b/frontend/packages/swarm-ui/src/pages/JobsPage.tsx @@ -3,7 +3,7 @@ // queue) pointed at swarm-controller's own GET /api/jobq/graph instead // of hive-c0re's — same wire shape (Vec), // different endpoint, no component fork. swarm-controller's graph has -// no real node kinds wired in yet (see swarm-controller/src/main.rs's +// no real node kinds wired in yet (see swarm-controller/src/jobq.rs's // SwarmNodeKind/SwarmResourceKind), so this renders an empty tree today // — the page exists so the wiring is in place before the first real // swarm-level job (e.g. CreateAgent) lands. diff --git a/frontend/packages/swarm-ui/src/pages/agents/AgentsPage.tsx b/frontend/packages/swarm-ui/src/pages/agents/AgentsPage.tsx index c082db11..14368eb0 100644 --- a/frontend/packages/swarm-ui/src/pages/agents/AgentsPage.tsx +++ b/frontend/packages/swarm-ui/src/pages/agents/AgentsPage.tsx @@ -11,7 +11,7 @@ // three separate fetches joined client-side by name; per operator // review feedback ("the view should be filled by a single backend // call") the join moved server-side instead — see -// `swarm-controller/src/main.rs`'s `get_agents_status` handler. No more +// `swarm-controller/src/routes/agents.rs`'s `get_agents_status` handler. No more // per-page joining left to do here: the wire row *is* the row this page renders. // // Owns the "+ agent" trigger too — the roster this populates is the diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index bcf2af07..28a8cf83 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -754,7 +754,7 @@ in # `AF_UNIX` for the socket this daemon serves on, plus what its # outbound clients need: it mints authelia tokens and calls the forge # over HTTPS (`auth.rs`, `forge.rs`) and reaches the queue over NATS - # (`main.rs`, `status.rs`). `AF_NETLINK` because glibc's + # (`startup.rs`, `status.rs`). `AF_NETLINK` because glibc's # `getaddrinfo` opens a netlink socket to enumerate local addresses # before it will return one. # @@ -789,7 +789,7 @@ in # The swarm's hive directory, JSON-encoded — the full directory # (this daemon has no "self" hive to exclude) rather than # peers-minus-self. Consumed by `GET /api/hives` - # (swarm-controller/src/main.rs::load_hives). + # (swarm-controller/src/startup.rs::load_hives). # The one blacklist of names an agent may not take, shared verbatim # with hive-c0re and with the collector's owner assertion — see # `nix/reserved-names.nix`. Whitespace-separated rather than JSON @@ -806,7 +806,7 @@ in ); # The merged links list — see `links`' description above for who # contributes to it. Consumed by `GET /api/links` - # (swarm-controller/src/main.rs::load_links). + # (swarm-controller/src/startup.rs::load_links). SWARM_CONTROLLER_LINKS = builtins.toJSON cfg.links; # Staleness threshold for `GET /api/hives/status` — see # `staleAfterSeconds`' description. Set unconditionally rather diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 5de9d826..25097220 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1555,7 +1555,7 @@ in for = "15m"; summary = "swarm-controller has failed the agent credential renewal pass on consecutive ticks"; }) - # `MINT_IDENTITY_FAILED` in swarm-controller's main.rs, + # `MINT_IDENTITY_FAILED` in swarm-controller's jobq.rs, # logged once per failed `MintAgentIdentity` job node, at # agent creation or on a certificate renewal; the window # is how long the rule stays firing after it. Host diff --git a/swarm-controller/src/agent_archetype.rs b/swarm-controller/src/agent_archetype.rs index ce8d1eb1..d793a150 100644 --- a/swarm-controller/src/agent_archetype.rs +++ b/swarm-controller/src/agent_archetype.rs @@ -9,8 +9,8 @@ use problem_details::ProblemDetails; use swarm_queue_client::agent_archetype::{self, ArchetypeRef}; use swarm_queue_client::archetypes::SourceRecord; +use crate::routes::{AppState, error_problem}; use crate::schedules::{queue, read_or_default, write_revision}; -use crate::{AppState, error_problem}; /// How many times a losing writer re-reads and re-applies, as in /// `wanted::MAX_ATTEMPTS`. diff --git a/swarm-controller/src/agent_control.rs b/swarm-controller/src/agent_control.rs index d1c8c1bf..2bc02f0c 100644 --- a/swarm-controller/src/agent_control.rs +++ b/swarm-controller/src/agent_control.rs @@ -22,7 +22,7 @@ use serde::Serialize; use swarm_queue_client::{AgentControl, AgentControlReply}; use utoipa::ToSchema; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// What became of one command. #[derive(Debug, PartialEq, Eq)] diff --git a/swarm-controller/src/agent_input.rs b/swarm-controller/src/agent_input.rs index 7fac03a0..aeaf8f7c 100644 --- a/swarm-controller/src/agent_input.rs +++ b/swarm-controller/src/agent_input.rs @@ -31,7 +31,7 @@ use serde::{Deserialize, Serialize}; use swarm_queue_client::AgentInput; use utoipa::ToSchema; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// The header the gateway names the session's user in. const SENDER_HEADER: &str = "remote-user"; diff --git a/swarm-controller/src/agent_limits.rs b/swarm-controller/src/agent_limits.rs index ac1b56f8..9c4bcfa9 100644 --- a/swarm-controller/src/agent_limits.rs +++ b/swarm-controller/src/agent_limits.rs @@ -19,8 +19,8 @@ use serde::{Deserialize, Serialize}; use swarm_queue_client::agent_limits::{self, AgentCaps}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{live, queue, read_or_default, refused, write_revision}; -use crate::{AppState, error_problem, wanted_error_status}; /// How many times a losing writer re-reads and re-applies, as in /// `wanted::MAX_ATTEMPTS`. @@ -478,9 +478,9 @@ pub(crate) mod tests { } /// One hive, `pr1ma`, on `client`. - pub(crate) fn state(client: &async_nats::Client) -> crate::AppState { - crate::AppState { - hives: Arc::new(vec![crate::HiveEntry { + pub(crate) fn state(client: &async_nats::Client) -> crate::routes::AppState { + crate::routes::AppState { + hives: Arc::new(vec![crate::routes::hives::HiveEntry { name: "pr1ma".to_owned(), domain: "pr1ma.example".to_owned(), dashboard: "pr1ma.hives.example".to_owned(), @@ -516,7 +516,7 @@ pub(crate) mod tests { } /// A state on a real server whose `pr1ma` declares `atlas`. - pub(crate) async fn declared(client: &async_nats::Client) -> crate::AppState { + pub(crate) async fn declared(client: &async_nats::Client) -> crate::routes::AppState { WantedWriter::new(client.clone()) .set("pr1ma", "atlas", AgentState::Up) .await @@ -532,7 +532,7 @@ pub(crate) mod tests { } async fn put( - state: &crate::AppState, + state: &crate::routes::AppState, agent: &str, value: Caps, ) -> Result, problem_details::ProblemDetails> { @@ -544,7 +544,7 @@ pub(crate) mod tests { .await } - async fn get(state: &crate::AppState, agent: &str) -> Caps { + async fn get(state: &crate::routes::AppState, agent: &str) -> Caps { get_agent_limits(State(state.clone()), Path(agent.to_owned())) .await .expect("caps are read") @@ -662,7 +662,7 @@ pub(crate) mod tests { let _ = put(&state, "atlas", caps(Some("400%"), Some("8G"))) .await .expect("cap atlas"); - crate::build_app(state) + crate::startup::build_app(state) } async fn put_raw(app: &axum::Router, body: &'static str) -> StatusCode { diff --git a/swarm-controller/src/agent_permissions.rs b/swarm-controller/src/agent_permissions.rs index b459d860..251f457e 100644 --- a/swarm-controller/src/agent_permissions.rs +++ b/swarm-controller/src/agent_permissions.rs @@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize}; use swarm_queue_client::agent_permissions::{self, AgentPermissions, Import, ImportReply}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{live, queue, read_or_default, refused, write, write_revision}; use crate::wanted::WantedWriter; -use crate::{AppState, error_problem, wanted_error_status}; /// How many times a losing writer re-reads and re-applies, as in /// `wanted::MAX_ATTEMPTS`. @@ -549,7 +549,7 @@ mod tests { } async fn put( - state: &crate::AppState, + state: &crate::routes::AppState, agent: &str, value: Permissions, ) -> Result, problem_details::ProblemDetails> { @@ -561,7 +561,7 @@ mod tests { .await } - async fn get(state: &crate::AppState, agent: &str) -> Permissions { + async fn get(state: &crate::routes::AppState, agent: &str) -> Permissions { get_agent_permissions(State(state.clone()), Path(agent.to_owned())) .await .expect("read") @@ -641,7 +641,7 @@ mod tests { let client = nats.client().await; let state = crate::agent_limits::tests::declared(&client).await; let _ = put(&state, "atlas", granted()).await.expect("set"); - let app = crate::build_app(state); + let app = crate::startup::build_app(state); for (body, status) in [ ( @@ -803,9 +803,9 @@ mod tests { let client = nats.client().await; let state = crate::agent_limits::tests::declared(&client).await; let create = |name: &str, permissions| { - crate::create_agent( + crate::routes::agents::create_agent( State(state.clone()), - axum::Json(crate::CreateAgentRequest { + axum::Json(crate::routes::agents::CreateAgentRequest { name: name.to_owned(), hive: "pr1ma".to_owned(), limits: None, @@ -855,18 +855,18 @@ mod tests { let state = crate::agent_limits::tests::declared(&client).await; let _ = put(&state, "atlas", granted()).await.expect("set"); - let _ = crate::set_agent_state( + let _ = crate::routes::agents::set_agent_state( State(state.clone()), Path(("pr1ma".to_owned(), "atlas".to_owned())), - axum::Json(crate::SetAgentStateRequest { + axum::Json(crate::routes::agents::SetAgentStateRequest { state: swarm_queue_client::wanted::AgentState::Destroyed, }), ) .await .expect("destroyed"); - let _ = crate::create_agent( + let _ = crate::routes::agents::create_agent( State(state.clone()), - axum::Json(crate::CreateAgentRequest { + axum::Json(crate::routes::agents::CreateAgentRequest { name: "atlas".to_owned(), hive: "pr1ma".to_owned(), limits: None, diff --git a/swarm-controller/src/agent_state_stream.rs b/swarm-controller/src/agent_state_stream.rs index 29a48fce..dc990428 100644 --- a/swarm-controller/src/agent_state_stream.rs +++ b/swarm-controller/src/agent_state_stream.rs @@ -29,7 +29,7 @@ use axum::extract::{Path, State}; use axum::response::sse::{Event, KeepAlive, Sse}; use futures_util::{Stream, StreamExt as _}; -use crate::AppState; +use crate::routes::AppState; #[utoipa::path( get, @@ -60,19 +60,21 @@ pub(crate) async fn stream_agent_state( Path(agent): Path, ) -> Result>>, problem_details::ProblemDetails> { let Some(status) = state.status.as_ref() else { - return Err(crate::error_problem( + return Err(crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, "no swarm queue is configured on this host", )); }; let Some(agent_status) = state.agent_status.as_ref() else { - return Err(crate::error_problem( + return Err(crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, "no agent-status reader is configured on this host", )); }; let agent = hive_types::Ident::parse(&agent) - .map_err(|reason| crate::error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .map_err(|reason| { + crate::routes::error_problem(axum::http::StatusCode::BAD_REQUEST, reason) + })? .into_string(); // One-agent "roster": reuses `AgentStatusReader::view`'s already-tested @@ -84,12 +86,12 @@ pub(crate) async fn stream_agent_state( .map_err(|e| { let detail = format!("{e:#}"); tracing::warn!(%agent, error = %detail, "state stream: reading agent status failed"); - crate::error_problem(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &detail) + crate::routes::error_problem(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &detail) })? .into_iter() .find_map(|row| row.hive) .ok_or_else(|| { - crate::error_problem( + crate::routes::error_problem( axum::http::StatusCode::NOT_FOUND, &format!("{agent:?} has never reported to the swarm"), ) @@ -101,7 +103,7 @@ pub(crate) async fn stream_agent_state( // against a queue that is not there — the honest answer is 503, the // same shape every other queue-backed route here already uses. swarm_queue_client::ensure_connected(&client).map_err(|e| { - crate::error_problem( + crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, &swarm_queue_client::chain(&e), ) diff --git a/swarm-controller/src/agents_md.rs b/swarm-controller/src/agents_md.rs index f082bb49..294f4da3 100644 --- a/swarm-controller/src/agents_md.rs +++ b/swarm-controller/src/agents_md.rs @@ -20,7 +20,7 @@ use swarm_queue_client::AgentControl; use swarm_queue_client::agents_md::{Apply, ApplyReply, Mirror}; use crate::agent_control::{ControlSink, Outcome}; -use crate::{AppState, error_problem, wanted_error_status}; +use crate::routes::{AppState, error_problem, wanted_error_status}; /// Reads the `agents-md` bucket, the way [`crate::agent_icon::AgentIconReader`] /// reads the icon bucket, and creates it: the agents writing it cannot. diff --git a/swarm-controller/src/alerts.rs b/swarm-controller/src/alerts.rs index 662b6387..d66d78dd 100644 --- a/swarm-controller/src/alerts.rs +++ b/swarm-controller/src/alerts.rs @@ -23,7 +23,7 @@ use problem_details::ProblemDetails; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// The env var `swarm-controller.nix` sets. const URL_ENV: &str = "SWARM_CONTROLLER_GRAFANA_URL"; @@ -282,12 +282,12 @@ mod tests { } async fn get_alerts_with(grafana: Option) -> (StatusCode, serde_json::Value) { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.alerts = grafana.map(Arc::new); let req = Request::get("/api/alerts") .body(Body::empty()) .expect("a request"); - let res = crate::build_app(state) + let res = crate::startup::build_app(state) .oneshot(req) .await .expect("infallible"); diff --git a/swarm-controller/src/archetypes.rs b/swarm-controller/src/archetypes.rs index 6c755a9f..3354c154 100644 --- a/swarm-controller/src/archetypes.rs +++ b/swarm-controller/src/archetypes.rs @@ -21,8 +21,9 @@ use swarm_wire::builder::{ }; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem}; use crate::schedules::{queue, read_or_default, refused, write}; -use crate::{AppState, agent_limits, agent_permissions, error_problem, inference}; +use crate::{agent_limits, agent_permissions, inference}; /// How often the builder is asked. Its source only changes when its host is /// redeployed, so this bounds how long a redeploy takes to reach the create diff --git a/swarm-controller/src/builder/runs.rs b/swarm-controller/src/builder/runs.rs index 5a669a0f..8dcb655c 100644 --- a/swarm-controller/src/builder/runs.rs +++ b/swarm-controller/src/builder/runs.rs @@ -18,8 +18,8 @@ use problem_details::ProblemDetails; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem}; use crate::schedules::{read_with, write}; -use crate::{AppState, error_problem}; /// The bucket holding the builds. Agents hold no grant on it. pub const BUCKET: &str = "builder-runs"; @@ -449,9 +449,9 @@ pub(crate) mod tests { push(&*doc, &record("b")).await.expect("stored"); let get_at = |builds: Builds, path: &'static str| async move { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.builds = builds; - let res = crate::build_app(state) + let res = crate::startup::build_app(state) .oneshot(Request::get(path).body(Body::empty()).expect("a request")) .await .expect("infallible"); diff --git a/swarm-controller/src/claude_login.rs b/swarm-controller/src/claude_login.rs index 857a9c58..dfd98f4c 100644 --- a/swarm-controller/src/claude_login.rs +++ b/swarm-controller/src/claude_login.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; use swarm_wire::claude_login::{REPLY_DEADLINE, Reply, Request, SUBJECT}; use utoipa::ToSchema; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// The header the gateway names the session's user in. const OPERATOR_HEADER: &str = "remote-user"; diff --git a/swarm-controller/src/forge_account.rs b/swarm-controller/src/forge_account.rs index 836a07f0..cb185e1a 100644 --- a/swarm-controller/src/forge_account.rs +++ b/swarm-controller/src/forge_account.rs @@ -13,7 +13,8 @@ use swarm_secret_client::forge; use utoipa::ToSchema; use super::linked_accounts::link; -use super::{AppState, error_problem, swarm_hive}; +use crate::routes::agents::swarm_hive; +use crate::routes::{AppState, error_problem}; /// The account to store for one agent's external forge. /// @@ -141,9 +142,9 @@ mod tests { } /// Bare-minimum `AppState`, as `matrix_account`'s tests build it. - fn state(store: &Arc) -> super::super::AppState { - super::super::AppState { - hives: std::sync::Arc::new(vec![super::super::HiveEntry { + fn state(store: &Arc) -> crate::routes::AppState { + crate::routes::AppState { + hives: std::sync::Arc::new(vec![crate::routes::hives::HiveEntry { name: "pr1ma".to_owned(), domain: "pr1ma.example".to_owned(), dashboard: "pr1ma.hives.example".to_owned(), diff --git a/swarm-controller/src/github_account.rs b/swarm-controller/src/github_account.rs index f560b105..b2c16be5 100644 --- a/swarm-controller/src/github_account.rs +++ b/swarm-controller/src/github_account.rs @@ -13,7 +13,8 @@ use swarm_secret_client::github; use utoipa::ToSchema; use super::linked_accounts::link; -use super::{AppState, error_problem, swarm_hive}; +use crate::routes::agents::swarm_hive; +use crate::routes::{AppState, error_problem}; /// The token to store for one agent. /// @@ -111,9 +112,9 @@ mod tests { } /// Bare-minimum `AppState`, as `forge_account`'s tests build it. - fn state(store: &Arc) -> super::super::AppState { - super::super::AppState { - hives: std::sync::Arc::new(vec![super::super::HiveEntry { + fn state(store: &Arc) -> crate::routes::AppState { + crate::routes::AppState { + hives: std::sync::Arc::new(vec![crate::routes::hives::HiveEntry { name: "pr1ma".to_owned(), domain: "pr1ma.example".to_owned(), dashboard: "pr1ma.hives.example".to_owned(), diff --git a/swarm-controller/src/inference.rs b/swarm-controller/src/inference.rs index 71130a9c..a09a0261 100644 --- a/swarm-controller/src/inference.rs +++ b/swarm-controller/src/inference.rs @@ -26,8 +26,8 @@ use serde::{Deserialize, Serialize}; use swarm_queue_client::agent_inference::{self, AgentInference, Effective, Effort, Source}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{live, queue, read_or_default, refused, write_revision}; -use crate::{AppState, error_problem, wanted_error_status}; /// How many times a losing writer re-reads and re-applies, as in /// `wanted::MAX_ATTEMPTS`. diff --git a/swarm-controller/src/issue_report.rs b/swarm-controller/src/issue_report.rs index da25f509..f4819d41 100644 --- a/swarm-controller/src/issue_report.rs +++ b/swarm-controller/src/issue_report.rs @@ -12,8 +12,8 @@ use axum::{ extract::{Path, State}, }; -use super::{AppState, StatusUnavailable, error_problem}; use crate::forge; +use crate::routes::{AppState, StatusUnavailable, error_problem}; /// The 503 detail when the forge did not answer within /// [`forge::FORGE_DEADLINE`]. @@ -21,7 +21,7 @@ const DEADLINE_DETAIL: &str = "the forge did not answer within the deadline"; /// Repos with at least one open issue, as `owner/name` full names — the /// data source for swarm-ui's repo-filter dropdown. Not scoped to -/// `forge::CONFIG_ORG` the way `main.rs`'s `get_config_prs` is: this +/// `forge::CONFIG_ORG` the way `routes/config_prs.rs`'s `get_config_prs` is: this /// report deliberately covers the whole forge instance, not one org, since /// the report itself is a general-purpose browsing tool rather than /// something agent-config-specific. @@ -156,7 +156,7 @@ pub async fn get_issue_report( /// /// `Client::issue_report` propagates through `anyhow`, so the concrete /// [`forgejo_api::ForgejoError`] survives underneath but is not the type in -/// hand — downcasting back to it, same pattern `main.rs`'s +/// hand — downcasting back to it, same pattern `routes`'s /// `wanted_error_status` uses for `swarm_queue_client::Error`. A 404 there /// (`forge::is_repo_not_found`) means the repo itself doesn't exist; /// anything else — including a 404 on some *other* call this function @@ -257,7 +257,7 @@ mod tests { "/api/issue-report", "/api/repos/o/r/issue-report", ] { - let (state, _) = crate::tests::state_with_roster(); + let (state, _) = crate::routes::tests::state_with_roster(); assert!( state .forge @@ -268,7 +268,7 @@ mod tests { ); let start = tokio::time::Instant::now(); let req = Request::get(uri).body(Body::empty()).expect("a request"); - let res = crate::build_app(state) + let res = crate::startup::build_app(state) .oneshot(req) .await .expect("infallible"); diff --git a/swarm-controller/src/jobq.rs b/swarm-controller/src/jobq.rs new file mode 100644 index 00000000..908da079 --- /dev/null +++ b/swarm-controller/src/jobq.rs @@ -0,0 +1,2398 @@ +//! The swarm-level job graph: its node and resource kinds, the worker that +//! runs claimed nodes, and the `queue_*` fns that insert them. + +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use swarm_authelia_bridge_sock::BridgeResponse; + +use crate::routes::AppState; +use crate::routes::agents::{claiming_hives, read_declarations}; +use crate::startup::PUBLIC_URL_ENV; +use crate::{ + agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, wanted, +}; + +/// Node payload for the swarm-level job graph. Named `Swarm*` rather than +/// the bare `NodeKind`/`Resource` `hive-c0re::job_queue::model` already +/// uses, so a grep for either doesn't land on both crates. +/// +/// `CreateIdentity` was the first real variant — "the minimal shape for +/// agent creation is creating the identity and wiring that in" (the design +/// thread's own framing for why that landed before the forge-node work +/// here). `CreateRepo`/`AddRepoMember`/`InitAgentConfigRepo` are three +/// separate nodes rather than one combined "create the repo" step so each +/// is independently retryable/observable in the job graph, same as every +/// other multi-step provisioning flow in this codebase (`hive-c0re`'s own +/// `NodeKind` never folds unrelated forge calls into one node either). +#[derive(Clone, Debug)] +pub(crate) enum SwarmNodeKind { + /// Ensure `agent` exists as an authelia subject at the swarm level. + CreateIdentity { agent: String }, + /// Create the agent's repo in `forge::CONFIG_ORG` with the operator + /// merge gate on its default branch. See `forge::Client::create_repo`. + CreateRepo { agent: String }, + /// Ensure `agent` exists as a Forgejo user account. Independent of + /// `CreateIdentity`/`CreateRepo` (a forge user needs neither an + /// authelia subject nor an existing repo) but a prerequisite for + /// `AddRepoMember`, which fails with "user does not exist" against an + /// account this node hasn't created yet. See + /// `forge::Client::ensure_agent_user`. + CreateForgeUser { agent: String }, + /// Add `agent` as a write collaborator on its own repo. See + /// `forge::Client::add_repo_member`. + AddRepoMember { agent: String }, + /// Add `agent` as a read collaborator on `internal/docs`. See + /// `forge::docs_grant`. + /// + /// Carries no hive: the grant is on a swarm-wide repo. + GrantDocsRead { agent: String }, + /// Seed the agent's repo with `agent.nix` + `flake.nix`. See + /// `forge::Client::seed_agent_config`. + /// + /// Deliberately carries no hive: seeding a config repo is the same + /// work whichever hive the agent is bound for, and an agent's config + /// states nothing about where it runs. The hive is an address the + /// swarm routes a deploy message to — it belongs on the node that + /// sends that message, not on this one. + InitAgentConfigRepo { agent: String }, + /// Mint `agent`'s own client certificate for the swarm secret store, + /// publish it there, grant it, and log in with it. See + /// `agent_identity::mint_and_verify` — including why this node does not + /// report success on a write. + /// + /// Carries no hive: neither the certificate, the queue credential nor + /// the agent's policy names one, so an agent keeps one store identity + /// whichever hive it runs on. + MintAgentIdentity { agent: String }, + /// Make sure `agent` holds a live forge access token in the swarm secret + /// store, minting one with the forge's admin API when it does not. See + /// `forge::agent_token` — including why a rotation is a delete then a + /// create. + /// + /// Carries no hive: the token's store path has no hive segment, and the + /// agent pulls it from wherever it runs. + MintAgentForgeToken { agent: String }, + /// Make sure `agent` holds a live token for its own account on the swarm's + /// homeserver, creating the account with the swarm's appservice token + /// when it does not. See `matrix_account::agent_token` — including why a + /// stored token is read back before anything is minted. + /// + /// Carries no hive: the path it writes has no hive segment, so an agent + /// that moves between hives keeps one matrix identity. + MintAgentMatrixAccount { agent: String }, + /// Make sure `hive`'s sender account (`@hive-:`) holds a live token + /// in the swarm store, where that hive's matrix sweep reads it. See + /// `matrix_account::hive_sender`. + MintHiveSenderToken { hive: String }, + /// Make sure `hive`'s read-only forge account (`hive-`) exists, is in + /// the `hives` team, and holds a live token in the swarm store, where + /// that hive's hive-c0re reads it. See `forge::hive_token`. + MintHiveForgeToken { hive: String }, + /// Make sure the review classifier's forge account exists and holds a + /// live token in the swarm store, where this daemon reads it to post the + /// classifier's reviews. See `forge::classifier_account`. + MintClassifierForgeToken, + /// Make sure the swarm builder's read-only forge account exists, is in + /// the `hives` team, and holds a live token in the swarm store, where the + /// builder reads it to fetch config repos. See `forge::builder_account`. + MintBuilderForgeToken, + /// Make sure the CI metrics poller's read-only forge account exists, is in + /// its team in every organisation, and holds a live token in the swarm + /// store, where the poller reads it to list CI runs. See + /// `forge::ci_metrics_account`. + MintCiMetricsForgeToken, + /// Build `agent`'s config at `rev`, the head of its config PR `pr`, on + /// the swarm builder, and write the result as a commit status on `rev`. + /// See `builder`. + /// + /// Carries no hive: the check builds on the builder's own agent base, + /// not on any hive. + BuildAgent { agent: String, rev: String, pr: u64 }, + /// Re-mint `agent`'s queue secret if it is still stored and old enough. + /// See `agent_renewal`. + /// + /// Carries no hive: the secret's store path has none. + RenewAgentQueueCredential { agent: String }, + /// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state + /// store, so a freshly created agent does not start driving turns the + /// moment it's deployed — the operator has to explicitly flip it to `Up`. + /// Carries the hive because the wanted-state bucket is keyed per hive. + SetAgentWanted { hive: String, agent: String }, + /// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy + /// subject. The one node kind whose effect leaves this host. + /// + /// Carries the hive, unlike every variant above — this is the node + /// `InitAgentConfigRepo`'s doc points at when it says the hive belongs + /// on the node that sends the deploy message. It names the subject the + /// message goes to, one per hive, so no other hive is woken by it. + TriggerDeploy { + hive: String, + agent: String, + /// The config commit to deploy, from a merged config PR. `None` from + /// agent creation, where the hive deploys what it seeds. + rev: Option, + }, +} + +impl hive_jobq_wire::WireNode for SwarmNodeKind { + fn label(&self) -> String { + match self { + SwarmNodeKind::CreateIdentity { .. } => "create_identity".to_owned(), + SwarmNodeKind::CreateRepo { .. } => "create_repo".to_owned(), + SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(), + SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(), + SwarmNodeKind::GrantDocsRead { .. } => "grant_docs_read".to_owned(), + SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(), + SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(), + SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(), + SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(), + SwarmNodeKind::MintHiveSenderToken { .. } => "mint_hive_sender_token".to_owned(), + SwarmNodeKind::MintHiveForgeToken { .. } => "mint_hive_forge_token".to_owned(), + SwarmNodeKind::MintClassifierForgeToken => "mint_classifier_forge_token".to_owned(), + SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(), + SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(), + SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(), + SwarmNodeKind::RenewAgentQueueCredential { .. } => { + "renew_agent_queue_credential".to_owned() + } + SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(), + SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), + } + } + + fn data(&self, _id: hive_jobq_wire::WireId) -> serde_json::Value { + // Spelled out as an or-pattern rather than a catch-all on purpose: a + // new variant then fails to compile here instead of silently + // rendering as an agent name. That gate has now fired once — + // `TriggerDeploy` is the first node that is about an agent *and a + // hive*, and a catch-all would have dropped the hive from the + // viewer without anyone noticing. + match self { + SwarmNodeKind::CreateIdentity { agent } + | SwarmNodeKind::CreateRepo { agent } + | SwarmNodeKind::CreateForgeUser { agent } + | SwarmNodeKind::AddRepoMember { agent } + | SwarmNodeKind::GrantDocsRead { agent } + | SwarmNodeKind::InitAgentConfigRepo { agent } + | SwarmNodeKind::MintAgentIdentity { agent } + | SwarmNodeKind::MintAgentForgeToken { agent } + | SwarmNodeKind::MintAgentMatrixAccount { agent } + | SwarmNodeKind::RenewAgentQueueCredential { agent } => { + serde_json::json!({ "agent": agent }) + } + SwarmNodeKind::MintHiveSenderToken { hive } + | SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }), + SwarmNodeKind::MintClassifierForgeToken + | SwarmNodeKind::MintBuilderForgeToken + | SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}), + SwarmNodeKind::BuildAgent { agent, rev, pr } => { + serde_json::json!({ "agent": agent, "rev": rev, "pr": pr }) + } + SwarmNodeKind::TriggerDeploy { hive, agent, rev } => { + serde_json::json!({ "agent": agent, "hive": hive, "rev": rev }) + } + SwarmNodeKind::SetAgentWanted { hive, agent } => { + serde_json::json!({ "agent": agent, "hive": hive }) + } + } + } +} + +/// A named counting semaphore a node acquires before it starts. Every name +/// has capacity 1: the scheduler's default, which no caller overrides. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum SwarmResourceKind { + /// The swarm builder: one `BuildAgent` node runs at a time, and the rest + /// wait pending, where a newer push can replace them. + Builder, +} + +impl hive_jobq_wire::WireResource for SwarmResourceKind { + fn name(&self) -> String { + match self { + SwarmResourceKind::Builder => "builder".to_owned(), + } + } +} + +/// A dependency this daemon may only gain after it starts serving: empty +/// until then, filled once. Readers take what is there at the moment they ask. +pub(crate) type Late = Arc>; + +/// Everything a claimed node's executor arm might need to reach outside +/// this process — bundled into one `Clone` struct rather than growing +/// `run_swarm_node`'s parameter list per node kind (three forge-shaped +/// node kinds landed in one slice; a fourth parameter each would have made +/// the signature the least readable part of this file). Each field is +/// built once at startup (see `main`) and is `None` exactly when that +/// dependency isn't configured on this host — every arm below treats +/// absence as *this node's* failure, not a reason to skip silently. +#[derive(Clone)] +pub(crate) struct WorkerDeps { + pub(crate) auth: Option>, + /// Empty until the forge token has been read (`forge::enable`), so a node + /// run before then fails as "no forge access", and one run after succeeds. + pub(crate) forge: Late>, + /// The swarm queue connection, for nodes whose effect is a published + /// event rather than an API call. A handle rather than the status + /// reader it is cloned from: a node that announces a deploy has no + /// business reading hive status, and `status.rs`'s own doc says the + /// connection living there is an accident of construction order, not a + /// claim that events are a kind of status. + pub(crate) queue: Option, + /// The wanted-state writer, for nodes that declare what a hive should + /// converge an agent to. Shares the status reader's queue connection — + /// see `wanted_writer`. `None` exactly when no swarm queue is configured + /// on this host, same as `queue`. + pub(crate) wanted: Option>, + /// Client-server API base of the swarm's homeserver, for the node that + /// mints agents' accounts on it. `None` when + /// `matrix_account::DEFAULT_HOMESERVER_ENV` is unset, which that node + /// **fails** on rather than skips, same as every other field here. + pub(crate) matrix_homeserver: Option>, +} + +/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be. +/// +/// A named constant rather than the literal inline, because this *is* the +/// decision the node exists to carry — writing a declaration at all is the +/// mechanism, `Paused` is the policy — and a test can pin a name where it +/// cannot pin a value written into a KV bucket no unit test has a server for. +const NEW_AGENT_WANTED_STATE: swarm_queue_client::wanted::AgentState = + swarm_queue_client::wanted::AgentState::Paused; + +/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ +/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind` +/// variant turns into a real effect. +pub(crate) async fn run_swarm_node( + _id: hive_jobq::NodeId, + kind: SwarmNodeKind, + builder: hive_jobq::builder::JobBuilder, + deps: WorkerDeps, +) -> ( + hive_jobq::builder::JobBuilder, + hive_jobq::scheduler::Outcome, +) { + use hive_jobq::scheduler::Outcome; + + let forge = deps.forge.get().cloned(); + let outcome = match kind { + SwarmNodeKind::CreateIdentity { agent } => match deps.auth { + None => { + Outcome::Failed("no swarm-authelia-bridge is configured on this host".to_owned()) + } + Some(bridge) => match bridge.ensure_agent_identity(&agent).await { + // Matched rather than discarded with `Ok(_)`: a heal writes + // to a subject this job did not create, and that has to + // reach a human. `Outcome` has no success-carrying variant, + // so the controller's own log is the only channel a + // succeeding node has today. + Ok(BridgeResponse::Healed) => { + tracing::warn!( + %agent, + "swarm jobq: create_identity marked an EXISTING identity as an \ + agent — verify this name did not belong to a person" + ); + Outcome::Done + } + Ok(_) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + }, + }, + SwarmNodeKind::CreateRepo { agent } => match forge { + None => Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store)" + .to_owned(), + ), + Some(client) => match forge_node("create_repo", client.create_repo(&agent)).await { + Ok(full_name) => { + tracing::info!(%full_name, "swarm jobq: create_repo done"); + Outcome::Done + } + Err(e) => Outcome::Failed(format!("{e:#}")), + }, + }, + SwarmNodeKind::CreateForgeUser { agent } => match forge { + None => Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store)" + .to_owned(), + ), + Some(client) => { + match forge_node("create_forge_user", client.ensure_agent_user(&agent)).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } + } + }, + SwarmNodeKind::AddRepoMember { agent } => match forge { + None => Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store)" + .to_owned(), + ), + Some(client) => { + match forge_node("add_repo_member", client.add_repo_member(&agent, &agent)).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } + } + }, + SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await, + SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await, + SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await, + SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await, + SwarmNodeKind::MintAgentMatrixAccount { agent } => { + mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await + } + SwarmNodeKind::MintHiveSenderToken { hive } => { + mint_hive_sender(deps.matrix_homeserver.as_deref(), &hive).await + } + SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await, + SwarmNodeKind::MintClassifierForgeToken => mint_classifier_forge_token(forge).await, + SwarmNodeKind::MintBuilderForgeToken => mint_builder_forge_token(forge).await, + SwarmNodeKind::MintCiMetricsForgeToken => mint_ci_metrics_forge_token(forge).await, + SwarmNodeKind::BuildAgent { agent, rev, pr } => { + build_agent(&deps, forge, agent, rev, pr).await + } + SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await, + SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted { + None => Outcome::Failed( + "no swarm queue is configured on this host, so no wanted-state \ + declaration can be written" + .to_owned(), + ), + Some(writer) => declare_new_agent(&writer, &hive, &agent).await, + }, + SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue { + None => Outcome::Failed( + "no swarm queue is configured on this host, so no hive can be told to deploy" + .to_owned(), + ), + Some(client) => publish_deploy(&client, &hive, &agent, rev).await, + }, + }; + (builder, outcome) +} + +/// A forge-calling node's work, run under [`forge::FORGE_DEADLINE`]. Elapsing +/// is the node's own error, naming `node`, so it fails like any other refused +/// step instead of holding its executor slot. +async fn forge_node( + node: &str, + work: impl std::future::Future>, +) -> Result { + forge::within_forge_deadline(node, work) + .await + .unwrap_or_else(|| { + Err(anyhow::anyhow!( + "{node}: no answer from the forge within {}s", + forge::FORGE_DEADLINE.as_secs() + )) + }) +} + +/// Logged at `warn`, with the agent and the error, when [`mint_identity`] +/// fails. The Grafana rule `hh-agent-identity-mint-failed` +/// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both +/// together. +const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed"; + +/// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under +/// `clippy::too_many_lines`. +pub(crate) async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + match agent_identity::mint_and_verify(agent).await { + Ok(()) => Outcome::Done, + Err(e) => { + let error = format!("{e:#}"); + tracing::warn!(%agent, %error, "{MINT_IDENTITY_FAILED}"); + Outcome::Failed(error) + } + } +} + +/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under +/// `clippy::too_many_lines`. +async fn mint_forge_token( + forge: Option>, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no agent forge token can be minted" + .to_owned(), + ); + }; + match forge_node( + "mint_agent_forge_token", + client.ensure_agent_forge_token(agent), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `GrantDocsRead` arm, lifted out for the same reason. +pub(crate) async fn grant_docs_read( + forge: Option>, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no docs grant can be made" + .to_owned(), + ); + }; + match forge_node("grant_docs_read", client.grant_docs_read(agent)).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `InitAgentConfigRepo` arm, lifted out for the same reason. +async fn init_agent_config_repo( + forge: Option>, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store)" + .to_owned(), + ); + }; + match forge_node( + "init_agent_config_repo", + client.seed_agent_config(agent, agent), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `RenewAgentQueueCredential` arm, lifted out for the same reason. +async fn renew_queue_credential(agent: &str) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + match agent_renewal::renew(agent).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `MintHiveForgeToken` arm, lifted out for the same reason. +async fn mint_hive_forge_token( + forge: Option>, + hive: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no hive forge token can be minted" + .to_owned(), + ); + }; + match forge_node( + "mint_hive_forge_token", + client.ensure_hive_forge_token(hive), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `BuildAgent` arm, lifted out for `clippy::too_many_lines`. +pub(crate) async fn build_agent( + deps: &WorkerDeps, + forge: Option>, + agent: String, + rev: String, + pr: u64, +) -> hive_jobq::scheduler::Outcome { + let job = swarm_wire::builder::BuildAgent { agent, rev, pr }; + let builds = builder::runs::Builds::Live(deps.queue.clone()); + let link_base = std::env::var(PUBLIC_URL_ENV).ok(); + builder::run( + deps.queue.as_ref(), + forge.as_deref(), + &builds, + link_base.as_deref(), + job, + ) + .await +} + +/// The `MintBuilderForgeToken` arm, lifted out for the same reason. +async fn mint_builder_forge_token( + forge: Option>, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no builder forge token can be minted" + .to_owned(), + ); + }; + match forge_node( + "mint_builder_forge_token", + client.ensure_builder_forge_token(), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `MintCiMetricsForgeToken` arm, lifted out for the same reason. +async fn mint_ci_metrics_forge_token( + forge: Option>, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no ci metrics forge token can be minted" + .to_owned(), + ); + }; + match forge_node( + "mint_ci_metrics_forge_token", + client.ensure_ci_metrics_forge_token(), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `MintClassifierForgeToken` arm, lifted out for the same reason. +async fn mint_classifier_forge_token( + forge: Option>, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(client) = forge else { + return Outcome::Failed( + "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ + not in the swarm secret store), so no review classifier forge token can be minted" + .to_owned(), + ); + }; + match forge_node( + "mint_classifier_forge_token", + client.ensure_classifier_forge_token(), + ) + .await + { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `MintAgentMatrixAccount` arm, lifted out for the same reason. +async fn mint_matrix_account( + homeserver: Option<&str>, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + // Loud on purpose: an agent quietly created without a matrix account is + // an agent nothing can talk to, and no hive mints one any more. + let Some(base) = homeserver else { + return Outcome::Failed(format!( + "no matrix homeserver configured on this host ({} unset), so no agent \ + matrix account can be minted", + matrix_account::DEFAULT_HOMESERVER_ENV + )); + }; + match matrix_account::agent_token::ensure_agent_matrix_account(base, agent).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// The `MintHiveSenderToken` arm, lifted out for the same reason. +async fn mint_hive_sender(homeserver: Option<&str>, hive: &str) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + let Some(base) = homeserver else { + return Outcome::Failed(format!( + "no matrix homeserver configured on this host ({} unset), so no hive \ + sender token can be minted", + matrix_account::DEFAULT_HOMESERVER_ENV + )); + }; + match matrix_account::hive_sender::ensure_hive_sender_token(base, hive).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns +/// out not to be new: an agent that already has a declaration (other than +/// `Destroyed`, which this treats as reusable) is left alone, so a retried +/// or migrating create is idempotent rather than pausing an already-running +/// agent. `wanted.rs` itself has no such carve-out, so this node makes the +/// call. +async fn declare_new_agent( + writer: &wanted::WantedWriter, + hive: &str, + agent: &str, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + use swarm_queue_client::wanted::AgentState; + + let existing = match writer.view(hive).await { + Ok(declaration) => declaration.and_then(|d| d.agents.get(agent).map(|a| a.state)), + Err(e) => return Outcome::Failed(format!("{e:#}")), + }; + if let Some(state) = existing + && state != AgentState::Destroyed + { + tracing::debug!( + hive, + agent, + ?state, + "agent already declared, leaving its wanted state alone" + ); + return Outcome::Done; + } + match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await { + Ok(_) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } +} + +/// Publish one deploy request and report whether it left this process. +/// +/// The flush is not belt-and-braces: `publish` hands the message to the +/// client's write buffer and returns, so a node that reported `Done` on +/// that alone would be claiming a delivery it has no evidence for — the +/// same ordering `webhook::announce_knowledge_change` documents. +async fn publish_deploy( + client: &async_nats::Client, + hive: &str, + agent: &str, + rev: Option, +) -> hive_jobq::scheduler::Outcome { + use hive_jobq::scheduler::Outcome; + + // One subject per hive, so the other hives are never woken by this. + let subject = swarm_queue_client::deploy_subject(hive); + let request = swarm_queue_client::DeployRequest { + agent: agent.to_owned(), + rev, + }; + let payload = match serde_json::to_vec(&request) { + Ok(payload) => payload, + Err(e) => return Outcome::Failed(format!("encoding the deploy request failed: {e}")), + }; + if let Err(e) = client.publish(subject.clone(), payload.into()).await { + return Outcome::Failed(format!("publishing to {subject} failed: {e}")); + } + if let Err(e) = client.flush().await { + return Outcome::Failed(format!( + "flushing the deploy event to {subject} failed: {e}" + )); + } + tracing::info!(%subject, %hive, %agent, rev = ?request.rev, "swarm jobq: deploy requested"); + Outcome::Done +} + +/// Spawn the swarm-level job-graph scheduler loop. Mirrors `hive-c0re/src/ +/// job_queue/scheduler.rs::run_worker`'s shape: claim one runnable node, +/// spawn the future that runs + completes it, loop again immediately if +/// something started (more may now be runnable), otherwise back off +/// briefly before re-polling. +/// +/// No shutdown signal to wire in — unlike `hive-c0re`'s `coord.shutdown_rx()`, +/// this daemon has no graceful-shutdown machinery at all yet (`main`'s +/// `axum::serve` runs unconditionally to process exit), so this loop +/// matches that: it rides the runtime down with the process, same as +/// every in-flight HTTP request does. +/// +/// Cheap to run with an empty graph: `claim_next` on a graph nothing was +/// ever inserted into just returns `None` every poll. +/// +/// `deps` is cloned per iteration (its fields are `Arc` clones, not +/// reconnects) and moved into the closure `claim_next` takes ownership +/// of — `run_swarm_node` needs its own owned copy since the claimed +/// future may outlive this loop iteration. +pub(crate) fn spawn_jobq_worker( + sched: Arc>>, + deps: WorkerDeps, +) { + tokio::spawn(async move { + loop { + let deps = deps.clone(); + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }); + match runner { + Some(runner) => { + tokio::spawn(async move { + let (id, grew) = runner.await; + if let Err(e) = grew { + tracing::warn!( + node = id.get(), + error = %e, + "swarm jobq: grown job rejected" + ); + } + }); + // Something just started — more may be runnable right + // now, so loop again immediately rather than sleeping. + } + None => { + // Nothing runnable. Bounded poll rather than an event + // wake (unlike hive-c0re's `notify.notify_one()`, + // there is no completion-signal channel here yet) — + // fine at this daemon's scale (one graph, no + // submitters yet); revisit if/when that stops holding. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + } + } + }); +} + +/// Queue a build check of a config PR's new head. Every head is built, on +/// whichever hive the agent runs: the check builds on the builder's own +/// agent base. +pub(crate) fn queue_config_build(state: &AppState, changed: config_pr::ChangedConfigPr) { + let (agent, rev, pr) = (changed.agent.clone(), changed.rev.clone(), changed.pr); + let queued = queue_build( + &mut state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + changed, + ); + match queued { + Ok(superseded) => { + tracing::info!(%agent, %rev, pr, superseded, "config-pr head: build queued"); + } + Err(e) => { + tracing::warn!(%agent, %rev, pr, error = %e, "config-pr head: queueing the build failed"); + } + } +} + +/// Cancel every pending build of the same PR, then insert the new one behind +/// [`SwarmResourceKind::Builder`]. A build already running is left to finish: +/// its status lands on its own commit, not on the new head. Returns how many +/// pending builds were cancelled. +pub(crate) fn queue_build( + sched: &mut hive_jobq::scheduler::Scheduler, + changed: config_pr::ChangedConfigPr, +) -> Result { + let config_pr::ChangedConfigPr { agent, rev, pr } = changed; + let stale: Vec = sched + .graph() + .nodes() + .filter(|n| n.state == hive_jobq::State::Pending) + .filter(|n| { + matches!(&n.payload, SwarmNodeKind::BuildAgent { agent: a, pr: p, .. } if *a == agent && *p == pr) + }) + .map(|n| n.id) + .collect(); + let superseded = stale + .into_iter() + .filter(|&id| sched.cancel_node(id)) + .count(); + sched + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::BuildAgent { agent, rev, pr }) + .needs(SwarmResourceKind::Builder) + .guid(), + ] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + Ok(superseded) +} + +/// Queue the deploy of a merged config PR on the one hive that runs its agent. +/// +/// The hive is found by scanning every hive's wanted state, since nothing +/// records an agent's hive (see `forge::initial_agent_nix`). Zero claimants or +/// several deploy nothing: there is no hive to send it to, or no telling which. +pub(crate) async fn queue_merged_config_deploy( + state: &AppState, + merged: config_pr::MergedConfigPr, +) { + let config_pr::MergedConfigPr { agent, rev } = merged; + let declared = match read_declarations(state, state.hives.iter()).await { + Ok(declared) => declared, + Err((hive, e)) => { + tracing::warn!( + %agent, %rev, %hive, error = %format!("{e:#}"), + "config-pr merge: cannot read where the agent is placed; nothing deployed" + ); + return; + } + }; + let claimants = claiming_hives(&agent, &declared); + let [hive] = claimants.as_slice() else { + tracing::warn!( + %agent, %rev, ?claimants, + "config-pr merge: not exactly one hive places this agent; nothing deployed" + ); + return; + }; + let hive = hive.clone(); + let inserted = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::TriggerDeploy { + hive: hive.clone(), + agent: agent.clone(), + rev: Some(rev.clone()), + }) + .guid(), + ] + }); + match inserted { + Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"), + Err(e) => { + tracing::warn!(%agent, %rev, %hive, error = %e, "config-pr merge: queueing the deploy failed"); + } + } +} + +/// The sub-DAG one agent creation is: the nodes, and the edges between them. +/// +/// A function rather than a closure inside [`create_agent`](crate::routes::agents::create_agent) so the endpoint's +/// validation and the graph's shape can each be read without scrolling past +/// the other — and so the one handle the response reports is returned from +/// the place that decides which node it is. +pub(crate) fn declare_agent_job( + b: &hive_jobq::builder::JobBuilder, + agent: &str, + hive: &str, + archetype: bool, +) -> Vec { + let create_identity = b.node(SwarmNodeKind::CreateIdentity { + agent: agent.to_owned(), + }); + // A second, independent root: a forge user needs neither an authelia + // subject nor an existing repo, so it does not chain off + // `create_identity` (see the doc comment above). + let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { + agent: agent.to_owned(), + }); + // An archetype agent has no config repo: its hive builds it from the + // archetype's module, so there is no repo to create, share or seed. + let init_config = (!archetype).then(|| { + let create_repo = b + .node(SwarmNodeKind::CreateRepo { + agent: agent.to_owned(), + }) + .after_ok(create_identity); + // `AddRepoMember` needs both parents: the repo to add a collaborator + // to, and the forge user to add as one — adding a nonexistent user is + // a Forgejo validation error, not an idempotent no-op. + // `InitAgentConfigRepo` needs only the repo — see the doc comment + // above for why. + let _add_repo_member = b + .node(SwarmNodeKind::AddRepoMember { + agent: agent.to_owned(), + }) + .after_ok(create_repo) + .after_ok(create_forge_user); + b.node(SwarmNodeKind::InitAgentConfigRepo { + agent: agent.to_owned(), + }) + .after_ok(create_repo) + .guid() + }); + // Read on `internal/docs`, a repo that exists whatever this agent is, so + // it needs only the user. Not a parent of the deploy: the agent reads docs + // once running, and a failed grant is retried by `forge::docs_grant::spawn`. + let _grant_docs = b + .node(SwarmNodeKind::GrantDocsRead { + agent: agent.to_owned(), + }) + .after_ok(create_forge_user); + // The agent's own identity at the swarm secret store, minted and + // published at swarm level so no hive ever needs the capability to mint + // one. After `create_identity` because the certificate names a principal + // the swarm has agreed exists — not because anything in the store reads + // authelia. + let mint_identity = b + .node(SwarmNodeKind::MintAgentIdentity { + agent: agent.to_owned(), + }) + .after_ok(create_identity); + // The agent's own forge token, in the store before the hive deploys + // the container that pulls it. After the forge user, because the token + // is minted for that user. + let mint_forge_token = b + .node(SwarmNodeKind::MintAgentForgeToken { + agent: agent.to_owned(), + }) + .after_ok(create_forge_user); + // The agent's account on the swarm's homeserver. A root of its own: + // creating it needs neither an authelia subject nor a forge user nor a + // repo, and chaining it behind one of those would make an unrelated + // failure look like a matrix failure. + let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount { + agent: agent.to_owned(), + }); + // Declared before the deploy trigger so the pause is visible in the + // wanted-state store before the hive brings the container up — see the + // node's own doc comment for why "before", not just "eventually". Needs + // no parent of its own: the declaration is written from the hive/agent + // names known at request time, same as `create_forge_user`. + let set_wanted = b.node(SwarmNodeKind::SetAgentWanted { + hive: hive.to_owned(), + agent: agent.to_owned(), + }); + // Last, and specifically after the config repo is seeded: the hive + // deploys by reading that repo, so a deploy asked for any earlier would + // find nothing to build. This is the edge that makes creating an agent at + // swarm level actually put it on a hive, rather than leaving a + // provisioned name nobody runs. An archetype agent has no repo, so its + // deploy waits on the mints and the declaration alone. + // + // `after_any` on the mint, not `after_ok`: a hive cannot pass down a + // certificate the swarm has not published, so the deploy must not + // overtake the mint — but a host whose store or agent PKI is not set up + // must still create agents. `after_ok` there would turn a store outage + // into an agent nobody runs. + // + // The forge-token mint gets `after_any` for the same reason: a host with + // no forge or no store must still create agents; only that node fails, + // by name, and the agent's container fetches the token on its next + // timer tick once one is minted. + // + // `after_ok` on the declaration, though: an agent deployed without its + // pause landing first is the race this node exists to close, so a + // declaration that did not land must not be deployed past. A host with no + // queue at all fails this node — but `TriggerDeploy` has nothing to + // publish to on such a host either, so the strict edge cancels a node + // that could not have succeeded anyway. + let trigger_deploy = b + .node(SwarmNodeKind::TriggerDeploy { + hive: hive.to_owned(), + agent: agent.to_owned(), + rev: None, + }) + .after_any(mint_identity) + .after_any(mint_forge_token) + // `after_any` for the same reason: the container reads this token + // rather than producing it, so the deploy must not overtake the mint — + // but a host with no homeserver configured must still create agents, + // and only this node fails, by name. + .after_any(mint_matrix) + .after_ok(set_wanted); + if let Some(init_config) = init_config { + let _ = trigger_deploy.after_ok(init_config); + } + vec![create_identity.guid()] +} + +/// Insert one job per agent: `CreateForgeUser`, then `MintAgentForgeToken` +/// `after_ok` it — the same edge [`declare_agent_job`] uses. Returns the mint +/// nodes' ids. +/// +/// The one place that node is queued outside agent creation: both the manual +/// route below and `forge::agent_token::spawn`'s periodic pass come through +/// here, so a backfilled mint is the same node a new agent gets. The user +/// node is what lets the pass reach an agent that has a store identity but +/// no forge account; for one that has an account it is a no-op, because +/// `forge::Client::ensure_agent_user` folds "already exists" into success. +pub(crate) fn queue_forge_token_mints( + sched: &Mutex>, + agents: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(agents.len()); + for agent in agents { + let queued = sched + .insert_job(None, |b| { + let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { + agent: agent.clone(), + }); + vec![ + b.node(SwarmNodeKind::MintAgentForgeToken { agent }) + .after_ok(create_forge_user) + .guid(), + ] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Insert one job per agent: `CreateForgeUser`, then `GrantDocsRead` `after_ok` +/// it — the same edge [`declare_agent_job`] uses. Returns the grant nodes' ids. +/// `forge::docs_grant::spawn`'s periodic pass comes through here, so a +/// backfilled grant is the same node a new agent gets; the user node is a +/// no-op for an agent that has an account. +pub(crate) fn queue_docs_grants( + sched: &Mutex>, + agents: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(agents.len()); + for agent in agents { + let queued = sched + .insert_job(None, |b| { + let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { + agent: agent.clone(), + }); + vec![ + b.node(SwarmNodeKind::GrantDocsRead { agent }) + .after_ok(create_forge_user) + .guid(), + ] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Insert one `MintAgentMatrixAccount` job per agent and return the nodes' +/// ids. `matrix_account::agent_token::spawn`'s periodic pass comes through +/// here, so a backfilled mint is the same node a new agent gets. +pub(crate) fn queue_matrix_account_mints( + sched: &Mutex>, + agents: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(agents.len()); + for agent in agents { + let queued = sched + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::MintAgentMatrixAccount { agent }) + .guid(), + ] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Insert one `MintHiveSenderToken` job per hive and return the nodes' ids. +/// `matrix_account::hive_sender::spawn`'s periodic pass is the one caller. +pub(crate) fn queue_hive_sender_mints( + sched: &Mutex>, + hives: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(hives.len()); + for hive in hives { + let queued = sched + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintHiveSenderToken { hive }).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Insert one `MintHiveForgeToken` job per hive and return the nodes' ids. +/// `forge::hive_token::spawn`'s periodic pass is the one caller. +pub(crate) fn queue_hive_forge_token_mints( + sched: &Mutex>, + hives: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(hives.len()); + for hive in hives { + let queued = sched + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintHiveForgeToken { hive }).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +/// Insert one `MintClassifierForgeToken` job and return its node's id. +/// `forge::classifier_account::spawn`'s periodic pass is the one caller. +pub(crate) fn queue_classifier_forge_token_mint( + sched: &Mutex>, +) -> Result> { + sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintClassifierForgeToken).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +/// Insert one `MintBuilderForgeToken` job and return its node's id. +/// `forge::builder_account::spawn`'s periodic pass is the one caller. +pub(crate) fn queue_builder_forge_token_mint( + sched: &Mutex>, +) -> Result> { + sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintBuilderForgeToken).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +/// Insert one `MintCiMetricsForgeToken` job and return its node's id. +/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller. +pub(crate) fn queue_ci_metrics_forge_token_mint( + sched: &Mutex>, +) -> Result> { + sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert_job(None, |b| { + vec![b.node(SwarmNodeKind::MintCiMetricsForgeToken).guid()] + }) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +/// Insert one job per renewal and return the ids of its nodes: +/// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a +/// queue secret, the second `after_any` the first when an agent needs both. +/// `agent_renewal::spawn`'s periodic pass comes through here. +/// +/// Chained rather than parallel because `mint_and_verify` reads the queue +/// secret back and compares it with the one it read first; a re-mint landing +/// in between fails that node. `after_any`, so a certificate that could not be +/// re-issued does not hold up the secret. +pub(crate) fn queue_agent_renewals( + sched: &Mutex>, + renewals: Vec, +) -> Result> { + let mut sched = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut ids = Vec::with_capacity(renewals.len()); + for renewal in renewals { + let queued = sched + .insert_job(None, |b| { + let mut asked = Vec::new(); + let cert = renewal.cert.then(|| { + b.node(SwarmNodeKind::MintAgentIdentity { + agent: renewal.agent.clone(), + }) + .guid() + }); + asked.extend(cert); + if renewal.secret { + let secret = b.node(SwarmNodeKind::RenewAgentQueueCredential { + agent: renewal.agent.clone(), + }); + let secret = match cert { + Some(cert) => secret.after_any(cert), + None => secret, + }; + asked.push(secret.guid()); + } + asked + }) + .map_err(|e| anyhow::anyhow!("{e}"))?; + ids.extend(queued); + } + Ok(ids) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routes::tests::state_with_roster; + + pub(crate) fn changed(agent: &str, rev: &str, pr: u64) -> crate::config_pr::ChangedConfigPr { + crate::config_pr::ChangedConfigPr { + agent: agent.to_owned(), + rev: rev.to_owned(), + pr, + } + } + + fn build_state( + sched: &hive_jobq::scheduler::Scheduler, + rev: &str, + ) -> hive_jobq::State { + sched + .graph() + .nodes() + .find(|n| matches!(&n.payload, SwarmNodeKind::BuildAgent { rev: r, .. } if r == rev)) + .map(|n| n.state) + .expect("a build of that rev") + } + + /// A newer head of the same PR replaces its pending build; another PR's + /// build, and the same PR's running build, are left alone. + #[tokio::test] + async fn a_newer_push_replaces_the_prs_pending_build_only() { + let sched = + std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + let queue = |c| super::queue_build(&mut sched.lock().unwrap(), c).expect("queued"); + assert_eq!(queue(changed("atlas", "r1", 3)), 0); + // r1 takes the builder and stays running: its future is never awaited. + let _running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { + (b, hive_jobq::scheduler::Outcome::Done) + }) + .expect("r1 is runnable"); + assert_eq!( + queue(changed("atlas", "r2", 3)), + 0, + "r1 is running, not pending" + ); + assert_eq!(queue(changed("argus", "o1", 4)), 0); + assert_eq!(queue(changed("atlas", "r3", 3)), 1, "r2 was pending"); + + let guard = sched.lock().unwrap(); + assert_eq!(build_state(&guard, "r1"), hive_jobq::State::Running); + assert_eq!(build_state(&guard, "r2"), hive_jobq::State::Cancelled); + assert_eq!(build_state(&guard, "r3"), hive_jobq::State::Pending); + assert_eq!(build_state(&guard, "o1"), hive_jobq::State::Pending); + } + + /// Builds share one builder slot: a second build is not claimable while + /// the first runs, and is once it has finished. + #[tokio::test] + async fn one_build_runs_at_a_time() { + let sched = + std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + for c in [changed("atlas", "a1", 1), changed("argus", "b1", 2)] { + super::queue_build(&mut sched.lock().unwrap(), c).expect("queued"); + } + let run = |sched| { + hive_jobq::scheduler::Scheduler::claim_next(sched, |_, _, b| async move { + (b, hive_jobq::scheduler::Outcome::Done) + }) + }; + let first = run(&sched).expect("one build is runnable"); + assert!( + run(&sched).is_none(), + "a second build started beside the first" + ); + first.await.1.expect("no growth"); + run(&sched) + .expect("the second build runs once the first is done") + .await + .1 + .expect("no growth"); + } + + /// Every node whose first request goes to the forge, run against a forge + /// that accepts the connection and never answers: each ends `Failed` + /// with an error naming the node instead of holding its executor slot. + /// `MintAgentForgeToken` is not in the list: it logs in to the secret + /// store before its first forge request, and no test here has a store. + #[tokio::test(start_paused = true)] + async fn a_forge_node_fails_when_the_forge_never_answers() { + // Bound but never accepted: the kernel completes the handshake and + // buffers the request, and no response ever comes. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let forge = super::Late::>::default(); + assert!( + forge + .set(std::sync::Arc::new(crate::forge::Client::for_test_forge( + url + ))) + .is_ok() + ); + let agent = || "atlas".to_owned(); + let nodes = [ + ("create_repo", SwarmNodeKind::CreateRepo { agent: agent() }), + ( + "create_forge_user", + SwarmNodeKind::CreateForgeUser { agent: agent() }, + ), + ( + "add_repo_member", + SwarmNodeKind::AddRepoMember { agent: agent() }, + ), + ( + "grant_docs_read", + SwarmNodeKind::GrantDocsRead { agent: agent() }, + ), + ( + "init_agent_config_repo", + SwarmNodeKind::InitAgentConfigRepo { agent: agent() }, + ), + ( + "mint_hive_forge_token", + SwarmNodeKind::MintHiveForgeToken { + hive: "pr1ma".to_owned(), + }, + ), + ( + "mint_classifier_forge_token", + SwarmNodeKind::MintClassifierForgeToken, + ), + ( + "mint_builder_forge_token", + SwarmNodeKind::MintBuilderForgeToken, + ), + ( + "mint_ci_metrics_forge_token", + SwarmNodeKind::MintCiMetricsForgeToken, + ), + ]; + for (label, kind) in nodes { + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched.append(kind, Vec::new(), None).expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + let deps = WorkerDeps { + auth: None, + forge: forge.clone(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + tokio::time::timeout(std::time::Duration::from_mins(10), runner) + .await + .unwrap_or_else(|_| panic!("{label}: node still running after 10 minutes")) + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed, "{label}"); + let error = node.error.as_deref().unwrap_or_default(); + assert!( + error.contains(label) && error.contains("no answer from the forge"), + "{label}: expected a deadline error naming the node, got {error:?}" + ); + } + drop(listener); + } + + #[tokio::test] + async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() { + unsafe { + std::env::remove_var("SWARM_CONTROLLER_FORGE_URL"); + } + + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::CreateRepo { + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: std::sync::Arc::default(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + assert!( + node.error.as_deref().unwrap_or_default().contains("forge"), + "expected a forge-not-configured error, got {:?}", + node.error + ); + } + + /// Sibling of the `CreateRepo` test above, same shape: drives + /// `SwarmNodeKind::CreateForgeUser` through the real scheduler with no + /// forge configured, asserting the graceful-absence-is-failure branch — + /// this node's happy path needs a live forge, same caveat as + /// `CreateRepo`'s. + #[tokio::test] + async fn create_forge_user_node_runs_end_to_end_and_fails_without_forge_configured() { + unsafe { + std::env::remove_var("SWARM_CONTROLLER_FORGE_URL"); + } + + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::CreateForgeUser { + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: std::sync::Arc::default(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + assert!( + node.error.as_deref().unwrap_or_default().contains("forge"), + "expected a forge-not-configured error, got {:?}", + node.error + ); + } + + /// Third sibling of the two above, and the same deliberate caveat: with + /// no agent PKI named this reaches only the graceful-absence-is-failure + /// branch, before any network call. The happy path is a live store and a + /// real login, which is exactly why it is `mint_and_verify`'s own job to + /// prove it at agent-creation time rather than a unit test's. + /// + /// What this does pin is the degrade: a host whose environment does not + /// name the agent PKI fails this one node with a reason that names the + /// variable, and creates the agent anyway. + #[tokio::test] + async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_the_agent_pki_named() { + let pki = [ + crate::agent_identity::ENV_AGENT_PKI_MOUNT, + crate::agent_identity::ENV_AGENT_PKI_ROLE, + ]; + assert!( + pki.iter().all(|var| std::env::var_os(var).is_none()), + "{} and {} must be unset: with them set this test would reach a live store", + pki[0], + pki[1] + ); + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::MintAgentIdentity { + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: std::sync::Arc::default(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + let error = node.error.as_deref().unwrap_or_default(); + assert!( + error.contains(crate::agent_identity::ENV_AGENT_PKI_MOUNT), + "the reason must name the variable an operator has to set, got {error:?}" + ); + } + + /// Fourth sibling, same shape and the same caveat once more: with no + /// queue configured this reaches only the graceful-absence-is-failure + /// branch. Writing the declaration for real needs a `JetStream` server, + /// which no test in this crate has — `wanted::apply`'s own unit tests + /// cover what gets written, and [`a_new_agent_is_declared_paused`] below + /// pins the state this node asks for. + #[tokio::test] + async fn set_agent_wanted_node_runs_end_to_end_and_fails_without_a_queue() { + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::SetAgentWanted { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: std::sync::Arc::default(), + queue: None, + wanted: None, + matrix_homeserver: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + assert!( + node.error.as_deref().unwrap_or_default().contains("queue"), + "expected a no-queue-configured error, got {:?}", + node.error + ); + } + + /// The whole point of the node: a freshly created agent is declared + /// `Paused`, so it does not start driving turns the moment its hive + /// brings it up. Any other state here is the bug this fixes. + #[test] + fn a_new_agent_is_declared_paused() { + assert_eq!( + super::NEW_AGENT_WANTED_STATE, + swarm_queue_client::wanted::AgentState::Paused + ); + } + + /// Same reasoning as `a_mint_node_renders_both_the_agent_and_the_hive`: + /// this variant carries a hive too, and has to have joined `data`'s + /// two-field or-pattern rather than the agent-only one. + #[test] + fn a_set_wanted_node_renders_both_the_agent_and_the_hive() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::SetAgentWanted { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "set_agent_wanted"); + let data = kind.data(1); + assert_eq!(data["agent"], "atlas"); + assert_eq!(data["hive"], "pr1ma"); + } + + /// The ordering the fix depends on: the pause has to be in the + /// wanted-state store *before* the hive is told to deploy, or a freshly + /// created agent runs undeclared for the window between the two. + /// + /// Asserted on the graph `create_agent` builds, same as the mint edge's + /// test above and for the same reason — the edge is one line in a + /// builder closure whose absence changes nothing observable until a real + /// agent boots and starts taking turns nobody asked for. + #[tokio::test] + async fn the_deploy_waits_for_the_pause_to_be_declared() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = crate::routes::agents::create_agent( + axum::extract::State(state), + axum::Json(crate::routes::agents::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let set_wanted = id_of("set_agent_wanted"); + let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); + + let when = deploy + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == set_wanted => Some(*when), + _ => None, + }) + .expect("the deploy waits for the declaration"); + assert!( + !when.accepts(hive_jobq::TerminalState::Failed), + "a declaration that never landed must not be deployed past; this \ + edge has to be `after_ok`, not `after_any`" + ); + } + + /// Node labels of the graph [`super::declare_agent_job`] builds, and the + /// labels the deploy node waits on. + fn creation_graph(archetype: bool) -> (Vec, Vec) { + use hive_jobq_wire::WireNode as _; + + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + sched + .insert_job(None, |b| { + super::declare_agent_job(b, "atlas", "pr1ma", archetype) + }) + .expect("the graph inserts"); + let graph = sched.graph(); + let labels = graph.nodes().map(|n| n.payload.label()).collect(); + let deploy = graph + .nodes() + .find(|n| n.payload.label() == "trigger_deploy") + .expect("every creation deploys"); + let waits = deploy + .deps + .iter() + .filter_map(|d| match d { + hive_jobq::Dep::Node { id, .. } => graph.node(*id).map(|n| n.payload.label()), + hive_jobq::Dep::Resource { .. } => None, + }) + .collect(); + (labels, waits) + } + + /// An archetype agent's graph has no repo nodes and its deploy waits on + /// no repo; a custom agent's has all three, and its deploy waits on the + /// seeded repo. + #[test] + fn an_archetype_agent_is_created_without_a_config_repo() { + const REPO_NODES: [&str; 3] = ["create_repo", "add_repo_member", "init_agent_config_repo"]; + + let (custom, custom_waits) = creation_graph(false); + for repo in REPO_NODES { + assert!(custom.iter().any(|l| l == repo), "custom lacks {repo}"); + } + assert!(custom_waits.iter().any(|l| l == "init_agent_config_repo")); + + let (archetype, archetype_waits) = creation_graph(true); + for repo in REPO_NODES { + assert!(!archetype.iter().any(|l| l == repo), "archetype has {repo}"); + } + for kept in [ + "create_identity", + "create_forge_user", + "grant_docs_read", + "mint_agent_identity", + "mint_agent_forge_token", + "mint_agent_matrix_account", + "set_agent_wanted", + ] { + assert!( + archetype.iter().any(|l| l == kept), + "archetype lacks {kept}" + ); + assert!(custom.iter().any(|l| l == kept), "custom lacks {kept}"); + } + assert!(archetype_waits.iter().any(|l| l == "set_agent_wanted")); + assert_eq!(archetype.len() + REPO_NODES.len(), custom.len()); + } + + /// The ordering the operator's ruling requires: a hive cannot pass down + /// a certificate the swarm has not published, so the deploy message must + /// not leave before the mint is terminal. + /// + /// Asserted on the graph `create_agent` builds, because the edge is one + /// line in a builder closure and its absence changes nothing observable + /// until a real agent boots without an identity. + #[tokio::test] + async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = crate::routes::agents::create_agent( + axum::extract::State(state), + axum::Json(crate::routes::agents::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let mint = id_of("mint_agent_identity"); + let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); + + let when = deploy + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when), + _ => None, + }) + .expect("the deploy waits for the mint"); + assert!( + when.accepts(hive_jobq::TerminalState::Failed), + "a failed or unconfigured agent PKI issue must not cancel the deploy; this edge \ + has to be `after_any`, not `after_ok`" + ); + } + + /// The forge-token mint sits between the forge user and the deploy: it + /// needs the user to exist, and the deploy must not overtake it — but a + /// host with no forge or no store must still deploy the agent. + #[tokio::test] + async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = crate::routes::agents::create_agent( + axum::extract::State(state), + axum::Json(crate::routes::agents::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let edge = |from: hive_jobq::NodeId, to: &str| { + graph + .node(id_of(to)) + .expect("just found") + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == from => Some(*when), + _ => None, + }) + .unwrap_or_else(|| panic!("{to} waits for node {}", from.get())) + }; + + let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token"); + assert!( + !user_to_mint.accepts(hive_jobq::TerminalState::Failed), + "a token for a user that was never created cannot be minted; this \ + edge has to be `after_ok`" + ); + let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy"); + assert!( + mint_to_deploy.accepts(hive_jobq::TerminalState::Failed), + "a host with no forge or store must not cancel the deploy; this edge \ + has to be `after_any`, not `after_ok`" + ); + } + + /// Agent creation grants `internal/docs` read once the forge user exists, + /// and a failed grant does not hold up the deploy (the pass retries it). + #[tokio::test] + async fn the_docs_grant_follows_the_user_and_is_not_a_deploy_parent() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = crate::routes::agents::create_agent( + axum::extract::State(state), + axum::Json(crate::routes::agents::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let grant = graph.node(id_of("grant_docs_read")).expect("just found"); + assert_eq!(grant.payload.data(grant.id.get())["agent"], "atlas"); + let when = grant + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == id_of("create_forge_user") => { + Some(*when) + } + _ => None, + }) + .expect("the grant waits for the forge user"); + assert!( + !when.accepts(hive_jobq::TerminalState::Failed), + "adding a user that was never created as a collaborator is a forge error; \ + this edge has to be `after_ok`" + ); + let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); + assert!( + !deploy.deps.iter().any(|d| matches!( + d, + hive_jobq::Dep::Node { id, .. } if *id == grant.id + )), + "the deploy must not wait on the docs grant" + ); + } + + /// The periodic pass queues the same pair of nodes agent creation does. + #[test] + fn a_queued_docs_grant_creates_the_forge_user_first() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = super::queue_docs_grants(&sched, vec!["a".to_owned(), "b".to_owned()]) + .expect("two jobs insert"); + assert_eq!(ids.len(), 2, "one returned id per agent: its grant node"); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let mut labels: Vec = graph.nodes().map(|n| n.payload.label()).collect(); + labels.sort(); + assert_eq!( + labels, + [ + "create_forge_user", + "create_forge_user", + "grant_docs_read", + "grant_docs_read" + ] + ); + for grant in graph + .nodes() + .filter(|n| n.payload.label() == "grant_docs_read") + { + assert!(ids.contains(&grant.id)); + assert!( + grant + .deps + .iter() + .any(|d| matches!(d, hive_jobq::Dep::Node { when, .. } if !when.accepts(hive_jobq::TerminalState::Failed))), + "each grant waits `after_ok` on its user" + ); + } + } + + #[test] + fn a_forge_token_node_renders_the_agent() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::MintAgentForgeToken { + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "mint_agent_forge_token"); + assert_eq!(kind.data(1)["agent"], "atlas"); + } + + #[test] + fn a_matrix_account_node_renders_the_agent() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::MintAgentMatrixAccount { + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "mint_agent_matrix_account"); + assert_eq!(kind.data(1)["agent"], "atlas"); + } + + /// The periodic pass's queueing: a certificate re-issue is the node agent + /// creation mints with, a secret re-mint its own node, and an agent due + /// both gets the secret `after_any` the certificate. + #[test] + fn a_queued_renewal_chains_the_secret_after_the_certificate() { + use crate::agent_renewal::Renewal; + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let r = |agent: &str, cert, secret| Renewal { + agent: agent.to_owned(), + cert, + secret, + }; + let ids = super::queue_agent_renewals( + &sched, + vec![ + r("both", true, true), + r("cert", true, false), + r("secret", false, true), + ], + ) + .expect("three jobs insert"); + assert_eq!(ids.len(), 4, "one returned id per node"); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let agent_of = |n: &hive_jobq::Node| { + n.payload.data(n.id.get())["agent"] + .as_str() + .expect("agent is a string") + .to_owned() + }; + let mut nodes: Vec<(String, String)> = graph + .nodes() + .map(|n| (agent_of(n), n.payload.label())) + .collect(); + nodes.sort(); + assert_eq!( + nodes, + [ + ("both".to_owned(), "mint_agent_identity".to_owned()), + ("both".to_owned(), "renew_agent_queue_credential".to_owned()), + ("cert".to_owned(), "mint_agent_identity".to_owned()), + ( + "secret".to_owned(), + "renew_agent_queue_credential".to_owned() + ), + ] + ); + let find = |agent: &str, label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label && agent_of(n) == agent) + .unwrap_or_else(|| panic!("{agent} has a {label} node")) + }; + let cert = find("both", "mint_agent_identity").id; + let secret = find("both", "renew_agent_queue_credential"); + let when = secret + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == cert => Some(*when), + _ => None, + }) + .expect("the secret waits for the certificate"); + assert!( + when.accepts(hive_jobq::TerminalState::Failed), + "a certificate that could not be re-issued must not hold up the secret" + ); + assert!( + find("secret", "renew_agent_queue_credential") + .deps + .is_empty(), + "a secret on its own waits for nothing" + ); + } + + /// Agent creation mints the matrix account as a root of its own, and the + /// deploy waits for it without being cancelled by it: a host with no + /// homeserver configured must still deploy the agent. + #[tokio::test] + async fn the_matrix_account_mint_is_a_root_and_does_not_block_the_deploy() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = crate::routes::agents::create_agent( + axum::extract::State(state), + axum::Json(crate::routes::agents::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let find = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + }; + let mint = find("mint_agent_matrix_account"); + assert!(mint.deps.is_empty(), "the mint waits for nothing"); + let when = find("trigger_deploy") + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == mint.id => Some(*when), + _ => None, + }) + .expect("the deploy waits for the matrix mint"); + assert!( + when.accepts(hive_jobq::TerminalState::Failed), + "a host with no homeserver must not cancel the deploy; this edge \ + has to be `after_any`, not `after_ok`" + ); + } + + /// The backfill's queueing: one mint node per agent, nothing else. + #[test] + fn a_queued_matrix_mint_is_one_node_per_agent() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = super::queue_matrix_account_mints(&sched, vec!["a".to_owned(), "b".to_owned()]) + .expect("two jobs insert"); + assert_eq!(ids.len(), 2); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut agents: Vec = guard + .graph() + .nodes() + .map(|n| { + assert_eq!(n.payload.label(), "mint_agent_matrix_account"); + n.payload.data(n.id.get())["agent"] + .as_str() + .expect("agent is a string") + .to_owned() + }) + .collect(); + agents.sort(); + assert_eq!(agents, ["a", "b"]); + } + + /// The hive sender-token pass's queueing: one mint node per hive, and + /// the node names the hive rather than an agent. + #[test] + fn a_queued_hive_sender_mint_is_one_node_per_hive() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = + super::queue_hive_sender_mints(&sched, vec!["alpha".to_owned(), "beta".to_owned()]) + .expect("two jobs insert"); + assert_eq!(ids.len(), 2); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut hives: Vec = guard + .graph() + .nodes() + .map(|n| { + assert_eq!(n.payload.label(), "mint_hive_sender_token"); + let data = n.payload.data(n.id.get()); + assert!(data.get("agent").is_none(), "{data}"); + data["hive"].as_str().expect("hive is a string").to_owned() + }) + .collect(); + hives.sort(); + assert_eq!(hives, ["alpha", "beta"]); + } + + /// The hive forge-token pass's queueing: one node per hive, naming the + /// hive rather than an agent. + #[test] + fn a_queued_hive_forge_token_mint_is_one_node_per_hive() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = super::queue_hive_forge_token_mints( + &sched, + vec!["alpha".to_owned(), "beta".to_owned()], + ) + .expect("two jobs insert"); + assert_eq!(ids.len(), 2); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut hives: Vec = guard + .graph() + .nodes() + .map(|n| { + assert_eq!(n.payload.label(), "mint_hive_forge_token"); + let data = n.payload.data(n.id.get()); + assert!(data.get("agent").is_none(), "{data}"); + data["hive"].as_str().expect("hive is a string").to_owned() + }) + .collect(); + hives.sort(); + assert_eq!(hives, ["alpha", "beta"]); + } + + /// The manual route and the periodic pass both go through + /// `queue_forge_token_mints`, so this is the assertion that a backfilled + /// mint is the same pair of nodes agent creation inserts: the forge user, + /// then the mint `after_ok` it, per agent. + #[test] + fn a_queued_mint_creates_the_forge_user_first() { + use hive_jobq_wire::WireNode as _; + + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()]) + .expect("two jobs insert"); + assert_eq!(ids.len(), 2, "one returned id per agent: its mint node"); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let agent_of = |n: &hive_jobq::Node| { + n.payload.data(n.id.get())["agent"] + .as_str() + .expect("agent is a string") + .to_owned() + }; + for agent in ["a", "b"] { + let find = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label && agent_of(n) == agent) + .unwrap_or_else(|| panic!("{agent} has a {label} node")) + }; + let user = find("create_forge_user").id; + let mint = find("mint_agent_forge_token"); + assert!(ids.contains(&mint.id), "the returned id is {agent}'s mint"); + let when = mint + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == user => Some(*when), + _ => None, + }) + .unwrap_or_else(|| panic!("{agent}'s mint waits for its own forge user")); + assert!( + !when.accepts(hive_jobq::TerminalState::Failed), + "a token cannot be minted for a user that was never created; this \ + edge has to be `after_ok`" + ); + } + assert_eq!( + graph.nodes().count(), + 4, + "two nodes per agent, nothing else" + ); + } + + /// The backfill end to end, minus the IO: an agent the pass observed with + /// no forge user is planned, and the job it gets creates that user before + /// it mints. + #[test] + fn an_agent_with_no_forge_user_gets_a_user_and_a_mint() { + use crate::forge::agent_token::{Observed, plan}; + use hive_jobq_wire::WireNode as _; + + let agents = plan(&[("ruth".to_owned(), Observed::NoForgeUser)]); + let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + )); + super::queue_forge_token_mints(&sched, agents).expect("one job inserts"); + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut labels: Vec = guard.graph().nodes().map(|n| n.payload.label()).collect(); + labels.sort(); + assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]); + } +} + +#[cfg(test)] +mod mint_identity_tests { + use std::sync::{Arc, Mutex}; + + use hive_jobq::scheduler::Outcome; + use tracing_subscriber::layer::SubscriberExt as _; + + use super::mint_identity; + + /// One event's level and its fields, each by name. + type Event = (tracing::Level, Vec<(String, String)>); + + /// Every event. + #[derive(Clone, Default)] + struct Events(Arc>>); + + impl tracing_subscriber::Layer for Events { + fn on_event( + &self, + event: &tracing::Event<'_>, + _: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut fields = Vec::new(); + event.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + fields.push((field.name().to_owned(), format!("{value:?}"))); + }, + ); + self.0 + .lock() + .expect("unpoisoned") + .push((*event.metadata().level(), fields)); + } + } + + /// An agent name the store would refuse fails the mint before any store + /// call, whatever this process's environment holds. The literal is the + /// `_msg` that `hh-agent-identity-mint-failed` in + /// `nix/host-modules/swarm-grafana.nix` matches. + #[tokio::test] + async fn a_failed_mint_logs_the_agent_and_the_error_on_the_line_the_alert_rule_matches() { + let events = Events::default(); + let _guard = + tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone())); + + let Outcome::Failed(error) = mint_identity("a/b").await else { + panic!("an illegal agent name cannot be minted"); + }; + + let got = events.0.lock().expect("unpoisoned").clone(); + let [(level, fields)] = got.as_slice() else { + panic!("one event, got {got:?}"); + }; + assert_eq!(*level, tracing::Level::WARN); + let field = |name: &str| { + fields + .iter() + .find(|(n, _)| n == name) + .map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone()) + }; + assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed"); + assert_eq!(field("agent"), "a/b"); + assert_eq!(field("error"), error); + } +} diff --git a/swarm-controller/src/linked_accounts.rs b/swarm-controller/src/linked_accounts.rs index f8787cd1..deeaea4b 100644 --- a/swarm-controller/src/linked_accounts.rs +++ b/swarm-controller/src/linked_accounts.rs @@ -29,7 +29,8 @@ use serde::{Deserialize, Serialize}; use swarm_secret_client::{Error, SecretStore, forge, github, matrix}; use utoipa::{IntoParams, ToSchema}; -use super::{AppState, error_problem, swarm_hive}; +use crate::routes::agents::swarm_hive; +use crate::routes::{AppState, error_problem}; /// The host shown for a GitHub token. The store keeps none: a token is only /// ever used against github.com. @@ -950,9 +951,9 @@ pub(crate) mod tests { } /// Bare-minimum `AppState`, as `matrix_account`'s tests build it. - fn state(store: &Arc) -> super::super::AppState { - super::super::AppState { - hives: std::sync::Arc::new(vec![super::super::HiveEntry { + fn state(store: &Arc) -> crate::routes::AppState { + crate::routes::AppState { + hives: std::sync::Arc::new(vec![crate::routes::hives::HiveEntry { name: "pr1ma".to_owned(), domain: "pr1ma.example".to_owned(), dashboard: "pr1ma.hives.example".to_owned(), diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index ed5499a3..59b114b2 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -5,7 +5,7 @@ //! //! Configuration is read-only and loaded once at startup from env vars the //! NixOS module sets (`services.hyperhive.swarm.controller`) — see -//! `load_hives`. A config change means a redeploy, same as every other +//! `startup::load_hives`. A config change means a redeploy, same as every other //! option this process reads. //! //! It persists `webhook-secret` under its `StateDirectory` (see `webhook`), @@ -15,34 +15,11 @@ //! //! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on //! one host, this owns what is true across hives. -//! -//! `OpenAPI` spec generation mirrors `hive-c0re/src/dashboard/mod.rs` -//! exactly: `#[utoipa::path(...)]` per handler, an `ApiDoc` root, and a -//! raw JSON route at `/api/openapi.json`. Swagger UI itself is -//! nginx-hosted from the nix store, same shape as the per-hive -//! dashboard's (`nix/host-modules/hive-c0re/vhosts.nix`'s -//! `swaggerUiLocations` — a swagger-ui-theme dist under `/api/docs/`, no -//! fallback to this daemon). Only annotated routes appear in the spec; -//! an unannotated one just doesn't show up, nothing breaks. use std::os::unix::fs::PermissionsExt as _; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; use anyhow::{Context, Result}; -use axum::{ - Json, - extract::{Path, State}, - routing::get, -}; -use backon::{ExponentialBuilder, Retryable as _}; -use hive_jobq_wire::GraphWire as _; -use serde::{Deserialize, Serialize}; -use swarm_authelia_bridge_sock::BridgeResponse; -use swarm_controller_api::agents::{CreateAgentResponse, MintAgentForgeTokenResponse}; -use swarm_controller_api::forge::{ForgeAdminChange, MakeForgeAdminResponse}; -use utoipa::{OpenApi, ToSchema}; -use utoipa_axum::{router::OpenApiRouter, routes}; mod agent_archetype; mod agent_control; @@ -67,6 +44,7 @@ mod forge_account; mod github_account; mod inference; mod issue_report; +mod jobq; mod linked_accounts; mod matrix_account; mod otel_http_client; @@ -74,7 +52,9 @@ mod pulls; mod queue_identity; mod read_policy; mod review_classifier; +mod routes; mod schedules; +mod startup; mod status; mod store; mod subagent_term; @@ -85,3407 +65,14 @@ mod vcs_metrics; mod wanted; mod webhook; -/// Node payload for the swarm-level job graph. Named `Swarm*` rather than -/// the bare `NodeKind`/`Resource` `hive-c0re::job_queue::model` already -/// uses, so a grep for either doesn't land on both crates. -/// -/// `CreateIdentity` was the first real variant — "the minimal shape for -/// agent creation is creating the identity and wiring that in" (the design -/// thread's own framing for why that landed before the forge-node work -/// here). `CreateRepo`/`AddRepoMember`/`InitAgentConfigRepo` are three -/// separate nodes rather than one combined "create the repo" step so each -/// is independently retryable/observable in the job graph, same as every -/// other multi-step provisioning flow in this codebase (`hive-c0re`'s own -/// `NodeKind` never folds unrelated forge calls into one node either). -#[derive(Clone, Debug)] -enum SwarmNodeKind { - /// Ensure `agent` exists as an authelia subject at the swarm level. - CreateIdentity { agent: String }, - /// Create the agent's repo in `forge::CONFIG_ORG` with the operator - /// merge gate on its default branch. See `forge::Client::create_repo`. - CreateRepo { agent: String }, - /// Ensure `agent` exists as a Forgejo user account. Independent of - /// `CreateIdentity`/`CreateRepo` (a forge user needs neither an - /// authelia subject nor an existing repo) but a prerequisite for - /// `AddRepoMember`, which fails with "user does not exist" against an - /// account this node hasn't created yet. See - /// `forge::Client::ensure_agent_user`. - CreateForgeUser { agent: String }, - /// Add `agent` as a write collaborator on its own repo. See - /// `forge::Client::add_repo_member`. - AddRepoMember { agent: String }, - /// Add `agent` as a read collaborator on `internal/docs`. See - /// `forge::docs_grant`. - /// - /// Carries no hive: the grant is on a swarm-wide repo. - GrantDocsRead { agent: String }, - /// Seed the agent's repo with `agent.nix` + `flake.nix`. See - /// `forge::Client::seed_agent_config`. - /// - /// Deliberately carries no hive: seeding a config repo is the same - /// work whichever hive the agent is bound for, and an agent's config - /// states nothing about where it runs. The hive is an address the - /// swarm routes a deploy message to — it belongs on the node that - /// sends that message, not on this one. - InitAgentConfigRepo { agent: String }, - /// Mint `agent`'s own client certificate for the swarm secret store, - /// publish it there, grant it, and log in with it. See - /// `agent_identity::mint_and_verify` — including why this node does not - /// report success on a write. - /// - /// Carries no hive: neither the certificate, the queue credential nor - /// the agent's policy names one, so an agent keeps one store identity - /// whichever hive it runs on. - MintAgentIdentity { agent: String }, - /// Make sure `agent` holds a live forge access token in the swarm secret - /// store, minting one with the forge's admin API when it does not. See - /// `forge::agent_token` — including why a rotation is a delete then a - /// create. - /// - /// Carries no hive: the token's store path has no hive segment, and the - /// agent pulls it from wherever it runs. - MintAgentForgeToken { agent: String }, - /// Make sure `agent` holds a live token for its own account on the swarm's - /// homeserver, creating the account with the swarm's appservice token - /// when it does not. See `matrix_account::agent_token` — including why a - /// stored token is read back before anything is minted. - /// - /// Carries no hive: the path it writes has no hive segment, so an agent - /// that moves between hives keeps one matrix identity. - MintAgentMatrixAccount { agent: String }, - /// Make sure `hive`'s sender account (`@hive-:`) holds a live token - /// in the swarm store, where that hive's matrix sweep reads it. See - /// `matrix_account::hive_sender`. - MintHiveSenderToken { hive: String }, - /// Make sure `hive`'s read-only forge account (`hive-`) exists, is in - /// the `hives` team, and holds a live token in the swarm store, where - /// that hive's hive-c0re reads it. See `forge::hive_token`. - MintHiveForgeToken { hive: String }, - /// Make sure the review classifier's forge account exists and holds a - /// live token in the swarm store, where this daemon reads it to post the - /// classifier's reviews. See `forge::classifier_account`. - MintClassifierForgeToken, - /// Make sure the swarm builder's read-only forge account exists, is in - /// the `hives` team, and holds a live token in the swarm store, where the - /// builder reads it to fetch config repos. See `forge::builder_account`. - MintBuilderForgeToken, - /// Make sure the CI metrics poller's read-only forge account exists, is in - /// its team in every organisation, and holds a live token in the swarm - /// store, where the poller reads it to list CI runs. See - /// `forge::ci_metrics_account`. - MintCiMetricsForgeToken, - /// Build `agent`'s config at `rev`, the head of its config PR `pr`, on - /// the swarm builder, and write the result as a commit status on `rev`. - /// See `builder`. - /// - /// Carries no hive: the check builds on the builder's own agent base, - /// not on any hive. - BuildAgent { agent: String, rev: String, pr: u64 }, - /// Re-mint `agent`'s queue secret if it is still stored and old enough. - /// See `agent_renewal`. - /// - /// Carries no hive: the secret's store path has none. - RenewAgentQueueCredential { agent: String }, - /// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state - /// store, so a freshly created agent does not start driving turns the - /// moment it's deployed — the operator has to explicitly flip it to `Up`. - /// Carries the hive because the wanted-state bucket is keyed per hive. - SetAgentWanted { hive: String, agent: String }, - /// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy - /// subject. The one node kind whose effect leaves this host. - /// - /// Carries the hive, unlike every variant above — this is the node - /// `InitAgentConfigRepo`'s doc points at when it says the hive belongs - /// on the node that sends the deploy message. It names the subject the - /// message goes to, one per hive, so no other hive is woken by it. - TriggerDeploy { - hive: String, - agent: String, - /// The config commit to deploy, from a merged config PR. `None` from - /// agent creation, where the hive deploys what it seeds. - rev: Option, - }, -} - -impl hive_jobq_wire::WireNode for SwarmNodeKind { - fn label(&self) -> String { - match self { - SwarmNodeKind::CreateIdentity { .. } => "create_identity".to_owned(), - SwarmNodeKind::CreateRepo { .. } => "create_repo".to_owned(), - SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(), - SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(), - SwarmNodeKind::GrantDocsRead { .. } => "grant_docs_read".to_owned(), - SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(), - SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(), - SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(), - SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(), - SwarmNodeKind::MintHiveSenderToken { .. } => "mint_hive_sender_token".to_owned(), - SwarmNodeKind::MintHiveForgeToken { .. } => "mint_hive_forge_token".to_owned(), - SwarmNodeKind::MintClassifierForgeToken => "mint_classifier_forge_token".to_owned(), - SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(), - SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(), - SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(), - SwarmNodeKind::RenewAgentQueueCredential { .. } => { - "renew_agent_queue_credential".to_owned() - } - SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(), - SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), - } - } - - fn data(&self, _id: hive_jobq_wire::WireId) -> serde_json::Value { - // Spelled out as an or-pattern rather than a catch-all on purpose: a - // new variant then fails to compile here instead of silently - // rendering as an agent name. That gate has now fired once — - // `TriggerDeploy` is the first node that is about an agent *and a - // hive*, and a catch-all would have dropped the hive from the - // viewer without anyone noticing. - match self { - SwarmNodeKind::CreateIdentity { agent } - | SwarmNodeKind::CreateRepo { agent } - | SwarmNodeKind::CreateForgeUser { agent } - | SwarmNodeKind::AddRepoMember { agent } - | SwarmNodeKind::GrantDocsRead { agent } - | SwarmNodeKind::InitAgentConfigRepo { agent } - | SwarmNodeKind::MintAgentIdentity { agent } - | SwarmNodeKind::MintAgentForgeToken { agent } - | SwarmNodeKind::MintAgentMatrixAccount { agent } - | SwarmNodeKind::RenewAgentQueueCredential { agent } => { - serde_json::json!({ "agent": agent }) - } - SwarmNodeKind::MintHiveSenderToken { hive } - | SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }), - SwarmNodeKind::MintClassifierForgeToken - | SwarmNodeKind::MintBuilderForgeToken - | SwarmNodeKind::MintCiMetricsForgeToken => serde_json::json!({}), - SwarmNodeKind::BuildAgent { agent, rev, pr } => { - serde_json::json!({ "agent": agent, "rev": rev, "pr": pr }) - } - SwarmNodeKind::TriggerDeploy { hive, agent, rev } => { - serde_json::json!({ "agent": agent, "hive": hive, "rev": rev }) - } - SwarmNodeKind::SetAgentWanted { hive, agent } => { - serde_json::json!({ "agent": agent, "hive": hive }) - } - } - } -} - -/// A named counting semaphore a node acquires before it starts. Every name -/// has capacity 1: the scheduler's default, which no caller overrides. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -enum SwarmResourceKind { - /// The swarm builder: one `BuildAgent` node runs at a time, and the rest - /// wait pending, where a newer push can replace them. - Builder, -} - -impl hive_jobq_wire::WireResource for SwarmResourceKind { - fn name(&self) -> String { - match self { - SwarmResourceKind::Builder => "builder".to_owned(), - } - } -} - -/// A dependency this daemon may only gain after it starts serving: empty -/// until then, filled once. Readers take what is there at the moment they ask. -type Late = Arc>; - -/// Everything a claimed node's executor arm might need to reach outside -/// this process — bundled into one `Clone` struct rather than growing -/// `run_swarm_node`'s parameter list per node kind (three forge-shaped -/// node kinds landed in one slice; a fourth parameter each would have made -/// the signature the least readable part of this file). Each field is -/// built once at startup (see `main`) and is `None` exactly when that -/// dependency isn't configured on this host — every arm below treats -/// absence as *this node's* failure, not a reason to skip silently. -#[derive(Clone)] -struct WorkerDeps { - auth: Option>, - /// Empty until the forge token has been read (`forge::enable`), so a node - /// run before then fails as "no forge access", and one run after succeeds. - forge: Late>, - /// The swarm queue connection, for nodes whose effect is a published - /// event rather than an API call. A handle rather than the status - /// reader it is cloned from: a node that announces a deploy has no - /// business reading hive status, and `status.rs`'s own doc says the - /// connection living there is an accident of construction order, not a - /// claim that events are a kind of status. - queue: Option, - /// The wanted-state writer, for nodes that declare what a hive should - /// converge an agent to. Shares the status reader's queue connection — - /// see `wanted_writer`. `None` exactly when no swarm queue is configured - /// on this host, same as `queue`. - wanted: Option>, - /// Client-server API base of the swarm's homeserver, for the node that - /// mints agents' accounts on it. `None` when - /// `matrix_account::DEFAULT_HOMESERVER_ENV` is unset, which that node - /// **fails** on rather than skips, same as every other field here. - matrix_homeserver: Option>, -} - -/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be. -/// -/// A named constant rather than the literal inline, because this *is* the -/// decision the node exists to carry — writing a declaration at all is the -/// mechanism, `Paused` is the policy — and a test can pin a name where it -/// cannot pin a value written into a KV bucket no unit test has a server for. -const NEW_AGENT_WANTED_STATE: swarm_queue_client::wanted::AgentState = - swarm_queue_client::wanted::AgentState::Paused; - -/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ -/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind` -/// variant turns into a real effect. -async fn run_swarm_node( - _id: hive_jobq::NodeId, - kind: SwarmNodeKind, - builder: hive_jobq::builder::JobBuilder, - deps: WorkerDeps, -) -> ( - hive_jobq::builder::JobBuilder, - hive_jobq::scheduler::Outcome, -) { - use hive_jobq::scheduler::Outcome; - - let forge = deps.forge.get().cloned(); - let outcome = match kind { - SwarmNodeKind::CreateIdentity { agent } => match deps.auth { - None => { - Outcome::Failed("no swarm-authelia-bridge is configured on this host".to_owned()) - } - Some(bridge) => match bridge.ensure_agent_identity(&agent).await { - // Matched rather than discarded with `Ok(_)`: a heal writes - // to a subject this job did not create, and that has to - // reach a human. `Outcome` has no success-carrying variant, - // so the controller's own log is the only channel a - // succeeding node has today. - Ok(BridgeResponse::Healed) => { - tracing::warn!( - %agent, - "swarm jobq: create_identity marked an EXISTING identity as an \ - agent — verify this name did not belong to a person" - ); - Outcome::Done - } - Ok(_) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - }, - }, - SwarmNodeKind::CreateRepo { agent } => match forge { - None => Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store)" - .to_owned(), - ), - Some(client) => match forge_node("create_repo", client.create_repo(&agent)).await { - Ok(full_name) => { - tracing::info!(%full_name, "swarm jobq: create_repo done"); - Outcome::Done - } - Err(e) => Outcome::Failed(format!("{e:#}")), - }, - }, - SwarmNodeKind::CreateForgeUser { agent } => match forge { - None => Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store)" - .to_owned(), - ), - Some(client) => { - match forge_node("create_forge_user", client.ensure_agent_user(&agent)).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } - } - }, - SwarmNodeKind::AddRepoMember { agent } => match forge { - None => Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store)" - .to_owned(), - ), - Some(client) => { - match forge_node("add_repo_member", client.add_repo_member(&agent, &agent)).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } - } - }, - SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await, - SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await, - SwarmNodeKind::MintAgentIdentity { agent } => mint_identity(&agent).await, - SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await, - SwarmNodeKind::MintAgentMatrixAccount { agent } => { - mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await - } - SwarmNodeKind::MintHiveSenderToken { hive } => { - mint_hive_sender(deps.matrix_homeserver.as_deref(), &hive).await - } - SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await, - SwarmNodeKind::MintClassifierForgeToken => mint_classifier_forge_token(forge).await, - SwarmNodeKind::MintBuilderForgeToken => mint_builder_forge_token(forge).await, - SwarmNodeKind::MintCiMetricsForgeToken => mint_ci_metrics_forge_token(forge).await, - SwarmNodeKind::BuildAgent { agent, rev, pr } => { - build_agent(&deps, forge, agent, rev, pr).await - } - SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await, - SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted { - None => Outcome::Failed( - "no swarm queue is configured on this host, so no wanted-state \ - declaration can be written" - .to_owned(), - ), - Some(writer) => declare_new_agent(&writer, &hive, &agent).await, - }, - SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue { - None => Outcome::Failed( - "no swarm queue is configured on this host, so no hive can be told to deploy" - .to_owned(), - ), - Some(client) => publish_deploy(&client, &hive, &agent, rev).await, - }, - }; - (builder, outcome) -} - -/// A forge-calling node's work, run under [`forge::FORGE_DEADLINE`]. Elapsing -/// is the node's own error, naming `node`, so it fails like any other refused -/// step instead of holding its executor slot. -async fn forge_node( - node: &str, - work: impl std::future::Future>, -) -> Result { - forge::within_forge_deadline(node, work) - .await - .unwrap_or_else(|| { - Err(anyhow::anyhow!( - "{node}: no answer from the forge within {}s", - forge::FORGE_DEADLINE.as_secs() - )) - }) -} - -/// Logged at `warn`, with the agent and the error, when [`mint_identity`] -/// fails. The Grafana rule `hh-agent-identity-mint-failed` -/// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both -/// together. -const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed"; - -/// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under -/// `clippy::too_many_lines`. -async fn mint_identity(agent: &str) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - match agent_identity::mint_and_verify(agent).await { - Ok(()) => Outcome::Done, - Err(e) => { - let error = format!("{e:#}"); - tracing::warn!(%agent, %error, "{MINT_IDENTITY_FAILED}"); - Outcome::Failed(error) - } - } -} - -/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under -/// `clippy::too_many_lines`. -async fn mint_forge_token( - forge: Option>, - agent: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no agent forge token can be minted" - .to_owned(), - ); - }; - match forge_node( - "mint_agent_forge_token", - client.ensure_agent_forge_token(agent), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `GrantDocsRead` arm, lifted out for the same reason. -async fn grant_docs_read( - forge: Option>, - agent: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no docs grant can be made" - .to_owned(), - ); - }; - match forge_node("grant_docs_read", client.grant_docs_read(agent)).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `InitAgentConfigRepo` arm, lifted out for the same reason. -async fn init_agent_config_repo( - forge: Option>, - agent: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store)" - .to_owned(), - ); - }; - match forge_node( - "init_agent_config_repo", - client.seed_agent_config(agent, agent), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `RenewAgentQueueCredential` arm, lifted out for the same reason. -async fn renew_queue_credential(agent: &str) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - match agent_renewal::renew(agent).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `MintHiveForgeToken` arm, lifted out for the same reason. -async fn mint_hive_forge_token( - forge: Option>, - hive: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no hive forge token can be minted" - .to_owned(), - ); - }; - match forge_node( - "mint_hive_forge_token", - client.ensure_hive_forge_token(hive), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `BuildAgent` arm, lifted out for `clippy::too_many_lines`. -async fn build_agent( - deps: &WorkerDeps, - forge: Option>, - agent: String, - rev: String, - pr: u64, -) -> hive_jobq::scheduler::Outcome { - let job = swarm_wire::builder::BuildAgent { agent, rev, pr }; - let builds = builder::runs::Builds::Live(deps.queue.clone()); - let link_base = std::env::var(PUBLIC_URL_ENV).ok(); - builder::run( - deps.queue.as_ref(), - forge.as_deref(), - &builds, - link_base.as_deref(), - job, - ) - .await -} - -/// The `MintBuilderForgeToken` arm, lifted out for the same reason. -async fn mint_builder_forge_token( - forge: Option>, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no builder forge token can be minted" - .to_owned(), - ); - }; - match forge_node( - "mint_builder_forge_token", - client.ensure_builder_forge_token(), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `MintCiMetricsForgeToken` arm, lifted out for the same reason. -async fn mint_ci_metrics_forge_token( - forge: Option>, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no ci metrics forge token can be minted" - .to_owned(), - ); - }; - match forge_node( - "mint_ci_metrics_forge_token", - client.ensure_ci_metrics_forge_token(), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `MintClassifierForgeToken` arm, lifted out for the same reason. -async fn mint_classifier_forge_token( - forge: Option>, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(client) = forge else { - return Outcome::Failed( - "no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \ - not in the swarm secret store), so no review classifier forge token can be minted" - .to_owned(), - ); - }; - match forge_node( - "mint_classifier_forge_token", - client.ensure_classifier_forge_token(), - ) - .await - { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `MintAgentMatrixAccount` arm, lifted out for the same reason. -async fn mint_matrix_account( - homeserver: Option<&str>, - agent: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - // Loud on purpose: an agent quietly created without a matrix account is - // an agent nothing can talk to, and no hive mints one any more. - let Some(base) = homeserver else { - return Outcome::Failed(format!( - "no matrix homeserver configured on this host ({} unset), so no agent \ - matrix account can be minted", - matrix_account::DEFAULT_HOMESERVER_ENV - )); - }; - match matrix_account::agent_token::ensure_agent_matrix_account(base, agent).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// The `MintHiveSenderToken` arm, lifted out for the same reason. -async fn mint_hive_sender(homeserver: Option<&str>, hive: &str) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - let Some(base) = homeserver else { - return Outcome::Failed(format!( - "no matrix homeserver configured on this host ({} unset), so no hive \ - sender token can be minted", - matrix_account::DEFAULT_HOMESERVER_ENV - )); - }; - match matrix_account::hive_sender::ensure_hive_sender_token(base, hive).await { - Ok(()) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns -/// out not to be new: an agent that already has a declaration (other than -/// `Destroyed`, which this treats as reusable) is left alone, so a retried -/// or migrating create is idempotent rather than pausing an already-running -/// agent. `wanted.rs` itself has no such carve-out, so this node makes the -/// call. -async fn declare_new_agent( - writer: &wanted::WantedWriter, - hive: &str, - agent: &str, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - use swarm_queue_client::wanted::AgentState; - - let existing = match writer.view(hive).await { - Ok(declaration) => declaration.and_then(|d| d.agents.get(agent).map(|a| a.state)), - Err(e) => return Outcome::Failed(format!("{e:#}")), - }; - if let Some(state) = existing - && state != AgentState::Destroyed - { - tracing::debug!( - hive, - agent, - ?state, - "agent already declared, leaving its wanted state alone" - ); - return Outcome::Done; - } - match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await { - Ok(_) => Outcome::Done, - Err(e) => Outcome::Failed(format!("{e:#}")), - } -} - -/// Publish one deploy request and report whether it left this process. -/// -/// The flush is not belt-and-braces: `publish` hands the message to the -/// client's write buffer and returns, so a node that reported `Done` on -/// that alone would be claiming a delivery it has no evidence for — the -/// same ordering `webhook::announce_knowledge_change` documents. -async fn publish_deploy( - client: &async_nats::Client, - hive: &str, - agent: &str, - rev: Option, -) -> hive_jobq::scheduler::Outcome { - use hive_jobq::scheduler::Outcome; - - // One subject per hive, so the other hives are never woken by this. - let subject = swarm_queue_client::deploy_subject(hive); - let request = swarm_queue_client::DeployRequest { - agent: agent.to_owned(), - rev, - }; - let payload = match serde_json::to_vec(&request) { - Ok(payload) => payload, - Err(e) => return Outcome::Failed(format!("encoding the deploy request failed: {e}")), - }; - if let Err(e) = client.publish(subject.clone(), payload.into()).await { - return Outcome::Failed(format!("publishing to {subject} failed: {e}")); - } - if let Err(e) = client.flush().await { - return Outcome::Failed(format!( - "flushing the deploy event to {subject} failed: {e}" - )); - } - tracing::info!(%subject, %hive, %agent, rev = ?request.rev, "swarm jobq: deploy requested"); - Outcome::Done -} - -/// Spawn the swarm-level job-graph scheduler loop. Mirrors `hive-c0re/src/ -/// job_queue/scheduler.rs::run_worker`'s shape: claim one runnable node, -/// spawn the future that runs + completes it, loop again immediately if -/// something started (more may now be runnable), otherwise back off -/// briefly before re-polling. -/// -/// No shutdown signal to wire in — unlike `hive-c0re`'s `coord.shutdown_rx()`, -/// this daemon has no graceful-shutdown machinery at all yet (`main`'s -/// `axum::serve` runs unconditionally to process exit), so this loop -/// matches that: it rides the runtime down with the process, same as -/// every in-flight HTTP request does. -/// -/// Cheap to run with an empty graph: `claim_next` on a graph nothing was -/// ever inserted into just returns `None` every poll. -/// -/// `deps` is cloned per iteration (its fields are `Arc` clones, not -/// reconnects) and moved into the closure `claim_next` takes ownership -/// of — `run_swarm_node` needs its own owned copy since the claimed -/// future may outlive this loop iteration. -fn spawn_jobq_worker( - sched: Arc>>, - deps: WorkerDeps, -) { - tokio::spawn(async move { - loop { - let deps = deps.clone(); - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }); - match runner { - Some(runner) => { - tokio::spawn(async move { - let (id, grew) = runner.await; - if let Err(e) = grew { - tracing::warn!( - node = id.get(), - error = %e, - "swarm jobq: grown job rejected" - ); - } - }); - // Something just started — more may be runnable right - // now, so loop again immediately rather than sleeping. - } - None => { - // Nothing runnable. Bounded poll rather than an event - // wake (unlike hive-c0re's `notify.notify_one()`, - // there is no completion-signal channel here yet) — - // fine at this daemon's scale (one graph, no - // submitters yet); revisit if/when that stops holding. - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - } - } - } - }); -} - -/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`. -/// -/// A compiled-in default is legitimate here and is *not* the mistake that -/// a hardcoded remote address would be: this is a path this process -/// **creates**, not an address it hopes to find something at. systemd's -/// `RuntimeDirectory=swarm-controller` makes the parent exist before -/// `ExecStart`, so the default names a directory the unit just produced. -/// -/// The directory is its own — deliberately not shared with hive-c0re's -/// `/run/hyperhive`. The socket is `0666`, so its directory is the only -/// access control it has; co-locating it with c0re's admin socket would -/// put both within reach of whatever can reach either. nginx runs on the -/// host, so nothing narrows its reach for you. -const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock"; - -fn socket_path() -> PathBuf { - std::env::var_os("SWARM_CONTROLLER_SOCKET") - .map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from) -} - -/// Root of the auto-generated `OpenAPI` spec, served raw at -/// `/api/openapi.json` — see the module doc comment above. Tag list -/// grows alongside the swarm-level surfaces this daemon picks up, same -/// as `hive-c0re::dashboard::ApiDoc`'s tag list did. -#[derive(OpenApi)] -#[openapi( - info( - title = "hyperhive swarm-controller API", - description = "swarm-controller's HTTP surface, served over its unix \ - socket behind the gateway's swarm-UI vhost." - ), - tags( - (name = "health", description = "liveness probe"), - (name = "hives", description = "the swarm's hive directory"), - (name = "links", description = "swarm service quick links"), - (name = "jobq", description = "the swarm-level job graph"), - (name = "agents", description = "creating agent identities at swarm level"), - (name = "webhook", description = "swarm-wide forge webhook receipt"), - (name = "repos", description = "forge repo browsing (swarm-ui's issue-report page)"), - (name = "forge", description = "forge site admins (`swarmctl forge make-admin`)"), - (name = "schedules", description = "scheduled prompts (`swarmctl schedule`)"), - (name = "review-classifier", description = "trying review classifications against a PR (swarm-ui's try-out page), and the classifier's swarm-wide settings"), - (name = "pulls", description = "open pull requests, for the try-out page's dropdowns"), - (name = "alerts", description = "the swarm Grafana's current alerts (swarm-ui's alerts page)"), - (name = "builder", description = "the swarm builder's finished builds of agent config PRs, which their commit statuses link to"), - ) -)] -struct ApiDoc; - -/// Liveness probe. Returns the build's version so an operator can tell -/// *which* controller answered without shelling onto the host. -#[utoipa::path( - get, - path = "/health", - responses((status = 200, description = "process is up, body is \"swarm-controller \"", body = String)), - tag = "health" -)] -async fn health() -> &'static str { - concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") -} - -/// One hive in the swarm's directory — the same `name`/`domain` pair -/// `services.hyperhive.swarm.hives` (nix/host-modules/swarm.nix) declares, -/// plus that hive's `swarm.dashboardDomains` entry, carried across -/// unchanged rather than reshaped, so this stays a direct mirror of the -/// nix source of truth instead of a second vocabulary for the same fields. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] -struct HiveEntry { - name: String, - domain: String, - /// The host the hive's dashboard answers on. - dashboard: String, -} - -#[derive(Clone)] -struct AppState { - /// Loaded once at startup (`load_hives`); never mutated, so an - /// `Arc` clone per request is the whole synchronization story. - hives: Arc>, - /// Loaded once at startup (`load_links`); same synchronization story - /// as `hives`. - links: Arc>, - /// `None` when this deployment wired up no swarm queue — the only - /// state in which `/api/hives/status` cannot answer at all. A queue - /// that is merely *unreachable* still yields a reader, because - /// `async-nats` reconnects underneath it. - status: Option>, - /// Publishes the agent set this swarm declares for each hive. - /// - /// `None` in exactly the state `status` is: no swarm queue was wired - /// up, so there is nowhere to publish a declaration to. Shares that - /// reader's connection rather than opening a second one. - wanted: Option>, - /// Per-agent status, sharing `status`'s connection — same "one queue - /// connection, several consumers" rationale as `wanted` above. `None` - /// in exactly the state `status` is. - agent_status: Option>, - /// Per-agent icons, sharing `status`'s connection — same "one queue - /// connection, several consumers" rationale as `agent_status` above, - /// and `None` in exactly the state `status` is. - agent_icons: Option>, - /// Each agent's published `AGENTS.md`, on `status`'s connection like - /// `agent_icons`, and `None` in exactly the state `status` is. - agents_md: Option>, - /// What each agent last reported running on, from its turn-state - /// headers on `status`'s connection. `None` in exactly the state - /// `status` is. - reported_inference: Option>, - /// Whether each agent last said it needs a login, from its turn-state - /// headers on `status`'s connection. `None` in exactly the state - /// `status` is. - agent_login: Option>, - /// The swarm-level job graph, wrapped in its - /// [`hive_jobq::scheduler::Scheduler`] now that something drives it - /// (`spawn_jobq_worker`) — the graph alone was enough for the - /// read-only endpoints, the scheduler is what a `claim_next` loop - /// needs. `std::sync::Mutex`, not `tokio`'s — every lock scope below - /// is synchronous (no `.await` while held). Always present, never - /// gated on the swarm queue: this is process state, not something - /// read over the network. - jobq: Arc>>, - /// The identity store's only reader, for `GET /api/agents`. - /// - /// `None` when no bridge is wired up, which is the one state in which - /// the roster cannot be answered at all — the same shape as `status` - /// above, and for the same reason: an empty roster and an unreadable - /// one are different facts. - /// - /// ⚠️ The two `/api/agents` verbs treat this differently on purpose. - /// **POST does not need it**: creation only queues a job, and - /// whether a bridge exists is `run_swarm_node`'s concern (it holds its - /// own clone from `spawn_jobq_worker`), so a request still queues - /// cleanly on a bridge-less host and fails loud once claimed. POST reads - /// it only for a name that breaks a naming rule, to tell a new agent - /// from an existing one, and warns when it cannot (`name_verdict`). **GET - /// has nowhere to defer to** — there is no job, only an answer it - /// either has or does not. `POST /api/forge/users/{name}/admin` reads - /// it for every name and refuses when it cannot (`admin_unless_agent`). - auth: Option>, - /// HMAC secret for swarm-wide forge webhooks, loaded once at startup. - /// `None` when it could not be read or created — the webhook endpoint - /// then refuses every delivery with 503 rather than admitting one it - /// cannot verify. Deliberately not fatal to startup: nothing is - /// registered against that endpoint yet, and the rest of this daemon's - /// surface is unaffected. See [`webhook::load_or_generate_secret`]. - /// - /// `Arc`, not `Arc`: the value is never mutated after - /// startup, and this way `as_deref()` yields the `&str` the verifier - /// takes without a second hop through `String`. - webhook_secret: Option>, - /// Last successful `agent-configs/*` scan, kept current by - /// [`config_pr::spawn`]. Empty while this daemon has no forge access — - /// same "absent means don't ask" shape as `status` and `forge`, not a - /// startup failure — and filled when the forge client is. - config_prs: Late>, - /// The swarm's human display name (`services.hyperhive.swarm.name`), - /// loaded once at startup (`load_swarm_name`). `None` when the - /// operator never set it — a swarm without a display name is a - /// supported, if less friendly, state, not a startup failure. Same - /// `Arc` rationale as `webhook_secret`: never mutated, so a - /// clone per request is just a refcount bump. - swarm_name: Option>, - /// The forge client itself, for the read-only repo/issue-report - /// routes (`GET /api/repos`, `GET /api/repos/{org}/{repo}/issue-report`) - /// and `POST /api/forge/users/{name}/admin` — distinct from `config_prs`, which holds a *cache* built off this - /// same client rather than the client. Empty while this daemon has no - /// forge access, and filled the moment `forge::enable` builds the client. - forge: Late>, - /// Held by `create_agent` from reading where a name is placed until its - /// graph is queued, and by `set_agent_state` from that read until its - /// declaration is written, so two placements of one name cannot both - /// find it unplaced. `tokio`'s mutex, unlike `jobq`'s: the read awaits - /// the queue. - create_gate: Arc>, - /// The store the linked-account routes read and write. - accounts: linked_accounts::Accounts, - /// The review classifier try-outs started here, until their results - /// expire. - try_outs: review_classifier::TryOuts, - /// The requested reviews running or waiting. - reviews: Arc, - /// The review classifier's finished runs. - runs: review_classifier::runs::Runs, - /// The swarm builder's finished builds. - builds: builder::runs::Builds, - /// Grafana's alerts API, for `GET /api/alerts`. `None` when no Grafana URL - /// is configured. - alerts: Option>, -} - -/// Env var the controller's NixOS module sets from -/// `services.hyperhive.swarm.hives`, JSON-encoded — the full directory -/// (this daemon has no "self" to exclude) rather than peers-minus-self. -const HIVES_ENV: &str = "SWARM_CONTROLLER_HIVES"; - -/// Parses [`HIVES_ENV`] into the swarm's hive directory. Unset or -/// unparseable both fall back to an empty list with a warning rather than -/// failing startup — a swarm-controller that can't yet see its own config -/// (dev run, a module not wired up yet) should still serve `/health`. -fn load_hives() -> Vec { - let Some(raw) = std::env::var_os(HIVES_ENV) else { - return Vec::new(); - }; - match serde_json::from_str(&raw.to_string_lossy()) { - Ok(hives) => hives, - Err(e) => { - tracing::warn!(error = %e, env = HIVES_ENV, "failed to parse hive directory, serving an empty list"); - Vec::new() - } - } -} - -/// The swarm's hive directory — every hive, including whichever one this -/// controller instance happens to run on (there is no "self" to exclude -/// at the swarm level, unlike `hive-c0re`'s peer list). -#[utoipa::path( - get, - path = "/api/hives", - responses((status = 200, description = "every hive in the swarm", body = Vec)), - tag = "hives" -)] -async fn get_hives(State(state): State) -> Json> { - Json((*state.hives).clone()) -} - -/// The swarm's agent roster — every agent the swarm holds an identity for. -/// -/// The identity store **is** the roster rather than one source to assemble -/// one from: an agent without a swarm identity is not a swarm agent, so -/// there is no hive-side list to merge in and no reconciliation to do. -/// -/// Deliberately says nothing about health. A roster is the set other views -/// are *complete against* — it is what makes "this agent has never reported" -/// expressible at all, and that distinction only survives while the declared -/// set and the reported set stay separate. -/// -/// A hive-less swarm answers `[]`; a swarm with no bridge answers 503. Those -/// are different facts and collapsing them would render an unreadable store -/// as an empty one. -#[utoipa::path( - get, - path = "/api/agents", - responses( - (status = 200, description = "every agent the swarm holds an identity for", body = Vec), - (status = 503, description = "no identity bridge is configured here, or its store could not be read", body = String), - ), - tag = "agents" -)] -async fn get_agents(State(state): State) -> Result>, StatusUnavailable> { - let Some(bridge) = state.auth.as_ref() else { - return Err(StatusUnavailable( - "no identity bridge is configured on this host".to_owned(), - )); - }; - match bridge.list_agent_identities().await { - Ok(agents) => Ok(Json(agents)), - Err(e) => { - let detail = format!("{e:#}"); - tracing::warn!(error = %detail, "reading the agent roster failed"); - Err(StatusUnavailable(detail)) - } - } -} - -/// One quick link to a swarm-wide service (authelia, matrix, forge, this -/// daemon's own swagger UI, …). Deliberately generic rather than named -/// fields per service: each service's own nix module contributes its own -/// entry to `services.hyperhive.swarm.controller.links` (same list-merge -/// idiom `services.hyperhive.gateway.localNames` already uses), so adding -/// a new one is a nix-only change — no new field here, no swarm-ui change. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] -struct ServiceLink { - label: String, - /// Emoji or short glyph. Empty string, not `Option`, when a - /// contributing module has none — one fewer null-vs-absent case for - /// the frontend to handle, and every real contributor sets one today. - icon: String, - url: String, -} - -/// Env var the controller's NixOS module sets from the merged -/// `services.hyperhive.swarm.controller.links` list, JSON-encoded — same -/// shape/rationale as [`HIVES_ENV`]. Consumed by `GET /api/links`. -const LINKS_ENV: &str = "SWARM_CONTROLLER_LINKS"; - -/// Parses [`LINKS_ENV`] into the swarm's service-link list. Same -/// fall-back-to-empty rationale as `load_hives`: a daemon that can't yet -/// see this config should still serve `/health` rather than fail startup. -fn load_links() -> Vec { - let Some(raw) = std::env::var_os(LINKS_ENV) else { - return Vec::new(); - }; - match serde_json::from_str(&raw.to_string_lossy()) { - Ok(links) => links, - Err(e) => { - tracing::warn!(error = %e, env = LINKS_ENV, "failed to parse service links, serving an empty list"); - Vec::new() - } - } -} - -/// Quick links to swarm-wide services (authelia, matrix, forge, this -/// daemon's own swagger UI, …), as contributed by each service's own nix -/// module. Empty when nothing was configured to contribute — a caller -/// renders 0 links the same way it renders any other count, no special -/// "not configured" case. -#[utoipa::path( - get, - path = "/api/links", - responses((status = 200, description = "swarm service quick links", body = Vec)), - tag = "links" -)] -async fn get_links(State(state): State) -> Json> { - Json((*state.links).clone()) -} - -/// Env var the controller's NixOS module sets from -/// `services.hyperhive.swarm.name` — unset (rather than an empty string) -/// when the operator never configured it. Consumed by `GET /api/swarm`. -const NAME_ENV: &str = "SWARM_CONTROLLER_NAME"; - -/// Reads [`NAME_ENV`]. `None` on absence — no fallback-and-warn shape like -/// `load_hives`/`load_links` because there is nothing to parse and fail: -/// an unset env var and an operator who never named the swarm are the same -/// state, not an error. -fn load_swarm_name() -> Option { - std::env::var(NAME_ENV).ok() -} - -/// Body of `GET /api/swarm`. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] -struct SwarmInfo { - /// `None` when the operator never set `services.hyperhive.swarm.name` - /// — a caller (swarm-ui's chrome) falls back to a generic label rather - /// than treating this as an error. - name: Option, -} - -/// The swarm's own display name, for UI chrome (page title, nav) that -/// wants to say which swarm it's showing — distinct from [`get_hives`], -/// which lists the *hives inside* the swarm, not the swarm itself. -#[utoipa::path( - get, - path = "/api/swarm", - responses((status = 200, description = "the swarm's display name, if the operator set one", body = SwarmInfo)), - tag = "hives" -)] -async fn get_swarm_info(State(state): State) -> Json { - Json(SwarmInfo { - name: state.swarm_name.as_deref().map(str::to_owned), - }) -} - -/// Why the status route answers 503 rather than an empty list. -/// -/// "I cannot reach the store" and "every hive is silent" are different -/// answers, and rendering the second when the first is true is exactly -/// the smoothing this endpoint exists to avoid — a caller would draw a -/// swarm-wide outage out of a local one. The cause is carried in the -/// body because a bare 503 on an operator-facing diagnostic is how a -/// misconfiguration costs an afternoon; it is a queue/JetStream error -/// string, and this surface is already behind the swarm's SSO. -/// -/// It travels in `detail` of an RFC 9457 `application/problem+json` body -/// rather than as a bare string, so the cause is an addressable field -/// instead of the whole payload — see `error_problem` below. -struct StatusUnavailable(String); - -impl axum::response::IntoResponse for StatusUnavailable { - fn into_response(self) -> axum::response::Response { - error_problem(axum::http::StatusCode::SERVICE_UNAVAILABLE, &self.0).into_response() - } -} - -/// Every error this daemon returns, in one shape. -/// -/// RFC 9457 `application/problem+json` is the hive-wide contract for HTTP -/// error bodies (`docs/process/conventions.md`), and the operator UIs read `detail` -/// for display. A bare string forces the reader to treat the entire body as -/// the message, which is the difference between a UI that can offer "copy the -/// cause" and one that can only dump a response. -fn error_problem(status: axum::http::StatusCode, detail: &str) -> problem_details::ProblemDetails { - problem_details::ProblemDetails::from_status_code(status).with_detail(detail) -} - -/// The status a [`wanted::WantedWriter`], [`agent_icon::AgentIconReader`] or -/// `schedules` queue failure should answer with. -/// -/// `WantedWriter::view`/`set` call `ensure_connected` internally and -/// propagate through `anyhow`, so the concrete -/// [`swarm_queue_client::Error`] survives underneath but is not the type in -/// hand — `swarm_queue_client`'s own doc says a caller "gets a type it can -/// match on", which means downcasting back to it rather than string-matching -/// the rendered message. `NotConnected` is a retryable condition (the queue -/// is reconnecting, or hasn't yet) and every other queue-backed route here -/// already answers 503 for it; anything else is a genuine failure to publish -/// or read, which stays 500. -pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode { - match e.downcast_ref::() { - Some(swarm_queue_client::Error::NotConnected(_)) => { - axum::http::StatusCode::SERVICE_UNAVAILABLE - } - _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, - } -} - -/// The state to declare for one agent. -#[derive(Debug, Deserialize, ToSchema)] -struct SetAgentStateRequest { - /// Typed as a string in the schema only — the parse is the real enum, so - /// a value this build does not know is a 400 rather than a field that - /// silently does nothing. - #[schema(value_type = String, example = "up")] - state: swarm_queue_client::wanted::AgentState, -} - -/// One agent's line in a hive's declaration. -#[derive(Clone, Debug, Serialize, ToSchema)] -struct AgentDeclaration { - agent: String, - state: String, -} - -fn render(declaration: &swarm_queue_client::wanted::HiveWanted) -> Vec { - declaration - .agents - .iter() - .map(|(agent, wanted)| AgentDeclaration { - agent: agent.clone(), - state: <&str>::from(wanted.state).to_owned(), - }) - .collect() -} - -/// The declaration writer, sharing the status reader's connection. -/// -/// The controller holds exactly one queue connection by design — see -/// `StatusReader::queue_client`. A second connect would double the -/// auth-callout traffic and give the two paths independent reconnect state, -/// so one could be serving while the other was still down. -fn wanted_writer(status: Option<&Arc>) -> Option> { - status.map(|s| Arc::new(wanted::WantedWriter::new(s.queue_client()))) -} - -/// The per-agent status reader, sharing the status reader's connection and -/// staleness threshold — same rationale as [`wanted_writer`], and the same -/// threshold on purpose: both buckets are fed by the same publisher cadence -/// (`hive-c0re`'s `PUBLISH_INTERVAL`), so two separately-configured -/// thresholds would just be two ways to get out of sync with one cadence. -fn agent_status_reader( - status: Option<&Arc>, -) -> Option> { - status.map(|s| { - Arc::new(agent_status::AgentStatusReader::new( - s.queue_client(), - status::StatusReader::stale_after_from_env(), - )) - }) -} - -/// The per-agent icon reader, sharing the status reader's connection — -/// same rationale as [`wanted_writer`]. No staleness threshold, unlike -/// [`agent_status_reader`]: an icon is a property of the agent rather than -/// a report about it, so there is no cadence it could be late against. -/// -/// Also starts creating the bucket, which the agents writing it cannot do. -fn agent_icon_reader( - status: Option<&Arc>, -) -> Option> { - let reader = Arc::new(agent_icon::AgentIconReader::new(status?.queue_client())); - let creating = Arc::clone(&reader); - tokio::spawn(async move { creating.create_when_connected().await }); - Some(reader) -} - -/// The `AGENTS.md` reader, sharing the status reader's connection, and -/// creating the bucket, as [`agent_icon_reader`] does for icons. -fn agents_md_reader( - status: Option<&Arc>, -) -> Option> { - let reader = Arc::new(agents_md::AgentsMdReader::new(status?.queue_client())); - let creating = Arc::clone(&reader); - tokio::spawn(async move { creating.create_when_connected().await }); - Some(reader) -} - -/// What each agent last reported running on, read on the status reader's -/// connection — same rationale as [`wanted_writer`]. -fn reported_inference( - status: Option<&Arc>, -) -> Option> { - Some(inference::Reported::spawn(status?.queue_client())) -} - -/// Whether each agent last said it needs a login, read on the status -/// reader's connection — same rationale as [`wanted_writer`]. -fn agent_login(status: Option<&Arc>) -> Option> { - Some(agent_login::Reported::spawn(status?.queue_client())) -} - -/// Same "log and carry on" shape as the queue/bridge/forge wiring in `main`. -/// A controller that cannot hold a webhook secret still serves every other -/// route; the webhook endpoint answers 503, which is the honest answer -/// rather than a silent accept. Lifted out of `main` for -/// `clippy::too_many_lines`. -fn load_webhook_secret() -> Option> { - match webhook::load_or_generate_secret() { - Ok(secret) => Some(Arc::from(secret)), - Err(e) => { - tracing::warn!( - error = %format!("{e:#}"), - "webhook secret unavailable; swarm-wide forge webhooks are off" - ); - None - } - } -} - -/// A hive name that is shaped like one and names a hive this swarm has. -/// -/// Reports the status and the detail rather than a rendered -/// `ProblemDetails`: that type is 232 bytes, which makes every `Result` in -/// this path pay for the error case it usually does not take. The handlers -/// render at the boundary, where the body is actually needed. -fn swarm_hive(state: &AppState, hive: &str) -> Result { - let hive = hive_types::Ident::parse(hive) - .map_err(|reason| (axum::http::StatusCode::BAD_REQUEST, reason.to_owned()))? - .into_string(); - // Shaped like a hive name, and actually one. Same two checks - // `create_agent` makes, for the same reason: a typo otherwise publishes a - // declaration under a key no hive will ever read. - if !state.hives.iter().any(|h| h.name == hive) { - return Err(( - axum::http::StatusCode::BAD_REQUEST, - format!("{hive:?} is not a hive in this swarm"), - )); - } - Ok(hive) -} - -/// Both declaration handlers below take the same hive name and reject it the -/// same way, and need the writer besides. -fn declaration_target( - state: &AppState, - hive: &str, -) -> Result<(Arc, String), (axum::http::StatusCode, String)> { - // Before the name checks, so a deployment with no queue answers 503 - // whatever the caller spelled. - let writer = state.wanted.clone().ok_or_else(|| { - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "this deployment wired up no swarm queue, so there is nowhere to publish a declaration" - .to_owned(), - ) - })?; - Ok((writer, swarm_hive(state, hive)?)) -} - -/// Declare what this swarm wants of one agent on one hive. -/// -/// The whole declaration is returned as published, because the value is the -/// hive's entire agent map and a caller that changed one agent still wants to -/// render the rest. -#[utoipa::path( - put, - path = "/api/hives/{hive}/agents/{agent}/state", - params( - ("hive" = String, Path, description = "hive whose declaration this is"), - ("agent" = String, Path, description = "agent to declare"), - ), - request_body = SetAgentStateRequest, - responses( - (status = 200, description = "the declaration as now published", body = Vec), - (status = 400, description = "a name is not an identifier, the hive is not in this swarm, the state is unknown, or `agent` is new and reserved or refused by the forge (problem+json)", body = String), - (status = 409, description = "the swarm has already placed `agent` on a different hive (problem+json)", body = String), - (status = 503, description = "no swarm queue is wired up, or it is not connected, or `agent` is new here and breaks a naming rule but whether it already exists could not be checked (problem+json)", body = String), - (status = 500, description = "the declaration could not be published, or another hive's wanted state could not be read (problem+json)", body = String), - ), - tag = "agents" -)] -async fn set_agent_state( - State(state): State, - axum::extract::Path((hive, agent)): axum::extract::Path<(String, String)>, - Json(req): Json, -) -> Result>, problem_details::ProblemDetails> { - let (writer, hive) = - declaration_target(&state, &hive).map_err(|(s, d)| error_problem(s, &d))?; - let agent = hive_types::Ident::parse(&agent) - .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? - .into_string(); - - // Only a first declaration of the name on this hive is checked, against - // what `create_agent` refuses: an agent already declared here changes - // state whatever its name. An undeclared name that passes is accepted: - // that is how the swarm adopts an agent that predates swarm-level - // creation. Deliberate, pending the operator's decision on whether a name - // the swarm never placed should be refused here instead. - // - // The gate is held from the checks through the write. A destroy places - // nothing, so it is not checked and does not wait on creations. - let _gate = if places_agent(req.state) { - let gate = state.create_gate.lock().await; - let here = writer.view(&hive).await.map_err(|e| { - let detail = format!( - "cannot tell whether {agent:?} is already declared on hive {hive:?}, so it was \ - not placed: {e:#}" - ); - error_problem(wanted_error_status(&e), &detail) - })?; - refuse_placement( - &agent, - &hive, - here.as_ref(), - hive_types::reserved_names_raw().as_deref(), - || in_roster(state.auth.as_deref(), &agent), - || queued_placements_now(&state), - || declarations_elsewhere(&state, &agent, &hive), - ) - .await?; - Some(gate) - } else { - None - }; - - let declaration = writer.set(&hive, &agent, req.state).await.map_err(|e| { - tracing::warn!(hive = %hive, agent = %agent, error = %format!("{e:#}"), "declaring agent state failed"); - error_problem(wanted_error_status(&e), &format!("{e:#}")) - })?; - - // After the declaration and never before it: the write above is the - // destroy order, and until it lands the agent is not being torn down, so - // its credential, permissions and archetype have to keep working. Sequenced - // rather than spawned so every attempt is finished — and their outcomes - // logged — by the time the operator's call returns. - if revokes_queue_credential(req.state) { - revoke_or_complain(&hive, &agent).await; - agent_permissions::clear_on_destroy(&state, &hive, &agent).await; - agent_archetype::clear_on_destroy(&state, &hive, &agent).await; - } - Ok(Json(render(&declaration))) -} - -/// Whether a declaration in `state` places the agent on its hive. -/// -/// Exhaustive for the reason [`revokes_queue_credential`] is. A `Destroyed` -/// agent is gone from that hive, and the hive deploys nothing for it. -fn places_agent(state: swarm_queue_client::wanted::AgentState) -> bool { - use swarm_queue_client::wanted::AgentState; - match state { - AgentState::Up | AgentState::Offline | AgentState::Paused => true, - AgentState::Destroyed => false, - } -} - -/// Refuse a first placement of `agent` on `hive` that `create_agent` -/// refuses: 400 for a new name that breaks a naming rule, 409 for a name the -/// swarm has placed on another hive. An agent `here`, the hive's current -/// declaration, already declares in a placing state is placed on this hive, -/// and passes. -/// -/// The roster is read only for a name that breaks a rule. Unlike in -/// `create_agent`, no later step refuses a bad name, so a roster that cannot -/// be read refuses (503) rather than warns. -async fn refuse_placement( - agent: &str, - hive: &str, - here: Option<&swarm_queue_client::wanted::HiveWanted>, - reserved: Option<&str>, - read_roster: impl FnOnce() -> R, - snapshot_queued: impl FnOnce() -> Vec<(String, String)>, - read_declared: impl FnOnce() -> D, -) -> Result<(), problem_details::ProblemDetails> -where - R: std::future::Future>, - D: std::future::Future< - Output = Result< - Vec<(String, swarm_queue_client::wanted::HiveWanted)>, - problem_details::ProblemDetails, - >, - >, -{ - let declared_here = here - .and_then(|declaration| declaration.agents.get(agent)) - .is_some_and(|wanted| places_agent(wanted.state)); - if declared_here { - return Ok(()); - } - // Both calls log what they push here; this response has nowhere to - // carry warnings. - let mut warnings = Vec::new(); - let broken = broken_name_rules(agent, reserved, &mut warnings); - if !broken.is_empty() { - let existing = read_roster().await.map_err(|why| { - let detail = format!( - "{}; whether {agent:?} is an existing agent, which would keep its name, could \ - not be checked, so it was not placed: {why}", - broken.join("; ") - ); - error_problem(axum::http::StatusCode::SERVICE_UNAVAILABLE, &detail) - })?; - name_verdict(agent, &broken, Ok(existing), &mut warnings) - .map_err(|detail| error_problem(axum::http::StatusCode::BAD_REQUEST, &detail))?; - } - let elsewhere = placements_elsewhere(agent, hive, snapshot_queued, read_declared).await?; - if !elsewhere.is_empty() { - let detail = format!( - "agent {agent:?} is placed on hive {}, not {hive:?} — agent names are unique across \ - the swarm; declare its state there, or destroy it there first to move it", - elsewhere.join(", ") - ); - return Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)); - } - Ok(()) -} - -/// Whether declaring an agent into `state` ends its queue credential's life. -/// -/// Exhaustive on purpose, like every other match on -/// [`AgentState`][swarm_queue_client::wanted::AgentState] in this -/// tree: a state added later has to say whether it revokes rather than -/// inheriting "no" from a catch-all. `Offline` and `Paused` deliberately do -/// not — both are states an agent comes back from, and revoking on either -/// would mean an agent that could be stopped but never restarted. -fn revokes_queue_credential(state: swarm_queue_client::wanted::AgentState) -> bool { - use swarm_queue_client::wanted::AgentState; - match state { - AgentState::Destroyed => true, - AgentState::Up | AgentState::Offline | AgentState::Paused => false, - } -} - -/// Revoke the credential, and make a failure impossible to miss without -/// letting it stop the teardown. -/// -/// **The destroy is not blocked by this.** The declaration is already -/// published, the hive will converge on it, and refusing the operator's call -/// at this point would leave them with an agent they cannot destroy — the -/// larger of the two harms. What is left is that the failure must not be -/// quiet, or the outcome is the silent orphan this whole path exists to -/// remove: hence `error`, naming the agent, with the cause chain attached. -async fn revoke_or_complain(hive: &str, agent: &str) { - if let Err(e) = agent_identity::revoke_queue_credential(agent).await { - tracing::error!( - hive, - agent, - error = %format!("{e:#}"), - "revoking this agent's queue credential FAILED; it was destroyed anyway, so \ - the credential outlives it and may still authenticate. Declaring the agent \ - destroyed again re-runs this revocation." - ); - } -} - -/// What this swarm currently declares for a hive. -/// -/// Read back from the bucket rather than from a second copy kept here: the -/// published value is the record. -#[utoipa::path( - get, - path = "/api/hives/{hive}/wanted", - params(("hive" = String, Path, description = "hive whose declaration to read")), - responses( - (status = 200, description = "the declaration, empty when nothing is published yet", body = Vec), - (status = 400, description = "not an identifier, or not a hive in this swarm (problem+json)", body = String), - (status = 503, description = "no swarm queue is wired up, or it is not connected (problem+json)", body = String), - (status = 500, description = "the declaration could not be read (problem+json)", body = String), - ), - tag = "agents" -)] -async fn get_hive_wanted( - State(state): State, - axum::extract::Path(hive): axum::extract::Path, -) -> Result>, problem_details::ProblemDetails> { - let (writer, hive) = - declaration_target(&state, &hive).map_err(|(s, d)| error_problem(s, &d))?; - let declaration = writer.view(&hive).await.map_err(|e| { - tracing::warn!(hive = %hive, error = %format!("{e:#}"), "reading the declaration failed"); - error_problem(wanted_error_status(&e), &format!("{e:#}")) - })?; - Ok(Json(declaration.as_ref().map(render).unwrap_or_default())) -} - -/// What each hive last said about itself, read from the swarm queue at -/// request time. -/// -/// Every hive in the roster gets a row whether or not it has ever -/// reported — see the `status` module for why absence, not presence, is -/// the case this is built around. -#[utoipa::path( - get, - path = "/api/hives/status", - responses( - (status = 200, description = "a row per hive, freshness derived now", body = Vec), - (status = 503, description = "no swarm queue is configured here, or its store could not be read", body = String), - ), - tag = "hives" -)] -async fn get_hives_status( - State(state): State, -) -> Result>, StatusUnavailable> { - let Some(reader) = state.status.as_ref() else { - return Err(StatusUnavailable( - "no swarm queue is configured on this host".to_owned(), - )); - }; - match reader - .view(&state.hives, std::time::SystemTime::now()) - .await - { - Ok(rows) => Ok(Json(rows)), - Err(e) => { - let detail = format!("{e:#}"); - tracing::warn!(error = %detail, "reading the swarm status bucket failed"); - Err(StatusUnavailable(detail)) - } - } -} - -/// What each agent last said about itself, its open config PR if any, and -/// its declared wanted state if any, read at request time — the single -/// call swarm-ui's agent roster page fills its whole table from (mara, -/// 2026-09-02: "the view should be filled by a single backend call"). -/// -/// Every agent in the roster gets a row whether or not it has ever -/// reported — see the `agent_status` module for why, and for why its hive -/// comes from its own bucket key rather than a second lookup. `config_pr` -/// and `wanted` are both merged in here rather than inside -/// `agent_status::AgentStatusReader`: that module has no forge client or -/// wanted-state writer and is not the place to add one just for a single -/// field, whereas this handler already holds `AppState::config_prs` and -/// `AppState::wanted` alongside the roster-shaped status view — the one -/// spot that has all three without adding an avoidable dependency to a -/// reader that stays deliberately narrow. Both fields are absent on rows -/// where the underlying store isn't configured or has nothing on record, -/// same "absence is the answer" shape `get_config_prs` uses — not a -/// reason to fail the whole response over a field that already knows how -/// to be missing. -#[utoipa::path( - get, - path = "/api/agents/status", - responses( - (status = 200, description = "a row per agent, freshness + config PR derived now", body = Vec), - (status = 503, description = "no swarm queue is configured here, no identity bridge is configured, or either store could not be read", body = String), - ), - tag = "agents" -)] -async fn get_agents_status( - State(state): State, -) -> Result>, StatusUnavailable> { - let Some(reader) = state.agent_status.as_ref() else { - return Err(StatusUnavailable( - "no swarm queue is configured on this host".to_owned(), - )); - }; - let Some(bridge) = state.auth.as_ref() else { - return Err(StatusUnavailable( - "no identity bridge is configured on this host".to_owned(), - )); - }; - let roster = bridge.list_agent_identities().await.map_err(|e| { - let detail = format!("{e:#}"); - tracing::warn!(error = %detail, "reading the agent roster failed"); - StatusUnavailable(detail) - })?; - match reader.view(&roster, std::time::SystemTime::now()).await { - Ok(mut rows) => { - if let Some(cache) = state.config_prs.get() { - let mut config_prs = cache.snapshot(); - for row in &mut rows { - row.config_pr = config_prs.remove(&row.name); - } - } - // `wanted` merges in the same way and for the same reason as - // `config_pr` above — one read per distinct hive, not one per - // agent, since a declaration is already a hive's whole agent - // map. - if let Some(writer) = state.wanted.as_ref() { - // Owned `String` keys, not `&str` borrowed from `rows` — - // `declarations` outlives the loop below that needs `rows` - // mutably, so it cannot hold a live borrow into it. - let hives: std::collections::BTreeSet = - rows.iter().filter_map(|r| r.hive.clone()).collect(); - let mut declarations: std::collections::HashMap< - String, - swarm_queue_client::wanted::HiveWanted, - > = std::collections::HashMap::new(); - for hive in hives { - match writer.view(&hive).await { - Ok(Some(declaration)) => { - declarations.insert(hive, declaration); - } - Ok(None) => {} - Err(e) => { - // Missing wanted state for one hive shouldn't - // fail the whole roster — same "absence is - // survivable" rule `config_pr` follows above. - tracing::warn!( - hive, error = %format!("{e:#}"), - "reading the wanted declaration failed; its agents show no wanted state" - ); - } - } - } - for row in &mut rows { - row.wanted = row.hive.as_deref().and_then(|hive| { - declarations - .get(hive)? - .agents - .get(&row.name) - .map(|w| <&str>::from(w.state).to_owned()) - }); - } - } - if let Some(login) = state.agent_login.as_ref() { - for row in &mut rows { - row.needs_login = login.get(&row.name); - } - } - Ok(Json(rows)) - } - Err(e) => { - let detail = format!("{e:#}"); - tracing::warn!(error = %detail, "reading the agent-status bucket failed"); - Err(StatusUnavailable(detail)) - } - } -} - -/// Body of `POST /api/agents` — the agent name to create, and the hive the -/// creation is aimed at. The repo name inside `forge::CONFIG_ORG` is the -/// same string as `name`: one repo per agent, named after it, same -/// convention `hive-c0re::forge` already uses for its own single-hive -/// `CreateRepo` path. -/// -/// `hive` is an **address**, not an attribute of the agent: it is where a -/// deploy message goes over the queue. Nothing here writes it into the -/// agent's config — a config naming its own hive would be a second -/// statement of where the agent lives, free to drift from the queue that -/// actually delivers to it. -/// -/// It is required rather than optional because it is only knowable here, -/// at creation, from the operator making the choice. An optional field -/// would have been the friendlier migration and would have left every -/// creation in the state this endpoint exists to avoid — one nothing can -/// address. -/// -/// Validated here and carried no further: none of the nodes this endpoint -/// queues talks to a hive, so none of them needs the address. It reaches -/// its consumer when the node that *sends* a deploy message exists, and -/// that node takes it from this field. Settling the request shape now is -/// the point — it is the breaking half, and doing it once is cheaper for -/// every caller than doing it again later. -#[derive(Clone, Debug, Deserialize, ToSchema)] -struct CreateAgentRequest { - name: String, - hive: String, - /// CPU and memory caps to set for the agent, as `PUT - /// /api/agents/{name}/limits` takes them. Absent leaves any caps already - /// stored for the name alone. - #[serde(default)] - limits: Option, - /// Inference settings to set for the agent, as `PUT - /// /api/agents/{name}/inference` takes them. Absent as `limits`. - #[serde(default)] - inference: Option, - /// The agent's permissions, stored before the job is queued. Left out, the - /// agent holds the default set. - #[serde(default)] - #[schema(value_type = Option>)] - permissions: Option>, - /// The archetype to build the agent from, one `GET /api/archetypes` - /// lists. The agent then gets no `agent-configs/` repo: its hive - /// builds it from the archetype's module. Absent is a custom agent. - #[serde(default)] - #[schema(value_type = Option, example = json!({"source": "hyperhive", "name": "reviewer"}))] - archetype: Option, -} - -/// Queue the whole agent-creation job graph for `name` — two independent -/// roots, `CreateIdentity` and `CreateForgeUser`, since an authelia -/// subject and a forge account need nothing from each other. `CreateRepo` -/// runs once `CreateIdentity` succeeds; once THAT succeeds, -/// `AddRepoMember` and `InitAgentConfigRepo` both run off it — except -/// `AddRepoMember` also waits on `CreateForgeUser`, because adding a -/// collaborator that does not yet exist as a forge user is a Forgejo -/// validation error, not an idempotent no-op. `InitAgentConfigRepo` has no -/// such second dependency: seeding files uses this daemon's own forge -/// token, never the agent's, so it only ever needed the repo (mara, design -/// review: it's a jobq graph, not a linear chain). An agent created from an -/// `archetype` has no config repo, so its graph has none of the three repo -/// nodes. Returns as soon as the graph is inserted — **not** once any of it -/// has run; `run_swarm_node` does that asynchronously off the scheduler loop -/// already running -/// (`spawn_jobq_worker`). The response reports `CreateIdentity`'s id, one -/// of the graph's two roots — a caller watches the whole thing settle via -/// `/api/jobq/graph`, which serves every root, not just this one. -/// -/// `name` is validated with [`hive_types::Ident::parse`] before it becomes -/// `agent`/`repo` anywhere downstream — not the *only* gate (`CreateIdentity` -/// validates server-side too), but a few hops removed from where an -/// unvalidated name would do damage (`forge::seed_agent_config` -/// interpolates `agent` into a nix comment line). Fragile if a second -/// caller of these nodes ever appears; validating here closes it locally. -#[utoipa::path( - post, - path = "/api/agents", - request_body = CreateAgentRequest, - responses( - (status = 200, description = "job chain queued", body = CreateAgentResponse), - (status = 400, description = "`name` or `hive` is not a valid identifier, `name` is new and reserved or refused by the forge, `hive` is not in this swarm, `limits` or `inference` holds a value the matching `PUT` refuses, a permission is listed twice, or `archetype` is not one the stored archetype list holds (problem+json)", body = String), - (status = 409, description = "the swarm has already placed `name` on a different hive (problem+json)", body = String), - (status = 422, description = "an unknown field in the body, in `limits`, or in `inference`, an `effort` that is not one of the five levels, or a `permissions` entry that is not a permission name"), - (status = 503, description = "the swarm queue is not connected, so whether `name` is placed on another hive is unknown, or `limits`, `inference`, `permissions` or `archetype` is set and cannot be stored (problem+json)", body = String), - (status = 500, description = "the job chain could not be queued, or another hive's wanted state could not be read (problem+json)", body = String), - ), - tag = "agents" -)] -async fn create_agent( - State(state): State, - Json(req): Json, -) -> Result, problem_details::ProblemDetails> { - let agent = hive_types::Ident::parse(&req.name) - .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? - .into_string(); - - // Settings are checked before anything is created, so a bad value stops - // the request rather than leaving an agent without them. - let settings = CreateSettings::parse(req.limits, req.inference, req.permissions, req.archetype) - .map_err(|why| error_problem(axum::http::StatusCode::BAD_REQUEST, &why))?; - - // Two checks, and the second is the one that makes the field worth - // having: `Ident::parse` says the string is *shaped* like a hive name, - // and the roster says it *is* one. Without the roster check a typo is - // accepted, the agent gets created, and nothing notices until a deploy - // message is addressed to a hive that does not exist — by which point - // the operator who could have corrected it in one keystroke is gone. - // - // `state.hives` is the swarm directory loaded from `SWARM_CONTROLLER_HIVES` - // at startup and never mutated, so this is a scan of a handful of - // entries against a value an operator just chose. - // The name is shaped like an identifier; the checks below ask whether it - // is *available*. An operator with agents already created under a - // colliding name must still be able to re-run creation, so a name that - // breaks a naming rule is refused only when it is new — see - // `name_verdict`. The hive collision further down still only warns, on - // its grace period, once the warning has had time to be seen. - // - // Collected rather than logged-and-dropped — see `CreateAgentResponse`. - let mut warnings = Vec::new(); - let broken = broken_name_rules( - &agent, - hive_types::reserved_names_raw().as_deref(), - &mut warnings, - ); - if !broken.is_empty() { - let existing = in_roster(state.auth.as_deref(), &agent).await; - name_verdict(&agent, &broken, existing, &mut warnings) - .map_err(|detail| error_problem(axum::http::StatusCode::BAD_REQUEST, &detail))?; - } - - let hive = hive_types::Ident::parse(&req.hive) - .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? - .into_string(); - - // The roster is the same source the `hive` field is validated against - // immediately below — one scan, asked of the other field. A hive and an - // agent sharing a name collide in the swarm's own directory keys, which - // is what makes a pipeline and a label vanish with no failed assertion. - if state.hives.iter().any(|h| h.name == agent) { - let detail = format!( - "agent name {agent:?} is also a hive in this swarm — they share the swarm's directory \ - keys; this will become an error" - ); - tracing::warn!(agent = %agent, "create_agent: name collides with a hive"); - warnings.push(detail); - } - if !state.hives.iter().any(|h| h.name == hive) { - let known: Vec<&str> = state.hives.iter().map(|h| h.name.as_str()).collect(); - // Name the hives that *would* work: the caller is an operator who - // just picked one, and "not in this swarm" without the roster - // leaves them guessing at a typo they cannot see. - let known = if known.is_empty() { - "(none configured)".to_owned() - } else { - known.join(", ") - }; - let detail = format!("hive {hive:?} is not in this swarm — known hives: {known}"); - return Err(error_problem(axum::http::StatusCode::BAD_REQUEST, &detail)); - } - // Checked after the roster, so a mistyped hive stays a 400 while the queue - // is down. - let settings_queue = settings.queue(&state).await?; - - // Agent names are one swarm-wide namespace: identity, forge user, store - // path and matrix id all carry the bare name. The same name on the same - // hive is that agent being re-created, and goes through. - let _gate = state.create_gate.lock().await; - let elsewhere = placements_elsewhere( - &agent, - &hive, - || queued_placements_now(&state), - || declarations_elsewhere(&state, &agent, &hive), - ) - .await?; - if !elsewhere.is_empty() { - let detail = format!( - "agent name {agent:?} is already placed on hive {} — agent names are unique across \ - the swarm; destroy it there first to move it, or choose another name", - elsewhere.join(", ") - ); - return Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)); - } - // Stored ahead of the job graph, not once the agent is declared: the hive - // reads these by agent name when it first runs the agent. A failure from - // here on puts back what was stored, so a refused creation leaves no - // settings behind for the name. - let stored = match &settings_queue { - Some(client) => { - StoredSettings::store( - client, - &agent, - settings.limits.as_ref(), - settings.inference.as_ref(), - settings.permissions.as_ref(), - settings.archetype.as_ref(), - ) - .await? - } - None => StoredSettings::default(), - }; - let queued = { - let mut sched = state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - sched.insert_job(None, |b| { - declare_agent_job(b, &agent, &hive, settings.archetype.is_some()) - }) - }; - let ids = match queued { - Ok(ids) => ids, - Err(e) => { - if let Some(client) = &settings_queue { - stored.roll_back(client, &agent).await; - } - return Err(error_problem( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - &e.to_string(), - )); - } - }; - let [id] = ids[..] else { - unreachable!("exactly one handle was asked for"); - }; - Ok(Json(CreateAgentResponse { - node_id: id.get(), - warnings, - })) -} - -/// The settings a `POST /api/agents` body gives, as they are stored. -struct CreateSettings { - limits: Option, - inference: Option, - permissions: Option, - archetype: Option, -} - -impl CreateSettings { - /// Each setting given, or why its `PUT` would refuse the first bad one. - /// An archetype is checked here for its shape only; whether the swarm - /// lists it is [`agent_archetype::check`]. - fn parse( - limits: Option, - inference: Option, - permissions: Option>, - archetype: Option, - ) -> Result { - if let Some(a) = &archetype { - for part in [&a.source, &a.name] { - hive_types::Ident::parse(part) - .map_err(|why| format!("archetype {part:?}: {why}"))?; - } - } - Ok(Self { - limits: limits.map(agent_limits::Caps::into_stored).transpose()?, - inference: inference - .map(inference::InferenceSettings::into_stored) - .transpose()?, - permissions: permissions.map(agent_permissions::at_create).transpose()?, - archetype, - }) - } - - /// Whether any setting is given. - fn any(&self) -> bool { - self.limits.is_some() - || self.inference.is_some() - || self.permissions.is_some() - || self.archetype.is_some() - } - - /// The queue to store the settings through, `None` when none is given. - /// An archetype is refused here unless the stored list holds it. - async fn queue( - &self, - state: &AppState, - ) -> Result, problem_details::ProblemDetails> { - if !self.any() { - return Ok(None); - } - let client = schedules::queue(state).map_err(schedules::refused)?; - if let Some(archetype) = &self.archetype { - agent_archetype::check(&client, archetype).await?; - } - Ok(Some(client)) - } -} - -/// What `create_agent` replaced in the settings stores, to put back if the -/// creation is refused after the write. -#[derive(Default)] -struct StoredSettings { - limits: agent_limits::Replaced, - inference: inference::Replaced, - permissions: agent_permissions::Replaced, - archetype: agent_archetype::Replaced, -} - -impl StoredSettings { - /// Store whichever of `limits`, `permissions`, `inference` and `archetype` are - /// given, in that order. A failure on one puts back the ones stored before it. - async fn store( - client: &async_nats::Client, - agent: &str, - limits: Option<&swarm_queue_client::agent_limits::AgentCaps>, - inference: Option<&swarm_queue_client::agent_inference::AgentInference>, - permissions: Option<&swarm_queue_client::agent_permissions::AgentPermissions>, - archetype: Option<&swarm_queue_client::agent_archetype::ArchetypeRef>, - ) -> Result { - let mut stored = Self::default(); - if let Some(limits) = limits { - stored.limits = agent_limits::store(client, agent, limits).await?; - } - if let Some(permissions) = permissions { - match agent_permissions::store(client, agent, permissions).await { - Ok(replaced) => stored.permissions = replaced, - Err(e) => { - stored.roll_back(client, agent).await; - return Err(e); - } - } - } - if let Some(inference) = inference { - match inference::store(client, agent, inference).await { - Ok(replaced) => stored.inference = replaced, - Err(e) => { - stored.roll_back(client, agent).await; - return Err(e); - } - } - } - if let Some(archetype) = archetype { - match agent_archetype::store(client, agent, archetype).await { - Ok(replaced) => stored.archetype = replaced, - Err(e) => { - stored.roll_back(client, agent).await; - return Err(e); - } - } - } - Ok(stored) - } - - async fn roll_back(self, client: &async_nats::Client, agent: &str) { - agent_limits::restore(client, agent, self.limits).await; - inference::restore(client, agent, self.inference).await; - agent_permissions::restore(client, agent, self.permissions).await; - agent_archetype::restore(client, agent, self.archetype).await; - } -} - -/// [`placed_elsewhere`] over a snapshot of the queued placements and a read -/// of the published ones, **in that order**. -/// -/// A `SetAgentWanted` node writes its declaration before it turns terminal. -/// Snapshotting the queue first means a node already terminal there has -/// written, so the read that follows sees its declaration; a node still -/// queued is in the snapshot. Reading first leaves a gap: a node finishing -/// between the two is in neither. -async fn placements_elsewhere( - agent: &str, - hive: &str, - snapshot_queued: impl FnOnce() -> Vec<(String, String)>, - read_declared: impl FnOnce() -> Fut, -) -> Result, E> -where - Fut: std::future::Future< - Output = Result, E>, - >, -{ - let queued = snapshot_queued(); - let declared = read_declared().await?; - Ok(placed_elsewhere(agent, hive, &declared, &queued)) -} - -/// Every hive but `hive`'s published wanted state, for [`placed_elsewhere`]. -/// -/// No queue wired up means no hive has been sent a declaration, so there is -/// nothing to collide with. A queue that cannot be read refuses: an unread -/// declaration and an absent one look the same, and reading "absent" is how -/// a duplicate would get through. -async fn declarations_elsewhere( - state: &AppState, - agent: &str, - hive: &str, -) -> Result, problem_details::ProblemDetails> -{ - read_declarations(state, state.hives.iter().filter(|h| h.name != hive)) - .await - .map_err(|(other, e)| { - let detail = format!( - "cannot tell whether {agent:?} already exists on hive {other:?}, so it was not \ - placed: {e:#}" - ); - error_problem(wanted_error_status(&e), &detail) - }) -} - -/// The published wanted state of each of `hives` that has one. No queue wired -/// up reads as nothing declared anywhere. The error names the hive whose read -/// failed. -async fn read_declarations<'a>( - state: &AppState, - hives: impl Iterator, -) -> Result, (String, anyhow::Error)> { - let Some(writer) = state.wanted.as_deref() else { - return Ok(Vec::new()); - }; - let mut declared = Vec::new(); - for hive in hives { - match writer.view(&hive.name).await { - Ok(Some(declaration)) => declared.push((hive.name.clone(), declaration)), - Ok(None) => {} - Err(e) => return Err((hive.name.clone(), e)), - } - } - Ok(declared) -} - -/// Every hive whose declaration places `agent` there. -fn claiming_hives( - agent: &str, - declared: &[(String, swarm_queue_client::wanted::HiveWanted)], -) -> Vec { - declared - .iter() - .filter(|(_, declaration)| { - declaration - .agents - .get(agent) - .is_some_and(|wanted| places_agent(wanted.state)) - }) - .map(|(hive, _)| hive.clone()) - .collect() -} - -/// Queue a build check of a config PR's new head. Every head is built, on -/// whichever hive the agent runs: the check builds on the builder's own -/// agent base. -fn queue_config_build(state: &AppState, changed: config_pr::ChangedConfigPr) { - let (agent, rev, pr) = (changed.agent.clone(), changed.rev.clone(), changed.pr); - let queued = queue_build( - &mut state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner), - changed, - ); - match queued { - Ok(superseded) => { - tracing::info!(%agent, %rev, pr, superseded, "config-pr head: build queued"); - } - Err(e) => { - tracing::warn!(%agent, %rev, pr, error = %e, "config-pr head: queueing the build failed"); - } - } -} - -/// Cancel every pending build of the same PR, then insert the new one behind -/// [`SwarmResourceKind::Builder`]. A build already running is left to finish: -/// its status lands on its own commit, not on the new head. Returns how many -/// pending builds were cancelled. -fn queue_build( - sched: &mut hive_jobq::scheduler::Scheduler, - changed: config_pr::ChangedConfigPr, -) -> Result { - let config_pr::ChangedConfigPr { agent, rev, pr } = changed; - let stale: Vec = sched - .graph() - .nodes() - .filter(|n| n.state == hive_jobq::State::Pending) - .filter(|n| { - matches!(&n.payload, SwarmNodeKind::BuildAgent { agent: a, pr: p, .. } if *a == agent && *p == pr) - }) - .map(|n| n.id) - .collect(); - let superseded = stale - .into_iter() - .filter(|&id| sched.cancel_node(id)) - .count(); - sched - .insert_job(None, |b| { - vec![ - b.node(SwarmNodeKind::BuildAgent { agent, rev, pr }) - .needs(SwarmResourceKind::Builder) - .guid(), - ] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - Ok(superseded) -} - -/// Queue the deploy of a merged config PR on the one hive that runs its agent. -/// -/// The hive is found by scanning every hive's wanted state, since nothing -/// records an agent's hive (see `forge::initial_agent_nix`). Zero claimants or -/// several deploy nothing: there is no hive to send it to, or no telling which. -async fn queue_merged_config_deploy(state: &AppState, merged: config_pr::MergedConfigPr) { - let config_pr::MergedConfigPr { agent, rev } = merged; - let declared = match read_declarations(state, state.hives.iter()).await { - Ok(declared) => declared, - Err((hive, e)) => { - tracing::warn!( - %agent, %rev, %hive, error = %format!("{e:#}"), - "config-pr merge: cannot read where the agent is placed; nothing deployed" - ); - return; - } - }; - let claimants = claiming_hives(&agent, &declared); - let [hive] = claimants.as_slice() else { - tracing::warn!( - %agent, %rev, ?claimants, - "config-pr merge: not exactly one hive places this agent; nothing deployed" - ); - return; - }; - let hive = hive.clone(); - let inserted = state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert_job(None, |b| { - vec![ - b.node(SwarmNodeKind::TriggerDeploy { - hive: hive.clone(), - agent: agent.clone(), - rev: Some(rev.clone()), - }) - .guid(), - ] - }); - match inserted { - Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"), - Err(e) => { - tracing::warn!(%agent, %rev, %hive, error = %e, "config-pr merge: queueing the deploy failed"); - } - } -} - -/// [`queued_placements`] of the swarm's own queue, as it is now. -fn queued_placements_now(state: &AppState) -> Vec<(String, String)> { - queued_placements( - state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph(), - ) -} - -/// `(hive, agent)` of every `SetAgentWanted` node not yet settled: a creation -/// accepted but not yet visible in the wanted state it is about to write. -fn queued_placements( - graph: &hive_jobq::Graph, -) -> Vec<(String, String)> { - graph - .nodes() - .filter(|node| !node.state.is_terminal()) - .filter_map(|node| match &node.payload { - SwarmNodeKind::SetAgentWanted { hive, agent } => Some((hive.clone(), agent.clone())), - _ => None, - }) - .collect() -} - -/// The hives other than `hive` the swarm has placed `agent` on, sorted. -/// -/// A placement is a declaration in a state that [places][places_agent] the -/// agent, or a queued one. A `Destroyed` agent is gone from that hive, so -/// placing it on another moves it. -fn placed_elsewhere( - agent: &str, - hive: &str, - declared: &[(String, swarm_queue_client::wanted::HiveWanted)], - queued: &[(String, String)], -) -> Vec { - let declared = declared.iter().filter(|(_, declaration)| { - declaration - .agents - .get(agent) - .is_some_and(|wanted| places_agent(wanted.state)) - }); - let queued = queued.iter().filter(|(_, queued)| queued == agent); - let hives: std::collections::BTreeSet<&str> = declared - .map(|(h, _)| h.as_str()) - .chain(queued.map(|(h, _)| h.as_str())) - .filter(|h| *h != hive) - .collect(); - hives.into_iter().map(str::to_owned).collect() -} - -/// Every naming rule `agent` breaks, each as a sentence naming the rule. -/// -/// `reserved` is [`hive_types::reserved_names_raw`]: the list nix owns in -/// `nix/reserved-names.nix`, read by this daemon, by hive-c0re and by the -/// swarm collector's own owner assertion. `None` means this process was never -/// told, which is not the same as "no name is reserved" — staying quiet there -/// would be a check that reports clean because it could not run, so it goes -/// onto `warnings` instead. -fn broken_name_rules( - agent: &str, - reserved: Option<&str>, - warnings: &mut Vec, -) -> Vec { - let mut broken = Vec::new(); - match reserved.map(hive_types::parse_reserved_names) { - None => { - tracing::error!( - var = hive_types::RESERVED_NAMES_ENV, - "reserved-name check could not run — variable not set" - ); - warnings.push(format!( - "the reserved-name check did not run: {} is unset, so {agent:?} was accepted \ - without being checked against the reserved names", - hive_types::RESERVED_NAMES_ENV - )); - } - Some(reserved) if hive_types::is_reserved_name(agent, &reserved) => { - broken.push(format!( - "agent name {agent:?} is reserved (nix/reserved-names.nix): hyperhive's message \ - layer already uses it, or the forge refuses it as a username" - )); - } - Some(_) => {} - } - if let Some(rule) = hive_types::forge_username_violation(agent) { - broken.push(format!( - "agent name {agent:?} {rule}: the forge refuses it as a username" - )); - } - if let Some(rule) = hive_types::reserved_agent_prefix_violation(agent) { - broken.push(format!("agent name {agent:?} {rule}")); - } - broken -} - -/// Whether `agent` is already in the swarm roster, or why that could not be -/// told. `create_agent` asks only of a name that breaks a rule, so an -/// ordinary creation still never waits on the bridge; `make_forge_admin` -/// asks of every name. -async fn in_roster(auth: Option<&auth::AuthBridge>, agent: &str) -> Result { - let Some(auth) = auth else { - return Err("no identity bridge is configured".to_owned()); - }; - auth.list_agent_identities() - .await - .map(|roster| roster.iter().any(|name| name == agent)) - .map_err(|e| format!("{e:#}")) -} - -/// Refuse a NEW agent whose name breaks a rule; warn for one that exists. -/// -/// Re-creating an agent under its own name is a migration path (mara, on -/// the duplicate-name refusal), so an agent already in the roster keeps its -/// name with a warning. When the roster cannot be read, a new name and an -/// existing one look the same, and this warns rather than refuse: refusing -/// would block that re-creation, and the forge still refuses the account one -/// node in, as it did before. -/// -/// # Errors -/// Every rule `agent` breaks, as one caller-ready sentence, when `agent` is -/// not in the roster. -fn name_verdict( - agent: &str, - broken: &[String], - existing: Result, - warnings: &mut Vec, -) -> Result<(), String> { - match existing { - Ok(false) => Err(format!("{}; choose another name", broken.join("; "))), - Ok(true) => { - tracing::warn!(agent, "existing agent breaks a naming rule"); - warnings.extend( - broken - .iter() - .map(|rule| format!("{rule} — accepted only because {agent:?} already exists")), - ); - Ok(()) - } - Err(why) => { - tracing::warn!(agent, why, "naming rule broken, roster unreadable"); - warnings.extend(broken.iter().map(|rule| { - format!( - "{rule} — not refused, because whether {agent:?} already exists could not \ - be checked: {why}" - ) - })); - Ok(()) - } - } -} - -/// The sub-DAG one agent creation is: the nodes, and the edges between them. -/// -/// A function rather than a closure inside [`create_agent`] so the endpoint's -/// validation and the graph's shape can each be read without scrolling past -/// the other — and so the one handle the response reports is returned from -/// the place that decides which node it is. -fn declare_agent_job( - b: &hive_jobq::builder::JobBuilder, - agent: &str, - hive: &str, - archetype: bool, -) -> Vec { - let create_identity = b.node(SwarmNodeKind::CreateIdentity { - agent: agent.to_owned(), - }); - // A second, independent root: a forge user needs neither an authelia - // subject nor an existing repo, so it does not chain off - // `create_identity` (see the doc comment above). - let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { - agent: agent.to_owned(), - }); - // An archetype agent has no config repo: its hive builds it from the - // archetype's module, so there is no repo to create, share or seed. - let init_config = (!archetype).then(|| { - let create_repo = b - .node(SwarmNodeKind::CreateRepo { - agent: agent.to_owned(), - }) - .after_ok(create_identity); - // `AddRepoMember` needs both parents: the repo to add a collaborator - // to, and the forge user to add as one — adding a nonexistent user is - // a Forgejo validation error, not an idempotent no-op. - // `InitAgentConfigRepo` needs only the repo — see the doc comment - // above for why. - let _add_repo_member = b - .node(SwarmNodeKind::AddRepoMember { - agent: agent.to_owned(), - }) - .after_ok(create_repo) - .after_ok(create_forge_user); - b.node(SwarmNodeKind::InitAgentConfigRepo { - agent: agent.to_owned(), - }) - .after_ok(create_repo) - .guid() - }); - // Read on `internal/docs`, a repo that exists whatever this agent is, so - // it needs only the user. Not a parent of the deploy: the agent reads docs - // once running, and a failed grant is retried by `forge::docs_grant::spawn`. - let _grant_docs = b - .node(SwarmNodeKind::GrantDocsRead { - agent: agent.to_owned(), - }) - .after_ok(create_forge_user); - // The agent's own identity at the swarm secret store, minted and - // published at swarm level so no hive ever needs the capability to mint - // one. After `create_identity` because the certificate names a principal - // the swarm has agreed exists — not because anything in the store reads - // authelia. - let mint_identity = b - .node(SwarmNodeKind::MintAgentIdentity { - agent: agent.to_owned(), - }) - .after_ok(create_identity); - // The agent's own forge token, in the store before the hive deploys - // the container that pulls it. After the forge user, because the token - // is minted for that user. - let mint_forge_token = b - .node(SwarmNodeKind::MintAgentForgeToken { - agent: agent.to_owned(), - }) - .after_ok(create_forge_user); - // The agent's account on the swarm's homeserver. A root of its own: - // creating it needs neither an authelia subject nor a forge user nor a - // repo, and chaining it behind one of those would make an unrelated - // failure look like a matrix failure. - let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount { - agent: agent.to_owned(), - }); - // Declared before the deploy trigger so the pause is visible in the - // wanted-state store before the hive brings the container up — see the - // node's own doc comment for why "before", not just "eventually". Needs - // no parent of its own: the declaration is written from the hive/agent - // names known at request time, same as `create_forge_user`. - let set_wanted = b.node(SwarmNodeKind::SetAgentWanted { - hive: hive.to_owned(), - agent: agent.to_owned(), - }); - // Last, and specifically after the config repo is seeded: the hive - // deploys by reading that repo, so a deploy asked for any earlier would - // find nothing to build. This is the edge that makes creating an agent at - // swarm level actually put it on a hive, rather than leaving a - // provisioned name nobody runs. An archetype agent has no repo, so its - // deploy waits on the mints and the declaration alone. - // - // `after_any` on the mint, not `after_ok`: a hive cannot pass down a - // certificate the swarm has not published, so the deploy must not - // overtake the mint — but a host whose store or agent PKI is not set up - // must still create agents. `after_ok` there would turn a store outage - // into an agent nobody runs. - // - // The forge-token mint gets `after_any` for the same reason: a host with - // no forge or no store must still create agents; only that node fails, - // by name, and the agent's container fetches the token on its next - // timer tick once one is minted. - // - // `after_ok` on the declaration, though: an agent deployed without its - // pause landing first is the race this node exists to close, so a - // declaration that did not land must not be deployed past. A host with no - // queue at all fails this node — but `TriggerDeploy` has nothing to - // publish to on such a host either, so the strict edge cancels a node - // that could not have succeeded anyway. - let trigger_deploy = b - .node(SwarmNodeKind::TriggerDeploy { - hive: hive.to_owned(), - agent: agent.to_owned(), - rev: None, - }) - .after_any(mint_identity) - .after_any(mint_forge_token) - // `after_any` for the same reason: the container reads this token - // rather than producing it, so the deploy must not overtake the mint — - // but a host with no homeserver configured must still create agents, - // and only this node fails, by name. - .after_any(mint_matrix) - .after_ok(set_wanted); - if let Some(init_config) = init_config { - let _ = trigger_deploy.after_ok(init_config); - } - vec![create_identity.guid()] -} - -/// Query params for `GET /api/jobq/graph` — `?states=` narrows to root -/// groups in the named states, same shape `hive_jobq_wire::parse_states` -/// parses. -#[derive(Deserialize, utoipa::IntoParams)] -struct JobqGraphQuery { - states: Option, -} - -/// Every node of every root group in the swarm-level job graph. Nothing -/// filters "done and old" here — with zero nodes ever submitted there is -/// nothing to bound yet, so `graph.roots()` (everything) is the whole -/// roster passed to [`hive_jobq_wire::GraphWire::wire_snapshot`]. -#[utoipa::path( - get, - path = "/api/jobq/graph", - params(JobqGraphQuery), - responses((status = 200, description = "every node of every root group, as generic \ - `hive_jobq` graph nodes. `?states=` narrows to root groups in the named states.", - body = Vec)), - tag = "jobq" -)] -async fn get_jobq_graph( - State(state): State, - axum::extract::Query(q): axum::extract::Query, -) -> Json> { - let states = hive_jobq_wire::parse_states(q.states.as_deref()); - let sched = state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = sched.graph(); - let roots: Vec = graph.roots().map(|n| n.id).collect(); - let nodes = graph.wire_snapshot(roots); - Json(hive_jobq_wire::filter_nodes_by_state( - nodes, - states.as_deref(), - )) -} - -/// Counts by lifecycle state over the same groups `/api/jobq/graph` serves. -#[utoipa::path( - get, - path = "/api/jobq/rollup", - responses((status = 200, description = "counts by lifecycle state", body = Vec)), - tag = "jobq" -)] -async fn get_jobq_rollup(State(state): State) -> Json> { - let sched = state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = sched.graph(); - let roots: Vec = graph.roots().map(|n| n.id).collect(); - Json(hive_jobq_wire::state_rollup(graph, roots)) -} - -/// `agent`'s open config PR, from [`config_pr::ConfigPrCache`] — kept -/// current by a webhook nudge (near-real-time) backstopped by a periodic -/// scan (worst case: up to one `config_pr::POLL_INTERVAL` stale if a -/// delivery was ever missed). Not a live forge read either way, so this -/// answers even when the forge itself is momentarily unreachable. -/// -/// `200` with a `null` body means "no open PR" (or "no scan has completed -/// yet") — not distinguished, for the same reason -/// [`config_pr::ConfigPrCache::get`] doesn't: both read as "nothing to show" -/// to the swarm-ui config-PR panel this feeds, and a cache is a best-effort -/// read, not a source of truth callers should expect to disambiguate -/// against. -#[utoipa::path( - get, - path = "/api/agents/{name}/config-pr", - params(("name" = String, Path, description = "agent name")), - responses( - (status = 200, description = "the agent's open config PR, or null if none / not scanned yet", body = Option), - (status = 503, description = "no forge is configured on this host", body = String), - ), - tag = "agents" -)] -async fn get_agent_config_pr( - State(state): State, - Path(name): Path, -) -> Result>, StatusUnavailable> { - let Some(cache) = state.config_prs.get() else { - return Err(StatusUnavailable( - "no forge is configured on this host".to_owned(), - )); - }; - Ok(Json(cache.get(&name))) -} - -/// Insert one job per agent: `CreateForgeUser`, then `MintAgentForgeToken` -/// `after_ok` it — the same edge [`declare_agent_job`] uses. Returns the mint -/// nodes' ids. -/// -/// The one place that node is queued outside agent creation: both the manual -/// route below and `forge::agent_token::spawn`'s periodic pass come through -/// here, so a backfilled mint is the same node a new agent gets. The user -/// node is what lets the pass reach an agent that has a store identity but -/// no forge account; for one that has an account it is a no-op, because -/// `forge::Client::ensure_agent_user` folds "already exists" into success. -fn queue_forge_token_mints( - sched: &Mutex>, - agents: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(agents.len()); - for agent in agents { - let queued = sched - .insert_job(None, |b| { - let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { - agent: agent.clone(), - }); - vec![ - b.node(SwarmNodeKind::MintAgentForgeToken { agent }) - .after_ok(create_forge_user) - .guid(), - ] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Insert one job per agent: `CreateForgeUser`, then `GrantDocsRead` `after_ok` -/// it — the same edge [`declare_agent_job`] uses. Returns the grant nodes' ids. -/// `forge::docs_grant::spawn`'s periodic pass comes through here, so a -/// backfilled grant is the same node a new agent gets; the user node is a -/// no-op for an agent that has an account. -fn queue_docs_grants( - sched: &Mutex>, - agents: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(agents.len()); - for agent in agents { - let queued = sched - .insert_job(None, |b| { - let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { - agent: agent.clone(), - }); - vec![ - b.node(SwarmNodeKind::GrantDocsRead { agent }) - .after_ok(create_forge_user) - .guid(), - ] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Insert one `MintAgentMatrixAccount` job per agent and return the nodes' -/// ids. `matrix_account::agent_token::spawn`'s periodic pass comes through -/// here, so a backfilled mint is the same node a new agent gets. -fn queue_matrix_account_mints( - sched: &Mutex>, - agents: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(agents.len()); - for agent in agents { - let queued = sched - .insert_job(None, |b| { - vec![ - b.node(SwarmNodeKind::MintAgentMatrixAccount { agent }) - .guid(), - ] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Insert one `MintHiveSenderToken` job per hive and return the nodes' ids. -/// `matrix_account::hive_sender::spawn`'s periodic pass is the one caller. -fn queue_hive_sender_mints( - sched: &Mutex>, - hives: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(hives.len()); - for hive in hives { - let queued = sched - .insert_job(None, |b| { - vec![b.node(SwarmNodeKind::MintHiveSenderToken { hive }).guid()] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Insert one `MintHiveForgeToken` job per hive and return the nodes' ids. -/// `forge::hive_token::spawn`'s periodic pass is the one caller. -fn queue_hive_forge_token_mints( - sched: &Mutex>, - hives: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(hives.len()); - for hive in hives { - let queued = sched - .insert_job(None, |b| { - vec![b.node(SwarmNodeKind::MintHiveForgeToken { hive }).guid()] - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Insert one `MintClassifierForgeToken` job and return its node's id. -/// `forge::classifier_account::spawn`'s periodic pass is the one caller. -fn queue_classifier_forge_token_mint( - sched: &Mutex>, -) -> Result> { - sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert_job(None, |b| { - vec![b.node(SwarmNodeKind::MintClassifierForgeToken).guid()] - }) - .map_err(|e| anyhow::anyhow!("{e}")) -} - -/// Insert one `MintBuilderForgeToken` job and return its node's id. -/// `forge::builder_account::spawn`'s periodic pass is the one caller. -fn queue_builder_forge_token_mint( - sched: &Mutex>, -) -> Result> { - sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert_job(None, |b| { - vec![b.node(SwarmNodeKind::MintBuilderForgeToken).guid()] - }) - .map_err(|e| anyhow::anyhow!("{e}")) -} - -/// Insert one `MintCiMetricsForgeToken` job and return its node's id. -/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller. -fn queue_ci_metrics_forge_token_mint( - sched: &Mutex>, -) -> Result> { - sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert_job(None, |b| { - vec![b.node(SwarmNodeKind::MintCiMetricsForgeToken).guid()] - }) - .map_err(|e| anyhow::anyhow!("{e}")) -} - -/// The homeserver agents' own matrix accounts are minted on. Unset is this -/// controller creating agents with no matrix account, which the mint node -/// reports by name rather than `main` refusing to start. -fn configured_matrix_homeserver() -> Option> { - matrix_account::configured_default_homeserver() - .filter(|v| !v.is_empty()) - .map(Arc::from) -} - -/// Start the recovery passphrase pass, which needs only the store; the -/// matrix-account backfill and the hive sender-token pass when a homeserver is -/// configured; and the operator admin pass when the bridge is too. Lifted out -/// of `main` for `clippy::too_many_lines`. -fn spawn_matrix_account_backfill( - jobq: &Arc>>, - homeserver: Option>, - hives: &[HiveEntry], - auth: Option>, -) { - matrix_account::recovery::spawn(); - let Some(base) = homeserver else { - return; - }; - if let Some(bridge) = auth { - matrix_account::operator_admin::spawn(bridge, Arc::clone(&base)); - } - let sched = Arc::clone(jobq); - matrix_account::agent_token::spawn(Arc::clone(&base), move |agents| { - if let Err(e) = queue_matrix_account_mints(&sched, agents) { - tracing::warn!(error = %format!("{e:#}"), "agent matrix accounts: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - let hive_names = hives.iter().map(|h| h.name.clone()).collect(); - matrix_account::hive_sender::spawn(base, hive_names, move |hives| { - if let Err(e) = queue_hive_sender_mints(&sched, hives) { - tracing::warn!(error = %format!("{e:#}"), "hive sender tokens: queueing failed"); - } - }); -} - -/// Start the forge's periodic passes — the swarm-wide objects, the runner -/// registration token, hives' and the builder's read tokens, agents' -/// `internal/docs` grants, and -/// agents' tokens (which also -/// sweeps legacy tokens; see `forge::legacy_tokens`) — -/// once there is a forge client, and the operators' profiles when the bridge -/// and the queue are wired up too. Lifted out of `main` for -/// `clippy::too_many_lines`. -fn spawn_forge_workers( - jobq: &Arc>>, - client: Arc, - hives: &[HiveEntry], - auth: Option>, - queue: Option, -) { - forge::objects::spawn(Arc::clone(&client)); - if let (Some(bridge), Some(queue)) = (auth, queue) { - forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue); - } - forge::runner_token::spawn(Arc::clone(&client)); - let sched = Arc::clone(jobq); - let hive_names = hives.iter().map(|h| h.name.clone()).collect(); - forge::hive_token::spawn(Arc::clone(&client), hive_names, move |hives| { - if let Err(e) = queue_hive_forge_token_mints(&sched, hives) { - tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - forge::classifier_account::spawn(Arc::clone(&client), move || { - if let Err(e) = queue_classifier_forge_token_mint(&sched) { - tracing::warn!(error = %format!("{e:#}"), "review classifier forge token: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - forge::builder_account::spawn(Arc::clone(&client), move || { - if let Err(e) = queue_builder_forge_token_mint(&sched) { - tracing::warn!(error = %format!("{e:#}"), "builder forge token: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - forge::ci_metrics_account::spawn(Arc::clone(&client), move || { - if let Err(e) = queue_ci_metrics_forge_token_mint(&sched) { - tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - forge::docs_grant::spawn(Arc::clone(&client), move |agents| { - if let Err(e) = queue_docs_grants(&sched, agents) { - tracing::warn!(error = %format!("{e:#}"), "docs grants: queueing failed"); - } - }); - let sched = Arc::clone(jobq); - forge::agent_token::spawn(client, move |agents| { - if let Err(e) = queue_forge_token_mints(&sched, agents) { - tracing::warn!(error = %format!("{e:#}"), "agent forge tokens: queueing failed"); - } - }); -} - -/// Insert one job per renewal and return the ids of its nodes: -/// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a -/// queue secret, the second `after_any` the first when an agent needs both. -/// `agent_renewal::spawn`'s periodic pass comes through here. -/// -/// Chained rather than parallel because `mint_and_verify` reads the queue -/// secret back and compares it with the one it read first; a re-mint landing -/// in between fails that node. `after_any`, so a certificate that could not be -/// re-issued does not hold up the secret. -fn queue_agent_renewals( - sched: &Mutex>, - renewals: Vec, -) -> Result> { - let mut sched = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut ids = Vec::with_capacity(renewals.len()); - for renewal in renewals { - let queued = sched - .insert_job(None, |b| { - let mut asked = Vec::new(); - let cert = renewal.cert.then(|| { - b.node(SwarmNodeKind::MintAgentIdentity { - agent: renewal.agent.clone(), - }) - .guid() - }); - asked.extend(cert); - if renewal.secret { - let secret = b.node(SwarmNodeKind::RenewAgentQueueCredential { - agent: renewal.agent.clone(), - }); - let secret = match cert { - Some(cert) => secret.after_any(cert), - None => secret, - }; - asked.push(secret.guid()); - } - asked - }) - .map_err(|e| anyhow::anyhow!("{e}"))?; - ids.extend(queued); - } - Ok(ids) -} - -/// Start the agent credential renewal pass when a swarm queue is wired up: -/// the pass reads the wanted-state declarations to tell live agents from -/// destroyed ones, and without a queue it has none. Lifted out of `main` for -/// `clippy::too_many_lines`. -fn spawn_agent_renewal( - jobq: &Arc>>, - wanted: Option>, - hives: &[HiveEntry], -) { - let Some(wanted) = wanted else { - return; - }; - let hives = hives.iter().map(|h| h.name.clone()).collect(); - let sched = Arc::clone(jobq); - agent_renewal::spawn(wanted, hives, move |renewals| { - if let Err(e) = queue_agent_renewals(&sched, renewals) { - tracing::warn!(error = %format!("{e:#}"), "agent credential renewal: queueing failed"); - } - }); -} - -/// Start creating every live agent's subagent and terminal streams when a swarm -/// queue is wired up. Lifted out of `main` for `clippy::too_many_lines`. -fn spawn_subagent_streams(status: Option<&Arc>, hives: &[HiveEntry]) { - let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { - return; - }; - let hives = hives.iter().map(|h| h.name.clone()).collect(); - subagent_term::spawn(status.queue_client(), wanted, hives); -} - -/// The per-agent background passes: subagent and terminal streams, and -/// schedule keys, and the permissions bucket and its import, and the -/// archetypes agents are created from. -/// Lifted out of `main` for `clippy::too_many_lines`. -fn spawn_per_agent_upkeep(status: Option<&Arc>, hives: &[HiveEntry]) { - spawn_subagent_streams(status, hives); - spawn_schedule_reconcile(status, hives); - spawn_agent_permissions(status, hives); - if let Some(status) = status { - archetypes::spawn(status.queue_client()); - } -} - -/// Create the agent-permissions bucket and answer hives' imports when a swarm -/// queue is wired up. Same shape as [`spawn_subagent_streams`]. -fn spawn_agent_permissions(status: Option<&Arc>, hives: &[HiveEntry]) { - let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { - return; - }; - let hives = hives.iter().map(|h| h.name.clone()).collect(); - agent_permissions::spawn(status.queue_client(), wanted, hives); -} - -/// Start rewriting every agent's schedule key from the definitions when a -/// swarm queue is wired up. Same shape as [`spawn_subagent_streams`]. -fn spawn_schedule_reconcile(status: Option<&Arc>, hives: &[HiveEntry]) { - let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { - return; - }; - let hives = hives.iter().map(|h| h.name.clone()).collect(); - schedules::spawn_reconcile(status.queue_client(), wanted, hives); -} - -/// Check an agent's forge token now, and mint one if it is missing or stale. -/// -/// The periodic pass (`forge::agent_token::spawn`) does the same every five -/// minutes for every agent with a store identity; this is for an operator who -/// does not want to wait, or for an agent the pass skipped. Queues the same -/// node agent creation does. Idempotent: a current token is left alone. -#[utoipa::path( - post, - path = "/api/agents/{name}/forge-token", - params(("name" = String, Path, description = "agent name")), - responses( - (status = 200, description = "mint queued", body = MintAgentForgeTokenResponse), - (status = 400, description = "`name` is not a valid identifier (problem+json)", body = String), - (status = 500, description = "the job could not be queued (problem+json)", body = String), - ), - tag = "agents" -)] -async fn mint_agent_forge_token( - State(state): State, - Path(name): Path, -) -> Result, problem_details::ProblemDetails> { - let agent = hive_types::Ident::parse(&name) - .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? - .into_string(); - let ids = queue_forge_token_mints(&state.jobq, vec![agent]).map_err(|e| { - error_problem( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - &e.to_string(), - ) - })?; - let [id] = ids[..] else { - unreachable!("exactly one handle was asked for"); - }; - Ok(Json(MintAgentForgeTokenResponse { node_id: id.get() })) -} - -/// Make an existing human forge account a site admin. Idempotent. -/// -/// Never creates the account: a human's is made by their first SSO login to -/// the forge, and until then this answers 404. Refuses an agent's name: a -/// site admin can create repos whatever its `max_repo_creation`, and an -/// agent's account is locked to 0 so its token cannot create a repo and -/// self-merge in it. -#[utoipa::path( - post, - path = "/api/forge/users/{name}/admin", - params(("name" = String, Path, description = "forge username")), - responses( - (status = 200, description = "the account is a site admin", body = MakeForgeAdminResponse), - (status = 400, description = "`name` is not a valid identifier (problem+json)", body = String), - (status = 404, description = "no such account: the user has not logged in via SSO yet (problem+json)", body = String), - (status = 409, description = "`name` is an agent (problem+json)", body = String), - (status = 500, description = "the forge refused the read or the edit (problem+json)", body = String), - (status = 503, description = "no forge configured, or the agent roster could not be read (problem+json)", body = String), - ), - tag = "forge" -)] -async fn make_forge_admin( - State(state): State, - Path(name): Path, -) -> Result, problem_details::ProblemDetails> { - let name = hive_types::Ident::parse(&name) - .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? - .into_string(); - let Some(forge) = state.forge.get().cloned() else { - return Err(error_problem( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "no forge is configured on this host", - )); - }; - let is_agent = in_roster(state.auth.as_deref(), &name).await; - admin_unless_agent(&name, is_agent, || async { - forge.make_site_admin(&name).await - }) - .await -} - -/// Run `promote` only if `name` is known not to be an agent, and answer with -/// what it found. -/// -/// An unreadable roster refuses too: a site admin is the one thing an agent's -/// account must never become, so "could not tell" is not "no". -async fn admin_unless_agent( - name: &str, - is_agent: Result, - promote: F, -) -> Result, problem_details::ProblemDetails> -where - F: FnOnce() -> Fut, - Fut: std::future::Future>, -{ - match is_agent { - Ok(false) => {} - Ok(true) => { - return Err(error_problem( - axum::http::StatusCode::CONFLICT, - &format!("{name:?} is an agent; an agent's forge account is never a site admin"), - )); - } - Err(e) => { - return Err(error_problem( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - &format!("cannot tell whether {name:?} is an agent, so refusing: {e}"), - )); - } - } - let plan = promote().await.map_err(|e| { - error_problem( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - &format!("{e:#}"), - ) - })?; - let change = match plan { - forge::site_admin::Plan::NoSuchUser => { - return Err(error_problem( - axum::http::StatusCode::NOT_FOUND, - &format!( - "the forge has no user {name:?}: the user has not logged in via SSO yet. \ - The forge creates the account on their first authelia login; run this \ - again after it" - ), - )); - } - forge::site_admin::Plan::AlreadyAdmin => ForgeAdminChange::AlreadyAdmin, - forge::site_admin::Plan::Promote => ForgeAdminChange::Promoted, - }; - Ok(Json(MakeForgeAdminResponse { change })) -} - -/// How long a served icon may be reused without asking again. -/// -/// An agent roster asks once per row, and an icon changes about as often -/// as an agent is reconfigured — so a few minutes of browser caching is -/// what keeps a page render from being one bucket read per agent, and -/// costs nothing anyone will notice. -const ICON_MAX_AGE: u32 = 300; - -/// `agent`'s icon — the swarm-level answer to "what does this agent look -/// like". -/// -/// **Nothing on this path names a hive.** The icon is a property of the -/// agent, held at swarm scope under the agent's own key, so it is -/// answerable for an agent that has moved hives or that no hive is -/// currently running. That is the whole point of it living here rather -/// than being fetched from wherever the agent happens to run. -/// -/// **404 means this agent has no icon**, which is the expected -/// unconfigured state and not an error — the same contract the per-agent -/// harness's own `GET /icon` already has, so a caller falls back -/// client-side on a failed load rather than probing first. -/// -/// 🩸 **The body is an untrusted document served from this daemon's own -/// origin**, and an SVG can carry script. The response headers say it may -/// not run any: `Content-Security-Policy: sandbox` with no `allow-scripts` -/// keeps a direct navigation to this URL from executing it, and `nosniff` -/// keeps a browser from deciding the bytes are something else. An `` -/// load — the one consumer — never runs script regardless; the headers are -/// for every other way a URL gets opened. -#[utoipa::path( - get, - path = "/api/agents/{name}/icon", - params(("name" = String, Path, description = "agent name")), - responses( - (status = 200, description = "the agent's icon, an SVG", content_type = "image/svg+xml"), - (status = 400, description = "not an identifier (problem+json)", body = String), - (status = 404, description = "this agent has no icon (problem+json)", body = String), - (status = 500, description = "the icon bucket could not be read (problem+json)", body = String), - (status = 503, description = "no swarm queue is wired up, or it is not connected (problem+json)", body = String), - ), - tag = "agents" -)] -async fn get_agent_icon( - State(state): State, - Path(name): Path, -) -> Result { - use axum::http::{StatusCode, header}; - use axum::response::IntoResponse as _; - - let Some(reader) = state.agent_icons.as_ref() else { - return Err(error_problem( - StatusCode::SERVICE_UNAVAILABLE, - "no swarm queue is wired up on this host", - )); - }; - let agent = hive_types::Ident::parse(&name) - .map_err(|reason| error_problem(StatusCode::BAD_REQUEST, reason))? - .into_string(); - let icon = reader.get(&agent).await.map_err(|e| { - tracing::warn!(agent = %agent, error = %format!("{e:#}"), "reading the agent icon failed"); - error_problem(wanted_error_status(&e), &format!("{e:#}")) - })?; - let Some(icon) = icon else { - return Err(error_problem( - StatusCode::NOT_FOUND, - "this agent has no icon", - )); - }; - Ok(( - [ - ( - header::CONTENT_TYPE, - swarm_queue_client::agent_icon::MEDIA_TYPE.to_owned(), - ), - ( - header::CACHE_CONTROL, - format!("public, max-age={ICON_MAX_AGE}"), - ), - (header::CONTENT_SECURITY_POLICY, "sandbox".to_owned()), - (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_owned()), - ], - icon, - ) - .into_response()) -} - -/// Every agent with an open config PR, in one response — the bulk -/// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs -/// every agent's status to render, and fetching them one at a time doesn't -/// scale and isn't how the rest of swarm-ui's single-fetch pages (jobq, -/// hives status) work. -/// -/// Deliberately **not** `/api/agents/config-prs`: agent names are -/// user-specified (any string `hive_types::Ident` accepts), so a literal -/// path segment sitting where a `{name}` capture could plausibly also want -/// to live is a real, not theoretical, clash risk the moment someone names -/// an agent `config-prs` — per mara's review call, closed off by -/// construction rather than relying on axum's static-route-priority -/// tiebreak to paper over it. -/// -/// Only agents with a currently-open PR are present — same "absence is the -/// answer" shape [`get_agent_config_pr`]'s `null` uses, just at map-entry -/// granularity instead of per-request. -#[utoipa::path( - get, - path = "/api/config-prs", - responses( - (status = 200, description = "agent name -> open config PR, only agents with one present", body = std::collections::HashMap), - (status = 503, description = "no forge is configured on this host", body = String), - ), - tag = "agents" -)] -async fn get_config_prs( - State(state): State, -) -> Result>, StatusUnavailable> { - let Some(cache) = state.config_prs.get() else { - return Err(StatusUnavailable( - "no forge is configured on this host".to_owned(), - )); - }; - Ok(Json(cache.snapshot())) -} - -/// The swarm's own public base URL, as the forge must address it. -/// -/// Set by `swarm-controller.nix` **only when this host actually serves the -/// swarm vhost** that carries the `/webhook/forge/` location. Absent means -/// "the endpoint is not reachable from outside", and the right response to -/// that is to register nothing: a hook pointing at a URL nothing answers is -/// worse than no hook, because forgejo records failed deliveries against a -/// registration that looks configured. -const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL"; - -/// Register the swarm-wide forge hooks against this controller, in the -/// background. -/// -/// Detached rather than awaited, and never fatal: the forge may be slow or -/// briefly down at boot, and none of the daemon's other routes depend on a -/// hook existing. -/// -/// Retried on a backoff rather than left to the next process start. Losing -/// that race is the *common* case, not an edge one — the controller and the -/// forge come up together on a rebuild, and a deploy was measured where both -/// consecutive starts got `502 Bad Gateway` from the gateway because forgejo -/// was not serving yet. Nothing schedules another start, so "the next restart -/// retries" can leave a swarm with no hooks registered for as long as the -/// daemon keeps running — and the failure is silent at both ends, because the -/// forge has nothing to report about a call that never arrived. -/// -/// Runs once there is a forge client, and silently does nothing when either -/// other precondition is missing — each is a legitimate deployment shape (no -/// state directory to hold a secret, no public vhost), and each is already -/// logged where it is discovered. -fn register_swarm_webhooks(forge: Arc, secret: Option>) { - let Some(secret) = secret else { return }; - let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else { - tracing::info!( - "{PUBLIC_URL_ENV} unset; not registering swarm-wide forge webhooks (the \ - endpoint is not published on this host)" - ); - return; - }; - tokio::spawn(async move { - // A bounded schedule, not a poll loop: 15s doubling to 240s, five - // retries. It exists to outlast a forge that is slow to start, not to - // re-register periodically. Registration is idempotent, so a repeat - // costs one API call and changes nothing. - const RETRIES: usize = 5; - - let mut failed = 0; - let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret)) - .retry( - ExponentialBuilder::default() - .with_min_delay(std::time::Duration::from_secs(15)) - .with_max_delay(std::time::Duration::from_mins(4)) - .with_max_times(RETRIES), - ) - .notify(|e, wait| { - failed += 1; - tracing::info!( - error = %format!("{e:#}"), - attempt = failed, - retry_in = ?wait, - "registering swarm-wide forge webhooks failed; retrying" - ); - }) - .await; - if let Err(e) = registered { - tracing::warn!( - error = %format!("{e:#}"), - attempts = RETRIES + 1, - "registering swarm-wide forge webhooks failed; giving up until the next start" - ); - } - }); -} - -/// Connect to the swarm queue when this deployment wired one up, extracted -/// out of `main` purely to keep that function under clippy's line-count -/// lint — no behavior split, every comment below is unchanged from where -/// it used to sit inline. -/// -/// Deliberately NOT fatal on failure: the controller's HTTP surface is -/// useful without the queue, and a hive that cannot be read from renders -/// as `unknown` rather than as an outage of this daemon. What IS fatal is -/// an identity that could not be resolved, which `queue_identity::init` -/// reports in `main` before this runs: silently behaving like an -/// unconfigured host is how every hive ends up reading `never_reported` -/// with nothing to point at. -async fn connect_status_reader() -> Result>> { - let Some(cfg) = queue_identity::get().cloned() else { - tracing::info!("no swarm queue configured; status aggregation is off"); - return Ok(None); - }; - match swarm_queue_client::connect(cfg).await { - Ok(client) => { - // NOT "connected": `retry_on_initial_connect` returns a client - // before any connection has been established, so claiming a - // connection here would put "connected to the swarm queue" in - // the journal moments before every request 503s with "not - // connected" — and a reader would rightly distrust the second - // line rather than the first. The connection's real state is - // reported by the status endpoint, which checks it per request. - tracing::info!("swarm queue configured; connecting in the background"); - Ok(Some(Arc::new(status::StatusReader::new( - client, - status::StatusReader::stale_after_from_env(), - )))) - } - Err(e) => { - // `chain`, not `{:#}`: this is the queue client's own - // error type, and thiserror's Display ignores the - // alternate flag — the source would be dropped silently. - tracing::warn!( - error = swarm_queue_client::chain(&e), - "swarm queue unreachable" - ); - Ok(None) - } - } -} - -/// Fill the forge slots and start everything that runs off the forge client — -/// its periodic passes, the config-PR poll and the webhook registration — now -/// if the token is in the store, otherwise when `forge::enable`'s background -/// re-read finds it. Nothing to start when no forge URL is configured. Lifted -/// out of `main` for `clippy::too_many_lines`. -async fn start_forge( - forge_client: Late>, - config_prs: Late>, - jobq: Arc>>, - hives: Vec, - webhook_secret: Option>, - auth: Option>, - queue: Option, -) { - let url = match forge::url_from_env() { - Ok(Some(url)) => url, - Ok(None) => return, - Err(e) => { - tracing::warn!(error = %format!("{e:#}"), "forge misconfigured; repo provisioning is off"); - return; - } - }; - forge::enable(url, forge_client, move |client| { - spawn_forge_workers(&jobq, Arc::clone(&client), &hives, auth, queue); - config_prs.get_or_init(|| config_pr::spawn(Arc::clone(&client))); - register_swarm_webhooks(client, webhook_secret); - }) - .await; -} +use jobq::{Late, WorkerDeps, spawn_jobq_worker}; +use routes::AppState; +use startup::{ + agent_icon_reader, agent_status_reader, agents_md_reader, build_app, + configured_matrix_homeserver, connect_status_reader, load_hives, load_links, load_swarm_name, + load_webhook_secret, reported_inference, socket_path, spawn_agent_renewal, + spawn_matrix_account_backfill, spawn_per_agent_upkeep, start_forge, wanted_writer, +}; #[tokio::main] async fn main() -> Result<()> { @@ -3629,7 +216,7 @@ async fn main() -> Result<()> { agent_icons: agent_icon_reader(status.as_ref()), agents_md: agents_md_reader(status.as_ref()), reported_inference: reported_inference(status.as_ref()), - agent_login: agent_login(status.as_ref()), + agent_login: startup::agent_login(status.as_ref()), runs: review_classifier::runs::Runs::Live(status.as_ref().map(|s| s.queue_client())), builds: builder::runs::Builds::Live(status.as_ref().map(|s| s.queue_client())), status, @@ -3650,3010 +237,3 @@ async fn main() -> Result<()> { .await .context("serving swarm-controller") } - -/// Every route this daemon serves, wired to its state. -/// -/// Split out of `main` because the route list is the part that grows, and -/// `main` sat exactly on the `too_many_lines` limit — adding an endpoint -/// tripped a lint about the startup sequence, which is not where the change -/// was. A new route now costs one line here and none there. -fn build_app(state: AppState) -> axum::Router { - let (router, api) = OpenApiRouter::::with_openapi(ApiDoc::openapi()) - .routes(routes!(health)) - .routes(routes!(alerts::get_alerts)) - .routes(routes!(get_hives)) - .routes(routes!(get_hives_status)) - .routes(routes!(get_links)) - .routes(routes!(get_swarm_info)) - .routes(routes!(get_jobq_graph)) - .routes(routes!(get_jobq_rollup)) - .routes(routes!(get_agent_config_pr)) - .routes(routes!(get_agent_icon)) - .routes(routes!(agents_md::get_agents_md, agents_md::put_agents_md)) - .routes(routes!( - inference::get_agent_inference, - inference::put_agent_inference, - inference::delete_agent_inference - )) - .routes(routes!( - agent_limits::get_agent_limits, - agent_limits::put_agent_limits, - agent_limits::delete_agent_limits - )) - .routes(routes!( - agent_permissions::get_agent_permissions, - agent_permissions::put_agent_permissions, - agent_permissions::delete_agent_permissions - )) - .routes(routes!(agent_permissions::get_permission_catalogue)) - .routes(routes!(archetypes::get_archetypes)) - .routes(routes!(get_config_prs)) - .routes(routes!(create_agent)) - .routes(routes!(mint_agent_forge_token)) - .routes(routes!(make_forge_admin)) - .routes(routes!(get_agents)) - .routes(routes!(get_agents_status)) - .routes(routes!(set_agent_state)) - .routes(routes!( - matrix_account::put_matrix_account, - linked_accounts::delete_matrix_account - )) - .routes(routes!( - forge_account::put_forge_account, - linked_accounts::delete_forge_account - )) - .routes(routes!( - github_account::put_github_account, - linked_accounts::delete_github_account - )) - .routes(routes!(linked_accounts::get_linked_accounts)) - .routes(routes!(get_hive_wanted)) - .routes(routes!(term_stream::stream_agent_term)) - .routes(routes!(subagent_term::list_subagents)) - .routes(routes!(subagent_term::stream_subagent_term)) - .routes(routes!(agent_state_stream::stream_agent_state)) - .routes(routes!(agent_input::post_agent_input)) - .routes(routes!(agent_input::post_agents_broadcast)) - .routes(routes!(agent_control::post_agent_cancel)) - .routes(routes!( - claude_login::post_claude_login, - claude_login::delete_claude_login - )) - .routes(routes!(claude_login::post_claude_login_code)) - .routes(routes!(schedules::list_schedules, schedules::add_schedule)) - .routes(routes!( - schedules::edit_schedule, - schedules::remove_schedule - )) - .routes(routes!(issue_report::get_repos)) - .routes(routes!(issue_report::get_issue_report_all)) - .routes(routes!(issue_report::get_issue_report)) - .routes(routes!(pulls::get_pull_repos)) - .routes(routes!(pulls::get_pulls)) - .routes(routes!(review_classifier::post_try_out)) - .routes(routes!(review_classifier::get_try_out)) - .routes(routes!(review_classifier::get_rules)) - .routes(routes!(review_classifier::open_pr::post_open_pr)) - .routes(routes!(review_classifier::runs::get_runs)) - .routes(routes!(builder::runs::get_runs)) - .routes(routes!(builder::runs::get_run)) - .routes(routes!( - review_classifier::settings::get_settings, - review_classifier::settings::put_settings - )) - .routes(routes!(webhook::post_webhook_forge)) - .split_for_parts(); - // Just the JSON, not the UI — Swagger UI itself is nginx-hosted from - // the nix store (see the module doc comment above). `api` is - // `Clone`; each request gets its own owned copy for `Json` to - // serialize, same as `hive-c0re::dashboard::serve`. - router - .route( - "/api/openapi.json", - get(move || async move { Json(api.clone()) }), - ) - .with_state(state) -} - -#[cfg(test)] -mod tests { - use super::{ - DEFAULT_SOCKET, HIVES_ENV, HiveEntry, LINKS_ENV, NAME_ENV, ServiceLink, - SetAgentStateRequest, StatusUnavailable, SwarmNodeKind, WorkerDeps, load_hives, load_links, - load_swarm_name, run_swarm_node, wanted, - }; - use std::path::Path; - - /// The 503 this route returns is what the operator UIs' shared error - /// component renders, so assert the RENDERED response rather than the - /// `problem_details` crate: the contract a UI depends on is the content - /// type plus a `detail` it can address, and a handler that built the - /// value and returned it as a bare string would satisfy any test - /// written against the type alone. - #[tokio::test] - async fn status_unavailable_renders_problem_json_with_the_cause_in_detail() { - use axum::response::IntoResponse as _; - - let cause = "listing status bucket keys: timed out"; - let resp = StatusUnavailable(cause.to_owned()).into_response(); - - assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); - let ct = resp - .headers() - .get(axum::http::header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_owned(); - assert!( - ct.starts_with("application/problem+json"), - "RFC 9457 media type, got {ct:?}" - ); - - let bytes = axum::body::to_bytes(resp.into_body(), 64 * 1024) - .await - .expect("body reads"); - let v: serde_json::Value = serde_json::from_slice(&bytes).expect("problem+json parses"); - assert_eq!(v["status"], 503); - // The cause is an addressable field, not the entire payload — that - // distinction is the point of the change, so it is what is asserted. - assert_eq!(v["detail"], cause); - } - - /// Drives `SwarmNodeKind::CreateRepo` through the real - /// `hive_jobq::scheduler::Scheduler` claim → run → complete path, - /// rather than only through `create_agent`'s endpoint test (there - /// isn't one — the endpoint itself is thin, insert-and-return; the - /// interesting behavior is in `run_swarm_node`'s executor arm, which - /// this exercises directly). - /// - /// Deliberately offline: with the forge URL unset, - /// `forge::Client::from_env` returns `Ok(None)` (see that module's doc - /// comment), so this exercises the whole claim → run → complete path - /// through `hive_jobq::scheduler::Scheduler` without a real forge - /// server — at the cost of only ever observing the - /// graceful-absence-is-failure branch here. The happy path needs an - /// actual forge instance and isn't something a unit test in this crate - /// can reach. - /// - /// SAFETY: single-threaded mutation of the `forge` env var this - /// test itself owns, restored before returning — no other test in this - /// crate reads them. - /// One roster, one agent name, two hives — the accepted one and a - /// typo. Built per test so neither can see the other's queue. - /// The scheduler handle these tests hold onto so they can assert on - /// what the endpoint queued. Aliased because the full type is three - /// nested generics deep and reads worse inline than named. - type SharedSched = std::sync::Arc< - std::sync::Mutex< - hive_jobq::scheduler::Scheduler, - >, - >; - - pub(crate) fn state_with_roster() -> (super::AppState, SharedSched) { - let sched = - std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - let state = super::AppState { - hives: std::sync::Arc::new(vec![HiveEntry { - name: "pr1ma".to_owned(), - domain: "pr1ma.example".to_owned(), - dashboard: "pr1ma.hives.example".to_owned(), - }]), - links: std::sync::Arc::new(Vec::new()), - status: None, - // No queue, for the same reason as `status`: these tests build - // the creation graph and read its shape, and never run the node - // that would write a declaration. - wanted: None, - agent_status: None, - agent_icons: None, - agents_md: None, - reported_inference: None, - agent_login: None, - jobq: std::sync::Arc::clone(&sched), - webhook_secret: None, - config_prs: std::sync::Arc::default(), - swarm_name: None, - // No bridge: these tests drive agent *creation*, which queues a - // job and consults one only for a rule-breaking name. The roster - // read is the verb that needs it, and it has its own test below. - auth: None, - forge: std::sync::Arc::default(), - create_gate: std::sync::Arc::default(), - accounts: super::linked_accounts::Accounts::Fake(std::sync::Arc::default()), - try_outs: super::review_classifier::TryOuts::default(), - reviews: std::sync::Arc::default(), - runs: super::review_classifier::runs::Runs::Live(None), - builds: super::builder::runs::Builds::Live(None), - alerts: None, - }; - (state, sched) - } - - /// A swarm with no identity bridge cannot answer the roster, and must - /// say so rather than answer `[]`. - /// - /// The distinction is the whole reason this endpoint exists: an empty - /// roster means *no agents*, and a UI that renders "no agents" for a - /// store it could not read is showing a fact nobody established. 503 - /// rather than 500 for the same reason the status route uses it — a - /// bridge is wired up per deployment, so a caller retrying is right. - #[tokio::test] - async fn a_roster_with_no_bridge_refuses_rather_than_answering_empty() { - use axum::response::IntoResponse as _; - - let (state, _sched) = state_with_roster(); - assert!(state.auth.is_none(), "the fixture must have no bridge"); - - let err = super::get_agents(axum::extract::State(state)) - .await - .expect_err("a bridge-less controller cannot produce a roster"); - let resp = err.into_response(); - assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); - - let bytes = axum::body::to_bytes(resp.into_body(), 64 * 1024) - .await - .expect("body reads"); - let v: serde_json::Value = serde_json::from_slice(&bytes).expect("problem+json parses"); - // Asserted on the rendered body rather than the error value: what a - // consumer can distinguish is what matters, and `[]` and this share - // a type on the Rust side. - assert_eq!(v["status"], 503); - assert!( - v["detail"] - .as_str() - .is_some_and(|d| d.contains("identity bridge")), - "the cause must name what is missing, got {:?}", - v["detail"] - ); - } - - /// An icon route with no bucket behind it must say it could not ask, - /// not that the agent has no icon. - /// - /// The two are one status code apart and read identically to an - /// `` — both end in the fallback glyph — which is exactly why the - /// distinction has to be asserted here: nothing downstream would ever - /// notice the 503 silently becoming a 404. - #[tokio::test] - async fn an_icon_with_no_queue_refuses_rather_than_answering_no_icon() { - use axum::response::IntoResponse as _; - - let (state, _sched) = state_with_roster(); - assert!( - state.agent_icons.is_none(), - "the fixture must have no queue" - ); - - let err = super::get_agent_icon( - axum::extract::State(state), - axum::extract::Path("iris".to_owned()), - ) - .await - .expect_err("a queue-less controller cannot answer an icon"); - let resp = err.into_response(); - assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); - } - - /// The roster check is the half that makes the recorded hive worth - /// having, so assert it by EFFECT rather than by the message: a hive - /// that is not in this swarm must be refused **before anything is - /// queued**. A version that queued the graph and then complained would - /// satisfy an assertion on the status code alone while still creating - /// the agent — which is the failure this exists to stop. - #[tokio::test] - async fn a_hive_outside_the_roster_is_refused_before_anything_is_queued() { - let (state, sched) = state_with_roster(); - - let err = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1maa".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect_err("a hive outside the roster must be refused"); - - // Names what would have worked: an operator who just mistyped a - // hive cannot see the roster from the error otherwise. - let rendered = format!("{err:?}"); - assert!( - rendered.contains("pr1ma"), - "the refusal should name the known hives, got: {rendered}" - ); - - let queued = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph() - .nodes() - .count(); - assert_eq!(queued, 0, "a refused creation must queue no work"); - } - - /// The control for the arm above: the same call with a hive that IS in - /// the roster gets through and queues the graph. Without this, "refused" - /// could equally mean the endpoint refuses everything. - #[tokio::test] - async fn a_hive_in_the_roster_is_accepted_and_queues_the_graph() { - let (state, sched) = state_with_roster(); - - // The response is bound rather than asserted on: node ids start at - // zero, so every property I reached for first ("> 0") was a claim - // about the id allocator rather than about this endpoint. What the - // accept arm is actually for is the queue count below. - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let queued = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph() - .nodes() - .count(); - assert!(queued > 0, "an accepted creation must queue work"); - } - - /// A request with no settings is the one callers sent before the fields - /// existed, and each field is checked by the same rules as its `PUT`. - #[test] - fn a_creation_request_takes_optional_settings() { - let plain: super::CreateAgentRequest = - serde_json::from_str(r#"{"name":"atlas","hive":"pr1ma"}"#).expect("plain body"); - assert!(plain.limits.is_none() && plain.inference.is_none()); - - let full: super::CreateAgentRequest = serde_json::from_str( - r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, - "inference":{"model":"opus","effort":"high"}}"#, - ) - .expect("body with settings"); - assert!(full.limits.is_some() && full.inference.is_some()); - - let typo = serde_json::from_str::( - r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu":"400%"}}"#, - ); - assert!(typo.is_err(), "an unknown cap field must be refused"); - } - - /// A bad cap or model must stop the creation before the graph is queued: - /// the point of taking settings in the creating request is that a refusal - /// leaves no agent without them. - #[tokio::test] - async fn bad_settings_are_refused_before_anything_is_queued() { - use axum::http::StatusCode; - let bad: [(&str, &str); 2] = [ - (r#"{"cpu_quota":"lots"}"#, "limits"), - (r#"{"model":"-opus"}"#, "inference"), - ]; - for (value, field) in bad { - let (state, sched) = state_with_roster(); - let body = format!(r#"{{"name":"atlas","hive":"pr1ma","{field}":{value}}}"#); - let req: super::CreateAgentRequest = serde_json::from_str(&body).expect("body parses"); - let err = super::create_agent(axum::extract::State(state), axum::Json(req)) - .await - .expect_err("a bad setting must be refused"); - assert_eq!( - axum::response::IntoResponse::into_response(err).status(), - StatusCode::BAD_REQUEST, - "{field}" - ); - assert_eq!(node_count(&sched), 0, "{field}: nothing may be queued"); - } - } - - /// Settings that cannot be stored, here because there is no queue, refuse - /// the creation rather than leaving an agent without them. - #[tokio::test] - async fn settings_that_cannot_be_stored_refuse_the_creation() { - let (state, sched) = state_with_roster(); - let req: super::CreateAgentRequest = serde_json::from_str( - r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}}"#, - ) - .expect("body parses"); - let err = super::create_agent(axum::extract::State(state), axum::Json(req)) - .await - .expect_err("no queue means the caps cannot be stored"); - assert_eq!( - axum::response::IntoResponse::into_response(err).status(), - axum::http::StatusCode::SERVICE_UNAVAILABLE - ); - assert_eq!(node_count(&sched), 0, "nothing may be queued"); - } - - /// Through the real handler and a real `nats-server`: both settings land - /// under the agent's key as the hive reads them, and the graph is queued. - #[tokio::test] - async fn a_creation_with_settings_stores_them_and_queues_the_graph() { - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - let state = crate::agent_limits::tests::state(&client); - let req: super::CreateAgentRequest = serde_json::from_str( - r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, - "inference":{"model":"opus","max_subagents":3}}"#, - ) - .expect("body parses"); - let _queued = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) - .await - .expect("a valid creation with settings goes through"); - - assert_eq!( - crate::agent_limits::tests::published(&client, "atlas") - .await - .as_deref(), - Some(r#"{"cpu_quota":"400%"}"#) - ); - let inference = swarm_queue_client::agent_inference::open_or_create(&client) - .await - .expect("bucket") - .get("atlas") - .await - .expect("read") - .expect("a value"); - assert_eq!(&inference[..], br#"{"model":"opus","max_subagents":3}"#); - assert!( - state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph() - .nodes() - .count() - > 0, - "the graph must be queued" - ); - } - - /// When the inference store fails after the caps are stored, the caps go - /// back to what they were: the earlier value, or no key. Without the - /// roll-back this leaves a cap behind for an agent that was never created. - /// The inference bucket is made with a one-byte value limit, so any write - /// to it fails. - #[tokio::test] - async fn a_failed_second_store_puts_the_first_back() { - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - async_nats::jetstream::new(client.clone()) - .create_key_value(async_nats::jetstream::kv::Config { - bucket: swarm_queue_client::agent_inference::BUCKET.to_owned(), - history: 1, - max_value_size: 1, - ..Default::default() - }) - .await - .expect("a bucket that refuses any value"); - let caps = |json: &str| { - serde_json::from_str::(json) - .expect("caps") - .into_stored() - .expect("valid") - }; - let inference = - serde_json::from_str::(r#"{"model":"opus"}"#) - .expect("settings") - .into_stored() - .expect("valid"); - crate::agent_limits::store(&client, "atlas", &caps(r#"{"cpu_quota":"100%"}"#)) - .await - .expect("an earlier cap"); - - for agent in ["atlas", "iris"] { - let stored = super::StoredSettings::store( - &client, - agent, - Some(&caps(r#"{"cpu_quota":"400%"}"#)), - Some(&inference), - None, - None, - ) - .await; - assert!(stored.is_err(), "{agent}: the inference write must fail"); - } - assert_eq!( - crate::agent_limits::tests::published(&client, "atlas") - .await - .as_deref(), - Some(r#"{"cpu_quota":"100%"}"#), - "the earlier cap comes back" - ); - assert_eq!( - crate::agent_limits::tests::published(&client, "iris").await, - None, - "a cap stored for a name with none is removed" - ); - } - - /// Through the real handler: when the inference store fails after the - /// permissions are stored, the creation is refused and the permissions go - /// back to what they were, the earlier value or no key. The inference - /// bucket refuses any value, as above, and the refusal is that bucket's. - #[tokio::test] - async fn a_failed_creation_leaves_no_permissions_behind() { - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - async_nats::jetstream::new(client.clone()) - .create_key_value(async_nats::jetstream::kv::Config { - bucket: swarm_queue_client::agent_inference::BUCKET.to_owned(), - history: 1, - max_value_size: 1, - ..Default::default() - }) - .await - .expect("a bucket that refuses any value"); - let state = crate::agent_limits::tests::state(&client); - let earlier = crate::agent_permissions::at_create(vec![ - hive_sh4re::permissions::Permission::Messaging, - ]) - .expect("valid"); - crate::agent_permissions::store(&client, "atlas", &earlier) - .await - .expect("an earlier grant"); - - for agent in ["atlas", "iris"] { - let req: super::CreateAgentRequest = serde_json::from_str(&format!( - r#"{{"name":"{agent}","hive":"pr1ma","permissions":["web_tools"], - "inference":{{"model":"opus"}}}}"# - )) - .expect("body parses"); - let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) - .await - .expect_err("the inference write must fail"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::INTERNAL_SERVER_ERROR), - "{agent}: {err:?}" - ); - assert!( - err.detail - .as_deref() - .is_some_and(|d| d.contains("message size exceeds maximum allowed")), - "{agent}: refused by the inference bucket, not an earlier check: {err:?}" - ); - } - let bucket = swarm_queue_client::agent_permissions::open_or_create(&client) - .await - .expect("bucket"); - let held = - |value: Option| value.map(|v| String::from_utf8_lossy(&v).into_owned()); - assert_eq!( - held(bucket.get("atlas").await.expect("read")).as_deref(), - Some(r#"{"permissions":["messaging"]}"#), - "the earlier grant comes back" - ); - assert_eq!( - held(bucket.get("iris").await.expect("read")), - None, - "a grant stored for a name with none is removed" - ); - assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); - } - - /// A valid cap beside an invalid model stores neither and queues nothing, - /// on a server that could have stored the cap. - #[tokio::test] - async fn a_bad_setting_stores_none_of_them() { - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - let state = crate::agent_limits::tests::state(&client); - let req: super::CreateAgentRequest = serde_json::from_str( - r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, - "inference":{"model":"-opus"}}"#, - ) - .expect("body parses"); - let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) - .await - .expect_err("a bad model must be refused"); - assert_eq!( - axum::response::IntoResponse::into_response(err).status(), - axum::http::StatusCode::BAD_REQUEST - ); - assert_eq!( - crate::agent_limits::tests::published(&client, "atlas").await, - None - ); - assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); - } - - /// `state_with_roster` with a second hive, `sec0nd`. - fn state_with_two_hives() -> (super::AppState, SharedSched) { - let (state, sched) = state_with_roster(); - let mut hives = (*state.hives).clone(); - hives.push(HiveEntry { - name: "sec0nd".to_owned(), - domain: "sec0nd.example".to_owned(), - dashboard: "sec0nd.hives.example".to_owned(), - }); - let state = super::AppState { - hives: std::sync::Arc::new(hives), - ..state - }; - (state, sched) - } - - async fn create( - state: &super::AppState, - name: &str, - hive: &str, - ) -> Result<(), problem_details::ProblemDetails> { - super::create_agent( - axum::extract::State(state.clone()), - axum::Json(super::CreateAgentRequest { - name: name.to_owned(), - hive: hive.to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .map(|_| ()) - } - - fn node_count(sched: &SharedSched) -> usize { - sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph() - .nodes() - .count() - } - - /// A name queued for one hive is refused on another, by effect: 409 and - /// nothing more queued. The queued-but-unwritten placement is the one a - /// wanted-state read alone would miss. - #[tokio::test] - async fn a_name_placed_on_another_hive_is_refused_before_anything_is_queued() { - let (state, sched) = state_with_two_hives(); - create(&state, "atlas", "pr1ma") - .await - .expect("the first creation of a name must be accepted"); - let before = node_count(&sched); - - let err = create(&state, "atlas", "sec0nd") - .await - .expect_err("the same name on another hive must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::CONFLICT), - "{err:?}" - ); - assert!( - format!("{err:?}").contains("pr1ma"), - "the refusal should name the hive holding the name, got: {err:?}" - ); - assert_eq!( - node_count(&sched), - before, - "a refused creation must queue no work" - ); - } - - /// The control for the refusal above: the same name on the SAME hive is - /// that agent being re-created, and queues again. - #[tokio::test] - async fn re_creating_an_agent_on_its_own_hive_is_accepted() { - let (state, sched) = state_with_two_hives(); - create(&state, "atlas", "pr1ma") - .await - .expect("the first creation of a name must be accepted"); - let before = node_count(&sched); - - create(&state, "atlas", "pr1ma") - .await - .expect("re-creating an agent on its own hive must be accepted"); - assert!(node_count(&sched) > before, "a re-creation must queue work"); - } - - /// A wanted state that cannot be read refuses rather than reading as - /// "placed nowhere", and queues nothing. - #[tokio::test] - async fn an_unreadable_wanted_state_refuses_creation() { - let (state, sched) = state_with_two_hives(); - let state = super::AppState { - wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( - disconnected_client().await, - ))), - ..state - }; - - let err = create(&state, "atlas", "pr1ma") - .await - .expect_err("an unreadable placement must refuse"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{err:?}" - ); - assert_eq!( - node_count(&sched), - 0, - "a refused creation must queue no work" - ); - } - - fn declaring( - hive: &str, - agent: &str, - state: swarm_queue_client::wanted::AgentState, - ) -> (String, swarm_queue_client::wanted::HiveWanted) { - let mut declaration = swarm_queue_client::wanted::HiveWanted::default(); - declaration.agents.insert( - agent.to_owned(), - swarm_queue_client::wanted::AgentWanted { state }, - ); - (hive.to_owned(), declaration) - } - - /// A `SetAgentWanted` node that finishes between the two reads: whichever - /// read runs first sees it queued and undeclared, the second sees it - /// terminal and declared. Only the queue-first order catches it. - #[tokio::test] - async fn a_placement_finishing_between_the_two_reads_is_still_caught() { - use swarm_queue_client::wanted::AgentState; - let reads = std::cell::Cell::new(0); - let snapshot_queued = || { - reads.set(reads.get() + 1); - if reads.get() == 1 { - vec![("sec0nd".to_owned(), "atlas".to_owned())] - } else { - Vec::new() - } - }; - let read_declared = || { - reads.set(reads.get() + 1); - let declared = if reads.get() == 1 { - Vec::new() - } else { - vec![declaring("sec0nd", "atlas", AgentState::Paused)] - }; - std::future::ready(Ok::<_, ()>(declared)) - }; - - let elsewhere = - super::placements_elsewhere("atlas", "pr1ma", snapshot_queued, read_declared) - .await - .expect("both reads succeed"); - assert_eq!(elsewhere, ["sec0nd"]); - } - - #[test] - fn a_live_declaration_on_another_hive_is_a_placement() { - use swarm_queue_client::wanted::AgentState; - for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { - let declared = [declaring("sec0nd", "atlas", state)]; - assert_eq!( - super::placed_elsewhere("atlas", "pr1ma", &declared, &[]), - ["sec0nd"], - "{state:?}" - ); - } - } - - /// A destroyed agent has left its hive, so creating it elsewhere moves it. - #[test] - fn a_destroyed_declaration_is_not_a_placement() { - let declared = [declaring( - "sec0nd", - "atlas", - swarm_queue_client::wanted::AgentState::Destroyed, - )]; - assert!(super::placed_elsewhere("atlas", "pr1ma", &declared, &[]).is_empty()); - } - - #[test] - fn placements_on_the_requested_hive_or_of_other_names_do_not_count() { - use swarm_queue_client::wanted::AgentState; - let declared = [ - declaring("pr1ma", "atlas", AgentState::Up), - declaring("sec0nd", "argus", AgentState::Up), - ]; - let queued = [ - ("pr1ma".to_owned(), "atlas".to_owned()), - ("sec0nd".to_owned(), "argus".to_owned()), - ]; - assert!(super::placed_elsewhere("atlas", "pr1ma", &declared, &queued).is_empty()); - } - - #[test] - fn the_hive_placing_an_agent_claims_it_and_no_other_does() { - use swarm_queue_client::wanted::AgentState; - let declared = [ - declaring("pr1ma", "atlas", AgentState::Paused), - declaring("sec0nd", "argus", AgentState::Up), - declaring("th1rd", "atlas", AgentState::Destroyed), - ]; - assert_eq!(super::claiming_hives("atlas", &declared), ["pr1ma"]); - assert!(super::claiming_hives("iris", &declared).is_empty()); - } - - #[test] - fn two_hives_placing_one_agent_both_claim_it() { - use swarm_queue_client::wanted::AgentState; - let declared = [ - declaring("pr1ma", "atlas", AgentState::Up), - declaring("sec0nd", "atlas", AgentState::Up), - ]; - assert_eq!( - super::claiming_hives("atlas", &declared), - ["pr1ma", "sec0nd"] - ); - } - - /// `refuse_placement` of `agent` on `pr1ma`, with `admin` reserved, - /// `roster` as the roster read, and `declared` as every hive's wanted - /// state, `pr1ma`'s included. - async fn refuse_on_pr1ma_with( - agent: &str, - declared: Vec<(String, swarm_queue_client::wanted::HiveWanted)>, - roster: Result, - ) -> Result<(), problem_details::ProblemDetails> { - let here = declared - .iter() - .find(|(hive, _)| hive == "pr1ma") - .map(|(_, declaration)| declaration.clone()); - super::refuse_placement( - agent, - "pr1ma", - here.as_ref(), - Some("admin"), - || std::future::ready(roster), - Vec::new, - || std::future::ready(Ok(declared)), - ) - .await - } - - /// [`refuse_on_pr1ma_with`] for an `agent` the roster does not hold. - async fn refuse_on_pr1ma( - agent: &str, - declared: Vec<(String, swarm_queue_client::wanted::HiveWanted)>, - ) -> Result<(), problem_details::ProblemDetails> { - refuse_on_pr1ma_with(agent, declared, Ok(false)).await - } - - const PLACING: [swarm_queue_client::wanted::AgentState; 3] = [ - swarm_queue_client::wanted::AgentState::Up, - swarm_queue_client::wanted::AgentState::Offline, - swarm_queue_client::wanted::AgentState::Paused, - ]; - - /// The handler checks exactly the placing states; a destroy is never - /// checked. - #[test] - fn only_a_destroy_places_nothing() { - for state in PLACING { - assert!(super::places_agent(state), "{state:?}"); - } - assert!(!super::places_agent( - swarm_queue_client::wanted::AgentState::Destroyed - )); - } - - #[tokio::test] - async fn placing_a_new_reserved_name_is_refused() { - let err = refuse_on_pr1ma("admin", Vec::new()) - .await - .expect_err("a new reserved name must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{err:?}" - ); - } - - /// `offline` first, then `up`: the first step is a first declaration, so - /// it is refused and leaves nothing on the hive for `up` to pass as. - /// `refuse_placement` takes no state: every placing state reaches it the - /// same way, which `only_a_destroy_places_nothing` pins. - #[tokio::test] - async fn offline_first_cannot_carry_a_reserved_name_into_up() { - let err = refuse_on_pr1ma("admin", Vec::new()) - .await - .expect_err("a first `offline` of a reserved name must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{err:?}" - ); - // Nothing was declared, so `up` is a first declaration as well. - let err = refuse_on_pr1ma("admin", Vec::new()) - .await - .expect_err("`up` after a refused `offline` must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{err:?}" - ); - } - - /// No later step refuses a bad name, so an unreadable roster refuses - /// rather than lets it through. An ordinary name never reads it. - #[tokio::test] - async fn an_unreadable_roster_refuses_a_rule_breaking_name() { - let unreadable = || Err("no identity bridge is configured".to_owned()); - let err = refuse_on_pr1ma_with("admin", Vec::new(), unreadable()) - .await - .expect_err("an unreadable roster must not let a reserved name through"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{err:?}" - ); - refuse_on_pr1ma_with("atlas", Vec::new(), unreadable()) - .await - .expect("an ordinary name does not depend on the roster"); - } - - /// A destroyed agent is gone from its hive, so declaring it again is a - /// first declaration and its name is checked. - #[tokio::test] - async fn a_destroyed_declaration_here_does_not_exempt_the_name() { - use swarm_queue_client::wanted::AgentState; - let err = refuse_on_pr1ma( - "admin", - vec![declaring("pr1ma", "admin", AgentState::Destroyed)], - ) - .await - .expect_err("re-placing a destroyed reserved name must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{err:?}" - ); - } - - /// An agent already declared here keeps a rule-breaking name through any - /// state change, whether or not the roster holds it or can be read. - #[tokio::test] - async fn an_agent_declared_here_is_not_refused_over_its_name() { - for current in PLACING { - for roster in [Ok(false), Err("bridge down".to_owned())] { - refuse_on_pr1ma_with( - "admin", - vec![declaring("pr1ma", "admin", current)], - roster.clone(), - ) - .await - .unwrap_or_else(|err| panic!("{current:?}, {roster:?}: {err:?}")); - } - } - } - - #[tokio::test] - async fn a_first_declaration_of_a_name_placed_on_another_hive_is_refused() { - use swarm_queue_client::wanted::AgentState; - let err = refuse_on_pr1ma("atlas", vec![declaring("sec0nd", "atlas", AgentState::Up)]) - .await - .expect_err("a name placed on another hive must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::CONFLICT), - "{err:?}" - ); - assert!(format!("{err:?}").contains("sec0nd"), "{err:?}"); - } - - /// An agent that runs on `pr1ma` but was never declared anywhere, and is - /// not in the roster, is one the swarm adopts by declaring it. - #[tokio::test] - async fn adopting_an_undeclared_agent_on_its_own_hive_is_accepted() { - refuse_on_pr1ma("legacy", Vec::new()) - .await - .expect("an undeclared agent must still be adoptable"); - } - - #[tokio::test] - async fn changing_the_state_of_an_agent_on_its_own_hive_is_accepted() { - use swarm_queue_client::wanted::AgentState; - refuse_on_pr1ma("atlas", vec![declaring("pr1ma", "atlas", AgentState::Up)]) - .await - .expect("an agent's own hive must accept a new state for it"); - } - - /// `set_agent_state` on `pr1ma`, in a two-hive swarm whose queue never - /// connected and with no identity bridge. - async fn set_on_disconnected_pr1ma( - agent: &str, - wanted_state: swarm_queue_client::wanted::AgentState, - ) -> problem_details::ProblemDetails { - let (state, _sched) = state_with_two_hives(); - let state = super::AppState { - wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( - disconnected_client().await, - ))), - ..state - }; - super::set_agent_state( - axum::extract::State(state), - axum::extract::Path(("pr1ma".to_owned(), agent.to_owned())), - axum::Json(SetAgentStateRequest { - state: wanted_state, - }), - ) - .await - .expect_err("a queue that never connected cannot publish") - } - - /// Through the handler: a wanted state that cannot be read refuses the - /// declaration before it is written, as it refuses creation. - #[tokio::test] - async fn set_agent_state_refuses_when_the_wanted_state_is_unreadable() { - let err = - set_on_disconnected_pr1ma("atlas", swarm_queue_client::wanted::AgentState::Up).await; - assert_eq!( - err.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{err:?}" - ); - assert!( - format!("{err:?}").contains("was not placed"), - "the refusal must come from the placement check, got: {err:?}" - ); - } - - /// Through the handler: a destroy of a reserved name reaches the write - /// without being checked. It fails there only because the queue never - /// connected; the placing control above fails in the check instead. - #[tokio::test] - async fn set_agent_state_does_not_check_a_destroy() { - let err = - set_on_disconnected_pr1ma("admin", swarm_queue_client::wanted::AgentState::Destroyed) - .await; - assert_eq!( - err.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{err:?}" - ); - assert!( - !format!("{err:?}").contains("not placed"), - "a destroy must not be refused by the placement check, got: {err:?}" - ); - } - - /// Forgejo v16.0.5's reserved usernames our charset can spell, plus the - /// bare `-` — the forge section of `nix/reserved-names.nix`. Read from - /// the list nix exports, so dropping one from the file reds this. - const FORGE_REFUSED_LITERALS: [&str; 23] = [ - "-", - "admin", - "api", - "assets", - "attachments", - "avatar", - "avatars", - "captcha", - "explore", - "forgejo-actions", - "ghost", - "gitea-actions", - "issues", - "login", - "metrics", - "milestones", - "notifications", - "org", - "pulls", - "repo", - "repo-avatars", - "user", - "v2", - ]; - - /// The list nix renders, panicking when absent for the reason - /// `hive-sh4re`'s drift test gives: a skip would read as green. - fn nix_reserved() -> String { - hive_types::reserved_names_raw() - .expect("HIVE_RESERVED_NAMES is exported by nix/checks.nix and nix/devshell.nix") - } - - #[test] - fn a_new_agent_named_a_forge_refused_literal_is_refused() { - let raw = nix_reserved(); - for name in FORGE_REFUSED_LITERALS { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); - let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); - assert!(err.contains("nix/reserved-names.nix"), "{name:?}: {err}"); - } - } - - fn changed(agent: &str, rev: &str, pr: u64) -> crate::config_pr::ChangedConfigPr { - crate::config_pr::ChangedConfigPr { - agent: agent.to_owned(), - rev: rev.to_owned(), - pr, - } - } - - fn build_state( - sched: &hive_jobq::scheduler::Scheduler, - rev: &str, - ) -> hive_jobq::State { - sched - .graph() - .nodes() - .find(|n| matches!(&n.payload, SwarmNodeKind::BuildAgent { rev: r, .. } if r == rev)) - .map(|n| n.state) - .expect("a build of that rev") - } - - /// A newer head of the same PR replaces its pending build; another PR's - /// build, and the same PR's running build, are left alone. - #[tokio::test] - async fn a_newer_push_replaces_the_prs_pending_build_only() { - let sched = - std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - let queue = |c| super::queue_build(&mut sched.lock().unwrap(), c).expect("queued"); - assert_eq!(queue(changed("atlas", "r1", 3)), 0); - // r1 takes the builder and stays running: its future is never awaited. - let _running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { - (b, hive_jobq::scheduler::Outcome::Done) - }) - .expect("r1 is runnable"); - assert_eq!( - queue(changed("atlas", "r2", 3)), - 0, - "r1 is running, not pending" - ); - assert_eq!(queue(changed("argus", "o1", 4)), 0); - assert_eq!(queue(changed("atlas", "r3", 3)), 1, "r2 was pending"); - - let guard = sched.lock().unwrap(); - assert_eq!(build_state(&guard, "r1"), hive_jobq::State::Running); - assert_eq!(build_state(&guard, "r2"), hive_jobq::State::Cancelled); - assert_eq!(build_state(&guard, "r3"), hive_jobq::State::Pending); - assert_eq!(build_state(&guard, "o1"), hive_jobq::State::Pending); - } - - /// Builds share one builder slot: a second build is not claimable while - /// the first runs, and is once it has finished. - #[tokio::test] - async fn one_build_runs_at_a_time() { - let sched = - std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ))); - for c in [changed("atlas", "a1", 1), changed("argus", "b1", 2)] { - super::queue_build(&mut sched.lock().unwrap(), c).expect("queued"); - } - let run = |sched| { - hive_jobq::scheduler::Scheduler::claim_next(sched, |_, _, b| async move { - (b, hive_jobq::scheduler::Outcome::Done) - }) - }; - let first = run(&sched).expect("one build is runnable"); - assert!( - run(&sched).is_none(), - "a second build started beside the first" - ); - first.await.1.expect("no growth"); - run(&sched) - .expect("the second build runs once the first is done") - .await - .1 - .expect("no growth"); - } - - #[test] - fn a_new_agent_named_like_a_service_forge_account_is_refused() { - let raw = nix_reserved(); - for name in [ - crate::forge::builder_account::LOGIN, - crate::forge::classifier_account::LOGIN, - crate::forge::ci_metrics_account::LOGIN, - ] { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); - let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); - assert!(err.contains("nix/reserved-names.nix"), "{name:?}: {err}"); - } - } - - /// One case per shape rule, each named in the refusal. Checked against a - /// list without them, so the literal half cannot carry this test. - #[test] - fn a_new_agent_breaking_a_forge_shape_rule_is_refused() { - let over = "a".repeat(hive_types::FORGE_USERNAME_MAX_LEN + 1); - for (name, rule) in [ - ("-agent", "must not start with '-'"), - ("my--agent", "must not contain '--'"), - ("agent-", "must not end with '-'"), - (over.as_str(), "must be 40 characters or fewer"), - ] { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules(name, Some("operator"), &mut warnings); - let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); - assert!(err.contains(rule), "{name:?}: {err}"); - } - } - - /// The control for both tests above, against the real list. - #[test] - fn an_ordinary_name_and_a_40_char_name_break_no_rule() { - let raw = nix_reserved(); - let at_cap = "a".repeat(hive_types::FORGE_USERNAME_MAX_LEN); - for name in ["my-agent", at_cap.as_str()] { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); - assert!(broken.is_empty(), "{name:?}: {broken:?}"); - assert!(warnings.is_empty(), "{name:?}: {warnings:?}"); - } - } - - /// The `hive-` prefix belongs to hives' forge accounts, so a new agent - /// may not take it. `hivemind` and `xhive-y` are the control. - #[test] - fn a_new_agent_with_the_hive_prefix_is_refused() { - let raw = nix_reserved(); - let mut warnings = Vec::new(); - let broken = super::broken_name_rules("hive-x", Some(&raw), &mut warnings); - let err = - super::name_verdict("hive-x", &broken, Ok(false), &mut warnings).expect_err("hive-x"); - assert!(err.contains("reserved for hive forge accounts"), "{err}"); - for ok in ["hivemind", "xhive-y"] { - let broken = super::broken_name_rules(ok, Some(&raw), &mut warnings); - assert!(broken.is_empty(), "{ok:?}: {broken:?}"); - } - assert!(warnings.is_empty(), "{warnings:?}"); - } - - /// The prefix the rule reserves is the one hive accounts are named with. - #[test] - fn the_reserved_prefix_is_the_hive_account_prefix() { - assert!( - swarm_secret_client::forge::hive_login("alpha") - .starts_with(hive_types::RESERVED_AGENT_PREFIX) - ); - } - - /// Re-creating an agent under its own name must keep working: a name - /// already in the roster is warned about, never refused. - #[test] - fn an_existing_agent_keeps_a_rule_breaking_name_with_a_warning() { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules("admin", Some("admin"), &mut warnings); - super::name_verdict("admin", &broken, Ok(true), &mut warnings) - .expect("an existing agent must not be refused"); - assert_eq!(warnings.len(), 1, "{warnings:?}"); - assert!(warnings[0].contains("already exists"), "{warnings:?}"); - } - - /// A roster that cannot be read cannot tell new from existing, so it - /// warns — and the warning says the check did not decide. - #[test] - fn an_unreadable_roster_warns_rather_than_refusing() { - let mut warnings = Vec::new(); - let broken = super::broken_name_rules("agent-", Some("operator"), &mut warnings); - super::name_verdict( - "agent-", - &broken, - Err("bridge down".to_owned()), - &mut warnings, - ) - .expect("an unreadable roster must not refuse"); - assert_eq!(warnings.len(), 1, "{warnings:?}"); - assert!(warnings[0].contains("bridge down"), "{warnings:?}"); - } - - /// `admin_unless_agent` with a `promote` that records whether it ran and - /// answers `plan`. - async fn forge_admin_route( - is_agent: Result, - plan: crate::forge::site_admin::Plan, - ) -> ( - Result, problem_details::ProblemDetails>, - bool, - ) { - let ran = std::sync::atomic::AtomicBool::new(false); - let result = super::admin_unless_agent("atlas", is_agent, || async { - ran.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(plan) - }) - .await; - (result, ran.into_inner()) - } - - /// An agent's name never reaches the forge. - #[tokio::test] - async fn the_forge_admin_route_refuses_an_agent() { - let (result, ran) = - forge_admin_route(Ok(true), crate::forge::site_admin::Plan::Promote).await; - let Err(problem) = result else { - panic!("an agent's name must be refused"); - }; - assert!(!ran, "the account must not be touched"); - assert_eq!(problem.status, Some(axum::http::StatusCode::CONFLICT)); - } - - /// A roster that can't be read can't vouch for the name, so the route - /// refuses rather than guessing "not an agent". - #[tokio::test] - async fn the_forge_admin_route_refuses_on_an_unreadable_roster() { - let (result, ran) = forge_admin_route( - Err("bridge down".to_owned()), - crate::forge::site_admin::Plan::Promote, - ) - .await; - let Err(problem) = result else { - panic!("an unreadable roster must refuse"); - }; - assert!(!ran, "the account must not be touched"); - assert_eq!( - problem.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE) - ); - } - - /// A missing account is a 404 that tells the operator what to wait for. - #[tokio::test] - async fn the_forge_admin_route_names_the_missing_sso_login() { - let (result, ran) = - forge_admin_route(Ok(false), crate::forge::site_admin::Plan::NoSuchUser).await; - let Err(problem) = result else { - panic!("a missing account must not report success"); - }; - assert!(ran); - assert_eq!(problem.status, Some(axum::http::StatusCode::NOT_FOUND)); - let rendered = format!("{problem:?}"); - assert!( - rendered.contains("has not logged in via SSO yet"), - "{rendered}" - ); - } - - #[tokio::test] - async fn the_forge_admin_route_promotes_a_non_agent() { - let (result, ran) = - forge_admin_route(Ok(false), crate::forge::site_admin::Plan::Promote).await; - let Ok(axum::Json(resp)) = result else { - panic!("a non-agent must be promoted"); - }; - assert!(ran); - assert_eq!(resp.change, super::ForgeAdminChange::Promoted); - } - - /// Idempotent: a second run succeeds and says nothing changed. - #[tokio::test] - async fn the_forge_admin_route_succeeds_for_an_admin() { - let (result, _) = - forge_admin_route(Ok(false), crate::forge::site_admin::Plan::AlreadyAdmin).await; - let Ok(axum::Json(resp)) = result else { - panic!("an existing admin must succeed"); - }; - assert_eq!(resp.change, super::ForgeAdminChange::AlreadyAdmin); - } - - /// Every node whose first request goes to the forge, run against a forge - /// that accepts the connection and never answers: each ends `Failed` - /// with an error naming the node instead of holding its executor slot. - /// `MintAgentForgeToken` is not in the list: it logs in to the secret - /// store before its first forge request, and no test here has a store. - #[tokio::test(start_paused = true)] - async fn a_forge_node_fails_when_the_forge_never_answers() { - // Bound but never accepted: the kernel completes the handshake and - // buffers the request, and no response ever comes. - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); - let forge = super::Late::>::default(); - assert!( - forge - .set(std::sync::Arc::new(crate::forge::Client::for_test_forge( - url - ))) - .is_ok() - ); - let agent = || "atlas".to_owned(); - let nodes = [ - ("create_repo", SwarmNodeKind::CreateRepo { agent: agent() }), - ( - "create_forge_user", - SwarmNodeKind::CreateForgeUser { agent: agent() }, - ), - ( - "add_repo_member", - SwarmNodeKind::AddRepoMember { agent: agent() }, - ), - ( - "grant_docs_read", - SwarmNodeKind::GrantDocsRead { agent: agent() }, - ), - ( - "init_agent_config_repo", - SwarmNodeKind::InitAgentConfigRepo { agent: agent() }, - ), - ( - "mint_hive_forge_token", - SwarmNodeKind::MintHiveForgeToken { - hive: "pr1ma".to_owned(), - }, - ), - ( - "mint_classifier_forge_token", - SwarmNodeKind::MintClassifierForgeToken, - ), - ( - "mint_builder_forge_token", - SwarmNodeKind::MintBuilderForgeToken, - ), - ( - "mint_ci_metrics_forge_token", - SwarmNodeKind::MintCiMetricsForgeToken, - ), - ]; - for (label, kind) in nodes { - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - let id = sched.append(kind, Vec::new(), None).expect("insert"); - let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); - let deps = WorkerDeps { - auth: None, - forge: forge.clone(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("the node just inserted is runnable"); - tokio::time::timeout(std::time::Duration::from_mins(10), runner) - .await - .unwrap_or_else(|_| panic!("{label}: node still running after 10 minutes")) - .1 - .expect("no growth declared, nothing to reject"); - - let guard = sched.lock().unwrap(); - let node = guard.graph().node(id).expect("node still present"); - assert_eq!(node.state, hive_jobq::State::Failed, "{label}"); - let error = node.error.as_deref().unwrap_or_default(); - assert!( - error.contains(label) && error.contains("no answer from the forge"), - "{label}: expected a deadline error naming the node, got {error:?}" - ); - } - drop(listener); - } - - #[tokio::test] - async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() { - unsafe { - std::env::remove_var("SWARM_CONTROLLER_FORGE_URL"); - } - - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - let id = sched - .append( - SwarmNodeKind::CreateRepo { - agent: "atlas".to_owned(), - }, - Vec::new(), - None, - ) - .expect("insert"); - let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); - - let deps = WorkerDeps { - auth: None, - forge: std::sync::Arc::default(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("the node just inserted is runnable"); - runner - .await - .1 - .expect("no growth declared, nothing to reject"); - - let guard = sched.lock().unwrap(); - let node = guard.graph().node(id).expect("node still present"); - assert_eq!(node.state, hive_jobq::State::Failed); - assert!( - node.error.as_deref().unwrap_or_default().contains("forge"), - "expected a forge-not-configured error, got {:?}", - node.error - ); - } - - /// Sibling of the `CreateRepo` test above, same shape: drives - /// `SwarmNodeKind::CreateForgeUser` through the real scheduler with no - /// forge configured, asserting the graceful-absence-is-failure branch — - /// this node's happy path needs a live forge, same caveat as - /// `CreateRepo`'s. - #[tokio::test] - async fn create_forge_user_node_runs_end_to_end_and_fails_without_forge_configured() { - unsafe { - std::env::remove_var("SWARM_CONTROLLER_FORGE_URL"); - } - - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - let id = sched - .append( - SwarmNodeKind::CreateForgeUser { - agent: "atlas".to_owned(), - }, - Vec::new(), - None, - ) - .expect("insert"); - let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); - - let deps = WorkerDeps { - auth: None, - forge: std::sync::Arc::default(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("the node just inserted is runnable"); - runner - .await - .1 - .expect("no growth declared, nothing to reject"); - - let guard = sched.lock().unwrap(); - let node = guard.graph().node(id).expect("node still present"); - assert_eq!(node.state, hive_jobq::State::Failed); - assert!( - node.error.as_deref().unwrap_or_default().contains("forge"), - "expected a forge-not-configured error, got {:?}", - node.error - ); - } - - /// Third sibling of the two above, and the same deliberate caveat: with - /// no agent PKI named this reaches only the graceful-absence-is-failure - /// branch, before any network call. The happy path is a live store and a - /// real login, which is exactly why it is `mint_and_verify`'s own job to - /// prove it at agent-creation time rather than a unit test's. - /// - /// What this does pin is the degrade: a host whose environment does not - /// name the agent PKI fails this one node with a reason that names the - /// variable, and creates the agent anyway. - #[tokio::test] - async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_the_agent_pki_named() { - let pki = [ - crate::agent_identity::ENV_AGENT_PKI_MOUNT, - crate::agent_identity::ENV_AGENT_PKI_ROLE, - ]; - assert!( - pki.iter().all(|var| std::env::var_os(var).is_none()), - "{} and {} must be unset: with them set this test would reach a live store", - pki[0], - pki[1] - ); - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - let id = sched - .append( - SwarmNodeKind::MintAgentIdentity { - agent: "atlas".to_owned(), - }, - Vec::new(), - None, - ) - .expect("insert"); - let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); - - let deps = WorkerDeps { - auth: None, - forge: std::sync::Arc::default(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("the node just inserted is runnable"); - runner - .await - .1 - .expect("no growth declared, nothing to reject"); - - let guard = sched.lock().unwrap(); - let node = guard.graph().node(id).expect("node still present"); - assert_eq!(node.state, hive_jobq::State::Failed); - let error = node.error.as_deref().unwrap_or_default(); - assert!( - error.contains(crate::agent_identity::ENV_AGENT_PKI_MOUNT), - "the reason must name the variable an operator has to set, got {error:?}" - ); - } - - /// Fourth sibling, same shape and the same caveat once more: with no - /// queue configured this reaches only the graceful-absence-is-failure - /// branch. Writing the declaration for real needs a `JetStream` server, - /// which no test in this crate has — `wanted::apply`'s own unit tests - /// cover what gets written, and [`a_new_agent_is_declared_paused`] below - /// pins the state this node asks for. - #[tokio::test] - async fn set_agent_wanted_node_runs_end_to_end_and_fails_without_a_queue() { - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - let id = sched - .append( - SwarmNodeKind::SetAgentWanted { - hive: "pr1ma".to_owned(), - agent: "atlas".to_owned(), - }, - Vec::new(), - None, - ) - .expect("insert"); - let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); - - let deps = WorkerDeps { - auth: None, - forge: std::sync::Arc::default(), - queue: None, - wanted: None, - matrix_homeserver: None, - }; - let runner = - hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { - run_swarm_node(id, kind, builder, deps) - }) - .expect("the node just inserted is runnable"); - runner - .await - .1 - .expect("no growth declared, nothing to reject"); - - let guard = sched.lock().unwrap(); - let node = guard.graph().node(id).expect("node still present"); - assert_eq!(node.state, hive_jobq::State::Failed); - assert!( - node.error.as_deref().unwrap_or_default().contains("queue"), - "expected a no-queue-configured error, got {:?}", - node.error - ); - } - - /// `Destroyed` is the one declaration that ends an agent's life, so it is - /// the one that revokes. The credential is a bearer secret: left behind, - /// anything that recovers it still authenticates as an agent that no - /// longer exists. - #[test] - fn declaring_an_agent_destroyed_revokes_its_queue_credential() { - assert!(crate::revokes_queue_credential( - swarm_queue_client::wanted::AgentState::Destroyed - )); - } - - /// The control, and the more expensive half to get wrong: a revocation on - /// `Offline` or `Paused` would give an agent that can be stopped and never - /// restarted. Every non-terminal state is listed rather than sampled, so - /// a state added later shows up here as well as in the match itself. - #[test] - fn no_state_an_agent_comes_back_from_revokes() { - use swarm_queue_client::wanted::AgentState; - for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { - assert!( - !crate::revokes_queue_credential(state), - "{state:?} is a state an agent returns from" - ); - } - } - - /// The whole point of the node: a freshly created agent is declared - /// `Paused`, so it does not start driving turns the moment its hive - /// brings it up. Any other state here is the bug this fixes. - #[test] - fn a_new_agent_is_declared_paused() { - assert_eq!( - crate::NEW_AGENT_WANTED_STATE, - swarm_queue_client::wanted::AgentState::Paused - ); - } - - /// Same reasoning as `a_mint_node_renders_both_the_agent_and_the_hive`: - /// this variant carries a hive too, and has to have joined `data`'s - /// two-field or-pattern rather than the agent-only one. - #[test] - fn a_set_wanted_node_renders_both_the_agent_and_the_hive() { - use hive_jobq_wire::WireNode as _; - - let kind = SwarmNodeKind::SetAgentWanted { - hive: "pr1ma".to_owned(), - agent: "atlas".to_owned(), - }; - assert_eq!(kind.label(), "set_agent_wanted"); - let data = kind.data(1); - assert_eq!(data["agent"], "atlas"); - assert_eq!(data["hive"], "pr1ma"); - } - - /// The ordering the fix depends on: the pause has to be in the - /// wanted-state store *before* the hive is told to deploy, or a freshly - /// created agent runs undeclared for the window between the two. - /// - /// Asserted on the graph `create_agent` builds, same as the mint edge's - /// test above and for the same reason — the edge is one line in a - /// builder closure whose absence changes nothing observable until a real - /// agent boots and starts taking turns nobody asked for. - #[tokio::test] - async fn the_deploy_waits_for_the_pause_to_be_declared() { - use hive_jobq_wire::WireNode as _; - - let (state, sched) = state_with_roster(); - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let id_of = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label) - .unwrap_or_else(|| panic!("the graph holds a {label} node")) - .id - }; - let set_wanted = id_of("set_agent_wanted"); - let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); - - let when = deploy - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == set_wanted => Some(*when), - _ => None, - }) - .expect("the deploy waits for the declaration"); - assert!( - !when.accepts(hive_jobq::TerminalState::Failed), - "a declaration that never landed must not be deployed past; this \ - edge has to be `after_ok`, not `after_any`" - ); - } - - /// Node labels of the graph [`super::declare_agent_job`] builds, and the - /// labels the deploy node waits on. - fn creation_graph(archetype: bool) -> (Vec, Vec) { - use hive_jobq_wire::WireNode as _; - - let mut sched = hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - ); - sched - .insert_job(None, |b| { - super::declare_agent_job(b, "atlas", "pr1ma", archetype) - }) - .expect("the graph inserts"); - let graph = sched.graph(); - let labels = graph.nodes().map(|n| n.payload.label()).collect(); - let deploy = graph - .nodes() - .find(|n| n.payload.label() == "trigger_deploy") - .expect("every creation deploys"); - let waits = deploy - .deps - .iter() - .filter_map(|d| match d { - hive_jobq::Dep::Node { id, .. } => graph.node(*id).map(|n| n.payload.label()), - hive_jobq::Dep::Resource { .. } => None, - }) - .collect(); - (labels, waits) - } - - /// An archetype agent's graph has no repo nodes and its deploy waits on - /// no repo; a custom agent's has all three, and its deploy waits on the - /// seeded repo. - #[test] - fn an_archetype_agent_is_created_without_a_config_repo() { - const REPO_NODES: [&str; 3] = ["create_repo", "add_repo_member", "init_agent_config_repo"]; - - let (custom, custom_waits) = creation_graph(false); - for repo in REPO_NODES { - assert!(custom.iter().any(|l| l == repo), "custom lacks {repo}"); - } - assert!(custom_waits.iter().any(|l| l == "init_agent_config_repo")); - - let (archetype, archetype_waits) = creation_graph(true); - for repo in REPO_NODES { - assert!(!archetype.iter().any(|l| l == repo), "archetype has {repo}"); - } - for kept in [ - "create_identity", - "create_forge_user", - "grant_docs_read", - "mint_agent_identity", - "mint_agent_forge_token", - "mint_agent_matrix_account", - "set_agent_wanted", - ] { - assert!( - archetype.iter().any(|l| l == kept), - "archetype lacks {kept}" - ); - assert!(custom.iter().any(|l| l == kept), "custom lacks {kept}"); - } - assert!(archetype_waits.iter().any(|l| l == "set_agent_wanted")); - assert_eq!(archetype.len() + REPO_NODES.len(), custom.len()); - } - - /// Seed the stored archetype list with `reviewer` under `hyperhive`. - async fn seed_archetypes(client: &async_nats::Client) { - let record = swarm_queue_client::archetypes::SourceRecord { - source: "/nix/store/h".to_owned(), - archetypes: std::collections::BTreeMap::from([( - "reviewer".to_owned(), - swarm_queue_client::archetypes::ArchetypeMeta { - description: "reviews".to_owned(), - ..Default::default() - }, - )]), - refused: std::collections::BTreeMap::new(), - }; - swarm_queue_client::archetypes::open_or_create(client) - .await - .expect("bucket") - .put( - swarm_queue_client::archetypes::BUILTIN_SOURCE, - serde_json::to_vec(&record).expect("encodes").into(), - ) - .await - .expect("seeded"); - } - - /// Through the real handler and a real `nats-server`: a listed archetype - /// lands under the agent's key as the hive reads it, and the queued graph - /// has no config repo. - #[tokio::test] - async fn a_creation_from_an_archetype_records_it_and_queues_no_repo() { - use hive_jobq_wire::WireNode as _; - - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - seed_archetypes(&client).await; - let state = crate::agent_limits::tests::state(&client); - let req: super::CreateAgentRequest = serde_json::from_str( - r#"{"name":"atlas","hive":"pr1ma", - "archetype":{"source":"hyperhive","name":"reviewer"}}"#, - ) - .expect("body parses"); - let _queued = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) - .await - .expect("a listed archetype goes through"); - - let stored = swarm_queue_client::agent_archetype::open_or_create(&client) - .await - .expect("bucket") - .get("atlas") - .await - .expect("read") - .expect("a value"); - assert_eq!(&stored[..], br#"{"source":"hyperhive","name":"reviewer"}"#); - let labels: Vec = state - .jobq - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .graph() - .nodes() - .map(|n| n.payload.label()) - .collect(); - assert!(labels.iter().any(|l| l == "trigger_deploy"), "{labels:?}"); - assert!(!labels.iter().any(|l| l == "create_repo"), "{labels:?}"); - } - - /// An archetype the stored list does not hold is a 400 that stores and - /// queues nothing, and so is a name that is not an identifier. - #[tokio::test] - async fn an_unlisted_archetype_is_refused_before_anything_is_stored() { - let nats = crate::test_nats::TestNats::start(); - let client = nats.client().await; - seed_archetypes(&client).await; - let state = crate::agent_limits::tests::state(&client); - for archetype in [ - r#"{"source":"hyperhive","name":"nope"}"#, - r#"{"source":"elsewhere","name":"reviewer"}"#, - r#"{"source":"hyperhive","name":"a.b"}"#, - ] { - let req: super::CreateAgentRequest = serde_json::from_str(&format!( - r#"{{"name":"atlas","hive":"pr1ma","permissions":["messaging"], - "archetype":{archetype}}}"# - )) - .expect("body parses"); - let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) - .await - .expect_err("an unlisted archetype must be refused"); - assert_eq!( - err.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{archetype}: {err:?}" - ); - } - assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); - let permissions = swarm_queue_client::agent_permissions::open_or_create(&client) - .await - .expect("bucket"); - assert_eq!(permissions.get("atlas").await.expect("read"), None); - } - - /// The ordering the operator's ruling requires: a hive cannot pass down - /// a certificate the swarm has not published, so the deploy message must - /// not leave before the mint is terminal. - /// - /// Asserted on the graph `create_agent` builds, because the edge is one - /// line in a builder closure and its absence changes nothing observable - /// until a real agent boots without an identity. - #[tokio::test] - async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() { - use hive_jobq_wire::WireNode as _; - - let (state, sched) = state_with_roster(); - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let id_of = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label) - .unwrap_or_else(|| panic!("the graph holds a {label} node")) - .id - }; - let mint = id_of("mint_agent_identity"); - let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); - - let when = deploy - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when), - _ => None, - }) - .expect("the deploy waits for the mint"); - assert!( - when.accepts(hive_jobq::TerminalState::Failed), - "a failed or unconfigured agent PKI issue must not cancel the deploy; this edge \ - has to be `after_any`, not `after_ok`" - ); - } - - /// The forge-token mint sits between the forge user and the deploy: it - /// needs the user to exist, and the deploy must not overtake it — but a - /// host with no forge or no store must still deploy the agent. - #[tokio::test] - async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() { - use hive_jobq_wire::WireNode as _; - - let (state, sched) = state_with_roster(); - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let id_of = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label) - .unwrap_or_else(|| panic!("the graph holds a {label} node")) - .id - }; - let edge = |from: hive_jobq::NodeId, to: &str| { - graph - .node(id_of(to)) - .expect("just found") - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == from => Some(*when), - _ => None, - }) - .unwrap_or_else(|| panic!("{to} waits for node {}", from.get())) - }; - - let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token"); - assert!( - !user_to_mint.accepts(hive_jobq::TerminalState::Failed), - "a token for a user that was never created cannot be minted; this \ - edge has to be `after_ok`" - ); - let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy"); - assert!( - mint_to_deploy.accepts(hive_jobq::TerminalState::Failed), - "a host with no forge or store must not cancel the deploy; this edge \ - has to be `after_any`, not `after_ok`" - ); - } - - /// Agent creation grants `internal/docs` read once the forge user exists, - /// and a failed grant does not hold up the deploy (the pass retries it). - #[tokio::test] - async fn the_docs_grant_follows_the_user_and_is_not_a_deploy_parent() { - use hive_jobq_wire::WireNode as _; - - let (state, sched) = state_with_roster(); - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let id_of = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label) - .unwrap_or_else(|| panic!("the graph holds a {label} node")) - .id - }; - let grant = graph.node(id_of("grant_docs_read")).expect("just found"); - assert_eq!(grant.payload.data(grant.id.get())["agent"], "atlas"); - let when = grant - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == id_of("create_forge_user") => { - Some(*when) - } - _ => None, - }) - .expect("the grant waits for the forge user"); - assert!( - !when.accepts(hive_jobq::TerminalState::Failed), - "adding a user that was never created as a collaborator is a forge error; \ - this edge has to be `after_ok`" - ); - let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); - assert!( - !deploy.deps.iter().any(|d| matches!( - d, - hive_jobq::Dep::Node { id, .. } if *id == grant.id - )), - "the deploy must not wait on the docs grant" - ); - } - - /// The periodic pass queues the same pair of nodes agent creation does. - #[test] - fn a_queued_docs_grant_creates_the_forge_user_first() { - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let ids = super::queue_docs_grants(&sched, vec!["a".to_owned(), "b".to_owned()]) - .expect("two jobs insert"); - assert_eq!(ids.len(), 2, "one returned id per agent: its grant node"); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let mut labels: Vec = graph.nodes().map(|n| n.payload.label()).collect(); - labels.sort(); - assert_eq!( - labels, - [ - "create_forge_user", - "create_forge_user", - "grant_docs_read", - "grant_docs_read" - ] - ); - for grant in graph - .nodes() - .filter(|n| n.payload.label() == "grant_docs_read") - { - assert!(ids.contains(&grant.id)); - assert!( - grant - .deps - .iter() - .any(|d| matches!(d, hive_jobq::Dep::Node { when, .. } if !when.accepts(hive_jobq::TerminalState::Failed))), - "each grant waits `after_ok` on its user" - ); - } - } - - #[test] - fn a_forge_token_node_renders_the_agent() { - use hive_jobq_wire::WireNode as _; - - let kind = SwarmNodeKind::MintAgentForgeToken { - agent: "atlas".to_owned(), - }; - assert_eq!(kind.label(), "mint_agent_forge_token"); - assert_eq!(kind.data(1)["agent"], "atlas"); - } - - #[test] - fn a_matrix_account_node_renders_the_agent() { - use hive_jobq_wire::WireNode as _; - - let kind = SwarmNodeKind::MintAgentMatrixAccount { - agent: "atlas".to_owned(), - }; - assert_eq!(kind.label(), "mint_agent_matrix_account"); - assert_eq!(kind.data(1)["agent"], "atlas"); - } - - /// The periodic pass's queueing: a certificate re-issue is the node agent - /// creation mints with, a secret re-mint its own node, and an agent due - /// both gets the secret `after_any` the certificate. - #[test] - fn a_queued_renewal_chains_the_secret_after_the_certificate() { - use crate::agent_renewal::Renewal; - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let r = |agent: &str, cert, secret| Renewal { - agent: agent.to_owned(), - cert, - secret, - }; - let ids = super::queue_agent_renewals( - &sched, - vec![ - r("both", true, true), - r("cert", true, false), - r("secret", false, true), - ], - ) - .expect("three jobs insert"); - assert_eq!(ids.len(), 4, "one returned id per node"); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let agent_of = |n: &hive_jobq::Node| { - n.payload.data(n.id.get())["agent"] - .as_str() - .expect("agent is a string") - .to_owned() - }; - let mut nodes: Vec<(String, String)> = graph - .nodes() - .map(|n| (agent_of(n), n.payload.label())) - .collect(); - nodes.sort(); - assert_eq!( - nodes, - [ - ("both".to_owned(), "mint_agent_identity".to_owned()), - ("both".to_owned(), "renew_agent_queue_credential".to_owned()), - ("cert".to_owned(), "mint_agent_identity".to_owned()), - ( - "secret".to_owned(), - "renew_agent_queue_credential".to_owned() - ), - ] - ); - let find = |agent: &str, label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label && agent_of(n) == agent) - .unwrap_or_else(|| panic!("{agent} has a {label} node")) - }; - let cert = find("both", "mint_agent_identity").id; - let secret = find("both", "renew_agent_queue_credential"); - let when = secret - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == cert => Some(*when), - _ => None, - }) - .expect("the secret waits for the certificate"); - assert!( - when.accepts(hive_jobq::TerminalState::Failed), - "a certificate that could not be re-issued must not hold up the secret" - ); - assert!( - find("secret", "renew_agent_queue_credential") - .deps - .is_empty(), - "a secret on its own waits for nothing" - ); - } - - /// Agent creation mints the matrix account as a root of its own, and the - /// deploy waits for it without being cancelled by it: a host with no - /// homeserver configured must still deploy the agent. - #[tokio::test] - async fn the_matrix_account_mint_is_a_root_and_does_not_block_the_deploy() { - use hive_jobq_wire::WireNode as _; - - let (state, sched) = state_with_roster(); - let _queued = super::create_agent( - axum::extract::State(state), - axum::Json(super::CreateAgentRequest { - name: "atlas".to_owned(), - hive: "pr1ma".to_owned(), - limits: None, - inference: None, - permissions: None, - archetype: None, - }), - ) - .await - .expect("a hive in the roster must be accepted"); - - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let find = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label) - .unwrap_or_else(|| panic!("the graph holds a {label} node")) - }; - let mint = find("mint_agent_matrix_account"); - assert!(mint.deps.is_empty(), "the mint waits for nothing"); - let when = find("trigger_deploy") - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == mint.id => Some(*when), - _ => None, - }) - .expect("the deploy waits for the matrix mint"); - assert!( - when.accepts(hive_jobq::TerminalState::Failed), - "a host with no homeserver must not cancel the deploy; this edge \ - has to be `after_any`, not `after_ok`" - ); - } - - /// The backfill's queueing: one mint node per agent, nothing else. - #[test] - fn a_queued_matrix_mint_is_one_node_per_agent() { - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let ids = super::queue_matrix_account_mints(&sched, vec!["a".to_owned(), "b".to_owned()]) - .expect("two jobs insert"); - assert_eq!(ids.len(), 2); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut agents: Vec = guard - .graph() - .nodes() - .map(|n| { - assert_eq!(n.payload.label(), "mint_agent_matrix_account"); - n.payload.data(n.id.get())["agent"] - .as_str() - .expect("agent is a string") - .to_owned() - }) - .collect(); - agents.sort(); - assert_eq!(agents, ["a", "b"]); - } - - /// The hive sender-token pass's queueing: one mint node per hive, and - /// the node names the hive rather than an agent. - #[test] - fn a_queued_hive_sender_mint_is_one_node_per_hive() { - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let ids = - super::queue_hive_sender_mints(&sched, vec!["alpha".to_owned(), "beta".to_owned()]) - .expect("two jobs insert"); - assert_eq!(ids.len(), 2); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut hives: Vec = guard - .graph() - .nodes() - .map(|n| { - assert_eq!(n.payload.label(), "mint_hive_sender_token"); - let data = n.payload.data(n.id.get()); - assert!(data.get("agent").is_none(), "{data}"); - data["hive"].as_str().expect("hive is a string").to_owned() - }) - .collect(); - hives.sort(); - assert_eq!(hives, ["alpha", "beta"]); - } - - /// The hive forge-token pass's queueing: one node per hive, naming the - /// hive rather than an agent. - #[test] - fn a_queued_hive_forge_token_mint_is_one_node_per_hive() { - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let ids = super::queue_hive_forge_token_mints( - &sched, - vec!["alpha".to_owned(), "beta".to_owned()], - ) - .expect("two jobs insert"); - assert_eq!(ids.len(), 2); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut hives: Vec = guard - .graph() - .nodes() - .map(|n| { - assert_eq!(n.payload.label(), "mint_hive_forge_token"); - let data = n.payload.data(n.id.get()); - assert!(data.get("agent").is_none(), "{data}"); - data["hive"].as_str().expect("hive is a string").to_owned() - }) - .collect(); - hives.sort(); - assert_eq!(hives, ["alpha", "beta"]); - } - - /// The manual route and the periodic pass both go through - /// `queue_forge_token_mints`, so this is the assertion that a backfilled - /// mint is the same pair of nodes agent creation inserts: the forge user, - /// then the mint `after_ok` it, per agent. - #[test] - fn a_queued_mint_creates_the_forge_user_first() { - use hive_jobq_wire::WireNode as _; - - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()]) - .expect("two jobs insert"); - assert_eq!(ids.len(), 2, "one returned id per agent: its mint node"); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let graph = guard.graph(); - let agent_of = |n: &hive_jobq::Node| { - n.payload.data(n.id.get())["agent"] - .as_str() - .expect("agent is a string") - .to_owned() - }; - for agent in ["a", "b"] { - let find = |label: &str| { - graph - .nodes() - .find(|n| n.payload.label() == label && agent_of(n) == agent) - .unwrap_or_else(|| panic!("{agent} has a {label} node")) - }; - let user = find("create_forge_user").id; - let mint = find("mint_agent_forge_token"); - assert!(ids.contains(&mint.id), "the returned id is {agent}'s mint"); - let when = mint - .deps - .iter() - .find_map(|d| match d { - hive_jobq::Dep::Node { id, when } if *id == user => Some(*when), - _ => None, - }) - .unwrap_or_else(|| panic!("{agent}'s mint waits for its own forge user")); - assert!( - !when.accepts(hive_jobq::TerminalState::Failed), - "a token cannot be minted for a user that was never created; this \ - edge has to be `after_ok`" - ); - } - assert_eq!( - graph.nodes().count(), - 4, - "two nodes per agent, nothing else" - ); - } - - /// The backfill end to end, minus the IO: an agent the pass observed with - /// no forge user is planned, and the job it gets creates that user before - /// it mints. - #[test] - fn an_agent_with_no_forge_user_gets_a_user_and_a_mint() { - use crate::forge::agent_token::{Observed, plan}; - use hive_jobq_wire::WireNode as _; - - let agents = plan(&[("ruth".to_owned(), Observed::NoForgeUser)]); - let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( - hive_jobq::Graph::new(), - hive_jobq::resources::ResourceTable::new(), - )); - super::queue_forge_token_mints(&sched, agents).expect("one job inserts"); - let guard = sched - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let mut labels: Vec = guard.graph().nodes().map(|n| n.payload.label()).collect(); - labels.sort(); - assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]); - } - - /// The socket must not share a directory with anything else, because - /// the socket is `0666` and the directory is therefore the only access - /// control it has. `/run/hyperhive` in particular holds hive-c0re's - /// **admin** socket, and nginx — a host service — is bounded by - /// nothing but the directory itself. - /// - /// A test rather than a comment: the failure this guards against is a - /// one-word edit that looks tidier and reads fine in review. - #[test] - fn socket_lives_in_its_own_runtime_dir() { - let parent = Path::new(DEFAULT_SOCKET) - .parent() - .expect("socket path has a parent directory"); - assert_eq!( - parent, - Path::new("/run/swarm-controller"), - "the socket's directory is its access control — moving it under a shared \ - directory (notably /run/hyperhive, which holds the host admin socket) \ - exposes everything else in that directory to the gateway's nginx" - ); - } - - /// One test, not three, deliberately: `HIVES_ENV` is a real process - /// env var, and `cargo test`'s default parallel runner would race - /// separate missing/malformed/valid tests against each other. Driving - /// all three states sequentially inside one test needs no mutex and - /// still proves each branch of `load_hives`. - /// - /// SAFETY: single-threaded mutation of a process env var no other - /// test in this crate reads; restored (removed) before returning. - #[test] - fn load_hives_covers_missing_malformed_and_valid() { - unsafe { - std::env::remove_var(HIVES_ENV); - } - assert_eq!( - load_hives(), - Vec::::new(), - "unset env var is an empty directory, not a startup failure" - ); - - unsafe { - std::env::set_var(HIVES_ENV, "not json"); - } - assert_eq!( - load_hives(), - Vec::::new(), - "unparseable env var falls back to empty rather than panicking — \ - /health must still answer even if this daemon's own config is wrong" - ); - - unsafe { - std::env::set_var( - HIVES_ENV, - r#"[{"name":"pr1ma","domain":"pr1ma.example.com","dashboard":"pr1ma.hives.example"},{"name":"umbra","domain":"umbra.example.com","dashboard":"umbra.hives.example"}]"#, - ); - } - assert_eq!( - load_hives(), - vec![ - HiveEntry { - name: "pr1ma".to_string(), - domain: "pr1ma.example.com".to_string(), - dashboard: "pr1ma.hives.example".to_string(), - }, - HiveEntry { - name: "umbra".to_string(), - domain: "umbra.example.com".to_string(), - dashboard: "umbra.hives.example".to_string(), - }, - ] - ); - - unsafe { - std::env::remove_var(HIVES_ENV); - } - } - - /// Same three-state coverage as `load_hives_covers_missing_malformed_and_valid`, - /// same reason for one test rather than three (a shared process env var). - /// - /// SAFETY: single-threaded mutation of a process env var no other test - /// in this crate reads; restored (removed) before returning. - #[test] - fn load_links_covers_missing_malformed_and_valid() { - unsafe { - std::env::remove_var(LINKS_ENV); - } - assert_eq!( - load_links(), - Vec::::new(), - "unset env var is an empty list, not a startup failure" - ); - - unsafe { - std::env::set_var(LINKS_ENV, "not json"); - } - assert_eq!( - load_links(), - Vec::::new(), - "unparseable env var falls back to empty rather than panicking" - ); - - unsafe { - std::env::set_var( - LINKS_ENV, - r#"[{"label":"Authelia","icon":"🔑","url":"https://auth.example.com/"}]"#, - ); - } - assert_eq!( - load_links(), - vec![ServiceLink { - label: "Authelia".to_string(), - icon: "🔑".to_string(), - url: "https://auth.example.com/".to_string(), - }] - ); - - unsafe { - std::env::remove_var(LINKS_ENV); - } - } - - /// Two states, not three like `load_hives`/`load_links` above: there is - /// nothing to parse here, so no malformed-input branch exists to cover. - /// - /// SAFETY: single-threaded mutation of a process env var no other test - /// in this crate reads; restored (removed) before returning. - #[test] - fn load_swarm_name_covers_missing_and_set() { - unsafe { - std::env::remove_var(NAME_ENV); - } - assert_eq!( - load_swarm_name(), - None, - "unset env var is an unnamed swarm, not a startup failure" - ); - - unsafe { - std::env::set_var(NAME_ENV, "constellat1on"); - } - assert_eq!(load_swarm_name(), Some("constellat1on".to_string())); - - unsafe { - std::env::remove_var(NAME_ENV); - } - } - - // ── NotConnected maps to 503, everything else stays 500 ───────────── - - /// `wanted_error_status` as a pure function first: build the exact - /// `swarm_queue_client::Error::NotConnected` `ensure_connected` raises, - /// wrap it the same way `?` inside `WantedWriter::view`/`set` does, and - /// confirm it downcasts back to the status those routes are meant to - /// answer. - #[test] - fn not_connected_downcasts_to_503() { - let e: anyhow::Error = - swarm_queue_client::Error::NotConnected(async_nats::connection::State::Pending).into(); - assert_eq!( - super::wanted_error_status(&e), - axum::http::StatusCode::SERVICE_UNAVAILABLE - ); - } - - /// Invert-proof for the test above: an `anyhow::Error` that does not - /// wrap a `swarm_queue_client::Error` at all (the shape a genuine write - /// or decode failure takes — `apply`'s "declaration is not decodable" - /// context, for instance) must NOT downcast to `NotConnected`, and must - /// stay 500. Without this, a `wanted_error_status` that answered 503 - /// unconditionally would still pass the test above. - #[test] - fn an_unrelated_error_stays_500() { - let e = anyhow::anyhow!("the hive's current declaration is not decodable"); - assert_eq!( - super::wanted_error_status(&e), - axum::http::StatusCode::INTERNAL_SERVER_ERROR - ); - } - - /// Second invert-proof, closer to the actual bug this closes: a - /// *different* `swarm_queue_client::Error` variant (a real connect - /// failure, not `NotConnected`) also stays 500 — proving the match - /// looks at the variant, not just at "was this crate's error type - /// involved at all". - #[test] - fn a_different_queue_client_error_variant_stays_500() { - let e: anyhow::Error = swarm_queue_client::Error::Connect { - url: "nats://queue.example:4222".to_owned(), - source: async_nats::ConnectErrorKind::TimedOut.into(), - } - .into(); - assert_eq!( - super::wanted_error_status(&e), - axum::http::StatusCode::INTERNAL_SERVER_ERROR - ); - } - - /// A client that exists but has never connected — same shape - /// `matrix_account`'s tests build, duplicated here rather than shared: - /// both are a handful of lines and neither crate has a test-support - /// module to put a shared helper in. - async fn disconnected_client() -> async_nats::Client { - async_nats::ConnectOptions::new() - .retry_on_initial_connect() - .connect("127.0.0.1:1") - .await - .expect("retry_on_initial_connect returns without waiting for a real connection") - } - - /// `state_with_roster` plus a `wanted` writer over a client that never - /// connected — the shape `set_agent_state`/`get_hive_wanted` see when - /// the queue is mid-reconnect. - async fn state_with_disconnected_wanted() -> super::AppState { - let (state, _sched) = state_with_roster(); - super::AppState { - wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( - disconnected_client().await, - ))), - ..state - } - } - - /// End to end through the real handler: `PUT .../state` against a - /// disconnected queue answers 503, not the 500 every other writer - /// error still gets. - #[tokio::test] - async fn set_agent_state_answers_503_on_a_disconnected_queue() { - let state = state_with_disconnected_wanted().await; - - let result = super::set_agent_state( - axum::extract::State(state), - axum::extract::Path(("pr1ma".to_owned(), "atlas".to_owned())), - axum::Json(SetAgentStateRequest { - state: swarm_queue_client::wanted::AgentState::Paused, - }), - ) - .await; - - let problem = result.expect_err("a disconnected queue must not read as success"); - assert_eq!( - problem.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{problem:?}" - ); - } - - /// Same fix, the read side: `GET .../wanted` against a disconnected - /// queue also answers 503. - #[tokio::test] - async fn get_hive_wanted_answers_503_on_a_disconnected_queue() { - let state = state_with_disconnected_wanted().await; - - let result = super::get_hive_wanted( - axum::extract::State(state), - axum::extract::Path("pr1ma".to_owned()), - ) - .await; - - let problem = result.expect_err("a disconnected queue must not read as success"); - assert_eq!( - problem.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{problem:?}" - ); - } - - /// Control for both tests above: a hive that is not in the roster is - /// still a plain 400, even with the same disconnected writer wired up — - /// so the 503 above is `ensure_connected` firing, not every error this - /// handler can produce collapsing to 503. - #[tokio::test] - async fn set_agent_state_still_answers_400_for_an_unknown_hive() { - let state = state_with_disconnected_wanted().await; - - let result = super::set_agent_state( - axum::extract::State(state), - axum::extract::Path(("not-a-hive".to_owned(), "atlas".to_owned())), - axum::Json(SetAgentStateRequest { - state: swarm_queue_client::wanted::AgentState::Paused, - }), - ) - .await; - - let problem = result.expect_err("an unknown hive is still a 400"); - assert_eq!( - problem.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{problem:?}" - ); - } - - /// `GET /api/agents/{name}/icon` against a disconnected queue answers - /// 503 like its siblings, not the 404 an `` would show identically - /// or the 500 of a failed read. - #[tokio::test] - async fn get_agent_icon_answers_503_on_a_disconnected_queue() { - let (state, _sched) = state_with_roster(); - let state = super::AppState { - agent_icons: Some(std::sync::Arc::new( - super::agent_icon::AgentIconReader::new(disconnected_client().await), - )), - ..state - }; - - let problem = super::get_agent_icon( - axum::extract::State(state), - axum::extract::Path("iris".to_owned()), - ) - .await - .expect_err("a disconnected queue must not read as an icon"); - assert_eq!( - problem.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{problem:?}" - ); - } - - /// Every inference verb refuses a name that is not an identifier with a - /// 400 and answers 503 without a queue; a PUT whose body is refused is a - /// 400 before the queue is asked. - #[tokio::test] - async fn the_inference_routes_check_the_name_and_body_before_the_queue() { - use super::inference::{delete_agent_inference, get_agent_inference, put_agent_inference}; - use axum::extract::{Path, State}; - use axum::http::StatusCode; - - fn status(r: Result) -> Option { - r.map(|_| ()).expect_err("must not succeed").status - } - let (state, _sched) = state_with_roster(); - let body = |json: &str| axum::Json(serde_json::from_str(json).expect("decodes")); - for name in ["iris", "Not An Ident"] { - let want = Some(if name == "iris" { - StatusCode::SERVICE_UNAVAILABLE - } else { - StatusCode::BAD_REQUEST - }); - let path = || Path(name.to_owned()); - assert_eq!( - status(get_agent_inference(State(state.clone()), path()).await), - want, - "GET {name}" - ); - assert_eq!( - status( - put_agent_inference(State(state.clone()), path(), body(r#"{"model":"opus"}"#)) - .await - ), - want, - "PUT {name}" - ); - assert_eq!( - status(delete_agent_inference(State(state.clone()), path()).await), - want, - "DELETE {name}" - ); - } - for bad in [r#"{"model":"opus 4"}"#, r#"{"model":"--print"}"#] { - assert_eq!( - status( - put_agent_inference(State(state.clone()), Path("iris".to_owned()), body(bad)) - .await - ), - Some(StatusCode::BAD_REQUEST), - "{bad}" - ); - } - } - - #[tokio::test] - async fn listing_subagents_needs_a_queue_and_an_identifier() { - let (state, _sched) = state_with_roster(); - let no_queue = super::subagent_term::list_subagents( - axum::extract::State(state.clone()), - axum::extract::Path("iris".to_owned()), - ) - .await - .expect_err("no queue is configured"); - assert_eq!( - no_queue.status, - Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), - "{no_queue:?}" - ); - let bad_name = super::subagent_term::list_subagents( - axum::extract::State(state), - axum::extract::Path("iris.sub".to_owned()), - ) - .await - .expect_err("a dot would widen the subject"); - assert_eq!( - bad_name.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{bad_name:?}" - ); - } - - /// A subagent name is a subject token, so a wildcard or a dot in it is - /// refused before anything subscribes. - #[tokio::test] - async fn a_subagent_stream_refuses_a_name_that_is_not_one_token() { - let (state, _sched) = state_with_roster(); - for subagent in ["*", ">", "a.b"] { - let Err(problem) = super::subagent_term::stream_subagent_term( - axum::extract::State(state.clone()), - axum::extract::Path(("iris".to_owned(), subagent.to_owned())), - axum::http::HeaderMap::new(), - ) - .await - else { - panic!("{subagent:?} must be refused"); - }; - assert_eq!( - problem.status, - Some(axum::http::StatusCode::BAD_REQUEST), - "{subagent:?}: {problem:?}" - ); - } - } -} - -#[cfg(test)] -mod mint_identity_tests { - use std::sync::{Arc, Mutex}; - - use hive_jobq::scheduler::Outcome; - use tracing_subscriber::layer::SubscriberExt as _; - - use super::mint_identity; - - /// One event's level and its fields, each by name. - type Event = (tracing::Level, Vec<(String, String)>); - - /// Every event. - #[derive(Clone, Default)] - struct Events(Arc>>); - - impl tracing_subscriber::Layer for Events { - fn on_event( - &self, - event: &tracing::Event<'_>, - _: tracing_subscriber::layer::Context<'_, S>, - ) { - let mut fields = Vec::new(); - event.record( - &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { - fields.push((field.name().to_owned(), format!("{value:?}"))); - }, - ); - self.0 - .lock() - .expect("unpoisoned") - .push((*event.metadata().level(), fields)); - } - } - - /// An agent name the store would refuse fails the mint before any store - /// call, whatever this process's environment holds. The literal is the - /// `_msg` that `hh-agent-identity-mint-failed` in - /// `nix/host-modules/swarm-grafana.nix` matches. - #[tokio::test] - async fn a_failed_mint_logs_the_agent_and_the_error_on_the_line_the_alert_rule_matches() { - let events = Events::default(); - let _guard = - tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone())); - - let Outcome::Failed(error) = mint_identity("a/b").await else { - panic!("an illegal agent name cannot be minted"); - }; - - let got = events.0.lock().expect("unpoisoned").clone(); - let [(level, fields)] = got.as_slice() else { - panic!("one event, got {got:?}"); - }; - assert_eq!(*level, tracing::Level::WARN); - let field = |name: &str| { - fields - .iter() - .find(|(n, _)| n == name) - .map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone()) - }; - assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed"); - assert_eq!(field("agent"), "a/b"); - assert_eq!(field("error"), error); - } -} diff --git a/swarm-controller/src/matrix_account.rs b/swarm-controller/src/matrix_account.rs index 0905682f..c11e5e9e 100644 --- a/swarm-controller/src/matrix_account.rs +++ b/swarm-controller/src/matrix_account.rs @@ -34,7 +34,8 @@ use swarm_secret_client::matrix; use utoipa::ToSchema; use super::linked_accounts::{AccountStore, Linking, link, refuse_linked}; -use super::{AppState, error_problem, swarm_hive}; +use crate::routes::agents::swarm_hive; +use crate::routes::{AppState, error_problem}; pub mod agent_token; pub mod hive_sender; @@ -445,9 +446,9 @@ mod tests { /// Bare-minimum `AppState` for a handler test: one hive, nothing /// queue-backed, an empty in-memory job graph the endpoint under test /// never touches, and `store` as the account store. - fn state(store: &Arc) -> super::super::AppState { - super::super::AppState { - hives: std::sync::Arc::new(vec![super::super::HiveEntry { + fn state(store: &Arc) -> crate::routes::AppState { + crate::routes::AppState { + hives: std::sync::Arc::new(vec![crate::routes::hives::HiveEntry { name: "pr1ma".to_owned(), domain: "pr1ma.example".to_owned(), dashboard: "pr1ma.hives.example".to_owned(), diff --git a/swarm-controller/src/pulls.rs b/swarm-controller/src/pulls.rs index 4aa20651..acb806bf 100644 --- a/swarm-controller/src/pulls.rs +++ b/swarm-controller/src/pulls.rs @@ -15,7 +15,7 @@ use utoipa::IntoParams; use crate::forge::{self, OpenPull}; use crate::issue_report::issue_report_error_status; use crate::review_classifier::split_repo; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// Whose open pull requests to list. #[derive(Debug, Deserialize, IntoParams)] @@ -117,9 +117,9 @@ mod tests { use tower::ServiceExt as _; async fn get(uri: &str) -> (StatusCode, serde_json::Value) { - let (state, _) = crate::tests::state_with_roster(); + let (state, _) = crate::routes::tests::state_with_roster(); let req = Request::get(uri).body(Body::empty()).expect("a request"); - let res = crate::build_app(state) + let res = crate::startup::build_app(state) .oneshot(req) .await .expect("infallible"); diff --git a/swarm-controller/src/review_classifier.rs b/swarm-controller/src/review_classifier.rs index 2a429bbe..9603257b 100644 --- a/swarm-controller/src/review_classifier.rs +++ b/swarm-controller/src/review_classifier.rs @@ -38,7 +38,7 @@ use utoipa::{IntoParams, ToSchema}; use self::settings::{Settings, SettingsSource}; use crate::forge::{self, DefaultBranchFiles, PullForReview}; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; pub(crate) mod open_pr; pub(crate) mod review; @@ -715,7 +715,7 @@ async fn start( fn settings_problem(e: &anyhow::Error) -> ProblemDetails { let detail = format!("{e:#}"); tracing::warn!(error = %detail, "reading the review classifier's settings for a try-out failed"); - error_problem(crate::wanted_error_status(e), &detail) + error_problem(crate::routes::wanted_error_status(e), &detail) } /// 404 when the forge knows no such repo or PR, 503 for anything else. @@ -915,9 +915,9 @@ pub(crate) mod tests { } fn app_with_files(reply: Reply, files: Vec<(&'static str, String)>) -> axum::Router { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.try_outs.fake = Some(Arc::new(Fake::new(reply, files))); - crate::build_app(state) + crate::startup::build_app(state) } fn body(repo: &str, pr: u64) -> String { @@ -1001,7 +1001,7 @@ pub(crate) mod tests { cache_write: None, }, }; - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); let fake = Arc::new(Fake { settings: Settings { model: "opus".to_owned(), @@ -1013,7 +1013,7 @@ pub(crate) mod tests { ..Fake::new(done(), Vec::new()) }); state.try_outs.fake = Some(Arc::clone(&fake)); - let app = crate::build_app(state); + let app = crate::startup::build_app(state); let (status, accepted) = post(&app, body("o/r", 7)).await; assert_eq!(status, StatusCode::ACCEPTED, "{accepted}"); settled(&app, accepted["id"].as_str().expect("an id")).await; @@ -1051,10 +1051,10 @@ pub(crate) mod tests { (failed, runs::RunOutcome::Failed, Some("no API key")), ] { let doc = Arc::new(runs::tests::FakeDoc::default()); - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.try_outs.fake = Some(Arc::new(Fake::new(reply.clone(), Vec::new()))); state.runs = runs::Runs::Fake(Arc::clone(&doc)); - let app = crate::build_app(state); + let app = crate::startup::build_app(state); let (_, accepted) = post(&app, body("o/r", 7)).await; let id = accepted["id"].as_str().expect("an id"); settled(&app, id).await; @@ -1481,8 +1481,8 @@ classifications: #[tokio::test] async fn rules_without_forge_access_on_this_host_are_503() { - let (state, _) = crate::tests::state_with_roster(); - let (status, problem) = get_rules(&crate::build_app(state), "?repo=o/r").await; + let (state, _) = crate::routes::tests::state_with_roster(); + let (status, problem) = get_rules(&crate::startup::build_app(state), "?repo=o/r").await; assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{problem}"); assert_eq!(problem["detail"], "no forge access on this host yet"); } diff --git a/swarm-controller/src/review_classifier/open_pr.rs b/swarm-controller/src/review_classifier/open_pr.rs index 03ba680a..294d0231 100644 --- a/swarm-controller/src/review_classifier/open_pr.rs +++ b/swarm-controller/src/review_classifier/open_pr.rs @@ -32,7 +32,7 @@ use utoipa::ToSchema; use super::{FileSource, RULES_PATHS, check_classifications, operator, parse_rules, split_repo}; use crate::forge::classifier_account::{BranchCommit, CommitAuthor, LOGIN}; use crate::forge::{self, BranchFile}; -use crate::{AppState, error_problem}; +use crate::routes::{AppState, error_problem}; /// Where a branch's name starts, so the forge's branch list groups them. const BRANCH_PREFIX: &str = "review-classifier/"; @@ -676,9 +676,9 @@ classifications: } fn app(fake: Fake) -> axum::Router { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.try_outs.fake = Some(Arc::new(fake)); - crate::build_app(state) + crate::startup::build_app(state) } fn fake() -> Fake { diff --git a/swarm-controller/src/review_classifier/review.rs b/swarm-controller/src/review_classifier/review.rs index f9ec0a3a..5c58a1c9 100644 --- a/swarm-controller/src/review_classifier/review.rs +++ b/swarm-controller/src/review_classifier/review.rs @@ -33,10 +33,10 @@ use tokio::task::JoinHandle; use super::runs::{self, RunOutcome, RunRecord, Runs, Trigger}; use super::settings::{Settings, SettingsSource}; use super::{FileSource, PullSource, RunSink, split_repo}; -use crate::AppState; use crate::agent_renewal::unix_now; use crate::forge::pull_diff::CONTEXT_LINES; use crate::forge::{self, classifier_account::LOGIN, classifier_account::Review}; +use crate::routes::AppState; /// The most review runs waiting for a slot. const MAX_WAITING_REVIEWS: usize = 8; diff --git a/swarm-controller/src/review_classifier/runs.rs b/swarm-controller/src/review_classifier/runs.rs index eb6c3864..b2d8aeed 100644 --- a/swarm-controller/src/review_classifier/runs.rs +++ b/swarm-controller/src/review_classifier/runs.rs @@ -23,8 +23,8 @@ use serde::{Deserialize, Serialize}; use swarm_queue_client::review_classifier::{Overall, Reply, Row}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem}; use crate::schedules::{read_with, write}; -use crate::{AppState, error_problem}; /// The bucket holding the runs. Agents hold no grant on it. pub const BUCKET: &str = "review-classifier-runs"; @@ -593,9 +593,9 @@ pub(crate) mod tests { push(&*doc, &reviewed).await.expect("stored"); let get = |runs: Runs| async move { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.runs = runs; - let res = crate::build_app(state) + let res = crate::startup::build_app(state) .oneshot( Request::get("/api/review-classifier/runs") .body(Body::empty()) diff --git a/swarm-controller/src/review_classifier/settings.rs b/swarm-controller/src/review_classifier/settings.rs index aa7d19f1..47478b14 100644 --- a/swarm-controller/src/review_classifier/settings.rs +++ b/swarm-controller/src/review_classifier/settings.rs @@ -24,8 +24,8 @@ use swarm_queue_client::agent_inference::Effort; use swarm_queue_client::review_classifier::{Acp, Runtime}; use utoipa::ToSchema; +use crate::routes::{AppState, error_problem, wanted_error_status}; use crate::schedules::{queue, read_with, refused, write}; -use crate::{AppState, error_problem, wanted_error_status}; /// The bucket holding the settings. No other service holds a grant on it. pub const BUCKET: &str = "review-classifier-settings"; @@ -598,12 +598,12 @@ mod tests { /// An app whose queue is `nats`. async fn app(nats: &TestNats) -> (axum::Router, AppState) { - let (mut state, _) = crate::tests::state_with_roster(); + let (mut state, _) = crate::routes::tests::state_with_roster(); state.status = Some(std::sync::Arc::new(crate::status::StatusReader::new( nats.client().await, std::time::Duration::from_mins(1), ))); - (crate::build_app(state.clone()), state) + (crate::startup::build_app(state.clone()), state) } async fn send( @@ -777,21 +777,21 @@ mod tests { #[tokio::test] async fn a_put_without_an_operator_is_401() { - let (state, _) = crate::tests::state_with_roster(); + let (state, _) = crate::routes::tests::state_with_roster(); let req = Request::put("/api/review-classifier/settings") .header("content-type", "application/json") .body(Body::from( serde_json::to_string(&Settings::default()).unwrap(), )) .expect("a request"); - let (status, _, _) = send(&crate::build_app(state), req).await; + let (status, _, _) = send(&crate::startup::build_app(state), req).await; assert_eq!(status, StatusCode::UNAUTHORIZED); } #[tokio::test] async fn without_a_queue_both_verbs_are_503() { - let (state, _) = crate::tests::state_with_roster(); - let app = crate::build_app(state); + let (state, _) = crate::routes::tests::state_with_roster(); + let app = crate::startup::build_app(state); let (status, _, _) = send(&app, get()).await; assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); let body = serde_json::to_string(&Settings::default()).unwrap(); diff --git a/swarm-controller/src/routes/agents.rs b/swarm-controller/src/routes/agents.rs new file mode 100644 index 00000000..49cff3aa --- /dev/null +++ b/swarm-controller/src/routes/agents.rs @@ -0,0 +1,2404 @@ +//! Creating agents, declaring their wanted state, and reading the roster, +//! their status and their icons. + +use std::sync::Arc; + +use anyhow::Result; +use axum::{ + Json, + extract::{Path, State}, +}; +use serde::{Deserialize, Serialize}; +use swarm_controller_api::agents::CreateAgentResponse; +use utoipa::ToSchema; + +use crate::jobq::{SwarmNodeKind, SwarmResourceKind, declare_agent_job}; +use crate::routes::hives::HiveEntry; +use crate::routes::{AppState, StatusUnavailable, error_problem, wanted_error_status}; +use crate::{ + agent_archetype, agent_identity, agent_limits, agent_permissions, agent_status, auth, + inference, schedules, wanted, +}; + +/// The swarm's agent roster — every agent the swarm holds an identity for. +/// +/// The identity store **is** the roster rather than one source to assemble +/// one from: an agent without a swarm identity is not a swarm agent, so +/// there is no hive-side list to merge in and no reconciliation to do. +/// +/// Deliberately says nothing about health. A roster is the set other views +/// are *complete against* — it is what makes "this agent has never reported" +/// expressible at all, and that distinction only survives while the declared +/// set and the reported set stay separate. +/// +/// A hive-less swarm answers `[]`; a swarm with no bridge answers 503. Those +/// are different facts and collapsing them would render an unreadable store +/// as an empty one. +#[utoipa::path( + get, + path = "/api/agents", + responses( + (status = 200, description = "every agent the swarm holds an identity for", body = Vec), + (status = 503, description = "no identity bridge is configured here, or its store could not be read", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_agents( + State(state): State, +) -> Result>, StatusUnavailable> { + let Some(bridge) = state.auth.as_ref() else { + return Err(StatusUnavailable( + "no identity bridge is configured on this host".to_owned(), + )); + }; + match bridge.list_agent_identities().await { + Ok(agents) => Ok(Json(agents)), + Err(e) => { + let detail = format!("{e:#}"); + tracing::warn!(error = %detail, "reading the agent roster failed"); + Err(StatusUnavailable(detail)) + } + } +} + +/// The state to declare for one agent. +#[derive(Debug, Deserialize, ToSchema)] +pub(crate) struct SetAgentStateRequest { + /// Typed as a string in the schema only — the parse is the real enum, so + /// a value this build does not know is a 400 rather than a field that + /// silently does nothing. + #[schema(value_type = String, example = "up")] + pub(crate) state: swarm_queue_client::wanted::AgentState, +} + +/// One agent's line in a hive's declaration. +#[derive(Clone, Debug, Serialize, ToSchema)] +pub(crate) struct AgentDeclaration { + agent: String, + state: String, +} + +pub(crate) fn render( + declaration: &swarm_queue_client::wanted::HiveWanted, +) -> Vec { + declaration + .agents + .iter() + .map(|(agent, wanted)| AgentDeclaration { + agent: agent.clone(), + state: <&str>::from(wanted.state).to_owned(), + }) + .collect() +} + +/// A hive name that is shaped like one and names a hive this swarm has. +/// +/// Reports the status and the detail rather than a rendered +/// `ProblemDetails`: that type is 232 bytes, which makes every `Result` in +/// this path pay for the error case it usually does not take. The handlers +/// render at the boundary, where the body is actually needed. +pub(crate) fn swarm_hive( + state: &AppState, + hive: &str, +) -> Result { + let hive = hive_types::Ident::parse(hive) + .map_err(|reason| (axum::http::StatusCode::BAD_REQUEST, reason.to_owned()))? + .into_string(); + // Shaped like a hive name, and actually one. Same two checks + // `create_agent` makes, for the same reason: a typo otherwise publishes a + // declaration under a key no hive will ever read. + if !state.hives.iter().any(|h| h.name == hive) { + return Err(( + axum::http::StatusCode::BAD_REQUEST, + format!("{hive:?} is not a hive in this swarm"), + )); + } + Ok(hive) +} + +/// Both declaration handlers below take the same hive name and reject it the +/// same way, and need the writer besides. +fn declaration_target( + state: &AppState, + hive: &str, +) -> Result<(Arc, String), (axum::http::StatusCode, String)> { + // Before the name checks, so a deployment with no queue answers 503 + // whatever the caller spelled. + let writer = state.wanted.clone().ok_or_else(|| { + ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "this deployment wired up no swarm queue, so there is nowhere to publish a declaration" + .to_owned(), + ) + })?; + Ok((writer, swarm_hive(state, hive)?)) +} + +/// Declare what this swarm wants of one agent on one hive. +/// +/// The whole declaration is returned as published, because the value is the +/// hive's entire agent map and a caller that changed one agent still wants to +/// render the rest. +#[utoipa::path( + put, + path = "/api/hives/{hive}/agents/{agent}/state", + params( + ("hive" = String, Path, description = "hive whose declaration this is"), + ("agent" = String, Path, description = "agent to declare"), + ), + request_body = SetAgentStateRequest, + responses( + (status = 200, description = "the declaration as now published", body = Vec), + (status = 400, description = "a name is not an identifier, the hive is not in this swarm, the state is unknown, or `agent` is new and reserved or refused by the forge (problem+json)", body = String), + (status = 409, description = "the swarm has already placed `agent` on a different hive (problem+json)", body = String), + (status = 503, description = "no swarm queue is wired up, or it is not connected, or `agent` is new here and breaks a naming rule but whether it already exists could not be checked (problem+json)", body = String), + (status = 500, description = "the declaration could not be published, or another hive's wanted state could not be read (problem+json)", body = String), + ), + tag = "agents" +)] +pub(crate) async fn set_agent_state( + State(state): State, + axum::extract::Path((hive, agent)): axum::extract::Path<(String, String)>, + Json(req): Json, +) -> Result>, problem_details::ProblemDetails> { + let (writer, hive) = + declaration_target(&state, &hive).map_err(|(s, d)| error_problem(s, &d))?; + let agent = hive_types::Ident::parse(&agent) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + + // Only a first declaration of the name on this hive is checked, against + // what `create_agent` refuses: an agent already declared here changes + // state whatever its name. An undeclared name that passes is accepted: + // that is how the swarm adopts an agent that predates swarm-level + // creation. Deliberate, pending the operator's decision on whether a name + // the swarm never placed should be refused here instead. + // + // The gate is held from the checks through the write. A destroy places + // nothing, so it is not checked and does not wait on creations. + let _gate = if places_agent(req.state) { + let gate = state.create_gate.lock().await; + let here = writer.view(&hive).await.map_err(|e| { + let detail = format!( + "cannot tell whether {agent:?} is already declared on hive {hive:?}, so it was \ + not placed: {e:#}" + ); + error_problem(wanted_error_status(&e), &detail) + })?; + refuse_placement( + &agent, + &hive, + here.as_ref(), + hive_types::reserved_names_raw().as_deref(), + || in_roster(state.auth.as_deref(), &agent), + || queued_placements_now(&state), + || declarations_elsewhere(&state, &agent, &hive), + ) + .await?; + Some(gate) + } else { + None + }; + + let declaration = writer.set(&hive, &agent, req.state).await.map_err(|e| { + tracing::warn!(hive = %hive, agent = %agent, error = %format!("{e:#}"), "declaring agent state failed"); + error_problem(wanted_error_status(&e), &format!("{e:#}")) + })?; + + // After the declaration and never before it: the write above is the + // destroy order, and until it lands the agent is not being torn down, so + // its credential, permissions and archetype have to keep working. Sequenced + // rather than spawned so every attempt is finished — and their outcomes + // logged — by the time the operator's call returns. + if revokes_queue_credential(req.state) { + revoke_or_complain(&hive, &agent).await; + agent_permissions::clear_on_destroy(&state, &hive, &agent).await; + agent_archetype::clear_on_destroy(&state, &hive, &agent).await; + } + Ok(Json(render(&declaration))) +} + +/// Whether a declaration in `state` places the agent on its hive. +/// +/// Exhaustive for the reason [`revokes_queue_credential`] is. A `Destroyed` +/// agent is gone from that hive, and the hive deploys nothing for it. +fn places_agent(state: swarm_queue_client::wanted::AgentState) -> bool { + use swarm_queue_client::wanted::AgentState; + match state { + AgentState::Up | AgentState::Offline | AgentState::Paused => true, + AgentState::Destroyed => false, + } +} + +/// Refuse a first placement of `agent` on `hive` that `create_agent` +/// refuses: 400 for a new name that breaks a naming rule, 409 for a name the +/// swarm has placed on another hive. An agent `here`, the hive's current +/// declaration, already declares in a placing state is placed on this hive, +/// and passes. +/// +/// The roster is read only for a name that breaks a rule. Unlike in +/// `create_agent`, no later step refuses a bad name, so a roster that cannot +/// be read refuses (503) rather than warns. +async fn refuse_placement( + agent: &str, + hive: &str, + here: Option<&swarm_queue_client::wanted::HiveWanted>, + reserved: Option<&str>, + read_roster: impl FnOnce() -> R, + snapshot_queued: impl FnOnce() -> Vec<(String, String)>, + read_declared: impl FnOnce() -> D, +) -> Result<(), problem_details::ProblemDetails> +where + R: std::future::Future>, + D: std::future::Future< + Output = Result< + Vec<(String, swarm_queue_client::wanted::HiveWanted)>, + problem_details::ProblemDetails, + >, + >, +{ + let declared_here = here + .and_then(|declaration| declaration.agents.get(agent)) + .is_some_and(|wanted| places_agent(wanted.state)); + if declared_here { + return Ok(()); + } + // Both calls log what they push here; this response has nowhere to + // carry warnings. + let mut warnings = Vec::new(); + let broken = broken_name_rules(agent, reserved, &mut warnings); + if !broken.is_empty() { + let existing = read_roster().await.map_err(|why| { + let detail = format!( + "{}; whether {agent:?} is an existing agent, which would keep its name, could \ + not be checked, so it was not placed: {why}", + broken.join("; ") + ); + error_problem(axum::http::StatusCode::SERVICE_UNAVAILABLE, &detail) + })?; + name_verdict(agent, &broken, Ok(existing), &mut warnings) + .map_err(|detail| error_problem(axum::http::StatusCode::BAD_REQUEST, &detail))?; + } + let elsewhere = placements_elsewhere(agent, hive, snapshot_queued, read_declared).await?; + if !elsewhere.is_empty() { + let detail = format!( + "agent {agent:?} is placed on hive {}, not {hive:?} — agent names are unique across \ + the swarm; declare its state there, or destroy it there first to move it", + elsewhere.join(", ") + ); + return Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)); + } + Ok(()) +} + +/// Whether declaring an agent into `state` ends its queue credential's life. +/// +/// Exhaustive on purpose, like every other match on +/// [`AgentState`][swarm_queue_client::wanted::AgentState] in this +/// tree: a state added later has to say whether it revokes rather than +/// inheriting "no" from a catch-all. `Offline` and `Paused` deliberately do +/// not — both are states an agent comes back from, and revoking on either +/// would mean an agent that could be stopped but never restarted. +fn revokes_queue_credential(state: swarm_queue_client::wanted::AgentState) -> bool { + use swarm_queue_client::wanted::AgentState; + match state { + AgentState::Destroyed => true, + AgentState::Up | AgentState::Offline | AgentState::Paused => false, + } +} + +/// Revoke the credential, and make a failure impossible to miss without +/// letting it stop the teardown. +/// +/// **The destroy is not blocked by this.** The declaration is already +/// published, the hive will converge on it, and refusing the operator's call +/// at this point would leave them with an agent they cannot destroy — the +/// larger of the two harms. What is left is that the failure must not be +/// quiet, or the outcome is the silent orphan this whole path exists to +/// remove: hence `error`, naming the agent, with the cause chain attached. +async fn revoke_or_complain(hive: &str, agent: &str) { + if let Err(e) = agent_identity::revoke_queue_credential(agent).await { + tracing::error!( + hive, + agent, + error = %format!("{e:#}"), + "revoking this agent's queue credential FAILED; it was destroyed anyway, so \ + the credential outlives it and may still authenticate. Declaring the agent \ + destroyed again re-runs this revocation." + ); + } +} + +/// What this swarm currently declares for a hive. +/// +/// Read back from the bucket rather than from a second copy kept here: the +/// published value is the record. +#[utoipa::path( + get, + path = "/api/hives/{hive}/wanted", + params(("hive" = String, Path, description = "hive whose declaration to read")), + responses( + (status = 200, description = "the declaration, empty when nothing is published yet", body = Vec), + (status = 400, description = "not an identifier, or not a hive in this swarm (problem+json)", body = String), + (status = 503, description = "no swarm queue is wired up, or it is not connected (problem+json)", body = String), + (status = 500, description = "the declaration could not be read (problem+json)", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_hive_wanted( + State(state): State, + axum::extract::Path(hive): axum::extract::Path, +) -> Result>, problem_details::ProblemDetails> { + let (writer, hive) = + declaration_target(&state, &hive).map_err(|(s, d)| error_problem(s, &d))?; + let declaration = writer.view(&hive).await.map_err(|e| { + tracing::warn!(hive = %hive, error = %format!("{e:#}"), "reading the declaration failed"); + error_problem(wanted_error_status(&e), &format!("{e:#}")) + })?; + Ok(Json(declaration.as_ref().map(render).unwrap_or_default())) +} + +/// What each agent last said about itself, its open config PR if any, and +/// its declared wanted state if any, read at request time — the single +/// call swarm-ui's agent roster page fills its whole table from (mara, +/// 2026-09-02: "the view should be filled by a single backend call"). +/// +/// Every agent in the roster gets a row whether or not it has ever +/// reported — see the `agent_status` module for why, and for why its hive +/// comes from its own bucket key rather than a second lookup. `config_pr` +/// and `wanted` are both merged in here rather than inside +/// `agent_status::AgentStatusReader`: that module has no forge client or +/// wanted-state writer and is not the place to add one just for a single +/// field, whereas this handler already holds `AppState::config_prs` and +/// `AppState::wanted` alongside the roster-shaped status view — the one +/// spot that has all three without adding an avoidable dependency to a +/// reader that stays deliberately narrow. Both fields are absent on rows +/// where the underlying store isn't configured or has nothing on record, +/// same "absence is the answer" shape `get_config_prs` uses — not a +/// reason to fail the whole response over a field that already knows how +/// to be missing. +#[utoipa::path( + get, + path = "/api/agents/status", + responses( + (status = 200, description = "a row per agent, freshness + config PR derived now", body = Vec), + (status = 503, description = "no swarm queue is configured here, no identity bridge is configured, or either store could not be read", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_agents_status( + State(state): State, +) -> Result>, StatusUnavailable> { + let Some(reader) = state.agent_status.as_ref() else { + return Err(StatusUnavailable( + "no swarm queue is configured on this host".to_owned(), + )); + }; + let Some(bridge) = state.auth.as_ref() else { + return Err(StatusUnavailable( + "no identity bridge is configured on this host".to_owned(), + )); + }; + let roster = bridge.list_agent_identities().await.map_err(|e| { + let detail = format!("{e:#}"); + tracing::warn!(error = %detail, "reading the agent roster failed"); + StatusUnavailable(detail) + })?; + match reader.view(&roster, std::time::SystemTime::now()).await { + Ok(mut rows) => { + if let Some(cache) = state.config_prs.get() { + let mut config_prs = cache.snapshot(); + for row in &mut rows { + row.config_pr = config_prs.remove(&row.name); + } + } + // `wanted` merges in the same way and for the same reason as + // `config_pr` above — one read per distinct hive, not one per + // agent, since a declaration is already a hive's whole agent + // map. + if let Some(writer) = state.wanted.as_ref() { + // Owned `String` keys, not `&str` borrowed from `rows` — + // `declarations` outlives the loop below that needs `rows` + // mutably, so it cannot hold a live borrow into it. + let hives: std::collections::BTreeSet = + rows.iter().filter_map(|r| r.hive.clone()).collect(); + let mut declarations: std::collections::HashMap< + String, + swarm_queue_client::wanted::HiveWanted, + > = std::collections::HashMap::new(); + for hive in hives { + match writer.view(&hive).await { + Ok(Some(declaration)) => { + declarations.insert(hive, declaration); + } + Ok(None) => {} + Err(e) => { + // Missing wanted state for one hive shouldn't + // fail the whole roster — same "absence is + // survivable" rule `config_pr` follows above. + tracing::warn!( + hive, error = %format!("{e:#}"), + "reading the wanted declaration failed; its agents show no wanted state" + ); + } + } + } + for row in &mut rows { + row.wanted = row.hive.as_deref().and_then(|hive| { + declarations + .get(hive)? + .agents + .get(&row.name) + .map(|w| <&str>::from(w.state).to_owned()) + }); + } + } + if let Some(login) = state.agent_login.as_ref() { + for row in &mut rows { + row.needs_login = login.get(&row.name); + } + } + Ok(Json(rows)) + } + Err(e) => { + let detail = format!("{e:#}"); + tracing::warn!(error = %detail, "reading the agent-status bucket failed"); + Err(StatusUnavailable(detail)) + } + } +} + +/// Body of `POST /api/agents` — the agent name to create, and the hive the +/// creation is aimed at. The repo name inside `forge::CONFIG_ORG` is the +/// same string as `name`: one repo per agent, named after it, same +/// convention `hive-c0re::forge` already uses for its own single-hive +/// `CreateRepo` path. +/// +/// `hive` is an **address**, not an attribute of the agent: it is where a +/// deploy message goes over the queue. Nothing here writes it into the +/// agent's config — a config naming its own hive would be a second +/// statement of where the agent lives, free to drift from the queue that +/// actually delivers to it. +/// +/// It is required rather than optional because it is only knowable here, +/// at creation, from the operator making the choice. An optional field +/// would have been the friendlier migration and would have left every +/// creation in the state this endpoint exists to avoid — one nothing can +/// address. +/// +/// Validated here and carried no further: none of the nodes this endpoint +/// queues talks to a hive, so none of them needs the address. It reaches +/// its consumer when the node that *sends* a deploy message exists, and +/// that node takes it from this field. Settling the request shape now is +/// the point — it is the breaking half, and doing it once is cheaper for +/// every caller than doing it again later. +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub(crate) struct CreateAgentRequest { + pub(crate) name: String, + pub(crate) hive: String, + /// CPU and memory caps to set for the agent, as `PUT + /// /api/agents/{name}/limits` takes them. Absent leaves any caps already + /// stored for the name alone. + #[serde(default)] + pub(crate) limits: Option, + /// Inference settings to set for the agent, as `PUT + /// /api/agents/{name}/inference` takes them. Absent as `limits`. + #[serde(default)] + pub(crate) inference: Option, + /// The agent's permissions, stored before the job is queued. Left out, the + /// agent holds the default set. + #[serde(default)] + #[schema(value_type = Option>)] + pub(crate) permissions: Option>, + /// The archetype to build the agent from, one `GET /api/archetypes` + /// lists. The agent then gets no `agent-configs/` repo: its hive + /// builds it from the archetype's module. Absent is a custom agent. + #[serde(default)] + #[schema(value_type = Option, example = json!({"source": "hyperhive", "name": "reviewer"}))] + pub(crate) archetype: Option, +} + +/// Queue the whole agent-creation job graph for `name` — two independent +/// roots, `CreateIdentity` and `CreateForgeUser`, since an authelia +/// subject and a forge account need nothing from each other. `CreateRepo` +/// runs once `CreateIdentity` succeeds; once THAT succeeds, +/// `AddRepoMember` and `InitAgentConfigRepo` both run off it — except +/// `AddRepoMember` also waits on `CreateForgeUser`, because adding a +/// collaborator that does not yet exist as a forge user is a Forgejo +/// validation error, not an idempotent no-op. `InitAgentConfigRepo` has no +/// such second dependency: seeding files uses this daemon's own forge +/// token, never the agent's, so it only ever needed the repo (mara, design +/// review: it's a jobq graph, not a linear chain). An agent created from an +/// `archetype` has no config repo, so its graph has none of the three repo +/// nodes. Returns as soon as the graph is inserted — **not** once any of it +/// has run; `run_swarm_node` does that asynchronously off the scheduler loop +/// already running +/// (`spawn_jobq_worker`). The response reports `CreateIdentity`'s id, one +/// of the graph's two roots — a caller watches the whole thing settle via +/// `/api/jobq/graph`, which serves every root, not just this one. +/// +/// `name` is validated with [`hive_types::Ident::parse`] before it becomes +/// `agent`/`repo` anywhere downstream — not the *only* gate (`CreateIdentity` +/// validates server-side too), but a few hops removed from where an +/// unvalidated name would do damage (`forge::seed_agent_config` +/// interpolates `agent` into a nix comment line). Fragile if a second +/// caller of these nodes ever appears; validating here closes it locally. +#[utoipa::path( + post, + path = "/api/agents", + request_body = CreateAgentRequest, + responses( + (status = 200, description = "job chain queued", body = CreateAgentResponse), + (status = 400, description = "`name` or `hive` is not a valid identifier, `name` is new and reserved or refused by the forge, `hive` is not in this swarm, `limits` or `inference` holds a value the matching `PUT` refuses, a permission is listed twice, or `archetype` is not one the stored archetype list holds (problem+json)", body = String), + (status = 409, description = "the swarm has already placed `name` on a different hive (problem+json)", body = String), + (status = 422, description = "an unknown field in the body, in `limits`, or in `inference`, an `effort` that is not one of the five levels, or a `permissions` entry that is not a permission name"), + (status = 503, description = "the swarm queue is not connected, so whether `name` is placed on another hive is unknown, or `limits`, `inference`, `permissions` or `archetype` is set and cannot be stored (problem+json)", body = String), + (status = 500, description = "the job chain could not be queued, or another hive's wanted state could not be read (problem+json)", body = String), + ), + tag = "agents" +)] +pub(crate) async fn create_agent( + State(state): State, + Json(req): Json, +) -> Result, problem_details::ProblemDetails> { + let agent = hive_types::Ident::parse(&req.name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + + // Settings are checked before anything is created, so a bad value stops + // the request rather than leaving an agent without them. + let settings = CreateSettings::parse(req.limits, req.inference, req.permissions, req.archetype) + .map_err(|why| error_problem(axum::http::StatusCode::BAD_REQUEST, &why))?; + + // Two checks, and the second is the one that makes the field worth + // having: `Ident::parse` says the string is *shaped* like a hive name, + // and the roster says it *is* one. Without the roster check a typo is + // accepted, the agent gets created, and nothing notices until a deploy + // message is addressed to a hive that does not exist — by which point + // the operator who could have corrected it in one keystroke is gone. + // + // `state.hives` is the swarm directory loaded from `SWARM_CONTROLLER_HIVES` + // at startup and never mutated, so this is a scan of a handful of + // entries against a value an operator just chose. + // The name is shaped like an identifier; the checks below ask whether it + // is *available*. An operator with agents already created under a + // colliding name must still be able to re-run creation, so a name that + // breaks a naming rule is refused only when it is new — see + // `name_verdict`. The hive collision further down still only warns, on + // its grace period, once the warning has had time to be seen. + // + // Collected rather than logged-and-dropped — see `CreateAgentResponse`. + let mut warnings = Vec::new(); + let broken = broken_name_rules( + &agent, + hive_types::reserved_names_raw().as_deref(), + &mut warnings, + ); + if !broken.is_empty() { + let existing = in_roster(state.auth.as_deref(), &agent).await; + name_verdict(&agent, &broken, existing, &mut warnings) + .map_err(|detail| error_problem(axum::http::StatusCode::BAD_REQUEST, &detail))?; + } + + let hive = hive_types::Ident::parse(&req.hive) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + + // The roster is the same source the `hive` field is validated against + // immediately below — one scan, asked of the other field. A hive and an + // agent sharing a name collide in the swarm's own directory keys, which + // is what makes a pipeline and a label vanish with no failed assertion. + if state.hives.iter().any(|h| h.name == agent) { + let detail = format!( + "agent name {agent:?} is also a hive in this swarm — they share the swarm's directory \ + keys; this will become an error" + ); + tracing::warn!(agent = %agent, "create_agent: name collides with a hive"); + warnings.push(detail); + } + if !state.hives.iter().any(|h| h.name == hive) { + let known: Vec<&str> = state.hives.iter().map(|h| h.name.as_str()).collect(); + // Name the hives that *would* work: the caller is an operator who + // just picked one, and "not in this swarm" without the roster + // leaves them guessing at a typo they cannot see. + let known = if known.is_empty() { + "(none configured)".to_owned() + } else { + known.join(", ") + }; + let detail = format!("hive {hive:?} is not in this swarm — known hives: {known}"); + return Err(error_problem(axum::http::StatusCode::BAD_REQUEST, &detail)); + } + // Checked after the roster, so a mistyped hive stays a 400 while the queue + // is down. + let settings_queue = settings.queue(&state).await?; + + // Agent names are one swarm-wide namespace: identity, forge user, store + // path and matrix id all carry the bare name. The same name on the same + // hive is that agent being re-created, and goes through. + let _gate = state.create_gate.lock().await; + let elsewhere = placements_elsewhere( + &agent, + &hive, + || queued_placements_now(&state), + || declarations_elsewhere(&state, &agent, &hive), + ) + .await?; + if !elsewhere.is_empty() { + let detail = format!( + "agent name {agent:?} is already placed on hive {} — agent names are unique across \ + the swarm; destroy it there first to move it, or choose another name", + elsewhere.join(", ") + ); + return Err(error_problem(axum::http::StatusCode::CONFLICT, &detail)); + } + // Stored ahead of the job graph, not once the agent is declared: the hive + // reads these by agent name when it first runs the agent. A failure from + // here on puts back what was stored, so a refused creation leaves no + // settings behind for the name. + let stored = match &settings_queue { + Some(client) => { + StoredSettings::store( + client, + &agent, + settings.limits.as_ref(), + settings.inference.as_ref(), + settings.permissions.as_ref(), + settings.archetype.as_ref(), + ) + .await? + } + None => StoredSettings::default(), + }; + let queued = { + let mut sched = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sched.insert_job(None, |b| { + declare_agent_job(b, &agent, &hive, settings.archetype.is_some()) + }) + }; + let ids = match queued { + Ok(ids) => ids, + Err(e) => { + if let Some(client) = &settings_queue { + stored.roll_back(client, &agent).await; + } + return Err(error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &e.to_string(), + )); + } + }; + let [id] = ids[..] else { + unreachable!("exactly one handle was asked for"); + }; + Ok(Json(CreateAgentResponse { + node_id: id.get(), + warnings, + })) +} + +/// The settings a `POST /api/agents` body gives, as they are stored. +struct CreateSettings { + limits: Option, + inference: Option, + permissions: Option, + archetype: Option, +} + +impl CreateSettings { + /// Each setting given, or why its `PUT` would refuse the first bad one. + /// An archetype is checked here for its shape only; whether the swarm + /// lists it is [`agent_archetype::check`]. + fn parse( + limits: Option, + inference: Option, + permissions: Option>, + archetype: Option, + ) -> Result { + if let Some(a) = &archetype { + for part in [&a.source, &a.name] { + hive_types::Ident::parse(part) + .map_err(|why| format!("archetype {part:?}: {why}"))?; + } + } + Ok(Self { + limits: limits.map(agent_limits::Caps::into_stored).transpose()?, + inference: inference + .map(inference::InferenceSettings::into_stored) + .transpose()?, + permissions: permissions.map(agent_permissions::at_create).transpose()?, + archetype, + }) + } + + /// Whether any setting is given. + fn any(&self) -> bool { + self.limits.is_some() + || self.inference.is_some() + || self.permissions.is_some() + || self.archetype.is_some() + } + + /// The queue to store the settings through, `None` when none is given. + /// An archetype is refused here unless the stored list holds it. + async fn queue( + &self, + state: &AppState, + ) -> Result, problem_details::ProblemDetails> { + if !self.any() { + return Ok(None); + } + let client = schedules::queue(state).map_err(schedules::refused)?; + if let Some(archetype) = &self.archetype { + agent_archetype::check(&client, archetype).await?; + } + Ok(Some(client)) + } +} + +/// What `create_agent` replaced in the settings stores, to put back if the +/// creation is refused after the write. +#[derive(Default)] +struct StoredSettings { + limits: agent_limits::Replaced, + inference: inference::Replaced, + permissions: agent_permissions::Replaced, + archetype: agent_archetype::Replaced, +} + +impl StoredSettings { + /// Store whichever of `limits`, `permissions`, `inference` and `archetype` are + /// given, in that order. A failure on one puts back the ones stored before it. + async fn store( + client: &async_nats::Client, + agent: &str, + limits: Option<&swarm_queue_client::agent_limits::AgentCaps>, + inference: Option<&swarm_queue_client::agent_inference::AgentInference>, + permissions: Option<&swarm_queue_client::agent_permissions::AgentPermissions>, + archetype: Option<&swarm_queue_client::agent_archetype::ArchetypeRef>, + ) -> Result { + let mut stored = Self::default(); + if let Some(limits) = limits { + stored.limits = agent_limits::store(client, agent, limits).await?; + } + if let Some(permissions) = permissions { + match agent_permissions::store(client, agent, permissions).await { + Ok(replaced) => stored.permissions = replaced, + Err(e) => { + stored.roll_back(client, agent).await; + return Err(e); + } + } + } + if let Some(inference) = inference { + match inference::store(client, agent, inference).await { + Ok(replaced) => stored.inference = replaced, + Err(e) => { + stored.roll_back(client, agent).await; + return Err(e); + } + } + } + if let Some(archetype) = archetype { + match agent_archetype::store(client, agent, archetype).await { + Ok(replaced) => stored.archetype = replaced, + Err(e) => { + stored.roll_back(client, agent).await; + return Err(e); + } + } + } + Ok(stored) + } + + async fn roll_back(self, client: &async_nats::Client, agent: &str) { + agent_limits::restore(client, agent, self.limits).await; + inference::restore(client, agent, self.inference).await; + agent_permissions::restore(client, agent, self.permissions).await; + agent_archetype::restore(client, agent, self.archetype).await; + } +} + +/// [`placed_elsewhere`] over a snapshot of the queued placements and a read +/// of the published ones, **in that order**. +/// +/// A `SetAgentWanted` node writes its declaration before it turns terminal. +/// Snapshotting the queue first means a node already terminal there has +/// written, so the read that follows sees its declaration; a node still +/// queued is in the snapshot. Reading first leaves a gap: a node finishing +/// between the two is in neither. +async fn placements_elsewhere( + agent: &str, + hive: &str, + snapshot_queued: impl FnOnce() -> Vec<(String, String)>, + read_declared: impl FnOnce() -> Fut, +) -> Result, E> +where + Fut: std::future::Future< + Output = Result, E>, + >, +{ + let queued = snapshot_queued(); + let declared = read_declared().await?; + Ok(placed_elsewhere(agent, hive, &declared, &queued)) +} + +/// Every hive but `hive`'s published wanted state, for [`placed_elsewhere`]. +/// +/// No queue wired up means no hive has been sent a declaration, so there is +/// nothing to collide with. A queue that cannot be read refuses: an unread +/// declaration and an absent one look the same, and reading "absent" is how +/// a duplicate would get through. +async fn declarations_elsewhere( + state: &AppState, + agent: &str, + hive: &str, +) -> Result, problem_details::ProblemDetails> +{ + read_declarations(state, state.hives.iter().filter(|h| h.name != hive)) + .await + .map_err(|(other, e)| { + let detail = format!( + "cannot tell whether {agent:?} already exists on hive {other:?}, so it was not \ + placed: {e:#}" + ); + error_problem(wanted_error_status(&e), &detail) + }) +} + +/// The published wanted state of each of `hives` that has one. No queue wired +/// up reads as nothing declared anywhere. The error names the hive whose read +/// failed. +pub(crate) async fn read_declarations<'a>( + state: &AppState, + hives: impl Iterator, +) -> Result, (String, anyhow::Error)> { + let Some(writer) = state.wanted.as_deref() else { + return Ok(Vec::new()); + }; + let mut declared = Vec::new(); + for hive in hives { + match writer.view(&hive.name).await { + Ok(Some(declaration)) => declared.push((hive.name.clone(), declaration)), + Ok(None) => {} + Err(e) => return Err((hive.name.clone(), e)), + } + } + Ok(declared) +} + +/// Every hive whose declaration places `agent` there. +pub(crate) fn claiming_hives( + agent: &str, + declared: &[(String, swarm_queue_client::wanted::HiveWanted)], +) -> Vec { + declared + .iter() + .filter(|(_, declaration)| { + declaration + .agents + .get(agent) + .is_some_and(|wanted| places_agent(wanted.state)) + }) + .map(|(hive, _)| hive.clone()) + .collect() +} + +/// [`queued_placements`] of the swarm's own queue, as it is now. +fn queued_placements_now(state: &AppState) -> Vec<(String, String)> { + queued_placements( + state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph(), + ) +} + +/// `(hive, agent)` of every `SetAgentWanted` node not yet settled: a creation +/// accepted but not yet visible in the wanted state it is about to write. +fn queued_placements( + graph: &hive_jobq::Graph, +) -> Vec<(String, String)> { + graph + .nodes() + .filter(|node| !node.state.is_terminal()) + .filter_map(|node| match &node.payload { + SwarmNodeKind::SetAgentWanted { hive, agent } => Some((hive.clone(), agent.clone())), + _ => None, + }) + .collect() +} + +/// The hives other than `hive` the swarm has placed `agent` on, sorted. +/// +/// A placement is a declaration in a state that [places][places_agent] the +/// agent, or a queued one. A `Destroyed` agent is gone from that hive, so +/// placing it on another moves it. +fn placed_elsewhere( + agent: &str, + hive: &str, + declared: &[(String, swarm_queue_client::wanted::HiveWanted)], + queued: &[(String, String)], +) -> Vec { + let declared = declared.iter().filter(|(_, declaration)| { + declaration + .agents + .get(agent) + .is_some_and(|wanted| places_agent(wanted.state)) + }); + let queued = queued.iter().filter(|(_, queued)| queued == agent); + let hives: std::collections::BTreeSet<&str> = declared + .map(|(h, _)| h.as_str()) + .chain(queued.map(|(h, _)| h.as_str())) + .filter(|h| *h != hive) + .collect(); + hives.into_iter().map(str::to_owned).collect() +} + +/// Every naming rule `agent` breaks, each as a sentence naming the rule. +/// +/// `reserved` is [`hive_types::reserved_names_raw`]: the list nix owns in +/// `nix/reserved-names.nix`, read by this daemon, by hive-c0re and by the +/// swarm collector's own owner assertion. `None` means this process was never +/// told, which is not the same as "no name is reserved" — staying quiet there +/// would be a check that reports clean because it could not run, so it goes +/// onto `warnings` instead. +fn broken_name_rules( + agent: &str, + reserved: Option<&str>, + warnings: &mut Vec, +) -> Vec { + let mut broken = Vec::new(); + match reserved.map(hive_types::parse_reserved_names) { + None => { + tracing::error!( + var = hive_types::RESERVED_NAMES_ENV, + "reserved-name check could not run — variable not set" + ); + warnings.push(format!( + "the reserved-name check did not run: {} is unset, so {agent:?} was accepted \ + without being checked against the reserved names", + hive_types::RESERVED_NAMES_ENV + )); + } + Some(reserved) if hive_types::is_reserved_name(agent, &reserved) => { + broken.push(format!( + "agent name {agent:?} is reserved (nix/reserved-names.nix): hyperhive's message \ + layer already uses it, or the forge refuses it as a username" + )); + } + Some(_) => {} + } + if let Some(rule) = hive_types::forge_username_violation(agent) { + broken.push(format!( + "agent name {agent:?} {rule}: the forge refuses it as a username" + )); + } + if let Some(rule) = hive_types::reserved_agent_prefix_violation(agent) { + broken.push(format!("agent name {agent:?} {rule}")); + } + broken +} + +/// Whether `agent` is already in the swarm roster, or why that could not be +/// told. `create_agent` asks only of a name that breaks a rule, so an +/// ordinary creation still never waits on the bridge; `make_forge_admin` +/// asks of every name. +pub(crate) async fn in_roster( + auth: Option<&auth::AuthBridge>, + agent: &str, +) -> Result { + let Some(auth) = auth else { + return Err("no identity bridge is configured".to_owned()); + }; + auth.list_agent_identities() + .await + .map(|roster| roster.iter().any(|name| name == agent)) + .map_err(|e| format!("{e:#}")) +} + +/// Refuse a NEW agent whose name breaks a rule; warn for one that exists. +/// +/// Re-creating an agent under its own name is a migration path (mara, on +/// the duplicate-name refusal), so an agent already in the roster keeps its +/// name with a warning. When the roster cannot be read, a new name and an +/// existing one look the same, and this warns rather than refuse: refusing +/// would block that re-creation, and the forge still refuses the account one +/// node in, as it did before. +/// +/// # Errors +/// Every rule `agent` breaks, as one caller-ready sentence, when `agent` is +/// not in the roster. +pub(crate) fn name_verdict( + agent: &str, + broken: &[String], + existing: Result, + warnings: &mut Vec, +) -> Result<(), String> { + match existing { + Ok(false) => Err(format!("{}; choose another name", broken.join("; "))), + Ok(true) => { + tracing::warn!(agent, "existing agent breaks a naming rule"); + warnings.extend( + broken + .iter() + .map(|rule| format!("{rule} — accepted only because {agent:?} already exists")), + ); + Ok(()) + } + Err(why) => { + tracing::warn!(agent, why, "naming rule broken, roster unreadable"); + warnings.extend(broken.iter().map(|rule| { + format!( + "{rule} — not refused, because whether {agent:?} already exists could not \ + be checked: {why}" + ) + })); + Ok(()) + } + } +} + +/// How long a served icon may be reused without asking again. +/// +/// An agent roster asks once per row, and an icon changes about as often +/// as an agent is reconfigured — so a few minutes of browser caching is +/// what keeps a page render from being one bucket read per agent, and +/// costs nothing anyone will notice. +const ICON_MAX_AGE: u32 = 300; + +/// `agent`'s icon — the swarm-level answer to "what does this agent look +/// like". +/// +/// **Nothing on this path names a hive.** The icon is a property of the +/// agent, held at swarm scope under the agent's own key, so it is +/// answerable for an agent that has moved hives or that no hive is +/// currently running. That is the whole point of it living here rather +/// than being fetched from wherever the agent happens to run. +/// +/// **404 means this agent has no icon**, which is the expected +/// unconfigured state and not an error — the same contract the per-agent +/// harness's own `GET /icon` already has, so a caller falls back +/// client-side on a failed load rather than probing first. +/// +/// 🩸 **The body is an untrusted document served from this daemon's own +/// origin**, and an SVG can carry script. The response headers say it may +/// not run any: `Content-Security-Policy: sandbox` with no `allow-scripts` +/// keeps a direct navigation to this URL from executing it, and `nosniff` +/// keeps a browser from deciding the bytes are something else. An `` +/// load — the one consumer — never runs script regardless; the headers are +/// for every other way a URL gets opened. +#[utoipa::path( + get, + path = "/api/agents/{name}/icon", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 200, description = "the agent's icon, an SVG", content_type = "image/svg+xml"), + (status = 400, description = "not an identifier (problem+json)", body = String), + (status = 404, description = "this agent has no icon (problem+json)", body = String), + (status = 500, description = "the icon bucket could not be read (problem+json)", body = String), + (status = 503, description = "no swarm queue is wired up, or it is not connected (problem+json)", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_agent_icon( + State(state): State, + Path(name): Path, +) -> Result { + use axum::http::{StatusCode, header}; + use axum::response::IntoResponse as _; + + let Some(reader) = state.agent_icons.as_ref() else { + return Err(error_problem( + StatusCode::SERVICE_UNAVAILABLE, + "no swarm queue is wired up on this host", + )); + }; + let agent = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(StatusCode::BAD_REQUEST, reason))? + .into_string(); + let icon = reader.get(&agent).await.map_err(|e| { + tracing::warn!(agent = %agent, error = %format!("{e:#}"), "reading the agent icon failed"); + error_problem(wanted_error_status(&e), &format!("{e:#}")) + })?; + let Some(icon) = icon else { + return Err(error_problem( + StatusCode::NOT_FOUND, + "this agent has no icon", + )); + }; + Ok(( + [ + ( + header::CONTENT_TYPE, + swarm_queue_client::agent_icon::MEDIA_TYPE.to_owned(), + ), + ( + header::CACHE_CONTROL, + format!("public, max-age={ICON_MAX_AGE}"), + ), + (header::CONTENT_SECURITY_POLICY, "sandbox".to_owned()), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_owned()), + ], + icon, + ) + .into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routes::tests::{SharedSched, state_with_roster}; + + /// A swarm with no identity bridge cannot answer the roster, and must + /// say so rather than answer `[]`. + /// + /// The distinction is the whole reason this endpoint exists: an empty + /// roster means *no agents*, and a UI that renders "no agents" for a + /// store it could not read is showing a fact nobody established. 503 + /// rather than 500 for the same reason the status route uses it — a + /// bridge is wired up per deployment, so a caller retrying is right. + #[tokio::test] + async fn a_roster_with_no_bridge_refuses_rather_than_answering_empty() { + use axum::response::IntoResponse as _; + + let (state, _sched) = state_with_roster(); + assert!(state.auth.is_none(), "the fixture must have no bridge"); + + let err = super::get_agents(axum::extract::State(state)) + .await + .expect_err("a bridge-less controller cannot produce a roster"); + let resp = err.into_response(); + assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); + + let bytes = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body reads"); + let v: serde_json::Value = serde_json::from_slice(&bytes).expect("problem+json parses"); + // Asserted on the rendered body rather than the error value: what a + // consumer can distinguish is what matters, and `[]` and this share + // a type on the Rust side. + assert_eq!(v["status"], 503); + assert!( + v["detail"] + .as_str() + .is_some_and(|d| d.contains("identity bridge")), + "the cause must name what is missing, got {:?}", + v["detail"] + ); + } + + /// An icon route with no bucket behind it must say it could not ask, + /// not that the agent has no icon. + /// + /// The two are one status code apart and read identically to an + /// `` — both end in the fallback glyph — which is exactly why the + /// distinction has to be asserted here: nothing downstream would ever + /// notice the 503 silently becoming a 404. + #[tokio::test] + async fn an_icon_with_no_queue_refuses_rather_than_answering_no_icon() { + use axum::response::IntoResponse as _; + + let (state, _sched) = state_with_roster(); + assert!( + state.agent_icons.is_none(), + "the fixture must have no queue" + ); + + let err = super::get_agent_icon( + axum::extract::State(state), + axum::extract::Path("iris".to_owned()), + ) + .await + .expect_err("a queue-less controller cannot answer an icon"); + let resp = err.into_response(); + assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); + } + + /// The roster check is the half that makes the recorded hive worth + /// having, so assert it by EFFECT rather than by the message: a hive + /// that is not in this swarm must be refused **before anything is + /// queued**. A version that queued the graph and then complained would + /// satisfy an assertion on the status code alone while still creating + /// the agent — which is the failure this exists to stop. + #[tokio::test] + async fn a_hive_outside_the_roster_is_refused_before_anything_is_queued() { + let (state, sched) = state_with_roster(); + + let err = super::create_agent( + axum::extract::State(state), + axum::Json(super::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1maa".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect_err("a hive outside the roster must be refused"); + + // Names what would have worked: an operator who just mistyped a + // hive cannot see the roster from the error otherwise. + let rendered = format!("{err:?}"); + assert!( + rendered.contains("pr1ma"), + "the refusal should name the known hives, got: {rendered}" + ); + + let queued = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count(); + assert_eq!(queued, 0, "a refused creation must queue no work"); + } + + /// The control for the arm above: the same call with a hive that IS in + /// the roster gets through and queues the graph. Without this, "refused" + /// could equally mean the endpoint refuses everything. + #[tokio::test] + async fn a_hive_in_the_roster_is_accepted_and_queues_the_graph() { + let (state, sched) = state_with_roster(); + + // The response is bound rather than asserted on: node ids start at + // zero, so every property I reached for first ("> 0") was a claim + // about the id allocator rather than about this endpoint. What the + // accept arm is actually for is the queue count below. + let _queued = super::create_agent( + axum::extract::State(state), + axum::Json(super::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let queued = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count(); + assert!(queued > 0, "an accepted creation must queue work"); + } + + /// A request with no settings is the one callers sent before the fields + /// existed, and each field is checked by the same rules as its `PUT`. + #[test] + fn a_creation_request_takes_optional_settings() { + let plain: super::CreateAgentRequest = + serde_json::from_str(r#"{"name":"atlas","hive":"pr1ma"}"#).expect("plain body"); + assert!(plain.limits.is_none() && plain.inference.is_none()); + + let full: super::CreateAgentRequest = serde_json::from_str( + r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, + "inference":{"model":"opus","effort":"high"}}"#, + ) + .expect("body with settings"); + assert!(full.limits.is_some() && full.inference.is_some()); + + let typo = serde_json::from_str::( + r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu":"400%"}}"#, + ); + assert!(typo.is_err(), "an unknown cap field must be refused"); + } + + /// A bad cap or model must stop the creation before the graph is queued: + /// the point of taking settings in the creating request is that a refusal + /// leaves no agent without them. + #[tokio::test] + async fn bad_settings_are_refused_before_anything_is_queued() { + use axum::http::StatusCode; + let bad: [(&str, &str); 2] = [ + (r#"{"cpu_quota":"lots"}"#, "limits"), + (r#"{"model":"-opus"}"#, "inference"), + ]; + for (value, field) in bad { + let (state, sched) = state_with_roster(); + let body = format!(r#"{{"name":"atlas","hive":"pr1ma","{field}":{value}}}"#); + let req: super::CreateAgentRequest = serde_json::from_str(&body).expect("body parses"); + let err = super::create_agent(axum::extract::State(state), axum::Json(req)) + .await + .expect_err("a bad setting must be refused"); + assert_eq!( + axum::response::IntoResponse::into_response(err).status(), + StatusCode::BAD_REQUEST, + "{field}" + ); + assert_eq!(node_count(&sched), 0, "{field}: nothing may be queued"); + } + } + + /// Settings that cannot be stored, here because there is no queue, refuse + /// the creation rather than leaving an agent without them. + #[tokio::test] + async fn settings_that_cannot_be_stored_refuse_the_creation() { + let (state, sched) = state_with_roster(); + let req: super::CreateAgentRequest = serde_json::from_str( + r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}}"#, + ) + .expect("body parses"); + let err = super::create_agent(axum::extract::State(state), axum::Json(req)) + .await + .expect_err("no queue means the caps cannot be stored"); + assert_eq!( + axum::response::IntoResponse::into_response(err).status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!(node_count(&sched), 0, "nothing may be queued"); + } + + /// Through the real handler and a real `nats-server`: both settings land + /// under the agent's key as the hive reads them, and the graph is queued. + #[tokio::test] + async fn a_creation_with_settings_stores_them_and_queues_the_graph() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::state(&client); + let req: super::CreateAgentRequest = serde_json::from_str( + r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, + "inference":{"model":"opus","max_subagents":3}}"#, + ) + .expect("body parses"); + let _queued = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + .await + .expect("a valid creation with settings goes through"); + + assert_eq!( + crate::agent_limits::tests::published(&client, "atlas") + .await + .as_deref(), + Some(r#"{"cpu_quota":"400%"}"#) + ); + let inference = swarm_queue_client::agent_inference::open_or_create(&client) + .await + .expect("bucket") + .get("atlas") + .await + .expect("read") + .expect("a value"); + assert_eq!(&inference[..], br#"{"model":"opus","max_subagents":3}"#); + assert!( + state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count() + > 0, + "the graph must be queued" + ); + } + + /// When the inference store fails after the caps are stored, the caps go + /// back to what they were: the earlier value, or no key. Without the + /// roll-back this leaves a cap behind for an agent that was never created. + /// The inference bucket is made with a one-byte value limit, so any write + /// to it fails. + #[tokio::test] + async fn a_failed_second_store_puts_the_first_back() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + async_nats::jetstream::new(client.clone()) + .create_key_value(async_nats::jetstream::kv::Config { + bucket: swarm_queue_client::agent_inference::BUCKET.to_owned(), + history: 1, + max_value_size: 1, + ..Default::default() + }) + .await + .expect("a bucket that refuses any value"); + let caps = |json: &str| { + serde_json::from_str::(json) + .expect("caps") + .into_stored() + .expect("valid") + }; + let inference = + serde_json::from_str::(r#"{"model":"opus"}"#) + .expect("settings") + .into_stored() + .expect("valid"); + crate::agent_limits::store(&client, "atlas", &caps(r#"{"cpu_quota":"100%"}"#)) + .await + .expect("an earlier cap"); + + for agent in ["atlas", "iris"] { + let stored = super::StoredSettings::store( + &client, + agent, + Some(&caps(r#"{"cpu_quota":"400%"}"#)), + Some(&inference), + None, + None, + ) + .await; + assert!(stored.is_err(), "{agent}: the inference write must fail"); + } + assert_eq!( + crate::agent_limits::tests::published(&client, "atlas") + .await + .as_deref(), + Some(r#"{"cpu_quota":"100%"}"#), + "the earlier cap comes back" + ); + assert_eq!( + crate::agent_limits::tests::published(&client, "iris").await, + None, + "a cap stored for a name with none is removed" + ); + } + + /// Through the real handler: when the inference store fails after the + /// permissions are stored, the creation is refused and the permissions go + /// back to what they were, the earlier value or no key. The inference + /// bucket refuses any value, as above, and the refusal is that bucket's. + #[tokio::test] + async fn a_failed_creation_leaves_no_permissions_behind() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + async_nats::jetstream::new(client.clone()) + .create_key_value(async_nats::jetstream::kv::Config { + bucket: swarm_queue_client::agent_inference::BUCKET.to_owned(), + history: 1, + max_value_size: 1, + ..Default::default() + }) + .await + .expect("a bucket that refuses any value"); + let state = crate::agent_limits::tests::state(&client); + let earlier = crate::agent_permissions::at_create(vec![ + hive_sh4re::permissions::Permission::Messaging, + ]) + .expect("valid"); + crate::agent_permissions::store(&client, "atlas", &earlier) + .await + .expect("an earlier grant"); + + for agent in ["atlas", "iris"] { + let req: super::CreateAgentRequest = serde_json::from_str(&format!( + r#"{{"name":"{agent}","hive":"pr1ma","permissions":["web_tools"], + "inference":{{"model":"opus"}}}}"# + )) + .expect("body parses"); + let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + .await + .expect_err("the inference write must fail"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::INTERNAL_SERVER_ERROR), + "{agent}: {err:?}" + ); + assert!( + err.detail + .as_deref() + .is_some_and(|d| d.contains("message size exceeds maximum allowed")), + "{agent}: refused by the inference bucket, not an earlier check: {err:?}" + ); + } + let bucket = swarm_queue_client::agent_permissions::open_or_create(&client) + .await + .expect("bucket"); + let held = + |value: Option| value.map(|v| String::from_utf8_lossy(&v).into_owned()); + assert_eq!( + held(bucket.get("atlas").await.expect("read")).as_deref(), + Some(r#"{"permissions":["messaging"]}"#), + "the earlier grant comes back" + ); + assert_eq!( + held(bucket.get("iris").await.expect("read")), + None, + "a grant stored for a name with none is removed" + ); + assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); + } + + /// A valid cap beside an invalid model stores neither and queues nothing, + /// on a server that could have stored the cap. + #[tokio::test] + async fn a_bad_setting_stores_none_of_them() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + let state = crate::agent_limits::tests::state(&client); + let req: super::CreateAgentRequest = serde_json::from_str( + r#"{"name":"atlas","hive":"pr1ma","limits":{"cpu_quota":"400%"}, + "inference":{"model":"-opus"}}"#, + ) + .expect("body parses"); + let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + .await + .expect_err("a bad model must be refused"); + assert_eq!( + axum::response::IntoResponse::into_response(err).status(), + axum::http::StatusCode::BAD_REQUEST + ); + assert_eq!( + crate::agent_limits::tests::published(&client, "atlas").await, + None + ); + assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); + } + + /// `state_with_roster` with a second hive, `sec0nd`. + fn state_with_two_hives() -> (crate::routes::AppState, SharedSched) { + let (state, sched) = state_with_roster(); + let mut hives = (*state.hives).clone(); + hives.push(HiveEntry { + name: "sec0nd".to_owned(), + domain: "sec0nd.example".to_owned(), + dashboard: "sec0nd.hives.example".to_owned(), + }); + let state = crate::routes::AppState { + hives: std::sync::Arc::new(hives), + ..state + }; + (state, sched) + } + + pub(crate) async fn create( + state: &crate::routes::AppState, + name: &str, + hive: &str, + ) -> Result<(), problem_details::ProblemDetails> { + super::create_agent( + axum::extract::State(state.clone()), + axum::Json(super::CreateAgentRequest { + name: name.to_owned(), + hive: hive.to_owned(), + limits: None, + inference: None, + permissions: None, + archetype: None, + }), + ) + .await + .map(|_| ()) + } + + fn node_count(sched: &SharedSched) -> usize { + sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count() + } + + /// A name queued for one hive is refused on another, by effect: 409 and + /// nothing more queued. The queued-but-unwritten placement is the one a + /// wanted-state read alone would miss. + #[tokio::test] + async fn a_name_placed_on_another_hive_is_refused_before_anything_is_queued() { + let (state, sched) = state_with_two_hives(); + create(&state, "atlas", "pr1ma") + .await + .expect("the first creation of a name must be accepted"); + let before = node_count(&sched); + + let err = create(&state, "atlas", "sec0nd") + .await + .expect_err("the same name on another hive must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::CONFLICT), + "{err:?}" + ); + assert!( + format!("{err:?}").contains("pr1ma"), + "the refusal should name the hive holding the name, got: {err:?}" + ); + assert_eq!( + node_count(&sched), + before, + "a refused creation must queue no work" + ); + } + + /// The control for the refusal above: the same name on the SAME hive is + /// that agent being re-created, and queues again. + #[tokio::test] + async fn re_creating_an_agent_on_its_own_hive_is_accepted() { + let (state, sched) = state_with_two_hives(); + create(&state, "atlas", "pr1ma") + .await + .expect("the first creation of a name must be accepted"); + let before = node_count(&sched); + + create(&state, "atlas", "pr1ma") + .await + .expect("re-creating an agent on its own hive must be accepted"); + assert!(node_count(&sched) > before, "a re-creation must queue work"); + } + + /// A wanted state that cannot be read refuses rather than reading as + /// "placed nowhere", and queues nothing. + #[tokio::test] + async fn an_unreadable_wanted_state_refuses_creation() { + let (state, sched) = state_with_two_hives(); + let state = crate::routes::AppState { + wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( + disconnected_client().await, + ))), + ..state + }; + + let err = create(&state, "atlas", "pr1ma") + .await + .expect_err("an unreadable placement must refuse"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{err:?}" + ); + assert_eq!( + node_count(&sched), + 0, + "a refused creation must queue no work" + ); + } + + pub(crate) fn declaring( + hive: &str, + agent: &str, + state: swarm_queue_client::wanted::AgentState, + ) -> (String, swarm_queue_client::wanted::HiveWanted) { + let mut declaration = swarm_queue_client::wanted::HiveWanted::default(); + declaration.agents.insert( + agent.to_owned(), + swarm_queue_client::wanted::AgentWanted { state }, + ); + (hive.to_owned(), declaration) + } + + /// A `SetAgentWanted` node that finishes between the two reads: whichever + /// read runs first sees it queued and undeclared, the second sees it + /// terminal and declared. Only the queue-first order catches it. + #[tokio::test] + async fn a_placement_finishing_between_the_two_reads_is_still_caught() { + use swarm_queue_client::wanted::AgentState; + let reads = std::cell::Cell::new(0); + let snapshot_queued = || { + reads.set(reads.get() + 1); + if reads.get() == 1 { + vec![("sec0nd".to_owned(), "atlas".to_owned())] + } else { + Vec::new() + } + }; + let read_declared = || { + reads.set(reads.get() + 1); + let declared = if reads.get() == 1 { + Vec::new() + } else { + vec![declaring("sec0nd", "atlas", AgentState::Paused)] + }; + std::future::ready(Ok::<_, ()>(declared)) + }; + + let elsewhere = + super::placements_elsewhere("atlas", "pr1ma", snapshot_queued, read_declared) + .await + .expect("both reads succeed"); + assert_eq!(elsewhere, ["sec0nd"]); + } + + #[test] + fn a_live_declaration_on_another_hive_is_a_placement() { + use swarm_queue_client::wanted::AgentState; + for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { + let declared = [declaring("sec0nd", "atlas", state)]; + assert_eq!( + super::placed_elsewhere("atlas", "pr1ma", &declared, &[]), + ["sec0nd"], + "{state:?}" + ); + } + } + + /// A destroyed agent has left its hive, so creating it elsewhere moves it. + #[test] + fn a_destroyed_declaration_is_not_a_placement() { + let declared = [declaring( + "sec0nd", + "atlas", + swarm_queue_client::wanted::AgentState::Destroyed, + )]; + assert!(super::placed_elsewhere("atlas", "pr1ma", &declared, &[]).is_empty()); + } + + #[test] + fn placements_on_the_requested_hive_or_of_other_names_do_not_count() { + use swarm_queue_client::wanted::AgentState; + let declared = [ + declaring("pr1ma", "atlas", AgentState::Up), + declaring("sec0nd", "argus", AgentState::Up), + ]; + let queued = [ + ("pr1ma".to_owned(), "atlas".to_owned()), + ("sec0nd".to_owned(), "argus".to_owned()), + ]; + assert!(super::placed_elsewhere("atlas", "pr1ma", &declared, &queued).is_empty()); + } + + #[test] + fn the_hive_placing_an_agent_claims_it_and_no_other_does() { + use swarm_queue_client::wanted::AgentState; + let declared = [ + declaring("pr1ma", "atlas", AgentState::Paused), + declaring("sec0nd", "argus", AgentState::Up), + declaring("th1rd", "atlas", AgentState::Destroyed), + ]; + assert_eq!(super::claiming_hives("atlas", &declared), ["pr1ma"]); + assert!(super::claiming_hives("iris", &declared).is_empty()); + } + + #[test] + fn two_hives_placing_one_agent_both_claim_it() { + use swarm_queue_client::wanted::AgentState; + let declared = [ + declaring("pr1ma", "atlas", AgentState::Up), + declaring("sec0nd", "atlas", AgentState::Up), + ]; + assert_eq!( + super::claiming_hives("atlas", &declared), + ["pr1ma", "sec0nd"] + ); + } + + /// `refuse_placement` of `agent` on `pr1ma`, with `admin` reserved, + /// `roster` as the roster read, and `declared` as every hive's wanted + /// state, `pr1ma`'s included. + async fn refuse_on_pr1ma_with( + agent: &str, + declared: Vec<(String, swarm_queue_client::wanted::HiveWanted)>, + roster: Result, + ) -> Result<(), problem_details::ProblemDetails> { + let here = declared + .iter() + .find(|(hive, _)| hive == "pr1ma") + .map(|(_, declaration)| declaration.clone()); + super::refuse_placement( + agent, + "pr1ma", + here.as_ref(), + Some("admin"), + || std::future::ready(roster), + Vec::new, + || std::future::ready(Ok(declared)), + ) + .await + } + + /// [`refuse_on_pr1ma_with`] for an `agent` the roster does not hold. + async fn refuse_on_pr1ma( + agent: &str, + declared: Vec<(String, swarm_queue_client::wanted::HiveWanted)>, + ) -> Result<(), problem_details::ProblemDetails> { + refuse_on_pr1ma_with(agent, declared, Ok(false)).await + } + + const PLACING: [swarm_queue_client::wanted::AgentState; 3] = [ + swarm_queue_client::wanted::AgentState::Up, + swarm_queue_client::wanted::AgentState::Offline, + swarm_queue_client::wanted::AgentState::Paused, + ]; + + /// The handler checks exactly the placing states; a destroy is never + /// checked. + #[test] + fn only_a_destroy_places_nothing() { + for state in PLACING { + assert!(super::places_agent(state), "{state:?}"); + } + assert!(!super::places_agent( + swarm_queue_client::wanted::AgentState::Destroyed + )); + } + + #[tokio::test] + async fn placing_a_new_reserved_name_is_refused() { + let err = refuse_on_pr1ma("admin", Vec::new()) + .await + .expect_err("a new reserved name must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{err:?}" + ); + } + + /// `offline` first, then `up`: the first step is a first declaration, so + /// it is refused and leaves nothing on the hive for `up` to pass as. + /// `refuse_placement` takes no state: every placing state reaches it the + /// same way, which `only_a_destroy_places_nothing` pins. + #[tokio::test] + async fn offline_first_cannot_carry_a_reserved_name_into_up() { + let err = refuse_on_pr1ma("admin", Vec::new()) + .await + .expect_err("a first `offline` of a reserved name must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{err:?}" + ); + // Nothing was declared, so `up` is a first declaration as well. + let err = refuse_on_pr1ma("admin", Vec::new()) + .await + .expect_err("`up` after a refused `offline` must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{err:?}" + ); + } + + /// No later step refuses a bad name, so an unreadable roster refuses + /// rather than lets it through. An ordinary name never reads it. + #[tokio::test] + async fn an_unreadable_roster_refuses_a_rule_breaking_name() { + let unreadable = || Err("no identity bridge is configured".to_owned()); + let err = refuse_on_pr1ma_with("admin", Vec::new(), unreadable()) + .await + .expect_err("an unreadable roster must not let a reserved name through"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{err:?}" + ); + refuse_on_pr1ma_with("atlas", Vec::new(), unreadable()) + .await + .expect("an ordinary name does not depend on the roster"); + } + + /// A destroyed agent is gone from its hive, so declaring it again is a + /// first declaration and its name is checked. + #[tokio::test] + async fn a_destroyed_declaration_here_does_not_exempt_the_name() { + use swarm_queue_client::wanted::AgentState; + let err = refuse_on_pr1ma( + "admin", + vec![declaring("pr1ma", "admin", AgentState::Destroyed)], + ) + .await + .expect_err("re-placing a destroyed reserved name must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{err:?}" + ); + } + + /// An agent already declared here keeps a rule-breaking name through any + /// state change, whether or not the roster holds it or can be read. + #[tokio::test] + async fn an_agent_declared_here_is_not_refused_over_its_name() { + for current in PLACING { + for roster in [Ok(false), Err("bridge down".to_owned())] { + refuse_on_pr1ma_with( + "admin", + vec![declaring("pr1ma", "admin", current)], + roster.clone(), + ) + .await + .unwrap_or_else(|err| panic!("{current:?}, {roster:?}: {err:?}")); + } + } + } + + #[tokio::test] + async fn a_first_declaration_of_a_name_placed_on_another_hive_is_refused() { + use swarm_queue_client::wanted::AgentState; + let err = refuse_on_pr1ma("atlas", vec![declaring("sec0nd", "atlas", AgentState::Up)]) + .await + .expect_err("a name placed on another hive must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::CONFLICT), + "{err:?}" + ); + assert!(format!("{err:?}").contains("sec0nd"), "{err:?}"); + } + + /// An agent that runs on `pr1ma` but was never declared anywhere, and is + /// not in the roster, is one the swarm adopts by declaring it. + #[tokio::test] + async fn adopting_an_undeclared_agent_on_its_own_hive_is_accepted() { + refuse_on_pr1ma("legacy", Vec::new()) + .await + .expect("an undeclared agent must still be adoptable"); + } + + #[tokio::test] + async fn changing_the_state_of_an_agent_on_its_own_hive_is_accepted() { + use swarm_queue_client::wanted::AgentState; + refuse_on_pr1ma("atlas", vec![declaring("pr1ma", "atlas", AgentState::Up)]) + .await + .expect("an agent's own hive must accept a new state for it"); + } + + /// `set_agent_state` on `pr1ma`, in a two-hive swarm whose queue never + /// connected and with no identity bridge. + async fn set_on_disconnected_pr1ma( + agent: &str, + wanted_state: swarm_queue_client::wanted::AgentState, + ) -> problem_details::ProblemDetails { + let (state, _sched) = state_with_two_hives(); + let state = crate::routes::AppState { + wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( + disconnected_client().await, + ))), + ..state + }; + super::set_agent_state( + axum::extract::State(state), + axum::extract::Path(("pr1ma".to_owned(), agent.to_owned())), + axum::Json(SetAgentStateRequest { + state: wanted_state, + }), + ) + .await + .expect_err("a queue that never connected cannot publish") + } + + /// Through the handler: a wanted state that cannot be read refuses the + /// declaration before it is written, as it refuses creation. + #[tokio::test] + async fn set_agent_state_refuses_when_the_wanted_state_is_unreadable() { + let err = + set_on_disconnected_pr1ma("atlas", swarm_queue_client::wanted::AgentState::Up).await; + assert_eq!( + err.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{err:?}" + ); + assert!( + format!("{err:?}").contains("was not placed"), + "the refusal must come from the placement check, got: {err:?}" + ); + } + + /// Through the handler: a destroy of a reserved name reaches the write + /// without being checked. It fails there only because the queue never + /// connected; the placing control above fails in the check instead. + #[tokio::test] + async fn set_agent_state_does_not_check_a_destroy() { + let err = + set_on_disconnected_pr1ma("admin", swarm_queue_client::wanted::AgentState::Destroyed) + .await; + assert_eq!( + err.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{err:?}" + ); + assert!( + !format!("{err:?}").contains("not placed"), + "a destroy must not be refused by the placement check, got: {err:?}" + ); + } + + /// Forgejo v16.0.5's reserved usernames our charset can spell, plus the + /// bare `-` — the forge section of `nix/reserved-names.nix`. Read from + /// the list nix exports, so dropping one from the file reds this. + const FORGE_REFUSED_LITERALS: [&str; 23] = [ + "-", + "admin", + "api", + "assets", + "attachments", + "avatar", + "avatars", + "captcha", + "explore", + "forgejo-actions", + "ghost", + "gitea-actions", + "issues", + "login", + "metrics", + "milestones", + "notifications", + "org", + "pulls", + "repo", + "repo-avatars", + "user", + "v2", + ]; + + /// The list nix renders, panicking when absent for the reason + /// `hive-sh4re`'s drift test gives: a skip would read as green. + fn nix_reserved() -> String { + hive_types::reserved_names_raw() + .expect("HIVE_RESERVED_NAMES is exported by nix/checks.nix and nix/devshell.nix") + } + + #[test] + fn a_new_agent_named_a_forge_refused_literal_is_refused() { + let raw = nix_reserved(); + for name in FORGE_REFUSED_LITERALS { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); + let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); + assert!(err.contains("nix/reserved-names.nix"), "{name:?}: {err}"); + } + } + + #[test] + fn a_new_agent_named_like_a_service_forge_account_is_refused() { + let raw = nix_reserved(); + for name in [ + crate::forge::builder_account::LOGIN, + crate::forge::classifier_account::LOGIN, + crate::forge::ci_metrics_account::LOGIN, + ] { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); + let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); + assert!(err.contains("nix/reserved-names.nix"), "{name:?}: {err}"); + } + } + + /// One case per shape rule, each named in the refusal. Checked against a + /// list without them, so the literal half cannot carry this test. + #[test] + fn a_new_agent_breaking_a_forge_shape_rule_is_refused() { + let over = "a".repeat(hive_types::FORGE_USERNAME_MAX_LEN + 1); + for (name, rule) in [ + ("-agent", "must not start with '-'"), + ("my--agent", "must not contain '--'"), + ("agent-", "must not end with '-'"), + (over.as_str(), "must be 40 characters or fewer"), + ] { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules(name, Some("operator"), &mut warnings); + let err = super::name_verdict(name, &broken, Ok(false), &mut warnings).expect_err(name); + assert!(err.contains(rule), "{name:?}: {err}"); + } + } + + /// The control for both tests above, against the real list. + #[test] + fn an_ordinary_name_and_a_40_char_name_break_no_rule() { + let raw = nix_reserved(); + let at_cap = "a".repeat(hive_types::FORGE_USERNAME_MAX_LEN); + for name in ["my-agent", at_cap.as_str()] { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules(name, Some(&raw), &mut warnings); + assert!(broken.is_empty(), "{name:?}: {broken:?}"); + assert!(warnings.is_empty(), "{name:?}: {warnings:?}"); + } + } + + /// The `hive-` prefix belongs to hives' forge accounts, so a new agent + /// may not take it. `hivemind` and `xhive-y` are the control. + #[test] + fn a_new_agent_with_the_hive_prefix_is_refused() { + let raw = nix_reserved(); + let mut warnings = Vec::new(); + let broken = super::broken_name_rules("hive-x", Some(&raw), &mut warnings); + let err = + super::name_verdict("hive-x", &broken, Ok(false), &mut warnings).expect_err("hive-x"); + assert!(err.contains("reserved for hive forge accounts"), "{err}"); + for ok in ["hivemind", "xhive-y"] { + let broken = super::broken_name_rules(ok, Some(&raw), &mut warnings); + assert!(broken.is_empty(), "{ok:?}: {broken:?}"); + } + assert!(warnings.is_empty(), "{warnings:?}"); + } + + /// The prefix the rule reserves is the one hive accounts are named with. + #[test] + fn the_reserved_prefix_is_the_hive_account_prefix() { + assert!( + swarm_secret_client::forge::hive_login("alpha") + .starts_with(hive_types::RESERVED_AGENT_PREFIX) + ); + } + + /// Re-creating an agent under its own name must keep working: a name + /// already in the roster is warned about, never refused. + #[test] + fn an_existing_agent_keeps_a_rule_breaking_name_with_a_warning() { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules("admin", Some("admin"), &mut warnings); + super::name_verdict("admin", &broken, Ok(true), &mut warnings) + .expect("an existing agent must not be refused"); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!(warnings[0].contains("already exists"), "{warnings:?}"); + } + + /// A roster that cannot be read cannot tell new from existing, so it + /// warns — and the warning says the check did not decide. + #[test] + fn an_unreadable_roster_warns_rather_than_refusing() { + let mut warnings = Vec::new(); + let broken = super::broken_name_rules("agent-", Some("operator"), &mut warnings); + super::name_verdict( + "agent-", + &broken, + Err("bridge down".to_owned()), + &mut warnings, + ) + .expect("an unreadable roster must not refuse"); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!(warnings[0].contains("bridge down"), "{warnings:?}"); + } + + /// `Destroyed` is the one declaration that ends an agent's life, so it is + /// the one that revokes. The credential is a bearer secret: left behind, + /// anything that recovers it still authenticates as an agent that no + /// longer exists. + #[test] + fn declaring_an_agent_destroyed_revokes_its_queue_credential() { + assert!(super::revokes_queue_credential( + swarm_queue_client::wanted::AgentState::Destroyed + )); + } + + /// The control, and the more expensive half to get wrong: a revocation on + /// `Offline` or `Paused` would give an agent that can be stopped and never + /// restarted. Every non-terminal state is listed rather than sampled, so + /// a state added later shows up here as well as in the match itself. + #[test] + fn no_state_an_agent_comes_back_from_revokes() { + use swarm_queue_client::wanted::AgentState; + for state in [AgentState::Up, AgentState::Offline, AgentState::Paused] { + assert!( + !super::revokes_queue_credential(state), + "{state:?} is a state an agent returns from" + ); + } + } + + /// Seed the stored archetype list with `reviewer` under `hyperhive`. + async fn seed_archetypes(client: &async_nats::Client) { + let record = swarm_queue_client::archetypes::SourceRecord { + source: "/nix/store/h".to_owned(), + archetypes: std::collections::BTreeMap::from([( + "reviewer".to_owned(), + swarm_queue_client::archetypes::ArchetypeMeta { + description: "reviews".to_owned(), + ..Default::default() + }, + )]), + refused: std::collections::BTreeMap::new(), + }; + swarm_queue_client::archetypes::open_or_create(client) + .await + .expect("bucket") + .put( + swarm_queue_client::archetypes::BUILTIN_SOURCE, + serde_json::to_vec(&record).expect("encodes").into(), + ) + .await + .expect("seeded"); + } + + /// Through the real handler and a real `nats-server`: a listed archetype + /// lands under the agent's key as the hive reads it, and the queued graph + /// has no config repo. + #[tokio::test] + async fn a_creation_from_an_archetype_records_it_and_queues_no_repo() { + use hive_jobq_wire::WireNode as _; + + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + seed_archetypes(&client).await; + let state = crate::agent_limits::tests::state(&client); + let req: super::CreateAgentRequest = serde_json::from_str( + r#"{"name":"atlas","hive":"pr1ma", + "archetype":{"source":"hyperhive","name":"reviewer"}}"#, + ) + .expect("body parses"); + let _queued = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + .await + .expect("a listed archetype goes through"); + + let stored = swarm_queue_client::agent_archetype::open_or_create(&client) + .await + .expect("bucket") + .get("atlas") + .await + .expect("read") + .expect("a value"); + assert_eq!(&stored[..], br#"{"source":"hyperhive","name":"reviewer"}"#); + let labels: Vec = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .map(|n| n.payload.label()) + .collect(); + assert!(labels.iter().any(|l| l == "trigger_deploy"), "{labels:?}"); + assert!(!labels.iter().any(|l| l == "create_repo"), "{labels:?}"); + } + + /// An archetype the stored list does not hold is a 400 that stores and + /// queues nothing, and so is a name that is not an identifier. + #[tokio::test] + async fn an_unlisted_archetype_is_refused_before_anything_is_stored() { + let nats = crate::test_nats::TestNats::start(); + let client = nats.client().await; + seed_archetypes(&client).await; + let state = crate::agent_limits::tests::state(&client); + for archetype in [ + r#"{"source":"hyperhive","name":"nope"}"#, + r#"{"source":"elsewhere","name":"reviewer"}"#, + r#"{"source":"hyperhive","name":"a.b"}"#, + ] { + let req: super::CreateAgentRequest = serde_json::from_str(&format!( + r#"{{"name":"atlas","hive":"pr1ma","permissions":["messaging"], + "archetype":{archetype}}}"# + )) + .expect("body parses"); + let err = super::create_agent(axum::extract::State(state.clone()), axum::Json(req)) + .await + .expect_err("an unlisted archetype must be refused"); + assert_eq!( + err.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{archetype}: {err:?}" + ); + } + assert_eq!(node_count(&state.jobq), 0, "nothing may be queued"); + let permissions = swarm_queue_client::agent_permissions::open_or_create(&client) + .await + .expect("bucket"); + assert_eq!(permissions.get("atlas").await.expect("read"), None); + } + + /// A client that exists but has never connected — same shape + /// `matrix_account`'s tests build, duplicated here rather than shared: + /// both are a handful of lines and neither crate has a test-support + /// module to put a shared helper in. + async fn disconnected_client() -> async_nats::Client { + async_nats::ConnectOptions::new() + .retry_on_initial_connect() + .connect("127.0.0.1:1") + .await + .expect("retry_on_initial_connect returns without waiting for a real connection") + } + + /// `state_with_roster` plus a `wanted` writer over a client that never + /// connected — the shape `set_agent_state`/`get_hive_wanted` see when + /// the queue is mid-reconnect. + async fn state_with_disconnected_wanted() -> crate::routes::AppState { + let (state, _sched) = state_with_roster(); + crate::routes::AppState { + wanted: Some(std::sync::Arc::new(wanted::WantedWriter::new( + disconnected_client().await, + ))), + ..state + } + } + + /// End to end through the real handler: `PUT .../state` against a + /// disconnected queue answers 503, not the 500 every other writer + /// error still gets. + #[tokio::test] + async fn set_agent_state_answers_503_on_a_disconnected_queue() { + let state = state_with_disconnected_wanted().await; + + let result = super::set_agent_state( + axum::extract::State(state), + axum::extract::Path(("pr1ma".to_owned(), "atlas".to_owned())), + axum::Json(SetAgentStateRequest { + state: swarm_queue_client::wanted::AgentState::Paused, + }), + ) + .await; + + let problem = result.expect_err("a disconnected queue must not read as success"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{problem:?}" + ); + } + + /// Same fix, the read side: `GET .../wanted` against a disconnected + /// queue also answers 503. + #[tokio::test] + async fn get_hive_wanted_answers_503_on_a_disconnected_queue() { + let state = state_with_disconnected_wanted().await; + + let result = super::get_hive_wanted( + axum::extract::State(state), + axum::extract::Path("pr1ma".to_owned()), + ) + .await; + + let problem = result.expect_err("a disconnected queue must not read as success"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{problem:?}" + ); + } + + /// Control for both tests above: a hive that is not in the roster is + /// still a plain 400, even with the same disconnected writer wired up — + /// so the 503 above is `ensure_connected` firing, not every error this + /// handler can produce collapsing to 503. + #[tokio::test] + async fn set_agent_state_still_answers_400_for_an_unknown_hive() { + let state = state_with_disconnected_wanted().await; + + let result = super::set_agent_state( + axum::extract::State(state), + axum::extract::Path(("not-a-hive".to_owned(), "atlas".to_owned())), + axum::Json(SetAgentStateRequest { + state: swarm_queue_client::wanted::AgentState::Paused, + }), + ) + .await; + + let problem = result.expect_err("an unknown hive is still a 400"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{problem:?}" + ); + } + + /// `GET /api/agents/{name}/icon` against a disconnected queue answers + /// 503 like its siblings, not the 404 an `` would show identically + /// or the 500 of a failed read. + #[tokio::test] + async fn get_agent_icon_answers_503_on_a_disconnected_queue() { + let (state, _sched) = state_with_roster(); + let state = crate::routes::AppState { + agent_icons: Some(std::sync::Arc::new( + crate::agent_icon::AgentIconReader::new(disconnected_client().await), + )), + ..state + }; + + let problem = super::get_agent_icon( + axum::extract::State(state), + axum::extract::Path("iris".to_owned()), + ) + .await + .expect_err("a disconnected queue must not read as an icon"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{problem:?}" + ); + } +} diff --git a/swarm-controller/src/routes/config_prs.rs b/swarm-controller/src/routes/config_prs.rs new file mode 100644 index 00000000..b0ac4004 --- /dev/null +++ b/swarm-controller/src/routes/config_prs.rs @@ -0,0 +1,81 @@ +//! Agents' open config PRs. + +use anyhow::Result; +use axum::{ + Json, + extract::{Path, State}, +}; + +use crate::forge; +use crate::routes::{AppState, StatusUnavailable}; + +/// `agent`'s open config PR, from [`ConfigPrCache`](crate::config_pr::ConfigPrCache) — kept +/// current by a webhook nudge (near-real-time) backstopped by a periodic +/// scan (worst case: up to one `config_pr::POLL_INTERVAL` stale if a +/// delivery was ever missed). Not a live forge read either way, so this +/// answers even when the forge itself is momentarily unreachable. +/// +/// `200` with a `null` body means "no open PR" (or "no scan has completed +/// yet") — not distinguished, for the same reason +/// [`ConfigPrCache::get`](crate::config_pr::ConfigPrCache::get) doesn't: both read as "nothing to show" +/// to the swarm-ui config-PR panel this feeds, and a cache is a best-effort +/// read, not a source of truth callers should expect to disambiguate +/// against. +#[utoipa::path( + get, + path = "/api/agents/{name}/config-pr", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 200, description = "the agent's open config PR, or null if none / not scanned yet", body = Option), + (status = 503, description = "no forge is configured on this host", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_agent_config_pr( + State(state): State, + Path(name): Path, +) -> Result>, StatusUnavailable> { + let Some(cache) = state.config_prs.get() else { + return Err(StatusUnavailable( + "no forge is configured on this host".to_owned(), + )); + }; + Ok(Json(cache.get(&name))) +} + +/// Every agent with an open config PR, in one response — the bulk +/// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs +/// every agent's status to render, and fetching them one at a time doesn't +/// scale and isn't how the rest of swarm-ui's single-fetch pages (jobq, +/// hives status) work. +/// +/// Deliberately **not** `/api/agents/config-prs`: agent names are +/// user-specified (any string `hive_types::Ident` accepts), so a literal +/// path segment sitting where a `{name}` capture could plausibly also want +/// to live is a real, not theoretical, clash risk the moment someone names +/// an agent `config-prs` — per mara's review call, closed off by +/// construction rather than relying on axum's static-route-priority +/// tiebreak to paper over it. +/// +/// Only agents with a currently-open PR are present — same "absence is the +/// answer" shape [`get_agent_config_pr`]'s `null` uses, just at map-entry +/// granularity instead of per-request. +#[utoipa::path( + get, + path = "/api/config-prs", + responses( + (status = 200, description = "agent name -> open config PR, only agents with one present", body = std::collections::HashMap), + (status = 503, description = "no forge is configured on this host", body = String), + ), + tag = "agents" +)] +pub(crate) async fn get_config_prs( + State(state): State, +) -> Result>, StatusUnavailable> { + let Some(cache) = state.config_prs.get() else { + return Err(StatusUnavailable( + "no forge is configured on this host".to_owned(), + )); + }; + Ok(Json(cache.snapshot())) +} diff --git a/swarm-controller/src/routes/forge_admin.rs b/swarm-controller/src/routes/forge_admin.rs new file mode 100644 index 00000000..c89a5579 --- /dev/null +++ b/swarm-controller/src/routes/forge_admin.rs @@ -0,0 +1,236 @@ +//! Forge site admins and an agent's forge token, on request. + +use anyhow::Result; +use axum::{ + Json, + extract::{Path, State}, +}; +use swarm_controller_api::agents::MintAgentForgeTokenResponse; +use swarm_controller_api::forge::{ForgeAdminChange, MakeForgeAdminResponse}; + +use crate::forge; +use crate::jobq::queue_forge_token_mints; +use crate::routes::agents::in_roster; +use crate::routes::{AppState, error_problem}; + +/// Check an agent's forge token now, and mint one if it is missing or stale. +/// +/// The periodic pass (`forge::agent_token::spawn`) does the same every five +/// minutes for every agent with a store identity; this is for an operator who +/// does not want to wait, or for an agent the pass skipped. Queues the same +/// node agent creation does. Idempotent: a current token is left alone. +#[utoipa::path( + post, + path = "/api/agents/{name}/forge-token", + params(("name" = String, Path, description = "agent name")), + responses( + (status = 200, description = "mint queued", body = MintAgentForgeTokenResponse), + (status = 400, description = "`name` is not a valid identifier (problem+json)", body = String), + (status = 500, description = "the job could not be queued (problem+json)", body = String), + ), + tag = "agents" +)] +pub(crate) async fn mint_agent_forge_token( + State(state): State, + Path(name): Path, +) -> Result, problem_details::ProblemDetails> { + let agent = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + let ids = queue_forge_token_mints(&state.jobq, vec![agent]).map_err(|e| { + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &e.to_string(), + ) + })?; + let [id] = ids[..] else { + unreachable!("exactly one handle was asked for"); + }; + Ok(Json(MintAgentForgeTokenResponse { node_id: id.get() })) +} + +/// Make an existing human forge account a site admin. Idempotent. +/// +/// Never creates the account: a human's is made by their first SSO login to +/// the forge, and until then this answers 404. Refuses an agent's name: a +/// site admin can create repos whatever its `max_repo_creation`, and an +/// agent's account is locked to 0 so its token cannot create a repo and +/// self-merge in it. +#[utoipa::path( + post, + path = "/api/forge/users/{name}/admin", + params(("name" = String, Path, description = "forge username")), + responses( + (status = 200, description = "the account is a site admin", body = MakeForgeAdminResponse), + (status = 400, description = "`name` is not a valid identifier (problem+json)", body = String), + (status = 404, description = "no such account: the user has not logged in via SSO yet (problem+json)", body = String), + (status = 409, description = "`name` is an agent (problem+json)", body = String), + (status = 500, description = "the forge refused the read or the edit (problem+json)", body = String), + (status = 503, description = "no forge configured, or the agent roster could not be read (problem+json)", body = String), + ), + tag = "forge" +)] +pub(crate) async fn make_forge_admin( + State(state): State, + Path(name): Path, +) -> Result, problem_details::ProblemDetails> { + let name = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + let Some(forge) = state.forge.get().cloned() else { + return Err(error_problem( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "no forge is configured on this host", + )); + }; + let is_agent = in_roster(state.auth.as_deref(), &name).await; + admin_unless_agent(&name, is_agent, || async { + forge.make_site_admin(&name).await + }) + .await +} + +/// Run `promote` only if `name` is known not to be an agent, and answer with +/// what it found. +/// +/// An unreadable roster refuses too: a site admin is the one thing an agent's +/// account must never become, so "could not tell" is not "no". +pub(crate) async fn admin_unless_agent( + name: &str, + is_agent: Result, + promote: F, +) -> Result, problem_details::ProblemDetails> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + match is_agent { + Ok(false) => {} + Ok(true) => { + return Err(error_problem( + axum::http::StatusCode::CONFLICT, + &format!("{name:?} is an agent; an agent's forge account is never a site admin"), + )); + } + Err(e) => { + return Err(error_problem( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + &format!("cannot tell whether {name:?} is an agent, so refusing: {e}"), + )); + } + } + let plan = promote().await.map_err(|e| { + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &format!("{e:#}"), + ) + })?; + let change = match plan { + forge::site_admin::Plan::NoSuchUser => { + return Err(error_problem( + axum::http::StatusCode::NOT_FOUND, + &format!( + "the forge has no user {name:?}: the user has not logged in via SSO yet. \ + The forge creates the account on their first authelia login; run this \ + again after it" + ), + )); + } + forge::site_admin::Plan::AlreadyAdmin => ForgeAdminChange::AlreadyAdmin, + forge::site_admin::Plan::Promote => ForgeAdminChange::Promoted, + }; + Ok(Json(MakeForgeAdminResponse { change })) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `admin_unless_agent` with a `promote` that records whether it ran and + /// answers `plan`. + async fn forge_admin_route( + is_agent: Result, + plan: crate::forge::site_admin::Plan, + ) -> ( + Result, problem_details::ProblemDetails>, + bool, + ) { + let ran = std::sync::atomic::AtomicBool::new(false); + let result = super::admin_unless_agent("atlas", is_agent, || async { + ran.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(plan) + }) + .await; + (result, ran.into_inner()) + } + + /// An agent's name never reaches the forge. + #[tokio::test] + async fn the_forge_admin_route_refuses_an_agent() { + let (result, ran) = + forge_admin_route(Ok(true), crate::forge::site_admin::Plan::Promote).await; + let Err(problem) = result else { + panic!("an agent's name must be refused"); + }; + assert!(!ran, "the account must not be touched"); + assert_eq!(problem.status, Some(axum::http::StatusCode::CONFLICT)); + } + + /// A roster that can't be read can't vouch for the name, so the route + /// refuses rather than guessing "not an agent". + #[tokio::test] + async fn the_forge_admin_route_refuses_on_an_unreadable_roster() { + let (result, ran) = forge_admin_route( + Err("bridge down".to_owned()), + crate::forge::site_admin::Plan::Promote, + ) + .await; + let Err(problem) = result else { + panic!("an unreadable roster must refuse"); + }; + assert!(!ran, "the account must not be touched"); + assert_eq!( + problem.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE) + ); + } + + /// A missing account is a 404 that tells the operator what to wait for. + #[tokio::test] + async fn the_forge_admin_route_names_the_missing_sso_login() { + let (result, ran) = + forge_admin_route(Ok(false), crate::forge::site_admin::Plan::NoSuchUser).await; + let Err(problem) = result else { + panic!("a missing account must not report success"); + }; + assert!(ran); + assert_eq!(problem.status, Some(axum::http::StatusCode::NOT_FOUND)); + let rendered = format!("{problem:?}"); + assert!( + rendered.contains("has not logged in via SSO yet"), + "{rendered}" + ); + } + + #[tokio::test] + async fn the_forge_admin_route_promotes_a_non_agent() { + let (result, ran) = + forge_admin_route(Ok(false), crate::forge::site_admin::Plan::Promote).await; + let Ok(axum::Json(resp)) = result else { + panic!("a non-agent must be promoted"); + }; + assert!(ran); + assert_eq!(resp.change, super::ForgeAdminChange::Promoted); + } + + /// Idempotent: a second run succeeds and says nothing changed. + #[tokio::test] + async fn the_forge_admin_route_succeeds_for_an_admin() { + let (result, _) = + forge_admin_route(Ok(false), crate::forge::site_admin::Plan::AlreadyAdmin).await; + let Ok(axum::Json(resp)) = result else { + panic!("an existing admin must succeed"); + }; + assert_eq!(resp.change, super::ForgeAdminChange::AlreadyAdmin); + } +} diff --git a/swarm-controller/src/routes/hives.rs b/swarm-controller/src/routes/hives.rs new file mode 100644 index 00000000..74301cf2 --- /dev/null +++ b/swarm-controller/src/routes/hives.rs @@ -0,0 +1,126 @@ +//! The swarm's hive directory, service links and display name. + +use anyhow::Result; +use axum::{Json, extract::State}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::routes::{AppState, StatusUnavailable}; +use crate::status; + +/// One hive in the swarm's directory — the same `name`/`domain` pair +/// `services.hyperhive.swarm.hives` (nix/host-modules/swarm.nix) declares, +/// plus that hive's `swarm.dashboardDomains` entry, carried across +/// unchanged rather than reshaped, so this stays a direct mirror of the +/// nix source of truth instead of a second vocabulary for the same fields. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub(crate) struct HiveEntry { + pub(crate) name: String, + pub(crate) domain: String, + /// The host the hive's dashboard answers on. + pub(crate) dashboard: String, +} + +/// The swarm's hive directory — every hive, including whichever one this +/// controller instance happens to run on (there is no "self" to exclude +/// at the swarm level, unlike `hive-c0re`'s peer list). +#[utoipa::path( + get, + path = "/api/hives", + responses((status = 200, description = "every hive in the swarm", body = Vec)), + tag = "hives" +)] +pub(crate) async fn get_hives(State(state): State) -> Json> { + Json((*state.hives).clone()) +} + +/// One quick link to a swarm-wide service (authelia, matrix, forge, this +/// daemon's own swagger UI, …). Deliberately generic rather than named +/// fields per service: each service's own nix module contributes its own +/// entry to `services.hyperhive.swarm.controller.links` (same list-merge +/// idiom `services.hyperhive.gateway.localNames` already uses), so adding +/// a new one is a nix-only change — no new field here, no swarm-ui change. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub(crate) struct ServiceLink { + pub(crate) label: String, + /// Emoji or short glyph. Empty string, not `Option`, when a + /// contributing module has none — one fewer null-vs-absent case for + /// the frontend to handle, and every real contributor sets one today. + pub(crate) icon: String, + pub(crate) url: String, +} + +/// Quick links to swarm-wide services (authelia, matrix, forge, this +/// daemon's own swagger UI, …), as contributed by each service's own nix +/// module. Empty when nothing was configured to contribute — a caller +/// renders 0 links the same way it renders any other count, no special +/// "not configured" case. +#[utoipa::path( + get, + path = "/api/links", + responses((status = 200, description = "swarm service quick links", body = Vec)), + tag = "links" +)] +pub(crate) async fn get_links(State(state): State) -> Json> { + Json((*state.links).clone()) +} + +/// Body of `GET /api/swarm`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] +pub(crate) struct SwarmInfo { + /// `None` when the operator never set `services.hyperhive.swarm.name` + /// — a caller (swarm-ui's chrome) falls back to a generic label rather + /// than treating this as an error. + name: Option, +} + +/// The swarm's own display name, for UI chrome (page title, nav) that +/// wants to say which swarm it's showing — distinct from [`get_hives`], +/// which lists the *hives inside* the swarm, not the swarm itself. +#[utoipa::path( + get, + path = "/api/swarm", + responses((status = 200, description = "the swarm's display name, if the operator set one", body = SwarmInfo)), + tag = "hives" +)] +pub(crate) async fn get_swarm_info(State(state): State) -> Json { + Json(SwarmInfo { + name: state.swarm_name.as_deref().map(str::to_owned), + }) +} + +/// What each hive last said about itself, read from the swarm queue at +/// request time. +/// +/// Every hive in the roster gets a row whether or not it has ever +/// reported — see the `status` module for why absence, not presence, is +/// the case this is built around. +#[utoipa::path( + get, + path = "/api/hives/status", + responses( + (status = 200, description = "a row per hive, freshness derived now", body = Vec), + (status = 503, description = "no swarm queue is configured here, or its store could not be read", body = String), + ), + tag = "hives" +)] +pub(crate) async fn get_hives_status( + State(state): State, +) -> Result>, StatusUnavailable> { + let Some(reader) = state.status.as_ref() else { + return Err(StatusUnavailable( + "no swarm queue is configured on this host".to_owned(), + )); + }; + match reader + .view(&state.hives, std::time::SystemTime::now()) + .await + { + Ok(rows) => Ok(Json(rows)), + Err(e) => { + let detail = format!("{e:#}"); + tracing::warn!(error = %detail, "reading the swarm status bucket failed"); + Err(StatusUnavailable(detail)) + } + } +} diff --git a/swarm-controller/src/routes/jobq_graph.rs b/swarm-controller/src/routes/jobq_graph.rs new file mode 100644 index 00000000..760029fa --- /dev/null +++ b/swarm-controller/src/routes/jobq_graph.rs @@ -0,0 +1,65 @@ +//! Read-only views of the swarm-level job graph. + +use axum::{Json, extract::State}; +use hive_jobq_wire::GraphWire as _; +use serde::Deserialize; + +use crate::routes::AppState; + +/// Query params for `GET /api/jobq/graph` — `?states=` narrows to root +/// groups in the named states, same shape `hive_jobq_wire::parse_states` +/// parses. +#[derive(Deserialize, utoipa::IntoParams)] +pub(crate) struct JobqGraphQuery { + states: Option, +} + +/// Every node of every root group in the swarm-level job graph. Nothing +/// filters "done and old" here — with zero nodes ever submitted there is +/// nothing to bound yet, so `graph.roots()` (everything) is the whole +/// roster passed to [`hive_jobq_wire::GraphWire::wire_snapshot`]. +#[utoipa::path( + get, + path = "/api/jobq/graph", + params(JobqGraphQuery), + responses((status = 200, description = "every node of every root group, as generic \ + `hive_jobq` graph nodes. `?states=` narrows to root groups in the named states.", + body = Vec)), + tag = "jobq" +)] +pub(crate) async fn get_jobq_graph( + State(state): State, + axum::extract::Query(q): axum::extract::Query, +) -> Json> { + let states = hive_jobq_wire::parse_states(q.states.as_deref()); + let sched = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = sched.graph(); + let roots: Vec = graph.roots().map(|n| n.id).collect(); + let nodes = graph.wire_snapshot(roots); + Json(hive_jobq_wire::filter_nodes_by_state( + nodes, + states.as_deref(), + )) +} + +/// Counts by lifecycle state over the same groups `/api/jobq/graph` serves. +#[utoipa::path( + get, + path = "/api/jobq/rollup", + responses((status = 200, description = "counts by lifecycle state", body = Vec)), + tag = "jobq" +)] +pub(crate) async fn get_jobq_rollup( + State(state): State, +) -> Json> { + let sched = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = sched.graph(); + let roots: Vec = graph.roots().map(|n| n.id).collect(); + Json(hive_jobq_wire::state_rollup(graph, roots)) +} diff --git a/swarm-controller/src/routes/mod.rs b/swarm-controller/src/routes/mod.rs new file mode 100644 index 00000000..47e55b74 --- /dev/null +++ b/swarm-controller/src/routes/mod.rs @@ -0,0 +1,365 @@ +//! The HTTP state every handler shares, the error shapes they answer with, +//! and the handlers that live outside a resource module of their own. + +pub(crate) mod agents; +pub(crate) mod config_prs; +pub(crate) mod forge_admin; +pub(crate) mod hives; +pub(crate) mod jobq_graph; + +use std::sync::{Arc, Mutex}; + +use crate::jobq::{Late, SwarmNodeKind, SwarmResourceKind}; +use crate::routes::hives::{HiveEntry, ServiceLink}; +use crate::{ + agent_icon, agent_login, agent_status, agents_md, alerts, auth, builder, config_pr, forge, + inference, linked_accounts, review_classifier, status, wanted, +}; + +/// Liveness probe. Returns the build's version so an operator can tell +/// *which* controller answered without shelling onto the host. +#[utoipa::path( + get, + path = "/health", + responses((status = 200, description = "process is up, body is \"swarm-controller \"", body = String)), + tag = "health" +)] +pub(crate) async fn health() -> &'static str { + concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") +} + +#[derive(Clone)] +pub(crate) struct AppState { + /// Loaded once at startup (`load_hives`); never mutated, so an + /// `Arc` clone per request is the whole synchronization story. + pub(crate) hives: Arc>, + /// Loaded once at startup (`load_links`); same synchronization story + /// as `hives`. + pub(crate) links: Arc>, + /// `None` when this deployment wired up no swarm queue — the only + /// state in which `/api/hives/status` cannot answer at all. A queue + /// that is merely *unreachable* still yields a reader, because + /// `async-nats` reconnects underneath it. + pub(crate) status: Option>, + /// Publishes the agent set this swarm declares for each hive. + /// + /// `None` in exactly the state `status` is: no swarm queue was wired + /// up, so there is nowhere to publish a declaration to. Shares that + /// reader's connection rather than opening a second one. + pub(crate) wanted: Option>, + /// Per-agent status, sharing `status`'s connection — same "one queue + /// connection, several consumers" rationale as `wanted` above. `None` + /// in exactly the state `status` is. + pub(crate) agent_status: Option>, + /// Per-agent icons, sharing `status`'s connection — same "one queue + /// connection, several consumers" rationale as `agent_status` above, + /// and `None` in exactly the state `status` is. + pub(crate) agent_icons: Option>, + /// Each agent's published `AGENTS.md`, on `status`'s connection like + /// `agent_icons`, and `None` in exactly the state `status` is. + pub(crate) agents_md: Option>, + /// What each agent last reported running on, from its turn-state + /// headers on `status`'s connection. `None` in exactly the state + /// `status` is. + pub(crate) reported_inference: Option>, + /// Whether each agent last said it needs a login, from its turn-state + /// headers on `status`'s connection. `None` in exactly the state + /// `status` is. + pub(crate) agent_login: Option>, + /// The swarm-level job graph, wrapped in its + /// [`hive_jobq::scheduler::Scheduler`] now that something drives it + /// (`spawn_jobq_worker`) — the graph alone was enough for the + /// read-only endpoints, the scheduler is what a `claim_next` loop + /// needs. `std::sync::Mutex`, not `tokio`'s — every lock scope below + /// is synchronous (no `.await` while held). Always present, never + /// gated on the swarm queue: this is process state, not something + /// read over the network. + pub(crate) jobq: Arc>>, + /// The identity store's only reader, for `GET /api/agents`. + /// + /// `None` when no bridge is wired up, which is the one state in which + /// the roster cannot be answered at all — the same shape as `status` + /// above, and for the same reason: an empty roster and an unreadable + /// one are different facts. + /// + /// ⚠️ The two `/api/agents` verbs treat this differently on purpose. + /// **POST does not need it**: creation only queues a job, and + /// whether a bridge exists is `run_swarm_node`'s concern (it holds its + /// own clone from `spawn_jobq_worker`), so a request still queues + /// cleanly on a bridge-less host and fails loud once claimed. POST reads + /// it only for a name that breaks a naming rule, to tell a new agent + /// from an existing one, and warns when it cannot (`name_verdict`). **GET + /// has nowhere to defer to** — there is no job, only an answer it + /// either has or does not. `POST /api/forge/users/{name}/admin` reads + /// it for every name and refuses when it cannot (`admin_unless_agent`). + pub(crate) auth: Option>, + /// HMAC secret for swarm-wide forge webhooks, loaded once at startup. + /// `None` when it could not be read or created — the webhook endpoint + /// then refuses every delivery with 503 rather than admitting one it + /// cannot verify. Deliberately not fatal to startup: nothing is + /// registered against that endpoint yet, and the rest of this daemon's + /// surface is unaffected. See [`load_or_generate_secret`](crate::webhook::load_or_generate_secret). + /// + /// `Arc`, not `Arc`: the value is never mutated after + /// startup, and this way `as_deref()` yields the `&str` the verifier + /// takes without a second hop through `String`. + pub(crate) webhook_secret: Option>, + /// Last successful `agent-configs/*` scan, kept current by + /// [`config_pr::spawn`]. Empty while this daemon has no forge access — + /// same "absent means don't ask" shape as `status` and `forge`, not a + /// startup failure — and filled when the forge client is. + pub(crate) config_prs: Late>, + /// The swarm's human display name (`services.hyperhive.swarm.name`), + /// loaded once at startup (`load_swarm_name`). `None` when the + /// operator never set it — a swarm without a display name is a + /// supported, if less friendly, state, not a startup failure. Same + /// `Arc` rationale as `webhook_secret`: never mutated, so a + /// clone per request is just a refcount bump. + pub(crate) swarm_name: Option>, + /// The forge client itself, for the read-only repo/issue-report + /// routes (`GET /api/repos`, `GET /api/repos/{org}/{repo}/issue-report`) + /// and `POST /api/forge/users/{name}/admin` — distinct from `config_prs`, which holds a *cache* built off this + /// same client rather than the client. Empty while this daemon has no + /// forge access, and filled the moment `forge::enable` builds the client. + pub(crate) forge: Late>, + /// Held by `create_agent` from reading where a name is placed until its + /// graph is queued, and by `set_agent_state` from that read until its + /// declaration is written, so two placements of one name cannot both + /// find it unplaced. `tokio`'s mutex, unlike `jobq`'s: the read awaits + /// the queue. + pub(crate) create_gate: Arc>, + /// The store the linked-account routes read and write. + pub(crate) accounts: linked_accounts::Accounts, + /// The review classifier try-outs started here, until their results + /// expire. + pub(crate) try_outs: review_classifier::TryOuts, + /// The requested reviews running or waiting. + pub(crate) reviews: Arc, + /// The review classifier's finished runs. + pub(crate) runs: review_classifier::runs::Runs, + /// The swarm builder's finished builds. + pub(crate) builds: builder::runs::Builds, + /// Grafana's alerts API, for `GET /api/alerts`. `None` when no Grafana URL + /// is configured. + pub(crate) alerts: Option>, +} + +/// Why the status route answers 503 rather than an empty list. +/// +/// "I cannot reach the store" and "every hive is silent" are different +/// answers, and rendering the second when the first is true is exactly +/// the smoothing this endpoint exists to avoid — a caller would draw a +/// swarm-wide outage out of a local one. The cause is carried in the +/// body because a bare 503 on an operator-facing diagnostic is how a +/// misconfiguration costs an afternoon; it is a queue/JetStream error +/// string, and this surface is already behind the swarm's SSO. +/// +/// It travels in `detail` of an RFC 9457 `application/problem+json` body +/// rather than as a bare string, so the cause is an addressable field +/// instead of the whole payload — see `error_problem` below. +pub(crate) struct StatusUnavailable(pub(crate) String); + +impl axum::response::IntoResponse for StatusUnavailable { + fn into_response(self) -> axum::response::Response { + error_problem(axum::http::StatusCode::SERVICE_UNAVAILABLE, &self.0).into_response() + } +} + +/// Every error this daemon returns, in one shape. +/// +/// RFC 9457 `application/problem+json` is the hive-wide contract for HTTP +/// error bodies (`docs/process/conventions.md`), and the operator UIs read `detail` +/// for display. A bare string forces the reader to treat the entire body as +/// the message, which is the difference between a UI that can offer "copy the +/// cause" and one that can only dump a response. +pub(crate) fn error_problem( + status: axum::http::StatusCode, + detail: &str, +) -> problem_details::ProblemDetails { + problem_details::ProblemDetails::from_status_code(status).with_detail(detail) +} + +/// The status a [`wanted::WantedWriter`], [`agent_icon::AgentIconReader`] or +/// `schedules` queue failure should answer with. +/// +/// `WantedWriter::view`/`set` call `ensure_connected` internally and +/// propagate through `anyhow`, so the concrete +/// [`swarm_queue_client::Error`] survives underneath but is not the type in +/// hand — `swarm_queue_client`'s own doc says a caller "gets a type it can +/// match on", which means downcasting back to it rather than string-matching +/// the rendered message. `NotConnected` is a retryable condition (the queue +/// is reconnecting, or hasn't yet) and every other queue-backed route here +/// already answers 503 for it; anything else is a genuine failure to publish +/// or read, which stays 500. +pub(crate) fn wanted_error_status(e: &anyhow::Error) -> axum::http::StatusCode { + match e.downcast_ref::() { + Some(swarm_queue_client::Error::NotConnected(_)) => { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } + _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// The 503 this route returns is what the operator UIs' shared error + /// component renders, so assert the RENDERED response rather than the + /// `problem_details` crate: the contract a UI depends on is the content + /// type plus a `detail` it can address, and a handler that built the + /// value and returned it as a bare string would satisfy any test + /// written against the type alone. + #[tokio::test] + async fn status_unavailable_renders_problem_json_with_the_cause_in_detail() { + use axum::response::IntoResponse as _; + + let cause = "listing status bucket keys: timed out"; + let resp = StatusUnavailable(cause.to_owned()).into_response(); + + assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE); + let ct = resp + .headers() + .get(axum::http::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_owned(); + assert!( + ct.starts_with("application/problem+json"), + "RFC 9457 media type, got {ct:?}" + ); + + let bytes = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body reads"); + let v: serde_json::Value = serde_json::from_slice(&bytes).expect("problem+json parses"); + assert_eq!(v["status"], 503); + // The cause is an addressable field, not the entire payload — that + // distinction is the point of the change, so it is what is asserted. + assert_eq!(v["detail"], cause); + } + + /// Drives `SwarmNodeKind::CreateRepo` through the real + /// `hive_jobq::scheduler::Scheduler` claim → run → complete path, + /// rather than only through `create_agent`'s endpoint test (there + /// isn't one — the endpoint itself is thin, insert-and-return; the + /// interesting behavior is in `run_swarm_node`'s executor arm, which + /// this exercises directly). + /// + /// Deliberately offline: with the forge URL unset, + /// `forge::Client::from_env` returns `Ok(None)` (see that module's doc + /// comment), so this exercises the whole claim → run → complete path + /// through `hive_jobq::scheduler::Scheduler` without a real forge + /// server — at the cost of only ever observing the + /// graceful-absence-is-failure branch here. The happy path needs an + /// actual forge instance and isn't something a unit test in this crate + /// can reach. + /// + /// SAFETY: single-threaded mutation of the `forge` env var this + /// test itself owns, restored before returning — no other test in this + /// crate reads them. + /// One roster, one agent name, two hives — the accepted one and a + /// typo. Built per test so neither can see the other's queue. + /// The scheduler handle these tests hold onto so they can assert on + /// what the endpoint queued. Aliased because the full type is three + /// nested generics deep and reads worse inline than named. + pub(crate) type SharedSched = std::sync::Arc< + std::sync::Mutex< + hive_jobq::scheduler::Scheduler< + crate::jobq::SwarmNodeKind, + crate::jobq::SwarmResourceKind, + >, + >, + >; + + pub(crate) fn state_with_roster() -> (super::AppState, SharedSched) { + let sched = + std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ))); + let state = super::AppState { + hives: std::sync::Arc::new(vec![HiveEntry { + name: "pr1ma".to_owned(), + domain: "pr1ma.example".to_owned(), + dashboard: "pr1ma.hives.example".to_owned(), + }]), + links: std::sync::Arc::new(Vec::new()), + status: None, + // No queue, for the same reason as `status`: these tests build + // the creation graph and read its shape, and never run the node + // that would write a declaration. + wanted: None, + agent_status: None, + agent_icons: None, + agents_md: None, + reported_inference: None, + agent_login: None, + jobq: std::sync::Arc::clone(&sched), + webhook_secret: None, + config_prs: std::sync::Arc::default(), + swarm_name: None, + // No bridge: these tests drive agent *creation*, which queues a + // job and consults one only for a rule-breaking name. The roster + // read is the verb that needs it, and it has its own test below. + auth: None, + forge: std::sync::Arc::default(), + create_gate: std::sync::Arc::default(), + accounts: crate::linked_accounts::Accounts::Fake(std::sync::Arc::default()), + try_outs: crate::review_classifier::TryOuts::default(), + reviews: std::sync::Arc::default(), + runs: crate::review_classifier::runs::Runs::Live(None), + builds: crate::builder::runs::Builds::Live(None), + alerts: None, + }; + (state, sched) + } + + /// `wanted_error_status` as a pure function first: build the exact + /// `swarm_queue_client::Error::NotConnected` `ensure_connected` raises, + /// wrap it the same way `?` inside `WantedWriter::view`/`set` does, and + /// confirm it downcasts back to the status those routes are meant to + /// answer. + #[test] + fn not_connected_downcasts_to_503() { + let e: anyhow::Error = + swarm_queue_client::Error::NotConnected(async_nats::connection::State::Pending).into(); + assert_eq!( + super::wanted_error_status(&e), + axum::http::StatusCode::SERVICE_UNAVAILABLE + ); + } + + /// Invert-proof for the test above: an `anyhow::Error` that does not + /// wrap a `swarm_queue_client::Error` at all (the shape a genuine write + /// or decode failure takes — `apply`'s "declaration is not decodable" + /// context, for instance) must NOT downcast to `NotConnected`, and must + /// stay 500. Without this, a `wanted_error_status` that answered 503 + /// unconditionally would still pass the test above. + #[test] + fn an_unrelated_error_stays_500() { + let e = anyhow::anyhow!("the hive's current declaration is not decodable"); + assert_eq!( + super::wanted_error_status(&e), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + } + + /// Second invert-proof, closer to the actual bug this closes: a + /// *different* `swarm_queue_client::Error` variant (a real connect + /// failure, not `NotConnected`) also stays 500 — proving the match + /// looks at the variant, not just at "was this crate's error type + /// involved at all". + #[test] + fn a_different_queue_client_error_variant_stays_500() { + let e: anyhow::Error = swarm_queue_client::Error::Connect { + url: "nats://queue.example:4222".to_owned(), + source: async_nats::ConnectErrorKind::TimedOut.into(), + } + .into(); + assert_eq!( + super::wanted_error_status(&e), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + } +} diff --git a/swarm-controller/src/schedules.rs b/swarm-controller/src/schedules.rs index c354a00c..a1299682 100644 --- a/swarm-controller/src/schedules.rs +++ b/swarm-controller/src/schedules.rs @@ -33,8 +33,8 @@ use swarm_controller_api::schedules::{ use swarm_queue_client::agent_schedules::{self, AgentSchedules, Schedule}; use crate::agent_renewal::unix_now; +use crate::routes::{AppState, error_problem}; use crate::wanted::WantedWriter; -use crate::{AppState, error_problem}; /// The bucket holding every schedule definition. Agents hold no grant on it. pub const DEFINITIONS_BUCKET: &str = "schedule-definitions"; @@ -384,7 +384,7 @@ fn problem(failure: Failure) -> ProblemDetails { } Failure::Queue(e) => { tracing::warn!(error = %format!("{e:#}"), "schedules: queue failure"); - error_problem(crate::wanted_error_status(&e), &format!("{e:#}")) + error_problem(crate::routes::wanted_error_status(&e), &format!("{e:#}")) } } } diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs new file mode 100644 index 00000000..ec4d5e11 --- /dev/null +++ b/swarm-controller/src/startup.rs @@ -0,0 +1,910 @@ +//! Process startup: the config this daemon loads once from env vars, the +//! background passes it starts, and the router it serves. +//! +//! `OpenAPI` spec generation mirrors `hive-c0re/src/dashboard/mod.rs` +//! exactly: `#[utoipa::path(...)]` per handler, an `ApiDoc` root, and a +//! raw JSON route at `/api/openapi.json`. Swagger UI itself is +//! nginx-hosted from the nix store, same shape as the per-hive +//! dashboard's (`nix/host-modules/hive-c0re/vhosts.nix`'s +//! `swaggerUiLocations` — a swagger-ui-theme dist under `/api/docs/`, no +//! fallback to this daemon). Only annotated routes appear in the spec; +//! an unannotated one just doesn't show up, nothing breaks. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use axum::{Json, routing::get}; +use backon::{ExponentialBuilder, Retryable as _}; +use utoipa::OpenApi; +use utoipa_axum::{router::OpenApiRouter, routes}; + +use crate::jobq::{ + Late, SwarmNodeKind, SwarmResourceKind, queue_agent_renewals, queue_builder_forge_token_mint, + queue_ci_metrics_forge_token_mint, queue_classifier_forge_token_mint, queue_docs_grants, + queue_forge_token_mints, queue_hive_forge_token_mints, queue_hive_sender_mints, + queue_matrix_account_mints, +}; +use crate::routes::hives::{HiveEntry, ServiceLink}; +use crate::routes::{self, AppState, agents, config_prs, forge_admin, hives, jobq_graph}; +use crate::{ + agent_control, agent_icon, agent_input, agent_limits, agent_login, agent_permissions, + agent_renewal, agent_state_stream, agent_status, agents_md, alerts, archetypes, auth, builder, + claude_login, config_pr, forge, forge_account, github_account, inference, issue_report, + linked_accounts, matrix_account, pulls, queue_identity, review_classifier, schedules, status, + subagent_term, term_stream, wanted, webhook, +}; + +/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`. +/// +/// A compiled-in default is legitimate here and is *not* the mistake that +/// a hardcoded remote address would be: this is a path this process +/// **creates**, not an address it hopes to find something at. systemd's +/// `RuntimeDirectory=swarm-controller` makes the parent exist before +/// `ExecStart`, so the default names a directory the unit just produced. +/// +/// The directory is its own — deliberately not shared with hive-c0re's +/// `/run/hyperhive`. The socket is `0666`, so its directory is the only +/// access control it has; co-locating it with c0re's admin socket would +/// put both within reach of whatever can reach either. nginx runs on the +/// host, so nothing narrows its reach for you. +const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock"; + +pub(crate) fn socket_path() -> PathBuf { + std::env::var_os("SWARM_CONTROLLER_SOCKET") + .map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from) +} + +/// Root of the auto-generated `OpenAPI` spec, served raw at +/// `/api/openapi.json` — see the module doc comment above. Tag list +/// grows alongside the swarm-level surfaces this daemon picks up, same +/// as `hive-c0re::dashboard::ApiDoc`'s tag list did. +#[derive(OpenApi)] +#[openapi( + info( + title = "hyperhive swarm-controller API", + description = "swarm-controller's HTTP surface, served over its unix \ + socket behind the gateway's swarm-UI vhost." + ), + tags( + (name = "health", description = "liveness probe"), + (name = "hives", description = "the swarm's hive directory"), + (name = "links", description = "swarm service quick links"), + (name = "jobq", description = "the swarm-level job graph"), + (name = "agents", description = "creating agent identities at swarm level"), + (name = "webhook", description = "swarm-wide forge webhook receipt"), + (name = "repos", description = "forge repo browsing (swarm-ui's issue-report page)"), + (name = "forge", description = "forge site admins (`swarmctl forge make-admin`)"), + (name = "schedules", description = "scheduled prompts (`swarmctl schedule`)"), + (name = "review-classifier", description = "trying review classifications against a PR (swarm-ui's try-out page), and the classifier's swarm-wide settings"), + (name = "pulls", description = "open pull requests, for the try-out page's dropdowns"), + (name = "alerts", description = "the swarm Grafana's current alerts (swarm-ui's alerts page)"), + (name = "builder", description = "the swarm builder's finished builds of agent config PRs, which their commit statuses link to"), + ) +)] +pub(crate) struct ApiDoc; + +/// Env var the controller's NixOS module sets from +/// `services.hyperhive.swarm.hives`, JSON-encoded — the full directory +/// (this daemon has no "self" to exclude) rather than peers-minus-self. +const HIVES_ENV: &str = "SWARM_CONTROLLER_HIVES"; + +/// Parses [`HIVES_ENV`] into the swarm's hive directory. Unset or +/// unparseable both fall back to an empty list with a warning rather than +/// failing startup — a swarm-controller that can't yet see its own config +/// (dev run, a module not wired up yet) should still serve `/health`. +pub(crate) fn load_hives() -> Vec { + let Some(raw) = std::env::var_os(HIVES_ENV) else { + return Vec::new(); + }; + match serde_json::from_str(&raw.to_string_lossy()) { + Ok(hives) => hives, + Err(e) => { + tracing::warn!(error = %e, env = HIVES_ENV, "failed to parse hive directory, serving an empty list"); + Vec::new() + } + } +} + +/// Env var the controller's NixOS module sets from the merged +/// `services.hyperhive.swarm.controller.links` list, JSON-encoded — same +/// shape/rationale as [`HIVES_ENV`]. Consumed by `GET /api/links`. +const LINKS_ENV: &str = "SWARM_CONTROLLER_LINKS"; + +/// Parses [`LINKS_ENV`] into the swarm's service-link list. Same +/// fall-back-to-empty rationale as `load_hives`: a daemon that can't yet +/// see this config should still serve `/health` rather than fail startup. +pub(crate) fn load_links() -> Vec { + let Some(raw) = std::env::var_os(LINKS_ENV) else { + return Vec::new(); + }; + match serde_json::from_str(&raw.to_string_lossy()) { + Ok(links) => links, + Err(e) => { + tracing::warn!(error = %e, env = LINKS_ENV, "failed to parse service links, serving an empty list"); + Vec::new() + } + } +} + +/// Env var the controller's NixOS module sets from +/// `services.hyperhive.swarm.name` — unset (rather than an empty string) +/// when the operator never configured it. Consumed by `GET /api/swarm`. +const NAME_ENV: &str = "SWARM_CONTROLLER_NAME"; + +/// Reads [`NAME_ENV`]. `None` on absence — no fallback-and-warn shape like +/// `load_hives`/`load_links` because there is nothing to parse and fail: +/// an unset env var and an operator who never named the swarm are the same +/// state, not an error. +pub(crate) fn load_swarm_name() -> Option { + std::env::var(NAME_ENV).ok() +} + +/// The declaration writer, sharing the status reader's connection. +/// +/// The controller holds exactly one queue connection by design — see +/// `StatusReader::queue_client`. A second connect would double the +/// auth-callout traffic and give the two paths independent reconnect state, +/// so one could be serving while the other was still down. +pub(crate) fn wanted_writer( + status: Option<&Arc>, +) -> Option> { + status.map(|s| Arc::new(wanted::WantedWriter::new(s.queue_client()))) +} + +/// The per-agent status reader, sharing the status reader's connection and +/// staleness threshold — same rationale as [`wanted_writer`], and the same +/// threshold on purpose: both buckets are fed by the same publisher cadence +/// (`hive-c0re`'s `PUBLISH_INTERVAL`), so two separately-configured +/// thresholds would just be two ways to get out of sync with one cadence. +pub(crate) fn agent_status_reader( + status: Option<&Arc>, +) -> Option> { + status.map(|s| { + Arc::new(agent_status::AgentStatusReader::new( + s.queue_client(), + status::StatusReader::stale_after_from_env(), + )) + }) +} + +/// The per-agent icon reader, sharing the status reader's connection — +/// same rationale as [`wanted_writer`]. No staleness threshold, unlike +/// [`agent_status_reader`]: an icon is a property of the agent rather than +/// a report about it, so there is no cadence it could be late against. +/// +/// Also starts creating the bucket, which the agents writing it cannot do. +pub(crate) fn agent_icon_reader( + status: Option<&Arc>, +) -> Option> { + let reader = Arc::new(agent_icon::AgentIconReader::new(status?.queue_client())); + let creating = Arc::clone(&reader); + tokio::spawn(async move { creating.create_when_connected().await }); + Some(reader) +} + +/// The `AGENTS.md` reader, sharing the status reader's connection, and +/// creating the bucket, as [`agent_icon_reader`] does for icons. +pub(crate) fn agents_md_reader( + status: Option<&Arc>, +) -> Option> { + let reader = Arc::new(agents_md::AgentsMdReader::new(status?.queue_client())); + let creating = Arc::clone(&reader); + tokio::spawn(async move { creating.create_when_connected().await }); + Some(reader) +} + +/// What each agent last reported running on, read on the status reader's +/// connection — same rationale as [`wanted_writer`]. +pub(crate) fn reported_inference( + status: Option<&Arc>, +) -> Option> { + Some(inference::Reported::spawn(status?.queue_client())) +} + +/// Whether each agent last said it needs a login, read on the status +/// reader's connection — same rationale as [`wanted_writer`]. +pub(crate) fn agent_login( + status: Option<&Arc>, +) -> Option> { + Some(agent_login::Reported::spawn(status?.queue_client())) +} + +/// Same "log and carry on" shape as the queue/bridge/forge wiring in `main`. +/// A controller that cannot hold a webhook secret still serves every other +/// route; the webhook endpoint answers 503, which is the honest answer +/// rather than a silent accept. Lifted out of `main` for +/// `clippy::too_many_lines`. +pub(crate) fn load_webhook_secret() -> Option> { + match webhook::load_or_generate_secret() { + Ok(secret) => Some(Arc::from(secret)), + Err(e) => { + tracing::warn!( + error = %format!("{e:#}"), + "webhook secret unavailable; swarm-wide forge webhooks are off" + ); + None + } + } +} + +/// The homeserver agents' own matrix accounts are minted on. Unset is this +/// controller creating agents with no matrix account, which the mint node +/// reports by name rather than `main` refusing to start. +pub(crate) fn configured_matrix_homeserver() -> Option> { + matrix_account::configured_default_homeserver() + .filter(|v| !v.is_empty()) + .map(Arc::from) +} + +/// Start the recovery passphrase pass, which needs only the store; the +/// matrix-account backfill and the hive sender-token pass when a homeserver is +/// configured; and the operator admin pass when the bridge is too. Lifted out +/// of `main` for `clippy::too_many_lines`. +pub(crate) fn spawn_matrix_account_backfill( + jobq: &Arc>>, + homeserver: Option>, + hives: &[HiveEntry], + auth: Option>, +) { + matrix_account::recovery::spawn(); + let Some(base) = homeserver else { + return; + }; + if let Some(bridge) = auth { + matrix_account::operator_admin::spawn(bridge, Arc::clone(&base)); + } + let sched = Arc::clone(jobq); + matrix_account::agent_token::spawn(Arc::clone(&base), move |agents| { + if let Err(e) = queue_matrix_account_mints(&sched, agents) { + tracing::warn!(error = %format!("{e:#}"), "agent matrix accounts: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + let hive_names = hives.iter().map(|h| h.name.clone()).collect(); + matrix_account::hive_sender::spawn(base, hive_names, move |hives| { + if let Err(e) = queue_hive_sender_mints(&sched, hives) { + tracing::warn!(error = %format!("{e:#}"), "hive sender tokens: queueing failed"); + } + }); +} + +/// Start the forge's periodic passes — the swarm-wide objects, the runner +/// registration token, hives' and the builder's read tokens, agents' +/// `internal/docs` grants, and +/// agents' tokens (which also +/// sweeps legacy tokens; see `forge::legacy_tokens`) — +/// once there is a forge client, and the operators' profiles when the bridge +/// and the queue are wired up too. Lifted out of `main` for +/// `clippy::too_many_lines`. +fn spawn_forge_workers( + jobq: &Arc>>, + client: Arc, + hives: &[HiveEntry], + auth: Option>, + queue: Option, +) { + forge::objects::spawn(Arc::clone(&client)); + if let (Some(bridge), Some(queue)) = (auth, queue) { + forge::operator_profiles::spawn(Arc::clone(&client), bridge, queue); + } + forge::runner_token::spawn(Arc::clone(&client)); + let sched = Arc::clone(jobq); + let hive_names = hives.iter().map(|h| h.name.clone()).collect(); + forge::hive_token::spawn(Arc::clone(&client), hive_names, move |hives| { + if let Err(e) = queue_hive_forge_token_mints(&sched, hives) { + tracing::warn!(error = %format!("{e:#}"), "hive forge tokens: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + forge::classifier_account::spawn(Arc::clone(&client), move || { + if let Err(e) = queue_classifier_forge_token_mint(&sched) { + tracing::warn!(error = %format!("{e:#}"), "review classifier forge token: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + forge::builder_account::spawn(Arc::clone(&client), move || { + if let Err(e) = queue_builder_forge_token_mint(&sched) { + tracing::warn!(error = %format!("{e:#}"), "builder forge token: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + forge::ci_metrics_account::spawn(Arc::clone(&client), move || { + if let Err(e) = queue_ci_metrics_forge_token_mint(&sched) { + tracing::warn!(error = %format!("{e:#}"), "ci metrics forge token: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + forge::docs_grant::spawn(Arc::clone(&client), move |agents| { + if let Err(e) = queue_docs_grants(&sched, agents) { + tracing::warn!(error = %format!("{e:#}"), "docs grants: queueing failed"); + } + }); + let sched = Arc::clone(jobq); + forge::agent_token::spawn(client, move |agents| { + if let Err(e) = queue_forge_token_mints(&sched, agents) { + tracing::warn!(error = %format!("{e:#}"), "agent forge tokens: queueing failed"); + } + }); +} + +/// Start the agent credential renewal pass when a swarm queue is wired up: +/// the pass reads the wanted-state declarations to tell live agents from +/// destroyed ones, and without a queue it has none. Lifted out of `main` for +/// `clippy::too_many_lines`. +pub(crate) fn spawn_agent_renewal( + jobq: &Arc>>, + wanted: Option>, + hives: &[HiveEntry], +) { + let Some(wanted) = wanted else { + return; + }; + let hives = hives.iter().map(|h| h.name.clone()).collect(); + let sched = Arc::clone(jobq); + agent_renewal::spawn(wanted, hives, move |renewals| { + if let Err(e) = queue_agent_renewals(&sched, renewals) { + tracing::warn!(error = %format!("{e:#}"), "agent credential renewal: queueing failed"); + } + }); +} + +/// Start creating every live agent's subagent and terminal streams when a swarm +/// queue is wired up. Lifted out of `main` for `clippy::too_many_lines`. +fn spawn_subagent_streams(status: Option<&Arc>, hives: &[HiveEntry]) { + let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { + return; + }; + let hives = hives.iter().map(|h| h.name.clone()).collect(); + subagent_term::spawn(status.queue_client(), wanted, hives); +} + +/// The per-agent background passes: subagent and terminal streams, and +/// schedule keys, and the permissions bucket and its import, and the +/// archetypes agents are created from. +/// Lifted out of `main` for `clippy::too_many_lines`. +pub(crate) fn spawn_per_agent_upkeep( + status: Option<&Arc>, + hives: &[HiveEntry], +) { + spawn_subagent_streams(status, hives); + spawn_schedule_reconcile(status, hives); + spawn_agent_permissions(status, hives); + if let Some(status) = status { + archetypes::spawn(status.queue_client()); + } +} + +/// Create the agent-permissions bucket and answer hives' imports when a swarm +/// queue is wired up. Same shape as [`spawn_subagent_streams`]. +fn spawn_agent_permissions(status: Option<&Arc>, hives: &[HiveEntry]) { + let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { + return; + }; + let hives = hives.iter().map(|h| h.name.clone()).collect(); + agent_permissions::spawn(status.queue_client(), wanted, hives); +} + +/// Start rewriting every agent's schedule key from the definitions when a +/// swarm queue is wired up. Same shape as [`spawn_subagent_streams`]. +fn spawn_schedule_reconcile(status: Option<&Arc>, hives: &[HiveEntry]) { + let (Some(status), Some(wanted)) = (status, wanted_writer(status)) else { + return; + }; + let hives = hives.iter().map(|h| h.name.clone()).collect(); + schedules::spawn_reconcile(status.queue_client(), wanted, hives); +} + +/// The swarm's own public base URL, as the forge must address it. +/// +/// Set by `swarm-controller.nix` **only when this host actually serves the +/// swarm vhost** that carries the `/webhook/forge/` location. Absent means +/// "the endpoint is not reachable from outside", and the right response to +/// that is to register nothing: a hook pointing at a URL nothing answers is +/// worse than no hook, because forgejo records failed deliveries against a +/// registration that looks configured. +pub(crate) const PUBLIC_URL_ENV: &str = "SWARM_CONTROLLER_PUBLIC_URL"; + +/// Register the swarm-wide forge hooks against this controller, in the +/// background. +/// +/// Detached rather than awaited, and never fatal: the forge may be slow or +/// briefly down at boot, and none of the daemon's other routes depend on a +/// hook existing. +/// +/// Retried on a backoff rather than left to the next process start. Losing +/// that race is the *common* case, not an edge one — the controller and the +/// forge come up together on a rebuild, and a deploy was measured where both +/// consecutive starts got `502 Bad Gateway` from the gateway because forgejo +/// was not serving yet. Nothing schedules another start, so "the next restart +/// retries" can leave a swarm with no hooks registered for as long as the +/// daemon keeps running — and the failure is silent at both ends, because the +/// forge has nothing to report about a call that never arrived. +/// +/// Runs once there is a forge client, and silently does nothing when either +/// other precondition is missing — each is a legitimate deployment shape (no +/// state directory to hold a secret, no public vhost), and each is already +/// logged where it is discovered. +fn register_swarm_webhooks(forge: Arc, secret: Option>) { + let Some(secret) = secret else { return }; + let Ok(public_url) = std::env::var(PUBLIC_URL_ENV) else { + tracing::info!( + "{PUBLIC_URL_ENV} unset; not registering swarm-wide forge webhooks (the \ + endpoint is not published on this host)" + ); + return; + }; + tokio::spawn(async move { + // A bounded schedule, not a poll loop: 15s doubling to 240s, five + // retries. It exists to outlast a forge that is slow to start, not to + // re-register periodically. Registration is idempotent, so a repeat + // costs one API call and changes nothing. + const RETRIES: usize = 5; + + let mut failed = 0; + let registered = (|| forge.ensure_swarm_webhooks(&public_url, &secret)) + .retry( + ExponentialBuilder::default() + .with_min_delay(std::time::Duration::from_secs(15)) + .with_max_delay(std::time::Duration::from_mins(4)) + .with_max_times(RETRIES), + ) + .notify(|e, wait| { + failed += 1; + tracing::info!( + error = %format!("{e:#}"), + attempt = failed, + retry_in = ?wait, + "registering swarm-wide forge webhooks failed; retrying" + ); + }) + .await; + if let Err(e) = registered { + tracing::warn!( + error = %format!("{e:#}"), + attempts = RETRIES + 1, + "registering swarm-wide forge webhooks failed; giving up until the next start" + ); + } + }); +} + +/// Connect to the swarm queue when this deployment wired one up, extracted +/// out of `main` purely to keep that function under clippy's line-count +/// lint — no behavior split, every comment below is unchanged from where +/// it used to sit inline. +/// +/// Deliberately NOT fatal on failure: the controller's HTTP surface is +/// useful without the queue, and a hive that cannot be read from renders +/// as `unknown` rather than as an outage of this daemon. What IS fatal is +/// an identity that could not be resolved, which `queue_identity::init` +/// reports in `main` before this runs: silently behaving like an +/// unconfigured host is how every hive ends up reading `never_reported` +/// with nothing to point at. +pub(crate) async fn connect_status_reader() -> Result>> { + let Some(cfg) = queue_identity::get().cloned() else { + tracing::info!("no swarm queue configured; status aggregation is off"); + return Ok(None); + }; + match swarm_queue_client::connect(cfg).await { + Ok(client) => { + // NOT "connected": `retry_on_initial_connect` returns a client + // before any connection has been established, so claiming a + // connection here would put "connected to the swarm queue" in + // the journal moments before every request 503s with "not + // connected" — and a reader would rightly distrust the second + // line rather than the first. The connection's real state is + // reported by the status endpoint, which checks it per request. + tracing::info!("swarm queue configured; connecting in the background"); + Ok(Some(Arc::new(status::StatusReader::new( + client, + status::StatusReader::stale_after_from_env(), + )))) + } + Err(e) => { + // `chain`, not `{:#}`: this is the queue client's own + // error type, and thiserror's Display ignores the + // alternate flag — the source would be dropped silently. + tracing::warn!( + error = swarm_queue_client::chain(&e), + "swarm queue unreachable" + ); + Ok(None) + } + } +} + +/// Fill the forge slots and start everything that runs off the forge client — +/// its periodic passes, the config-PR poll and the webhook registration — now +/// if the token is in the store, otherwise when `forge::enable`'s background +/// re-read finds it. Nothing to start when no forge URL is configured. Lifted +/// out of `main` for `clippy::too_many_lines`. +pub(crate) async fn start_forge( + forge_client: Late>, + config_prs: Late>, + jobq: Arc>>, + hives: Vec, + webhook_secret: Option>, + auth: Option>, + queue: Option, +) { + let url = match forge::url_from_env() { + Ok(Some(url)) => url, + Ok(None) => return, + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "forge misconfigured; repo provisioning is off"); + return; + } + }; + forge::enable(url, forge_client, move |client| { + spawn_forge_workers(&jobq, Arc::clone(&client), &hives, auth, queue); + config_prs.get_or_init(|| config_pr::spawn(Arc::clone(&client))); + register_swarm_webhooks(client, webhook_secret); + }) + .await; +} + +/// Every route this daemon serves, wired to its state. +/// +/// Split out of `main` because the route list is the part that grows, and +/// `main` sat exactly on the `too_many_lines` limit — adding an endpoint +/// tripped a lint about the startup sequence, which is not where the change +/// was. A new route now costs one line here and none there. +pub(crate) fn build_app(state: AppState) -> axum::Router { + let (router, api) = OpenApiRouter::::with_openapi(ApiDoc::openapi()) + .routes(routes!(routes::health)) + .routes(routes!(alerts::get_alerts)) + .routes(routes!(hives::get_hives)) + .routes(routes!(hives::get_hives_status)) + .routes(routes!(hives::get_links)) + .routes(routes!(hives::get_swarm_info)) + .routes(routes!(jobq_graph::get_jobq_graph)) + .routes(routes!(jobq_graph::get_jobq_rollup)) + .routes(routes!(config_prs::get_agent_config_pr)) + .routes(routes!(agents::get_agent_icon)) + .routes(routes!(agents_md::get_agents_md, agents_md::put_agents_md)) + .routes(routes!( + inference::get_agent_inference, + inference::put_agent_inference, + inference::delete_agent_inference + )) + .routes(routes!( + agent_limits::get_agent_limits, + agent_limits::put_agent_limits, + agent_limits::delete_agent_limits + )) + .routes(routes!( + agent_permissions::get_agent_permissions, + agent_permissions::put_agent_permissions, + agent_permissions::delete_agent_permissions + )) + .routes(routes!(agent_permissions::get_permission_catalogue)) + .routes(routes!(archetypes::get_archetypes)) + .routes(routes!(config_prs::get_config_prs)) + .routes(routes!(agents::create_agent)) + .routes(routes!(forge_admin::mint_agent_forge_token)) + .routes(routes!(forge_admin::make_forge_admin)) + .routes(routes!(agents::get_agents)) + .routes(routes!(agents::get_agents_status)) + .routes(routes!(agents::set_agent_state)) + .routes(routes!( + matrix_account::put_matrix_account, + linked_accounts::delete_matrix_account + )) + .routes(routes!( + forge_account::put_forge_account, + linked_accounts::delete_forge_account + )) + .routes(routes!( + github_account::put_github_account, + linked_accounts::delete_github_account + )) + .routes(routes!(linked_accounts::get_linked_accounts)) + .routes(routes!(agents::get_hive_wanted)) + .routes(routes!(term_stream::stream_agent_term)) + .routes(routes!(subagent_term::list_subagents)) + .routes(routes!(subagent_term::stream_subagent_term)) + .routes(routes!(agent_state_stream::stream_agent_state)) + .routes(routes!(agent_input::post_agent_input)) + .routes(routes!(agent_input::post_agents_broadcast)) + .routes(routes!(agent_control::post_agent_cancel)) + .routes(routes!( + claude_login::post_claude_login, + claude_login::delete_claude_login + )) + .routes(routes!(claude_login::post_claude_login_code)) + .routes(routes!(schedules::list_schedules, schedules::add_schedule)) + .routes(routes!( + schedules::edit_schedule, + schedules::remove_schedule + )) + .routes(routes!(issue_report::get_repos)) + .routes(routes!(issue_report::get_issue_report_all)) + .routes(routes!(issue_report::get_issue_report)) + .routes(routes!(pulls::get_pull_repos)) + .routes(routes!(pulls::get_pulls)) + .routes(routes!(review_classifier::post_try_out)) + .routes(routes!(review_classifier::get_try_out)) + .routes(routes!(review_classifier::get_rules)) + .routes(routes!(review_classifier::open_pr::post_open_pr)) + .routes(routes!(review_classifier::runs::get_runs)) + .routes(routes!(builder::runs::get_runs)) + .routes(routes!(builder::runs::get_run)) + .routes(routes!( + review_classifier::settings::get_settings, + review_classifier::settings::put_settings + )) + .routes(routes!(webhook::post_webhook_forge)) + .split_for_parts(); + // Just the JSON, not the UI — Swagger UI itself is nginx-hosted from + // the nix store (see the module doc comment above). `api` is + // `Clone`; each request gets its own owned copy for `Json` to + // serialize, same as `hive-c0re::dashboard::serve`. + router + .route( + "/api/openapi.json", + get(move || async move { Json(api.clone()) }), + ) + .with_state(state) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + use crate::routes::tests::state_with_roster; + + /// The socket must not share a directory with anything else, because + /// the socket is `0666` and the directory is therefore the only access + /// control it has. `/run/hyperhive` in particular holds hive-c0re's + /// **admin** socket, and nginx — a host service — is bounded by + /// nothing but the directory itself. + /// + /// A test rather than a comment: the failure this guards against is a + /// one-word edit that looks tidier and reads fine in review. + #[test] + fn socket_lives_in_its_own_runtime_dir() { + let parent = Path::new(DEFAULT_SOCKET) + .parent() + .expect("socket path has a parent directory"); + assert_eq!( + parent, + Path::new("/run/swarm-controller"), + "the socket's directory is its access control — moving it under a shared \ + directory (notably /run/hyperhive, which holds the host admin socket) \ + exposes everything else in that directory to the gateway's nginx" + ); + } + + /// One test, not three, deliberately: `HIVES_ENV` is a real process + /// env var, and `cargo test`'s default parallel runner would race + /// separate missing/malformed/valid tests against each other. Driving + /// all three states sequentially inside one test needs no mutex and + /// still proves each branch of `load_hives`. + /// + /// SAFETY: single-threaded mutation of a process env var no other + /// test in this crate reads; restored (removed) before returning. + #[test] + fn load_hives_covers_missing_malformed_and_valid() { + unsafe { + std::env::remove_var(HIVES_ENV); + } + assert_eq!( + load_hives(), + Vec::::new(), + "unset env var is an empty directory, not a startup failure" + ); + + unsafe { + std::env::set_var(HIVES_ENV, "not json"); + } + assert_eq!( + load_hives(), + Vec::::new(), + "unparseable env var falls back to empty rather than panicking — \ + /health must still answer even if this daemon's own config is wrong" + ); + + unsafe { + std::env::set_var( + HIVES_ENV, + r#"[{"name":"pr1ma","domain":"pr1ma.example.com","dashboard":"pr1ma.hives.example"},{"name":"umbra","domain":"umbra.example.com","dashboard":"umbra.hives.example"}]"#, + ); + } + assert_eq!( + load_hives(), + vec![ + HiveEntry { + name: "pr1ma".to_string(), + domain: "pr1ma.example.com".to_string(), + dashboard: "pr1ma.hives.example".to_string(), + }, + HiveEntry { + name: "umbra".to_string(), + domain: "umbra.example.com".to_string(), + dashboard: "umbra.hives.example".to_string(), + }, + ] + ); + + unsafe { + std::env::remove_var(HIVES_ENV); + } + } + + /// Same three-state coverage as `load_hives_covers_missing_malformed_and_valid`, + /// same reason for one test rather than three (a shared process env var). + /// + /// SAFETY: single-threaded mutation of a process env var no other test + /// in this crate reads; restored (removed) before returning. + #[test] + fn load_links_covers_missing_malformed_and_valid() { + unsafe { + std::env::remove_var(LINKS_ENV); + } + assert_eq!( + load_links(), + Vec::::new(), + "unset env var is an empty list, not a startup failure" + ); + + unsafe { + std::env::set_var(LINKS_ENV, "not json"); + } + assert_eq!( + load_links(), + Vec::::new(), + "unparseable env var falls back to empty rather than panicking" + ); + + unsafe { + std::env::set_var( + LINKS_ENV, + r#"[{"label":"Authelia","icon":"🔑","url":"https://auth.example.com/"}]"#, + ); + } + assert_eq!( + load_links(), + vec![ServiceLink { + label: "Authelia".to_string(), + icon: "🔑".to_string(), + url: "https://auth.example.com/".to_string(), + }] + ); + + unsafe { + std::env::remove_var(LINKS_ENV); + } + } + + /// Two states, not three like `load_hives`/`load_links` above: there is + /// nothing to parse here, so no malformed-input branch exists to cover. + /// + /// SAFETY: single-threaded mutation of a process env var no other test + /// in this crate reads; restored (removed) before returning. + #[test] + fn load_swarm_name_covers_missing_and_set() { + unsafe { + std::env::remove_var(NAME_ENV); + } + assert_eq!( + load_swarm_name(), + None, + "unset env var is an unnamed swarm, not a startup failure" + ); + + unsafe { + std::env::set_var(NAME_ENV, "constellat1on"); + } + assert_eq!(load_swarm_name(), Some("constellat1on".to_string())); + + unsafe { + std::env::remove_var(NAME_ENV); + } + } + + // ── NotConnected maps to 503, everything else stays 500 ───────────── + + /// Every inference verb refuses a name that is not an identifier with a + /// 400 and answers 503 without a queue; a PUT whose body is refused is a + /// 400 before the queue is asked. + #[tokio::test] + async fn the_inference_routes_check_the_name_and_body_before_the_queue() { + use crate::inference::{delete_agent_inference, get_agent_inference, put_agent_inference}; + use axum::extract::{Path, State}; + use axum::http::StatusCode; + + fn status(r: Result) -> Option { + r.map(|_| ()).expect_err("must not succeed").status + } + let (state, _sched) = state_with_roster(); + let body = |json: &str| axum::Json(serde_json::from_str(json).expect("decodes")); + for name in ["iris", "Not An Ident"] { + let want = Some(if name == "iris" { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::BAD_REQUEST + }); + let path = || Path(name.to_owned()); + assert_eq!( + status(get_agent_inference(State(state.clone()), path()).await), + want, + "GET {name}" + ); + assert_eq!( + status( + put_agent_inference(State(state.clone()), path(), body(r#"{"model":"opus"}"#)) + .await + ), + want, + "PUT {name}" + ); + assert_eq!( + status(delete_agent_inference(State(state.clone()), path()).await), + want, + "DELETE {name}" + ); + } + for bad in [r#"{"model":"opus 4"}"#, r#"{"model":"--print"}"#] { + assert_eq!( + status( + put_agent_inference(State(state.clone()), Path("iris".to_owned()), body(bad)) + .await + ), + Some(StatusCode::BAD_REQUEST), + "{bad}" + ); + } + } + + #[tokio::test] + async fn listing_subagents_needs_a_queue_and_an_identifier() { + let (state, _sched) = state_with_roster(); + let no_queue = crate::subagent_term::list_subagents( + axum::extract::State(state.clone()), + axum::extract::Path("iris".to_owned()), + ) + .await + .expect_err("no queue is configured"); + assert_eq!( + no_queue.status, + Some(axum::http::StatusCode::SERVICE_UNAVAILABLE), + "{no_queue:?}" + ); + let bad_name = crate::subagent_term::list_subagents( + axum::extract::State(state), + axum::extract::Path("iris.sub".to_owned()), + ) + .await + .expect_err("a dot would widen the subject"); + assert_eq!( + bad_name.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{bad_name:?}" + ); + } + + /// A subagent name is a subject token, so a wildcard or a dot in it is + /// refused before anything subscribes. + #[tokio::test] + async fn a_subagent_stream_refuses_a_name_that_is_not_one_token() { + let (state, _sched) = state_with_roster(); + for subagent in ["*", ">", "a.b"] { + let Err(problem) = crate::subagent_term::stream_subagent_term( + axum::extract::State(state.clone()), + axum::extract::Path(("iris".to_owned(), subagent.to_owned())), + axum::http::HeaderMap::new(), + ) + .await + else { + panic!("{subagent:?} must be refused"); + }; + assert_eq!( + problem.status, + Some(axum::http::StatusCode::BAD_REQUEST), + "{subagent:?}: {problem:?}" + ); + } + } +} diff --git a/swarm-controller/src/status.rs b/swarm-controller/src/status.rs index 98c398c0..ac87f394 100644 --- a/swarm-controller/src/status.rs +++ b/swarm-controller/src/status.rs @@ -36,7 +36,7 @@ use futures_util::TryStreamExt as _; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use crate::HiveEntry; +use crate::routes::hives::HiveEntry; /// Default age past which a snapshot is reported stale. /// @@ -355,7 +355,7 @@ pub(crate) fn row( #[cfg(test)] mod tests { use super::{DEFAULT_STALE_AFTER, Freshness, Offered, STALE_AFTER_ENV, StatusReader, render}; - use crate::HiveEntry; + use crate::routes::hives::HiveEntry; use std::collections::BTreeMap; use std::time::{Duration, SystemTime}; diff --git a/swarm-controller/src/subagent_term.rs b/swarm-controller/src/subagent_term.rs index 06c9791e..f93f27e1 100644 --- a/swarm-controller/src/subagent_term.rs +++ b/swarm-controller/src/subagent_term.rs @@ -37,7 +37,7 @@ use serde::Serialize; use swarm_queue_client::{agent_term, subagent_term}; use utoipa::ToSchema; -use crate::AppState; +use crate::routes::AppState; use crate::wanted::WantedWriter; /// How often [`spawn`] checks every live agent's stream. Also how long a newly @@ -264,11 +264,11 @@ fn ident(name: &str) -> Result { } fn bad_request(reason: &str) -> problem_details::ProblemDetails { - crate::error_problem(StatusCode::BAD_REQUEST, reason) + crate::routes::error_problem(StatusCode::BAD_REQUEST, reason) } fn unavailable(detail: &str) -> problem_details::ProblemDetails { - crate::error_problem(StatusCode::SERVICE_UNAVAILABLE, detail) + crate::routes::error_problem(StatusCode::SERVICE_UNAVAILABLE, detail) } fn is_stream_not_found(e: &async_nats::jetstream::context::GetStreamError) -> bool { @@ -282,7 +282,7 @@ fn is_stream_not_found(e: &async_nats::jetstream::context::GetStreamError) -> bo fn read_failed(agent: &str, e: &dyn std::error::Error) -> problem_details::ProblemDetails { let detail = swarm_queue_client::chain(e); tracing::warn!(%agent, error = %detail, "listing subagents: reading the stream failed"); - crate::error_problem(StatusCode::INTERNAL_SERVER_ERROR, &detail) + crate::routes::error_problem(StatusCode::INTERNAL_SERVER_ERROR, &detail) } #[cfg(test)] diff --git a/swarm-controller/src/term_stream.rs b/swarm-controller/src/term_stream.rs index e3cdf710..8a40aa9d 100644 --- a/swarm-controller/src/term_stream.rs +++ b/swarm-controller/src/term_stream.rs @@ -37,7 +37,7 @@ use futures_util::stream::BoxStream; use futures_util::{Stream, StreamExt as _}; use swarm_queue_client::agent_term; -use crate::AppState; +use crate::routes::AppState; #[utoipa::path( get, @@ -68,19 +68,21 @@ pub(crate) async fn stream_agent_term( headers: HeaderMap, ) -> Result>>, problem_details::ProblemDetails> { let Some(status) = state.status.as_ref() else { - return Err(crate::error_problem( + return Err(crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, "no swarm queue is configured on this host", )); }; let Some(agent_status) = state.agent_status.as_ref() else { - return Err(crate::error_problem( + return Err(crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, "no agent-status reader is configured on this host", )); }; let agent = hive_types::Ident::parse(&agent) - .map_err(|reason| crate::error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .map_err(|reason| { + crate::routes::error_problem(axum::http::StatusCode::BAD_REQUEST, reason) + })? .into_string(); // One-agent "roster": reuses `AgentStatusReader::view`'s already-tested @@ -92,12 +94,12 @@ pub(crate) async fn stream_agent_term( .map_err(|e| { let detail = format!("{e:#}"); tracing::warn!(%agent, error = %detail, "term stream: reading agent status failed"); - crate::error_problem(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &detail) + crate::routes::error_problem(axum::http::StatusCode::INTERNAL_SERVER_ERROR, &detail) })? .into_iter() .find_map(|row| row.hive) .ok_or_else(|| { - crate::error_problem( + crate::routes::error_problem( axum::http::StatusCode::NOT_FOUND, &format!("{agent:?} has never reported to the swarm"), ) @@ -110,7 +112,7 @@ pub(crate) async fn stream_agent_term( // answer is 503, the same shape every other queue-backed route // here already uses. swarm_queue_client::ensure_connected(&client).map_err(|e| { - crate::error_problem( + crate::routes::error_problem( axum::http::StatusCode::SERVICE_UNAVAILABLE, &swarm_queue_client::chain(&e), ) @@ -277,7 +279,7 @@ pub(crate) async fn subscribe( ) -> Result { let subscriber = client.subscribe(subject.clone()).await.map_err(|e| { tracing::warn!(%subject, error = %e, "{what} stream: subscribe failed"); - crate::error_problem( + crate::routes::error_problem( axum::http::StatusCode::INTERNAL_SERVER_ERROR, &format!("subscribing to {subject} failed: {e}"), ) diff --git a/swarm-controller/src/webhook.rs b/swarm-controller/src/webhook.rs index ae595d21..031e92f7 100644 --- a/swarm-controller/src/webhook.rs +++ b/swarm-controller/src/webhook.rs @@ -21,7 +21,7 @@ use axum::{ use hmac::{Hmac, KeyInit as _, Mac as _}; use sha2::Sha256; -use super::AppState; +use crate::routes::AppState; /// Where the HMAC secret is kept, relative to the daemon's state directory. /// @@ -416,10 +416,10 @@ pub(super) async fn post_webhook_forge( cache.apply_webhook_delivery(&body); } if let Some(merged) = crate::config_pr::merged(&body) { - super::queue_merged_config_deploy(&state, merged).await; + crate::jobq::queue_merged_config_deploy(&state, merged).await; } if let Some(changed) = crate::config_pr::changed_head(&body) { - super::queue_config_build(&state, changed); + crate::jobq::queue_config_build(&state, changed); } } DeliveryKind::VcsActivity => {