swarm-controller: CreateRepo/AddRepoMember/InitAgentConfigRepo forge nodes
This commit is contained in:
damocles
mara
parent
f287ff1ee8
commit
1d31bb6e80
4 changed files with 647 additions and 46 deletions
|
|
@ -33,28 +33,43 @@ use utoipa::{OpenApi, ToSchema};
|
|||
use utoipa_axum::{router::OpenApiRouter, routes};
|
||||
|
||||
mod auth;
|
||||
mod forge;
|
||||
mod status;
|
||||
|
||||
/// Node payload for the swarm-level job graph. Named `Swarm*` rather than
|
||||
/// the bare `NodeKind`/`Resource` `hive-c0re::job_queue::model` already
|
||||
/// uses, so a grep for either doesn't land on both crates.
|
||||
///
|
||||
/// `CreateIdentity` is the first real variant — "the minimal shape for
|
||||
/// `CreateIdentity` was the first real variant — "the minimal shape for
|
||||
/// agent creation is creating the identity and wiring that in" (the design
|
||||
/// thread's own framing for why this landed before the forge-node work a
|
||||
/// standalone `CreateRepo` variant would have started with). Its only
|
||||
/// effect is calling `swarm-controller::auth`, which calls
|
||||
/// `swarm-authelia-bridge`; nothing forge- or deploy-shaped happens yet.
|
||||
/// thread's own framing for why that landed before the forge-node work
|
||||
/// here). `CreateRepo`/`AddRepoMember`/`InitAgentConfigRepo` are three
|
||||
/// separate nodes rather than one combined "create the repo" step so each
|
||||
/// is independently retryable/observable in the job graph, same as every
|
||||
/// other multi-step provisioning flow in this codebase (`hive-c0re`'s own
|
||||
/// `NodeKind` never folds unrelated forge calls into one node either).
|
||||
#[derive(Clone, Debug)]
|
||||
enum SwarmNodeKind {
|
||||
/// Ensure `agent` exists as an authelia subject at the swarm level.
|
||||
CreateIdentity { agent: String },
|
||||
/// Create `repo` in `forge::AGENTS_ORG` with the operator merge gate
|
||||
/// on its default branch. See `forge::Client::create_repo`.
|
||||
CreateRepo { repo: String },
|
||||
/// Add `agent` as a write collaborator on `repo`. See
|
||||
/// `forge::Client::add_repo_member`.
|
||||
AddRepoMember { repo: String, agent: String },
|
||||
/// Seed `repo` with `agent.nix` + `flake.nix`. See
|
||||
/// `forge::Client::seed_agent_config`.
|
||||
InitAgentConfigRepo { repo: String, agent: String },
|
||||
}
|
||||
|
||||
impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
||||
fn label(&self) -> String {
|
||||
match self {
|
||||
SwarmNodeKind::CreateIdentity { .. } => "create_identity".to_owned(),
|
||||
SwarmNodeKind::CreateRepo { .. } => "create_repo".to_owned(),
|
||||
SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
|
||||
SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -63,6 +78,13 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|||
SwarmNodeKind::CreateIdentity { agent } => {
|
||||
serde_json::json!({ "agent": agent })
|
||||
}
|
||||
SwarmNodeKind::CreateRepo { repo } => {
|
||||
serde_json::json!({ "repo": repo })
|
||||
}
|
||||
SwarmNodeKind::AddRepoMember { repo, agent }
|
||||
| SwarmNodeKind::InitAgentConfigRepo { repo, agent } => {
|
||||
serde_json::json!({ "repo": repo, "agent": agent })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -78,31 +100,78 @@ impl hive_jobq_wire::WireResource for SwarmResourceKind {
|
|||
}
|
||||
}
|
||||
|
||||
/// Everything a claimed node's executor arm might need to reach outside
|
||||
/// this process — bundled into one `Clone` struct rather than growing
|
||||
/// `run_swarm_node`'s parameter list per node kind (three forge-shaped
|
||||
/// node kinds landed in one slice; a fourth parameter each would have made
|
||||
/// the signature the least readable part of this file). Each field is
|
||||
/// built once at startup (see `main`) and is `None` exactly when that
|
||||
/// dependency isn't configured on this host — every arm below treats
|
||||
/// absence as *this node's* failure, not a reason to skip silently.
|
||||
#[derive(Clone)]
|
||||
struct WorkerDeps {
|
||||
auth: Option<std::sync::Arc<auth::AuthBridge>>,
|
||||
forge: Option<std::sync::Arc<forge::Client>>,
|
||||
}
|
||||
|
||||
/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/
|
||||
/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind`
|
||||
/// variant turns into a real effect.
|
||||
///
|
||||
/// `auth` is `None` on a host that runs a controller split from
|
||||
/// `swarm-authelia` (no bridge configured there) — `CreateIdentity` fails
|
||||
/// explicitly in that case rather than this fn papering over a
|
||||
/// misconfigured deployment.
|
||||
async fn run_swarm_node(
|
||||
_id: hive_jobq::NodeId,
|
||||
kind: SwarmNodeKind,
|
||||
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
||||
auth: Option<std::sync::Arc<auth::AuthBridge>>,
|
||||
deps: WorkerDeps,
|
||||
) -> (
|
||||
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
||||
hive_jobq::scheduler::Outcome,
|
||||
) {
|
||||
use hive_jobq::scheduler::Outcome;
|
||||
|
||||
let outcome = match kind {
|
||||
SwarmNodeKind::CreateIdentity { agent } => match auth {
|
||||
None => hive_jobq::scheduler::Outcome::Failed(
|
||||
"no swarm-authelia-bridge is configured on this host".to_owned(),
|
||||
),
|
||||
SwarmNodeKind::CreateIdentity { agent } => match deps.auth {
|
||||
None => {
|
||||
Outcome::Failed("no swarm-authelia-bridge is configured on this host".to_owned())
|
||||
}
|
||||
Some(bridge) => match bridge.ensure_agent_identity(&agent).await {
|
||||
Ok(_) => hive_jobq::scheduler::Outcome::Done,
|
||||
Err(e) => hive_jobq::scheduler::Outcome::Failed(format!("{e:#}")),
|
||||
Ok(_) => Outcome::Done,
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::CreateRepo { repo } => match deps.forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge configured on this host (SWARM_CONTROLLER_FORGE_URL / \
|
||||
SWARM_CONTROLLER_FORGE_TOKEN_FILE unset)"
|
||||
.to_owned(),
|
||||
),
|
||||
Some(client) => match client.create_repo(&repo).await {
|
||||
Ok(full_name) => {
|
||||
tracing::info!(%full_name, "swarm jobq: create_repo done");
|
||||
Outcome::Done
|
||||
}
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::AddRepoMember { repo, agent } => match deps.forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge configured on this host (SWARM_CONTROLLER_FORGE_URL / \
|
||||
SWARM_CONTROLLER_FORGE_TOKEN_FILE unset)"
|
||||
.to_owned(),
|
||||
),
|
||||
Some(client) => match client.add_repo_member(&repo, &agent).await {
|
||||
Ok(()) => Outcome::Done,
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::InitAgentConfigRepo { repo, agent } => match deps.forge {
|
||||
None => Outcome::Failed(
|
||||
"no forge configured on this host (SWARM_CONTROLLER_FORGE_URL / \
|
||||
SWARM_CONTROLLER_FORGE_TOKEN_FILE unset)"
|
||||
.to_owned(),
|
||||
),
|
||||
Some(client) => match client.seed_agent_config(&repo, &agent).await {
|
||||
Ok(()) => Outcome::Done,
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -124,20 +193,20 @@ async fn run_swarm_node(
|
|||
/// Cheap to run with an empty graph: `claim_next` on a graph nothing was
|
||||
/// ever inserted into just returns `None` every poll.
|
||||
///
|
||||
/// `auth` is cloned per iteration (an `Arc` clone, not a reconnect) and
|
||||
/// moved into the closure `claim_next` takes ownership of — `run_swarm_node`
|
||||
/// needs its own owned copy since the claimed future may outlive this loop
|
||||
/// iteration.
|
||||
/// `deps` is cloned per iteration (its fields are `Arc` clones, not
|
||||
/// reconnects) and moved into the closure `claim_next` takes ownership
|
||||
/// of — `run_swarm_node` needs its own owned copy since the claimed
|
||||
/// future may outlive this loop iteration.
|
||||
fn spawn_jobq_worker(
|
||||
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
||||
auth: Option<Arc<auth::AuthBridge>>,
|
||||
deps: WorkerDeps,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let auth = auth.clone();
|
||||
let deps = deps.clone();
|
||||
let runner =
|
||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||
run_swarm_node(id, kind, builder, auth)
|
||||
run_swarm_node(id, kind, builder, deps)
|
||||
});
|
||||
match runner {
|
||||
Some(runner) => {
|
||||
|
|
@ -403,10 +472,10 @@ async fn get_hives_status(
|
|||
}
|
||||
}
|
||||
|
||||
/// Body of `POST /api/agents` — the agent name to create a swarm-level
|
||||
/// identity for. No other fields: this endpoint is deliberately narrow —
|
||||
/// "identity in authelia only, no forge or deploy yet" — so it asks for
|
||||
/// nothing a `CreateIdentity` node doesn't use.
|
||||
/// Body of `POST /api/agents` — the agent name to create. The repo name
|
||||
/// inside `forge::AGENTS_ORG` is the same string: one repo per agent,
|
||||
/// named after it, same convention `hive-c0re::forge` already uses for its
|
||||
/// own single-hive `CreateRepo` path.
|
||||
#[derive(Clone, Debug, Deserialize, ToSchema)]
|
||||
struct CreateAgentRequest {
|
||||
name: String,
|
||||
|
|
@ -420,20 +489,32 @@ struct CreateAgentResponse {
|
|||
node_id: u64,
|
||||
}
|
||||
|
||||
/// Queue a `CreateIdentity` job for `name`. Returns as soon as the node is
|
||||
/// inserted — **not** once the identity exists; `run_swarm_node` does that
|
||||
/// Queue the whole agent-creation job graph for `name` — `CreateIdentity`
|
||||
/// then, once that succeeds, `CreateRepo`; once THAT succeeds,
|
||||
/// `AddRepoMember` and `InitAgentConfigRepo` both run off it — a fan-out,
|
||||
/// not a chain, since adding a collaborator and seeding config files are
|
||||
/// independent operations against the same already-created, already-
|
||||
/// protected repo and have no ordering requirement on each other (mara,
|
||||
/// design review: `InitAgentConfigRepo` does not depend on
|
||||
/// `AddRepoMember` — it's a jobq graph, not a linear chain). Returns as
|
||||
/// soon as the graph is inserted — **not** once any of it has run; `run_swarm_node` does that
|
||||
/// work asynchronously off the scheduler loop already running
|
||||
/// (`spawn_jobq_worker`), same as every other node kind. This is also the
|
||||
/// first genuine non-test caller `SwarmNodeKind::CreateIdentity` has: the
|
||||
/// node kind's `dead_code` bound was the whole reason this endpoint had to
|
||||
/// land in the same change as the variant, not as a follow-up.
|
||||
/// (`spawn_jobq_worker`), same as every other node kind. The response
|
||||
/// reports `CreateIdentity`'s id, the graph's entry point — a caller
|
||||
/// watches the whole thing settle via `/api/jobq/graph`, which serves
|
||||
/// every root, not just this one.
|
||||
///
|
||||
/// This endpoint is also the first genuine non-test caller all four
|
||||
/// `SwarmNodeKind` variants have: each node kind's `dead_code` bound is
|
||||
/// the whole reason the executor arm and this endpoint had to land in the
|
||||
/// same change as the variant, not as a follow-up.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/agents",
|
||||
request_body = CreateAgentRequest,
|
||||
responses(
|
||||
(status = 200, description = "job queued", body = CreateAgentResponse),
|
||||
(status = 500, description = "the job could not be queued", body = String),
|
||||
(status = 200, description = "job chain queued", body = CreateAgentResponse),
|
||||
(status = 500, description = "the job chain could not be queued", body = String),
|
||||
),
|
||||
tag = "agents"
|
||||
)]
|
||||
|
|
@ -445,12 +526,29 @@ async fn create_agent(
|
|||
.jobq
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let agent = req.name;
|
||||
let repo = agent.clone();
|
||||
let ids = sched
|
||||
.insert_job(None, |b| {
|
||||
vec![
|
||||
b.node(SwarmNodeKind::CreateIdentity { agent: req.name })
|
||||
.guid(),
|
||||
]
|
||||
let create_identity = b.node(SwarmNodeKind::CreateIdentity {
|
||||
agent: agent.clone(),
|
||||
});
|
||||
let create_repo = b
|
||||
.node(SwarmNodeKind::CreateRepo { repo: repo.clone() })
|
||||
.after_ok(create_identity);
|
||||
// Both fan out from `create_repo` directly — independent
|
||||
// operations on the same repo, no ordering requirement on
|
||||
// each other (see the doc comment above).
|
||||
let _add_repo_member = b
|
||||
.node(SwarmNodeKind::AddRepoMember {
|
||||
repo: repo.clone(),
|
||||
agent: agent.clone(),
|
||||
})
|
||||
.after_ok(create_repo);
|
||||
let _init_config = b
|
||||
.node(SwarmNodeKind::InitAgentConfigRepo { repo, agent })
|
||||
.after_ok(create_repo);
|
||||
vec![create_identity.guid()]
|
||||
})
|
||||
.map_err(|e| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
let [id] = ids[..] else {
|
||||
|
|
@ -605,12 +703,26 @@ async fn main() -> Result<()> {
|
|||
None
|
||||
}
|
||||
};
|
||||
// Same shape again: a controller with no forge configured still serves
|
||||
// everything else, and the forge-shaped node kinds give an honest
|
||||
// per-job failure rather than this fn refusing to start.
|
||||
let forge_client = match forge::Client::from_env() {
|
||||
Ok(client) => client.map(Arc::new),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "forge misconfigured; repo provisioning is off");
|
||||
None
|
||||
}
|
||||
};
|
||||
let deps = WorkerDeps {
|
||||
auth,
|
||||
forge: forge_client,
|
||||
};
|
||||
|
||||
let jobq = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
||||
hive_jobq::Graph::new(),
|
||||
hive_jobq::resources::ResourceTable::new(),
|
||||
)));
|
||||
spawn_jobq_worker(Arc::clone(&jobq), auth.clone());
|
||||
spawn_jobq_worker(Arc::clone(&jobq), deps);
|
||||
|
||||
let state = AppState {
|
||||
hives: Arc::new(load_hives()),
|
||||
|
|
@ -646,10 +758,76 @@ async fn main() -> Result<()> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DEFAULT_SOCKET, HIVES_ENV, HiveEntry, LINKS_ENV, ServiceLink, load_hives, load_links,
|
||||
DEFAULT_SOCKET, HIVES_ENV, HiveEntry, LINKS_ENV, ServiceLink, SwarmNodeKind, WorkerDeps,
|
||||
load_hives, load_links, run_swarm_node,
|
||||
};
|
||||
use std::path::Path;
|
||||
|
||||
/// Drives `SwarmNodeKind::CreateRepo` through the real
|
||||
/// `hive_jobq::scheduler::Scheduler` claim → run → complete path,
|
||||
/// rather than only through `create_agent`'s endpoint test (there
|
||||
/// isn't one — the endpoint itself is thin, insert-and-return; the
|
||||
/// interesting behavior is in `run_swarm_node`'s executor arm, which
|
||||
/// this exercises directly).
|
||||
///
|
||||
/// Deliberately offline: with both forge env vars unset,
|
||||
/// `forge::Client::from_env` returns `Ok(None)` (see that module's doc
|
||||
/// comment), so this exercises the whole claim → run → complete path
|
||||
/// through `hive_jobq::scheduler::Scheduler` without a real forge
|
||||
/// server — at the cost of only ever observing the
|
||||
/// graceful-absence-is-failure branch here. The happy path needs an
|
||||
/// actual forge instance and isn't something a unit test in this crate
|
||||
/// can reach.
|
||||
///
|
||||
/// SAFETY: single-threaded mutation of the two `forge` env vars this
|
||||
/// test itself owns, restored before returning — no other test in this
|
||||
/// crate reads them.
|
||||
#[tokio::test]
|
||||
async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() {
|
||||
unsafe {
|
||||
std::env::remove_var("SWARM_CONTROLLER_FORGE_URL");
|
||||
std::env::remove_var("SWARM_CONTROLLER_FORGE_TOKEN_FILE");
|
||||
}
|
||||
|
||||
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
||||
hive_jobq::Graph::new(),
|
||||
hive_jobq::resources::ResourceTable::new(),
|
||||
);
|
||||
let id = sched
|
||||
.append(
|
||||
SwarmNodeKind::CreateRepo {
|
||||
repo: "atlas".to_owned(),
|
||||
},
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.expect("insert");
|
||||
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
||||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
};
|
||||
let runner =
|
||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||
run_swarm_node(id, kind, builder, deps)
|
||||
})
|
||||
.expect("the node just inserted is runnable");
|
||||
runner
|
||||
.await
|
||||
.1
|
||||
.expect("no growth declared, nothing to reject");
|
||||
|
||||
let guard = sched.lock().unwrap();
|
||||
let node = guard.graph().node(id).expect("node still present");
|
||||
assert_eq!(node.state, hive_jobq::State::Failed);
|
||||
assert!(
|
||||
node.error.as_deref().unwrap_or_default().contains("forge"),
|
||||
"expected a forge-not-configured error, got {:?}",
|
||||
node.error
|
||||
);
|
||||
}
|
||||
|
||||
/// The socket must not share a directory with anything else, because
|
||||
/// the socket is `0666` and the directory is therefore the only access
|
||||
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
|
||||
|
|
|
|||
Loading…
Reference in a new issue