b74011bb54
build_meta, the build of each agent from internal/meta before a deploy, now writes its result into the same list as the config PR builds: the agent, the internal/meta commit, how it ended and what nix printed. The builds card titles them by agent and shows them with the PR builds.
4299 lines
168 KiB
Rust
4299 lines
168 KiB
Rust
//! The swarm-level job graph: its node and resource kinds, the worker that
|
|
//! runs claimed nodes, and the `queue_*` fns that insert them.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use anyhow::Result;
|
|
use swarm_authelia_bridge_sock::BridgeResponse;
|
|
|
|
use crate::routes::AppState;
|
|
use crate::routes::agents::{claiming_hives, read_declarations};
|
|
use crate::startup::PUBLIC_URL_ENV;
|
|
use crate::{
|
|
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_build,
|
|
meta_fanout, meta_flake, wanted,
|
|
};
|
|
|
|
/// Node payload for the swarm-level job graph. Named `Swarm*` rather than
|
|
/// the bare `NodeKind`/`Resource` `hive-c0re::job_queue::model` already
|
|
/// uses, so a grep for either doesn't land on both crates.
|
|
///
|
|
/// `CreateIdentity` was the first real variant — "the minimal shape for
|
|
/// agent creation is creating the identity and wiring that in" (the design
|
|
/// thread's own framing for why that landed before the forge-node work
|
|
/// here). `CreateRepo`/`AddRepoMember`/`InitAgentConfigRepo` are three
|
|
/// separate nodes rather than one combined "create the repo" step so each
|
|
/// is independently retryable/observable in the job graph, same as every
|
|
/// other multi-step provisioning flow in this codebase (`hive-c0re`'s own
|
|
/// `NodeKind` never folds unrelated forge calls into one node either).
|
|
#[derive(Clone, Debug)]
|
|
pub(crate) enum SwarmNodeKind {
|
|
/// Ensure `agent` exists as an authelia subject at the swarm level.
|
|
CreateIdentity { agent: String },
|
|
/// Create the agent's repo in `forge::CONFIG_ORG` with the operator
|
|
/// merge gate on its default branch. See `forge::Client::create_repo`.
|
|
CreateRepo { agent: String },
|
|
/// Ensure `agent` exists as a Forgejo user account. Independent of
|
|
/// `CreateIdentity`/`CreateRepo` (a forge user needs neither an
|
|
/// authelia subject nor an existing repo) but a prerequisite for
|
|
/// `AddRepoMember`, which fails with "user does not exist" against an
|
|
/// account this node hasn't created yet. See
|
|
/// `forge::Client::ensure_agent_user`.
|
|
CreateForgeUser { agent: String },
|
|
/// Add `agent` as a write collaborator on its own repo. See
|
|
/// `forge::Client::add_repo_member`.
|
|
AddRepoMember { agent: String },
|
|
/// Add `agent` as a read collaborator on `internal/docs`. See
|
|
/// `forge::docs_grant`.
|
|
///
|
|
/// Carries no hive: the grant is on a swarm-wide repo.
|
|
GrantDocsRead { agent: String },
|
|
/// Seed the agent's repo with `agent.nix` + `flake.nix`. See
|
|
/// `forge::Client::seed_agent_config`.
|
|
///
|
|
/// Deliberately carries no hive: seeding a config repo is the same
|
|
/// work whichever hive the agent is bound for, and an agent's config
|
|
/// states nothing about where it runs. The hive is an address the
|
|
/// swarm routes a deploy message to — it belongs on the node that
|
|
/// sends that message, not on this one.
|
|
InitAgentConfigRepo { agent: String },
|
|
/// Mint `agent`'s own client certificate for the swarm secret store,
|
|
/// publish it there, grant it, and log in with it. See
|
|
/// `agent_identity::mint_and_verify` — including why this node does not
|
|
/// report success on a write.
|
|
///
|
|
/// Carries no hive: neither the certificate, the queue credential nor
|
|
/// the agent's policy names one, so an agent keeps one store identity
|
|
/// whichever hive it runs on.
|
|
MintAgentIdentity { agent: String },
|
|
/// Make sure `agent` holds a live forge access token in the swarm secret
|
|
/// store, minting one with the forge's admin API when it does not. See
|
|
/// `forge::agent_token` — including why a rotation is a delete then a
|
|
/// create.
|
|
///
|
|
/// Carries no hive: the token's store path has no hive segment, and the
|
|
/// agent pulls it from wherever it runs.
|
|
MintAgentForgeToken { agent: String },
|
|
/// Make sure `agent` holds a live token for its own account on the swarm's
|
|
/// homeserver, creating the account with the swarm's appservice token
|
|
/// when it does not. See `matrix_account::agent_token` — including why a
|
|
/// stored token is read back before anything is minted.
|
|
///
|
|
/// Carries no hive: the path it writes has no hive segment, so an agent
|
|
/// that moves between hives keeps one matrix identity.
|
|
MintAgentMatrixAccount { agent: String },
|
|
/// Make sure `hive`'s sender account (`@hive-<hive>:`) holds a live token
|
|
/// in the swarm store, where that hive's matrix sweep reads it. See
|
|
/// `matrix_account::hive_sender`.
|
|
MintHiveSenderToken { hive: String },
|
|
/// Make sure `hive`'s read-only forge account (`hive-<hive>`) exists, is in
|
|
/// the `hives` team, and holds a live token in the swarm store, where
|
|
/// that hive's hive-c0re reads it. See `forge::hive_token`.
|
|
MintHiveForgeToken { hive: String },
|
|
/// Make sure the review classifier's forge account exists and holds a
|
|
/// live token in the swarm store, where this daemon reads it to post the
|
|
/// classifier's reviews. See `forge::classifier_account`.
|
|
MintClassifierForgeToken,
|
|
/// Make sure the swarm builder's read-only forge account exists, is in
|
|
/// the `hives` team, and holds a live token in the swarm store, where the
|
|
/// builder reads it to fetch config repos. See `forge::builder_account`.
|
|
MintBuilderForgeToken,
|
|
/// Make sure the CI metrics poller's read-only forge account exists, is in
|
|
/// its team in every organisation, and holds a live token in the swarm
|
|
/// store, where the poller reads it to list CI runs. See
|
|
/// `forge::ci_metrics_account`.
|
|
MintCiMetricsForgeToken,
|
|
/// Build `agent`'s config at `rev`, the head of its config PR `pr`, on
|
|
/// the swarm builder, and write the result as a commit status on `rev`.
|
|
/// See `builder`.
|
|
///
|
|
/// Carries no hive: the check builds on the builder's own agent base,
|
|
/// not on any hive.
|
|
BuildAgent { agent: String, rev: String, pr: u64 },
|
|
/// Build `agent` from `internal/meta`'s head on the swarm builder, and
|
|
/// record the closure in `agent-closures`. The `TriggerDeploy` behind it
|
|
/// runs only when the build is green. See `meta_build`.
|
|
///
|
|
/// Carries no rev: it reads the head when it runs, which in most DAGs is
|
|
/// after the `UpdateMeta` that commits it.
|
|
///
|
|
/// `pr` is the merged config PR the build belongs to, if any: a red `nix`
|
|
/// build is reported on it once.
|
|
BuildMeta { agent: String, pr: Option<u64> },
|
|
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
|
|
/// See `agent_renewal`.
|
|
///
|
|
/// Carries no hive: the secret's store path has none.
|
|
RenewAgentQueueCredential { agent: String },
|
|
/// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state
|
|
/// store, so a freshly created agent does not start driving turns the
|
|
/// moment it's deployed — the operator has to explicitly flip it to `Up`.
|
|
/// Carries the hive because the wanted-state bucket is keyed per hive.
|
|
SetAgentWanted { hive: String, agent: String },
|
|
/// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy
|
|
/// subject. The one node kind whose effect leaves this host.
|
|
///
|
|
/// Carries the hive, unlike every variant above — this is the node
|
|
/// `InitAgentConfigRepo`'s doc points at when it says the hive belongs
|
|
/// on the node that sends the deploy message. It names the subject the
|
|
/// message goes to, one per hive, so no other hive is woken by it.
|
|
TriggerDeploy {
|
|
hive: String,
|
|
agent: String,
|
|
/// The config commit to deploy, from a merged config PR. `None` from
|
|
/// agent creation, where the hive deploys what it seeds.
|
|
rev: Option<String>,
|
|
},
|
|
/// Pin the swarm meta flake (`internal/meta`): render, lock on the swarm
|
|
/// builder, commit. See `meta_flake`. When the pin `internal/meta` holds is
|
|
/// not the one `meta_fanout` recorded, a `BuildMeta` and a `TriggerDeploy`
|
|
/// per placed agent follow, then a `RecordFanout`.
|
|
///
|
|
/// Carries nothing: it reads every head when it runs, so one pending node
|
|
/// covers every trigger queued behind it.
|
|
UpdateMeta,
|
|
/// Record `pin` in `meta_fanout` as sent to the hives. Runs once every
|
|
/// `TriggerDeploy` declared beside it is done or skipped by its build gate.
|
|
/// A deploy that failed or was lost leaves the record stale, and the next
|
|
/// `UpdateMeta` sends them again.
|
|
RecordFanout { pin: String },
|
|
}
|
|
|
|
impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|
fn label(&self) -> String {
|
|
match self {
|
|
SwarmNodeKind::CreateIdentity { .. } => "create_identity".to_owned(),
|
|
SwarmNodeKind::CreateRepo { .. } => "create_repo".to_owned(),
|
|
SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(),
|
|
SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
|
|
SwarmNodeKind::GrantDocsRead { .. } => "grant_docs_read".to_owned(),
|
|
SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
|
|
SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
|
|
SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(),
|
|
SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(),
|
|
SwarmNodeKind::MintHiveSenderToken { .. } => "mint_hive_sender_token".to_owned(),
|
|
SwarmNodeKind::MintHiveForgeToken { .. } => "mint_hive_forge_token".to_owned(),
|
|
SwarmNodeKind::MintClassifierForgeToken => "mint_classifier_forge_token".to_owned(),
|
|
SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(),
|
|
SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(),
|
|
SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(),
|
|
SwarmNodeKind::BuildMeta { .. } => "build_meta".to_owned(),
|
|
SwarmNodeKind::RenewAgentQueueCredential { .. } => {
|
|
"renew_agent_queue_credential".to_owned()
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
|
|
SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
|
|
SwarmNodeKind::UpdateMeta => "update_meta".to_owned(),
|
|
SwarmNodeKind::RecordFanout { .. } => "record_fanout".to_owned(),
|
|
}
|
|
}
|
|
|
|
fn data(&self, _id: hive_jobq_wire::WireId) -> serde_json::Value {
|
|
// Spelled out as an or-pattern rather than a catch-all on purpose: a
|
|
// new variant then fails to compile here instead of silently
|
|
// rendering as an agent name. That gate has now fired once —
|
|
// `TriggerDeploy` is the first node that is about an agent *and a
|
|
// hive*, and a catch-all would have dropped the hive from the
|
|
// viewer without anyone noticing.
|
|
match self {
|
|
SwarmNodeKind::CreateIdentity { agent }
|
|
| SwarmNodeKind::CreateRepo { agent }
|
|
| SwarmNodeKind::CreateForgeUser { agent }
|
|
| SwarmNodeKind::AddRepoMember { agent }
|
|
| SwarmNodeKind::GrantDocsRead { agent }
|
|
| SwarmNodeKind::InitAgentConfigRepo { agent }
|
|
| SwarmNodeKind::MintAgentIdentity { agent }
|
|
| SwarmNodeKind::MintAgentForgeToken { agent }
|
|
| SwarmNodeKind::MintAgentMatrixAccount { agent }
|
|
| SwarmNodeKind::RenewAgentQueueCredential { agent } => {
|
|
serde_json::json!({ "agent": agent })
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, pr } => {
|
|
serde_json::json!({ "agent": agent, "pr": pr })
|
|
}
|
|
SwarmNodeKind::MintHiveSenderToken { hive }
|
|
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
|
|
SwarmNodeKind::MintClassifierForgeToken
|
|
| SwarmNodeKind::MintBuilderForgeToken
|
|
| SwarmNodeKind::MintCiMetricsForgeToken
|
|
| SwarmNodeKind::UpdateMeta => serde_json::json!({}),
|
|
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
|
|
serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
|
|
}
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
|
|
serde_json::json!({ "agent": agent, "hive": hive, "rev": rev })
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { hive, agent } => {
|
|
serde_json::json!({ "agent": agent, "hive": hive })
|
|
}
|
|
SwarmNodeKind::RecordFanout { pin } => serde_json::json!({ "pin": pin }),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A named counting semaphore a node acquires before it starts. Every name
|
|
/// has capacity 1: the scheduler's default, which no caller overrides.
|
|
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
|
pub(crate) enum SwarmResourceKind {
|
|
/// The swarm builder: one `BuildAgent` or `BuildMeta` node runs at a time,
|
|
/// and the rest wait pending. A newer push replaces its PR's pending
|
|
/// `BuildAgent`.
|
|
Builder,
|
|
/// `internal/meta`: one `UpdateMeta` commits at a time, so two never race
|
|
/// on the same file's blob.
|
|
Meta,
|
|
}
|
|
|
|
impl hive_jobq_wire::WireResource for SwarmResourceKind {
|
|
fn name(&self) -> String {
|
|
match self {
|
|
SwarmResourceKind::Builder => "builder".to_owned(),
|
|
SwarmResourceKind::Meta => "meta".to_owned(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A dependency this daemon may only gain after it starts serving: empty
|
|
/// until then, filled once. Readers take what is there at the moment they ask.
|
|
pub(crate) type Late<T> = Arc<std::sync::OnceLock<T>>;
|
|
|
|
/// Everything a claimed node's executor arm might need to reach outside
|
|
/// this process — bundled into one `Clone` struct rather than growing
|
|
/// `run_swarm_node`'s parameter list per node kind (three forge-shaped
|
|
/// node kinds landed in one slice; a fourth parameter each would have made
|
|
/// the signature the least readable part of this file). Each field is
|
|
/// built once at startup (see `main`) and is `None` exactly when that
|
|
/// dependency isn't configured on this host — every arm below treats
|
|
/// absence as *this node's* failure, not a reason to skip silently.
|
|
#[derive(Clone)]
|
|
pub(crate) struct WorkerDeps {
|
|
pub(crate) auth: Option<std::sync::Arc<auth::AuthBridge>>,
|
|
/// Empty until the forge token has been read (`forge::enable`), so a node
|
|
/// run before then fails as "no forge access", and one run after succeeds.
|
|
pub(crate) forge: Late<Arc<forge::Client>>,
|
|
/// The swarm queue connection, for nodes whose effect is a published
|
|
/// event rather than an API call. A handle rather than the status
|
|
/// reader it is cloned from: a node that announces a deploy has no
|
|
/// business reading hive status, and `status.rs`'s own doc says the
|
|
/// connection living there is an accident of construction order, not a
|
|
/// claim that events are a kind of status.
|
|
pub(crate) queue: Option<async_nats::Client>,
|
|
/// The wanted-state writer, for nodes that declare what a hive should
|
|
/// converge an agent to. Shares the status reader's queue connection —
|
|
/// see `wanted_writer`. `None` exactly when no swarm queue is configured
|
|
/// on this host, same as `queue`.
|
|
pub(crate) wanted: Option<std::sync::Arc<wanted::WantedWriter>>,
|
|
/// Client-server API base of the swarm's homeserver, for the node that
|
|
/// mints agents' accounts on it. `None` when
|
|
/// `matrix_account::DEFAULT_HOMESERVER_ENV` is unset, which that node
|
|
/// **fails** on rather than skips, same as every other field here.
|
|
pub(crate) matrix_homeserver: Option<std::sync::Arc<str>>,
|
|
}
|
|
|
|
/// What `UpdateMeta` renders from, fixed for the process.
|
|
struct MetaDeps {
|
|
settings: meta_flake::Settings,
|
|
/// Every hive's name: each places agents, and each hive's forge account
|
|
/// gets read on the repo.
|
|
hives: Vec<String>,
|
|
}
|
|
|
|
/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be.
|
|
///
|
|
/// A named constant rather than the literal inline, because this *is* the
|
|
/// decision the node exists to carry — writing a declaration at all is the
|
|
/// mechanism, `Paused` is the policy — and a test can pin a name where it
|
|
/// cannot pin a value written into a KV bucket no unit test has a server for.
|
|
const NEW_AGENT_WANTED_STATE: swarm_wire::wanted::AgentState =
|
|
swarm_wire::wanted::AgentState::Paused;
|
|
|
|
/// Run a claimed node and log one warn when it fails, whatever its kind.
|
|
pub(crate) async fn run_swarm_node(
|
|
id: hive_jobq::NodeId,
|
|
kind: SwarmNodeKind,
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
deps: WorkerDeps,
|
|
) -> (
|
|
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive_jobq::scheduler::Outcome,
|
|
) {
|
|
let node_kind = hive_jobq_wire::WireNode::label(&kind);
|
|
let (builder, outcome) = run_node_work(kind, builder, deps).await;
|
|
if let hive_jobq::scheduler::Outcome::Failed(error) = &outcome {
|
|
tracing::warn!(node = id.get(), kind = %node_kind, %error, "swarm jobq: node failed");
|
|
}
|
|
(builder, outcome)
|
|
}
|
|
|
|
/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/
|
|
/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind`
|
|
/// variant turns into a real effect.
|
|
async fn run_node_work(
|
|
kind: SwarmNodeKind,
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
deps: WorkerDeps,
|
|
) -> (
|
|
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive_jobq::scheduler::Outcome,
|
|
) {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let forge = deps.forge.get().cloned();
|
|
let outcome = match kind {
|
|
SwarmNodeKind::CreateIdentity { agent } => match deps.auth {
|
|
None => {
|
|
Outcome::Failed("no swarm-authelia-bridge is configured on this host".to_owned())
|
|
}
|
|
Some(bridge) => match bridge.ensure_agent_identity(&agent).await {
|
|
// Matched rather than discarded with `Ok(_)`: a heal writes
|
|
// to a subject this job did not create, and that has to
|
|
// reach a human. `Outcome` has no success-carrying variant,
|
|
// so the controller's own log is the only channel a
|
|
// succeeding node has today.
|
|
Ok(BridgeResponse::Healed) => {
|
|
tracing::warn!(
|
|
%agent,
|
|
"swarm jobq: create_identity marked an EXISTING identity as an \
|
|
agent — verify this name did not belong to a person"
|
|
);
|
|
Outcome::Done
|
|
}
|
|
Ok(_) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
},
|
|
},
|
|
SwarmNodeKind::CreateRepo { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => match forge_node("create_repo", client.create_repo(&agent)).await {
|
|
Ok(full_name) => {
|
|
tracing::info!(%full_name, "swarm jobq: create_repo done");
|
|
Outcome::Done
|
|
}
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
},
|
|
},
|
|
SwarmNodeKind::CreateForgeUser { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => {
|
|
match forge_node("create_forge_user", client.ensure_agent_user(&agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
},
|
|
SwarmNodeKind::AddRepoMember { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => {
|
|
match forge_node("add_repo_member", client.add_repo_member(&agent, &agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
},
|
|
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
|
|
SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
|
|
SwarmNodeKind::MintAgentIdentity { agent } => {
|
|
mint_identity(deps.wanted.as_deref(), &agent).await
|
|
}
|
|
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
|
|
SwarmNodeKind::MintAgentMatrixAccount { agent } => {
|
|
mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
|
|
}
|
|
SwarmNodeKind::MintHiveSenderToken { hive } => {
|
|
mint_hive_sender(deps.matrix_homeserver.as_deref(), &hive).await
|
|
}
|
|
SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await,
|
|
SwarmNodeKind::MintClassifierForgeToken => mint_classifier_forge_token(forge).await,
|
|
SwarmNodeKind::MintBuilderForgeToken => mint_builder_forge_token(forge).await,
|
|
SwarmNodeKind::MintCiMetricsForgeToken => mint_ci_metrics_forge_token(forge).await,
|
|
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
|
|
build_agent(&deps, forge, agent, rev, pr).await
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, pr } => build_meta(&deps, forge, agent, pr).await,
|
|
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
|
|
SwarmNodeKind::SetAgentWanted { hive, agent } => {
|
|
set_agent_wanted(deps.wanted.as_deref(), &hive, &agent).await
|
|
}
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue {
|
|
None => Outcome::Failed(
|
|
"no swarm queue is configured on this host, so no hive can be told to deploy"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
|
|
},
|
|
SwarmNodeKind::UpdateMeta => finish_update_meta(update_meta(&deps, forge).await, &builder),
|
|
SwarmNodeKind::RecordFanout { pin } => record_fanout(deps.queue.as_ref(), &pin).await,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
|
|
/// A forge-calling node's work, run under [`forge::FORGE_DEADLINE`]. Elapsing
|
|
/// is the node's own error, naming `node`, so it fails like any other refused
|
|
/// step instead of holding its executor slot.
|
|
pub(crate) async fn forge_node<T>(
|
|
node: &str,
|
|
work: impl std::future::Future<Output = Result<T>>,
|
|
) -> Result<T> {
|
|
forge::within_forge_deadline(node, work)
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
Err(anyhow::anyhow!(
|
|
"{node}: no answer from the forge within {}s",
|
|
forge::FORGE_DEADLINE.as_secs()
|
|
))
|
|
})
|
|
}
|
|
|
|
/// Logged at `warn`, with the agent, the step and the error, when
|
|
/// [`mint_identity`] fails. The Grafana rule `hh-agent-identity-mint-failed`
|
|
/// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both
|
|
/// together.
|
|
const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
|
|
|
|
/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
|
|
/// `clippy::too_many_lines`.
|
|
///
|
|
/// Re-reads the wanted-state declarations first (see [`mint_if_live`]); with
|
|
/// no wanted-state writer there is nothing to read and the mint runs.
|
|
pub(crate) async fn mint_identity(
|
|
wanted: Option<&wanted::WantedWriter>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let declarations = match wanted {
|
|
Some(writer) => {
|
|
let hives: Vec<String> = crate::startup::load_hives()
|
|
.into_iter()
|
|
.map(|h| h.name)
|
|
.collect();
|
|
Some(agent_renewal::read_declarations(writer, &hives).await)
|
|
}
|
|
None => None,
|
|
};
|
|
mint_if_live(agent, declarations, agent_identity::mint_and_verify(agent)).await
|
|
}
|
|
|
|
/// Run `mint` unless `declarations` (every hive's, read when the node runs)
|
|
/// leave `agent` not live: declared `Destroyed` everywhere, or nowhere. A mint
|
|
/// queued before a destroy then writes no credential for the destroyed name,
|
|
/// and the node is `Done`. Creation orders its mint after the declaration, so a
|
|
/// created agent is always declared. A failed read fails the node rather than
|
|
/// minting blind.
|
|
async fn mint_if_live(
|
|
agent: &str,
|
|
declarations: Option<anyhow::Result<Vec<swarm_wire::wanted::HiveWanted>>>,
|
|
mint: impl std::future::Future<Output = anyhow::Result<()>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match declarations {
|
|
Some(Ok(declarations)) if !agent_renewal::live_agents(&declarations).contains(agent) => {
|
|
tracing::info!(agent, "identity mint skipped: {agent} is destroyed");
|
|
return Outcome::Done;
|
|
}
|
|
Some(Err(e)) => {
|
|
let error = format!("{e:#}");
|
|
tracing::warn!(%agent, step = "declaration", %error, "{MINT_IDENTITY_FAILED}");
|
|
return Outcome::Failed(error);
|
|
}
|
|
Some(Ok(_)) | None => {}
|
|
}
|
|
match mint.await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => {
|
|
let error = format!("{e:#}");
|
|
let step =
|
|
agent_identity::failed_step(&e).map_or("unknown", agent_identity::MintStep::as_str);
|
|
tracing::warn!(%agent, %step, %error, "{MINT_IDENTITY_FAILED}");
|
|
Outcome::Failed(error)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The `MintAgentForgeToken` arm, lifted out so `run_node_work` stays under
|
|
/// `clippy::too_many_lines`.
|
|
async fn mint_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no agent forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_agent_forge_token",
|
|
client.ensure_agent_forge_token(agent),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `GrantDocsRead` arm, lifted out for the same reason.
|
|
pub(crate) async fn grant_docs_read(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no docs grant can be made"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node("grant_docs_read", client.grant_docs_read(agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `InitAgentConfigRepo` arm, lifted out for the same reason.
|
|
async fn init_agent_config_repo(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"init_agent_config_repo",
|
|
client.seed_agent_config(agent, agent),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `RenewAgentQueueCredential` arm, lifted out for the same reason.
|
|
async fn renew_queue_credential(agent: &str) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match agent_renewal::renew(agent).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintHiveForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_hive_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
hive: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no hive forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_hive_forge_token",
|
|
client.ensure_hive_forge_token(hive),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `BuildAgent` arm, lifted out for `clippy::too_many_lines`.
|
|
pub(crate) async fn build_agent(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: String,
|
|
rev: String,
|
|
pr: u64,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let job = swarm_wire::builder::BuildAgent { agent, rev, pr };
|
|
let builds = builder::runs::Builds::Live(deps.queue.clone());
|
|
let link_base = std::env::var(PUBLIC_URL_ENV).ok();
|
|
builder::run(
|
|
deps.queue.as_ref(),
|
|
forge.as_deref(),
|
|
&builds,
|
|
link_base.as_deref(),
|
|
job,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// The `BuildMeta` arm, lifted out for `clippy::too_many_lines`.
|
|
async fn build_meta(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: String,
|
|
pr: Option<u64>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let builds = builder::runs::Builds::Live(deps.queue.clone());
|
|
meta_build::run(forge.as_deref(), deps.queue.as_ref(), &builds, agent, pr).await
|
|
}
|
|
|
|
/// What an `UpdateMeta` pass decided: the deploys to send, and the placements
|
|
/// to send them to.
|
|
struct MetaPass {
|
|
fan_out: FanOut,
|
|
placements: std::collections::BTreeSet<(String, String)>,
|
|
}
|
|
|
|
/// What a pass sends the placed agents.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
enum FanOut {
|
|
/// Nothing: the pin is the one recorded.
|
|
None,
|
|
/// A deploy each, then record `pin`.
|
|
Record { pin: String },
|
|
/// A deploy each, with nothing to record into: the record cannot be read.
|
|
Unrecorded,
|
|
}
|
|
|
|
/// The fan-out for the pin `report` leaves, given the pin `meta_fanout` has
|
|
/// recorded. With no readable record, only a move of the pin sends deploys,
|
|
/// and nothing can be recorded.
|
|
fn decide_fan_out(report: &meta_flake::Report, recorded: Result<Option<String>>) -> FanOut {
|
|
match recorded {
|
|
Ok(Some(pin)) if pin == report.pin => FanOut::None,
|
|
Ok(_) => FanOut::Record {
|
|
pin: report.pin.clone(),
|
|
},
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = %format!("{e:#}"),
|
|
moved = report.hyperhive_moved,
|
|
"update_meta: the recorded fan-out pin is unreadable; deploying only on a moved pin"
|
|
);
|
|
if report.hyperhive_moved {
|
|
FanOut::Unrecorded
|
|
} else {
|
|
FanOut::None
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The `UpdateMeta` arm, lifted out for the same reason: what the update
|
|
/// decided to send, and the placements it rendered for. `Err` is the node's
|
|
/// failure.
|
|
async fn update_meta(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> Result<MetaPass, String> {
|
|
let Some(client) = forge else {
|
|
return Err(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so internal/meta cannot be written"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
let Some(meta) = meta_deps() else {
|
|
return Err(format!(
|
|
"{} is unset, so there are no swarm values to render internal/meta from",
|
|
meta_flake::SETTINGS_ENV
|
|
));
|
|
};
|
|
let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else {
|
|
return Err(
|
|
"no swarm queue is configured on this host, so neither the placed agents \
|
|
nor a lock can be had"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
let placements = placements(wanted, &meta.hives)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
let agents = placed_agents(queue, &placements)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
let report = forge_node(
|
|
"update_meta",
|
|
meta_flake::update(client.as_ref(), queue, &meta.settings, &agents, &meta.hives),
|
|
)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
tracing::info!(?report, agents = agents.len(), "update_meta: done");
|
|
let fan_out = decide_fan_out(&report, meta_fanout::recorded(queue).await);
|
|
Ok(MetaPass {
|
|
fan_out,
|
|
placements,
|
|
})
|
|
}
|
|
|
|
/// The `UpdateMeta` node's outcome from what [`update_meta`] decided, with the
|
|
/// deploys declared into `builder` (the scheduler inserts them under the node
|
|
/// when it completes). Synchronous: the builder is not `Sync`, so it cannot be
|
|
/// held across an await.
|
|
fn finish_update_meta(
|
|
result: Result<MetaPass, String>,
|
|
builder: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match result {
|
|
Ok(pass) => {
|
|
declare_fan_out(builder, &pass.fan_out, &pass.placements);
|
|
Outcome::Done
|
|
}
|
|
Err(e) => Outcome::Failed(e),
|
|
}
|
|
}
|
|
|
|
/// The `RecordFanout` arm, lifted out so `run_node_work` stays under the line limit.
|
|
async fn record_fanout(
|
|
queue: Option<&async_nats::Client>,
|
|
pin: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = queue else {
|
|
return Outcome::Failed(
|
|
"no swarm queue is configured on this host, so the fan-out cannot be recorded"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match meta_fanout::record(client, pin).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The build and the deploy every path declares for one agent: a `BuildMeta`
|
|
/// of `agent`, then the `TriggerDeploy`, which runs only when the build is
|
|
/// green (`after_ok`), so a red build leaves it `Skipped`.
|
|
///
|
|
/// `update` is the `UpdateMeta` the pair follows when it is not this pair's
|
|
/// parent. The build waits on it, since the build reads the head that update
|
|
/// commits. The deploy keeps its own edge to it too: `skipped_deploys` finds
|
|
/// the deploys a failed update skipped by that edge.
|
|
fn gated_deploy<'a>(
|
|
b: &'a hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: Option<String>,
|
|
pr: Option<u64>,
|
|
update: Option<hive_jobq::builder::NodeGuid>,
|
|
) -> hive_jobq::builder::NodeRef<'a, SwarmNodeKind, SwarmResourceKind> {
|
|
let mut build = b
|
|
.node(SwarmNodeKind::BuildMeta {
|
|
agent: agent.to_owned(),
|
|
pr,
|
|
})
|
|
.needs(SwarmResourceKind::Builder);
|
|
if let Some(update) = update {
|
|
build = build.after_ok(update);
|
|
}
|
|
let mut deploy = b
|
|
.node(SwarmNodeKind::TriggerDeploy {
|
|
hive: hive.to_owned(),
|
|
agent: agent.to_owned(),
|
|
rev,
|
|
})
|
|
.after_ok(build);
|
|
if let Some(update) = update {
|
|
deploy = deploy.after_ok(update);
|
|
}
|
|
deploy
|
|
}
|
|
|
|
/// Declare one build and rebuild of each `(hive, agent)` placement, which
|
|
/// nothing else tells to build again, and for a [`FanOut::Record`] the
|
|
/// `RecordFanout` that waits on every one of them. A deploy the build gate
|
|
/// skipped counts as sent: the pin is recorded, so one failing agent does
|
|
/// not resend every healthy one on each retry. A deploy skipped by a failed
|
|
/// build is re-declared by `queue_meta_update` on every round.
|
|
fn declare_fan_out(
|
|
grown: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
fan_out: &FanOut,
|
|
placements: &std::collections::BTreeSet<(String, String)>,
|
|
) {
|
|
if *fan_out == FanOut::None {
|
|
return;
|
|
}
|
|
let deploys: Vec<hive_jobq::builder::NodeGuid> = placements
|
|
.iter()
|
|
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None, None).guid())
|
|
.collect();
|
|
if let FanOut::Record { pin } = fan_out {
|
|
let mut record = grown.node(SwarmNodeKind::RecordFanout { pin: pin.clone() });
|
|
for deploy in deploys {
|
|
record = record.on_outcome(
|
|
deploy,
|
|
&[
|
|
hive_jobq::TerminalState::Done,
|
|
hive_jobq::TerminalState::Skipped,
|
|
],
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Every `(hive, agent)` a hive in `hives` places.
|
|
pub(crate) async fn placements(
|
|
wanted: &wanted::WantedWriter,
|
|
hives: &[String],
|
|
) -> Result<std::collections::BTreeSet<(String, String)>> {
|
|
let mut placed = std::collections::BTreeSet::new();
|
|
for hive in hives {
|
|
let Some(declared) = wanted
|
|
.view(hive)
|
|
.await
|
|
.map_err(|e| e.context(format!("read where hive {hive} places agents")))?
|
|
else {
|
|
continue;
|
|
};
|
|
placed.extend(
|
|
declared
|
|
.agents
|
|
.into_iter()
|
|
.filter(|(_, w)| crate::routes::agents::places_agent(w.state))
|
|
.map(|(name, _)| (hive.clone(), name)),
|
|
);
|
|
}
|
|
Ok(placed)
|
|
}
|
|
|
|
/// Every agent in `placements`, with its archetype.
|
|
async fn placed_agents(
|
|
queue: &async_nats::Client,
|
|
placements: &std::collections::BTreeSet<(String, String)>,
|
|
) -> Result<Vec<meta_flake::Placed>> {
|
|
let names: std::collections::BTreeSet<&String> =
|
|
placements.iter().map(|(_, agent)| agent).collect();
|
|
let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue)
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?;
|
|
let mut placed = Vec::with_capacity(names.len());
|
|
for name in names {
|
|
let (archetype, _, _) = crate::schedules::read_or_default::<
|
|
Option<swarm_wire::agent_archetype::ArchetypeRef>,
|
|
>(&archetypes, name)
|
|
.await
|
|
.map_err(|e| e.context(format!("read the archetype of {name}")))?;
|
|
placed.push(meta_flake::Placed {
|
|
name: name.clone(),
|
|
archetype,
|
|
});
|
|
}
|
|
Ok(placed)
|
|
}
|
|
|
|
/// The `MintBuilderForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_builder_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no builder forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_builder_forge_token",
|
|
client.ensure_builder_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintCiMetricsForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_ci_metrics_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no ci metrics forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_ci_metrics_forge_token",
|
|
client.ensure_ci_metrics_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintClassifierForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_classifier_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no review classifier forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_classifier_forge_token",
|
|
client.ensure_classifier_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintAgentMatrixAccount` arm, lifted out for the same reason.
|
|
async fn mint_matrix_account(
|
|
homeserver: Option<&str>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
// Loud on purpose: an agent quietly created without a matrix account is
|
|
// an agent nothing can talk to, and no hive mints one any more.
|
|
let Some(base) = homeserver else {
|
|
return Outcome::Failed(format!(
|
|
"no matrix homeserver configured on this host ({} unset), so no agent \
|
|
matrix account can be minted",
|
|
matrix_account::DEFAULT_HOMESERVER_ENV
|
|
));
|
|
};
|
|
match matrix_account::agent_token::ensure_agent_matrix_account(base, agent).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintHiveSenderToken` arm, lifted out for the same reason.
|
|
async fn mint_hive_sender(homeserver: Option<&str>, hive: &str) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(base) = homeserver else {
|
|
return Outcome::Failed(format!(
|
|
"no matrix homeserver configured on this host ({} unset), so no hive \
|
|
sender token can be minted",
|
|
matrix_account::DEFAULT_HOMESERVER_ENV
|
|
));
|
|
};
|
|
match matrix_account::hive_sender::ensure_hive_sender_token(base, hive).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `SetAgentWanted` arm, lifted out so `run_node_work` stays under the line
|
|
/// limit: it fails without a wanted-state writer.
|
|
async fn set_agent_wanted(
|
|
wanted: Option<&wanted::WantedWriter>,
|
|
hive: &str,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
match wanted {
|
|
None => hive_jobq::scheduler::Outcome::Failed(
|
|
"no swarm queue is configured on this host, so no wanted-state \
|
|
declaration can be written"
|
|
.to_owned(),
|
|
),
|
|
Some(writer) => declare_new_agent(writer, hive, agent).await,
|
|
}
|
|
}
|
|
|
|
/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns
|
|
/// out not to be new: an agent that already has a declaration (other than
|
|
/// `Destroyed`, which this treats as reusable) is left alone, so a retried
|
|
/// or migrating create is idempotent rather than pausing an already-running
|
|
/// agent. `wanted.rs` itself has no such carve-out, so this node makes the
|
|
/// call.
|
|
async fn declare_new_agent(
|
|
writer: &wanted::WantedWriter,
|
|
hive: &str,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
use swarm_wire::wanted::AgentState;
|
|
|
|
let existing = match writer.view(hive).await {
|
|
Ok(declaration) => declaration.and_then(|d| d.agents.get(agent).map(|a| a.state)),
|
|
Err(e) => return Outcome::Failed(format!("{e:#}")),
|
|
};
|
|
if let Some(state) = existing
|
|
&& state != AgentState::Destroyed
|
|
{
|
|
tracing::debug!(
|
|
hive,
|
|
agent,
|
|
?state,
|
|
"agent already declared, leaving its wanted state alone"
|
|
);
|
|
return Outcome::Done;
|
|
}
|
|
match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await {
|
|
Ok(_) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// Publish one deploy request and report whether it left this process.
|
|
///
|
|
/// The flush is not belt-and-braces: `publish` hands the message to the
|
|
/// client's write buffer and returns, so a node that reported `Done` on
|
|
/// that alone would be claiming a delivery it has no evidence for — the
|
|
/// same ordering `webhook::announce_knowledge_change` documents.
|
|
async fn publish_deploy(
|
|
client: &async_nats::Client,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: Option<String>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
// One subject per hive, so the other hives are never woken by this.
|
|
let subject = swarm_queue_client::deploy_subject(hive);
|
|
let request = swarm_queue_client::DeployRequest {
|
|
agent: agent.to_owned(),
|
|
rev,
|
|
};
|
|
let payload = match serde_json::to_vec(&request) {
|
|
Ok(payload) => payload,
|
|
Err(e) => return Outcome::Failed(format!("encoding the deploy request failed: {e}")),
|
|
};
|
|
if let Err(e) = client.publish(subject.clone(), payload.into()).await {
|
|
return Outcome::Failed(format!("publishing to {subject} failed: {e}"));
|
|
}
|
|
if let Err(e) = client.flush().await {
|
|
return Outcome::Failed(format!(
|
|
"flushing the deploy event to {subject} failed: {e}"
|
|
));
|
|
}
|
|
tracing::info!(%subject, %hive, %agent, rev = ?request.rev, "swarm jobq: deploy requested");
|
|
Outcome::Done
|
|
}
|
|
|
|
/// Spawn the swarm-level job-graph scheduler loop. Mirrors `hive-c0re/src/
|
|
/// job_queue/scheduler.rs::run_worker`'s shape: claim one runnable node,
|
|
/// spawn the future that runs + completes it, loop again immediately if
|
|
/// something started (more may now be runnable), otherwise back off
|
|
/// briefly before re-polling.
|
|
///
|
|
/// No shutdown signal to wire in — unlike `hive-c0re`'s `coord.shutdown_rx()`,
|
|
/// this daemon has no graceful-shutdown machinery at all yet (`main`'s
|
|
/// `axum::serve` runs unconditionally to process exit), so this loop
|
|
/// matches that: it rides the runtime down with the process, same as
|
|
/// every in-flight HTTP request does.
|
|
///
|
|
/// Cheap to run with an empty graph: `claim_next` on a graph nothing was
|
|
/// ever inserted into just returns `None` every poll.
|
|
///
|
|
/// `deps` is cloned per iteration (its fields are `Arc` clones, not
|
|
/// reconnects) and moved into the closure `claim_next` takes ownership
|
|
/// of — `run_swarm_node` needs its own owned copy since the claimed
|
|
/// future may outlive this loop iteration.
|
|
pub(crate) fn spawn_jobq_worker(
|
|
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
|
deps: WorkerDeps,
|
|
) {
|
|
tokio::spawn(async move {
|
|
loop {
|
|
let deps = deps.clone();
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
});
|
|
match runner {
|
|
Some(runner) => {
|
|
tokio::spawn(async move {
|
|
let (id, grew) = runner.await;
|
|
if let Err(e) = grew {
|
|
tracing::warn!(
|
|
node = id.get(),
|
|
error = %e,
|
|
"swarm jobq: grown job rejected"
|
|
);
|
|
}
|
|
});
|
|
// Something just started — more may be runnable right
|
|
// now, so loop again immediately rather than sleeping.
|
|
}
|
|
None => {
|
|
// Nothing runnable. Bounded poll rather than an event
|
|
// wake (unlike hive-c0re's `notify.notify_one()`,
|
|
// there is no completion-signal channel here yet) —
|
|
// fine at this daemon's scale (one graph, no
|
|
// submitters yet); revisit if/when that stops holding.
|
|
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Queue a build check of a config PR's new head. Every head is built, on
|
|
/// whichever hive the agent runs: the check builds on the builder's own
|
|
/// agent base.
|
|
pub(crate) fn queue_config_build(state: &AppState, changed: config_pr::ChangedConfigPr) {
|
|
let (agent, rev, pr) = (changed.agent.clone(), changed.rev.clone(), changed.pr);
|
|
let queued = queue_build(
|
|
&mut state
|
|
.jobq
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
|
changed,
|
|
);
|
|
match queued {
|
|
Ok(superseded) => {
|
|
tracing::info!(%agent, %rev, pr, superseded, "config-pr head: build queued");
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%agent, %rev, pr, error = %e, "config-pr head: queueing the build failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Cancel every pending build of the same PR, then insert the new one behind
|
|
/// [`SwarmResourceKind::Builder`]. A build already running is left to finish:
|
|
/// its status lands on its own commit, not on the new head. Returns how many
|
|
/// pending builds were cancelled.
|
|
pub(crate) fn queue_build(
|
|
sched: &mut hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>,
|
|
changed: config_pr::ChangedConfigPr,
|
|
) -> Result<usize> {
|
|
let config_pr::ChangedConfigPr { agent, rev, pr } = changed;
|
|
let stale: Vec<hive_jobq::NodeId> = sched
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.state == hive_jobq::State::Pending)
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::BuildAgent { agent: a, pr: p, .. } if *a == agent && *p == pr)
|
|
})
|
|
.map(|n| n.id)
|
|
.collect();
|
|
let superseded = stale
|
|
.into_iter()
|
|
.filter(|&id| sched.cancel_node(id))
|
|
.count();
|
|
sched
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::BuildAgent { agent, rev, pr })
|
|
.needs(SwarmResourceKind::Builder)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
Ok(superseded)
|
|
}
|
|
|
|
/// Queue the deploy of a merged config PR on the one hive that runs its agent.
|
|
///
|
|
/// The hive is found by scanning every hive's wanted state, since nothing
|
|
/// records an agent's hive (see `forge::initial_agent_nix`). Zero claimants or
|
|
/// several deploy nothing: there is no hive to send it to, or no telling which.
|
|
pub(crate) async fn queue_merged_config_deploy(
|
|
state: &AppState,
|
|
merged: config_pr::MergedConfigPr,
|
|
) {
|
|
let config_pr::MergedConfigPr { agent, rev, pr } = merged;
|
|
let declared = match read_declarations(state, state.hives.iter()).await {
|
|
Ok(declared) => declared,
|
|
Err((hive, e)) => {
|
|
tracing::warn!(
|
|
%agent, %rev, %hive, error = %format!("{e:#}"),
|
|
"config-pr merge: cannot read where the agent is placed; nothing deployed"
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
let claimants = claiming_hives(&agent, &declared);
|
|
let [hive] = claimants.as_slice() else {
|
|
tracing::warn!(
|
|
%agent, %rev, ?claimants,
|
|
"config-pr merge: not exactly one hive places this agent; nothing deployed"
|
|
);
|
|
return;
|
|
};
|
|
let hive = hive.clone();
|
|
let inserted = state
|
|
.jobq
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| merged_deploy_job(b, &hive, &agent, &rev, pr));
|
|
match inserted {
|
|
Ok(_) => tracing::info!(%agent, %rev, %hive, "config-pr merge: deploy queued"),
|
|
Err(e) => {
|
|
tracing::warn!(%agent, %rev, %hive, error = %e, "config-pr merge: queueing the deploy failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The sub-DAG a merged config PR is: the meta update, the build of the agent
|
|
/// from the updated flake, then the deploy of `rev` on `hive`.
|
|
///
|
|
/// `after_ok`: a hive builds only from `internal/meta`, so a deploy asked for
|
|
/// after a failed update (no builder, an unreadable config repo) could only
|
|
/// fail its build, and one asked for after a red build would fail the same
|
|
/// way. The failed node is `Failed` in the graph and the deploy `Skipped`.
|
|
fn merged_deploy_job(
|
|
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: &str,
|
|
pr: u64,
|
|
) -> Vec<hive_jobq::builder::NodeGuid> {
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta);
|
|
let root = update_meta.guid();
|
|
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), Some(pr), Some(root));
|
|
vec![root]
|
|
}
|
|
|
|
/// The sub-DAG one agent creation is: the nodes, and the edges between them.
|
|
///
|
|
/// A function rather than a closure inside [`create_agent`](crate::routes::agents::create_agent) so the endpoint's
|
|
/// validation and the graph's shape can each be read without scrolling past
|
|
/// the other — and so the one handle the response reports is returned from
|
|
/// the place that decides which node it is.
|
|
pub(crate) fn declare_agent_job(
|
|
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
agent: &str,
|
|
hive: &str,
|
|
archetype: bool,
|
|
) -> Vec<hive_jobq::builder::NodeGuid> {
|
|
let create_identity = b.node(SwarmNodeKind::CreateIdentity {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// A second, independent root: a forge user needs neither an authelia
|
|
// subject nor an existing repo, so it does not chain off
|
|
// `create_identity` (see the doc comment above).
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// An archetype agent has no config repo: its hive builds it from the
|
|
// archetype's module, so there is no repo to create, share or seed.
|
|
let init_config = (!archetype).then(|| {
|
|
let create_repo = b
|
|
.node(SwarmNodeKind::CreateRepo {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_identity);
|
|
// `AddRepoMember` needs both parents: the repo to add a collaborator
|
|
// to, and the forge user to add as one — adding a nonexistent user is
|
|
// a Forgejo validation error, not an idempotent no-op.
|
|
// `InitAgentConfigRepo` needs only the repo — see the doc comment
|
|
// above for why.
|
|
let _add_repo_member = b
|
|
.node(SwarmNodeKind::AddRepoMember {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_repo)
|
|
.after_ok(create_forge_user);
|
|
b.node(SwarmNodeKind::InitAgentConfigRepo {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_repo)
|
|
.guid()
|
|
});
|
|
// Read on `internal/docs`, a repo that exists whatever this agent is, so
|
|
// it needs only the user. Not a parent of the deploy: the agent reads docs
|
|
// once running, and a failed grant is retried by `forge::docs_grant::spawn`.
|
|
let _grant_docs = b
|
|
.node(SwarmNodeKind::GrantDocsRead {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_forge_user);
|
|
// Declared before the deploy trigger so the pause is visible in the
|
|
// wanted-state store before the hive brings the container up — see the
|
|
// node's own doc comment for why "before", not just "eventually". Needs
|
|
// no parent of its own: the declaration is written from the hive/agent
|
|
// names known at request time, same as `create_forge_user`.
|
|
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
|
|
hive: hive.to_owned(),
|
|
agent: agent.to_owned(),
|
|
});
|
|
// The agent's own identity at the swarm secret store, minted and
|
|
// published at swarm level so no hive ever needs the capability to mint
|
|
// one. After `create_identity` because the certificate names a principal
|
|
// the swarm has agreed exists — not because anything in the store reads
|
|
// authelia. After `set_wanted` because the node re-reads the declaration
|
|
// and writes nothing for an agent declared nowhere or `Destroyed`.
|
|
let mint_identity = b
|
|
.node(SwarmNodeKind::MintAgentIdentity {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_identity)
|
|
.after_ok(set_wanted);
|
|
// The agent's own forge token, in the store before the hive deploys
|
|
// the container that pulls it. After the forge user, because the token
|
|
// is minted for that user.
|
|
let mint_forge_token = b
|
|
.node(SwarmNodeKind::MintAgentForgeToken {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_forge_user);
|
|
// The agent's account on the swarm's homeserver. A root of its own:
|
|
// creating it needs neither an authelia subject nor a forge user nor a
|
|
// repo, and chaining it behind one of those would make an unrelated
|
|
// failure look like a matrix failure.
|
|
let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// The hive builds the agent from `internal/meta` only, so the deploy needs
|
|
// the update and the swarm's build from it to succeed (`after_ok`): after a
|
|
// failed update (no builder, an unreadable config repo) or a red build, the
|
|
// hive's build could only fail.
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta)
|
|
.after_ok(set_wanted);
|
|
// Last, and specifically after the config repo is seeded: the hive
|
|
// deploys by reading that repo, so a deploy asked for any earlier would
|
|
// find nothing to build. This is the edge that makes creating an agent at
|
|
// swarm level actually put it on a hive, rather than leaving a
|
|
// provisioned name nobody runs. An archetype agent has no repo, so its
|
|
// deploy waits on the mints and the declaration alone.
|
|
//
|
|
// `after_any` on the mint, not `after_ok`: a hive cannot pass down a
|
|
// certificate the swarm has not published, so the deploy must not
|
|
// overtake the mint — but a host whose store or agent PKI is not set up
|
|
// must still create agents. `after_ok` there would turn a store outage
|
|
// into an agent nobody runs.
|
|
//
|
|
// The forge-token mint gets `after_any` for the same reason: a host with
|
|
// no forge or no store must still create agents; only that node fails,
|
|
// by name, and the agent's container fetches the token on its next
|
|
// timer tick once one is minted.
|
|
//
|
|
// `after_ok` on the declaration, though: an agent deployed without its
|
|
// pause landing first is the race this node exists to close, so a
|
|
// declaration that did not land must not be deployed past. A host with no
|
|
// queue at all fails this node — but `TriggerDeploy` has nothing to
|
|
// publish to on such a host either, so the strict edge cancels a node
|
|
// that could not have succeeded anyway.
|
|
let trigger_deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()))
|
|
.after_any(mint_identity)
|
|
.after_any(mint_forge_token)
|
|
// `after_any` for the same reason: the container reads this token
|
|
// rather than producing it, so the deploy must not overtake the mint —
|
|
// but a host with no homeserver configured must still create agents,
|
|
// and only this node fails, by name.
|
|
.after_any(mint_matrix)
|
|
.after_ok(set_wanted);
|
|
if let Some(init_config) = init_config {
|
|
let _ = trigger_deploy.after_ok(init_config);
|
|
let _ = update_meta.after_ok(init_config);
|
|
}
|
|
vec![create_identity.guid()]
|
|
}
|
|
|
|
/// Insert one job per agent: `CreateForgeUser`, then `MintAgentForgeToken`
|
|
/// `after_ok` it — the same edge [`declare_agent_job`] uses. Returns the mint
|
|
/// nodes' ids.
|
|
///
|
|
/// The one place that node is queued outside agent creation: both the manual
|
|
/// route below and `forge::agent_token::spawn`'s periodic pass come through
|
|
/// here, so a backfilled mint is the same node a new agent gets. The user
|
|
/// node is what lets the pass reach an agent that has a store identity but
|
|
/// no forge account; for one that has an account it is a no-op, because
|
|
/// `forge::Client::ensure_agent_user` folds "already exists" into success.
|
|
pub(crate) fn queue_forge_token_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.clone(),
|
|
});
|
|
vec![
|
|
b.node(SwarmNodeKind::MintAgentForgeToken { agent })
|
|
.after_ok(create_forge_user)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one job per agent: `CreateForgeUser`, then `GrantDocsRead` `after_ok`
|
|
/// it — the same edge [`declare_agent_job`] uses. Returns the grant nodes' ids.
|
|
/// `forge::docs_grant::spawn`'s periodic pass comes through here, so a
|
|
/// backfilled grant is the same node a new agent gets; the user node is a
|
|
/// no-op for an agent that has an account.
|
|
pub(crate) fn queue_docs_grants(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.clone(),
|
|
});
|
|
vec![
|
|
b.node(SwarmNodeKind::GrantDocsRead { agent })
|
|
.after_ok(create_forge_user)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintAgentMatrixAccount` job per agent and return the nodes'
|
|
/// ids. `matrix_account::agent_token::spawn`'s periodic pass comes through
|
|
/// here, so a backfilled mint is the same node a new agent gets.
|
|
pub(crate) fn queue_matrix_account_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::MintAgentMatrixAccount { agent })
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintHiveSenderToken` job per hive and return the nodes' ids.
|
|
/// `matrix_account::hive_sender::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_hive_sender_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
hives: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(hives.len());
|
|
for hive in hives {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintHiveSenderToken { hive }).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintHiveForgeToken` job per hive and return the nodes' ids.
|
|
/// `forge::hive_token::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_hive_forge_token_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
hives: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(hives.len());
|
|
for hive in hives {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintHiveForgeToken { hive }).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintClassifierForgeToken` job and return its node's id.
|
|
/// `forge::classifier_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_classifier_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintClassifierForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// Insert one `MintBuilderForgeToken` job and return its node's id.
|
|
/// `forge::builder_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_builder_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintBuilderForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// How often [`spawn_meta_update_timer`] looks at the last `UpdateMeta`.
|
|
const META_UPDATE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(10);
|
|
|
|
/// Look at the last `UpdateMeta` now and every [`META_UPDATE_INTERVAL`] after,
|
|
/// and queue one when none has run yet or the latest did not succeed, so a
|
|
/// boot run that failed or hung is retried. The first tick is the boot run.
|
|
/// Once a run has succeeded a tick adds nothing: the graph keeps every node,
|
|
/// so a clock-driven node per tick would grow it without bound.
|
|
pub(crate) fn spawn_meta_update_timer(
|
|
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
|
) {
|
|
tokio::spawn(async move {
|
|
let mut ticker = tokio::time::interval(META_UPDATE_INTERVAL);
|
|
loop {
|
|
ticker.tick().await;
|
|
if let Err(e) = queue_meta_update(&sched) {
|
|
tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed");
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Why a deploy is `Skipped` and the timer should declare it again.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum SkipCause {
|
|
/// It waited on an `UpdateMeta` that `Failed`.
|
|
FailedUpdate,
|
|
/// It waited on a `BuildMeta` that `Failed`, whatever the reason.
|
|
FailedBuild,
|
|
}
|
|
|
|
/// A deploy [`skipped_deploys`] found.
|
|
struct SkippedDeploy {
|
|
hive: String,
|
|
agent: String,
|
|
rev: Option<String>,
|
|
/// The merged config PR the failed build belonged to, so the build
|
|
/// declared again reports on it too.
|
|
pr: Option<u64>,
|
|
cause: SkipCause,
|
|
}
|
|
|
|
/// The deploys the timer declares again: for each agent, its latest
|
|
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta` or a
|
|
/// `BuildMeta` it waited on `Failed`. A failed build is built again on every
|
|
/// round until one succeeds. A deploy skipped for another reason is left alone:
|
|
/// the pause declaration or the config seeding failing means the update that
|
|
/// would unblock the deploy was skipped too, and deploying past those is the
|
|
/// race those edges close.
|
|
fn skipped_deploys(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
) -> Vec<SkippedDeploy> {
|
|
let mut latest = std::collections::BTreeMap::new();
|
|
for n in graph.nodes() {
|
|
if let SwarmNodeKind::TriggerDeploy { hive, agent, .. } = &n.payload {
|
|
let key = (hive.clone(), agent.clone());
|
|
if latest
|
|
.get(&key)
|
|
.is_none_or(|l: &&hive_jobq::Node<_, _>| l.id < n.id)
|
|
{
|
|
latest.insert(key, n);
|
|
}
|
|
}
|
|
}
|
|
latest
|
|
.into_values()
|
|
.filter_map(|n| {
|
|
let SwarmNodeKind::TriggerDeploy { hive, agent, rev } = &n.payload else {
|
|
return None;
|
|
};
|
|
let dep_nodes = || {
|
|
n.deps.iter().filter_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, .. } => graph.node(*id),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
})
|
|
};
|
|
let failed = |is: fn(&SwarmNodeKind) -> bool| {
|
|
dep_nodes().any(|dep| is(&dep.payload) && dep.state == hive_jobq::State::Failed)
|
|
};
|
|
let cause = if failed(|p| matches!(p, SwarmNodeKind::UpdateMeta)) {
|
|
SkipCause::FailedUpdate
|
|
} else if failed(|p| matches!(p, SwarmNodeKind::BuildMeta { .. })) {
|
|
SkipCause::FailedBuild
|
|
} else {
|
|
return None;
|
|
};
|
|
let pr = dep_nodes().find_map(|dep| match &dep.payload {
|
|
SwarmNodeKind::BuildMeta { pr, .. } => *pr,
|
|
_ => None,
|
|
});
|
|
Some(SkippedDeploy {
|
|
hive: hive.clone(),
|
|
agent: agent.clone(),
|
|
rev: rev.clone(),
|
|
pr,
|
|
cause,
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Insert one job when the latest `UpdateMeta`, by node id, is not `Done` (or
|
|
/// none exists) and none is in flight: an `UpdateMeta`, with a fresh build and
|
|
/// `TriggerDeploy` behind it ([`gated_deploy`]) for every deploy the failed
|
|
/// run skipped ([`skipped_deploys`]), so an agent created or merged while the
|
|
/// update was failing is deployed by the run that succeeds.
|
|
///
|
|
/// When the latest `UpdateMeta` is `Done`, a deploy skipped by a failed build
|
|
/// still gets a fresh build and deploy, with no update, so a build that failed
|
|
/// is tried again on every round until it succeeds. That adds two nodes per
|
|
/// failing agent on every tick. Returns whether a job was inserted.
|
|
fn queue_meta_update(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<bool> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut update_metas = sched
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
.collect::<Vec<_>>();
|
|
if update_metas.iter().any(|n| {
|
|
matches!(
|
|
n.state,
|
|
hive_jobq::State::Pending | hive_jobq::State::Running | hive_jobq::State::Finishing
|
|
)
|
|
}) {
|
|
return Ok(false);
|
|
}
|
|
update_metas.sort_by_key(|n| n.id);
|
|
let update_due = update_metas
|
|
.last()
|
|
.is_none_or(|n| n.state != hive_jobq::State::Done);
|
|
let skipped: Vec<SkippedDeploy> = skipped_deploys(sched.graph())
|
|
.into_iter()
|
|
.filter(|s| update_due || s.cause == SkipCause::FailedBuild)
|
|
.collect();
|
|
if !update_due && skipped.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
sched
|
|
.insert_job(None, |b| {
|
|
let update = update_due.then(|| {
|
|
b.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta)
|
|
.guid()
|
|
});
|
|
let deploys: Vec<_> = skipped
|
|
.into_iter()
|
|
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, s.pr, update).guid())
|
|
.collect();
|
|
update.map_or(deploys, |update| vec![update])
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
Ok(true)
|
|
}
|
|
|
|
/// [`MetaDeps`] from the environment, read on first use; `None`, logged, when
|
|
/// the settings are unset or do not parse, which `UpdateMeta` then fails on
|
|
/// by name.
|
|
fn meta_deps() -> Option<&'static MetaDeps> {
|
|
static DEPS: std::sync::OnceLock<Option<MetaDeps>> = std::sync::OnceLock::new();
|
|
DEPS.get_or_init(read_meta_deps).as_ref()
|
|
}
|
|
|
|
fn read_meta_deps() -> Option<MetaDeps> {
|
|
let settings = match meta_flake::Settings::from_env() {
|
|
Ok(Some(s)) => s,
|
|
Ok(None) => {
|
|
tracing::warn!(
|
|
"{} unset; internal/meta will not be written",
|
|
meta_flake::SETTINGS_ENV
|
|
);
|
|
return None;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(error = %format!("{e:#}"), "internal/meta will not be written");
|
|
return None;
|
|
}
|
|
};
|
|
Some(MetaDeps {
|
|
settings,
|
|
hives: crate::startup::load_hives()
|
|
.into_iter()
|
|
.map(|h| h.name)
|
|
.collect(),
|
|
})
|
|
}
|
|
|
|
/// Insert one `MintCiMetricsForgeToken` job and return its node's id.
|
|
/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_ci_metrics_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintCiMetricsForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// Whether a `MintAgentIdentity` for `agent` is queued or running.
|
|
fn identity_mint_in_flight(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
agent: &str,
|
|
) -> bool {
|
|
graph.nodes().any(|n| {
|
|
!n.state.is_terminal()
|
|
&& matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
|
|
})
|
|
}
|
|
|
|
/// Insert one job per renewal and return the ids of its nodes:
|
|
/// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a
|
|
/// queue secret, the second `after_any` the first when an agent needs both.
|
|
/// `agent_renewal::spawn`'s periodic pass comes through here.
|
|
///
|
|
/// An agent whose `MintAgentIdentity` has not finished (agent creation's, or an
|
|
/// earlier pass's) is skipped whole until the next pass: a mint issues a new
|
|
/// leaf each run, and a pass comes every five minutes whether or not the last
|
|
/// mint has run yet. Its secret renewal waits too, since the chain below
|
|
/// cannot reach a mint it did not queue.
|
|
///
|
|
/// Chained rather than parallel because `mint_and_verify` reads the queue
|
|
/// secret back and compares it with the one it read first; a re-mint landing
|
|
/// in between fails that node. `after_any`, so a certificate that could not be
|
|
/// re-issued does not hold up the secret.
|
|
pub(crate) fn queue_agent_renewals(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
renewals: Vec<agent_renewal::Renewal>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(renewals.len());
|
|
for renewal in renewals {
|
|
if renewal.cert && identity_mint_in_flight(sched.graph(), &renewal.agent) {
|
|
tracing::info!(
|
|
agent = %renewal.agent,
|
|
"agent identity mint already in flight; not queueing another"
|
|
);
|
|
continue;
|
|
}
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let mut asked = Vec::new();
|
|
let cert = renewal.cert.then(|| {
|
|
b.node(SwarmNodeKind::MintAgentIdentity {
|
|
agent: renewal.agent.clone(),
|
|
})
|
|
.guid()
|
|
});
|
|
asked.extend(cert);
|
|
if renewal.secret {
|
|
let secret = b.node(SwarmNodeKind::RenewAgentQueueCredential {
|
|
agent: renewal.agent.clone(),
|
|
});
|
|
let secret = match cert {
|
|
Some(cert) => secret.after_any(cert),
|
|
None => secret,
|
|
};
|
|
asked.push(secret.guid());
|
|
}
|
|
asked
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::routes::tests::state_with_roster_queued;
|
|
|
|
pub(crate) fn changed(agent: &str, rev: &str, pr: u64) -> crate::config_pr::ChangedConfigPr {
|
|
crate::config_pr::ChangedConfigPr {
|
|
agent: agent.to_owned(),
|
|
rev: rev.to_owned(),
|
|
pr,
|
|
}
|
|
}
|
|
|
|
fn build_state(
|
|
sched: &hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
|
|
rev: &str,
|
|
) -> hive_jobq::State {
|
|
sched
|
|
.graph()
|
|
.nodes()
|
|
.find(|n| matches!(&n.payload, SwarmNodeKind::BuildAgent { rev: r, .. } if r == rev))
|
|
.map(|n| n.state)
|
|
.expect("a build of that rev")
|
|
}
|
|
|
|
/// A newer head of the same PR replaces its pending build; another PR's
|
|
/// build, and the same PR's running build, are left alone.
|
|
#[tokio::test]
|
|
async fn a_newer_push_replaces_the_prs_pending_build_only() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
let queue = |c| super::queue_build(&mut sched.lock().unwrap(), c).expect("queued");
|
|
assert_eq!(queue(changed("atlas", "r1", 3)), 0);
|
|
// r1 takes the builder and stays running: its future is never awaited.
|
|
let _running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
.expect("r1 is runnable");
|
|
assert_eq!(
|
|
queue(changed("atlas", "r2", 3)),
|
|
0,
|
|
"r1 is running, not pending"
|
|
);
|
|
assert_eq!(queue(changed("argus", "o1", 4)), 0);
|
|
assert_eq!(queue(changed("atlas", "r3", 3)), 1, "r2 was pending");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
assert_eq!(build_state(&guard, "r1"), hive_jobq::State::Running);
|
|
assert_eq!(build_state(&guard, "r2"), hive_jobq::State::Cancelled);
|
|
assert_eq!(build_state(&guard, "r3"), hive_jobq::State::Pending);
|
|
assert_eq!(build_state(&guard, "o1"), hive_jobq::State::Pending);
|
|
}
|
|
|
|
/// Builds share one builder slot: a second build is not claimable while
|
|
/// the first runs, and is once it has finished.
|
|
#[tokio::test]
|
|
async fn one_build_runs_at_a_time() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
for c in [changed("atlas", "a1", 1), changed("argus", "b1", 2)] {
|
|
super::queue_build(&mut sched.lock().unwrap(), c).expect("queued");
|
|
}
|
|
let run = |sched| {
|
|
hive_jobq::scheduler::Scheduler::claim_next(sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
};
|
|
let first = run(&sched).expect("one build is runnable");
|
|
assert!(
|
|
run(&sched).is_none(),
|
|
"a second build started beside the first"
|
|
);
|
|
first.await.1.expect("no growth");
|
|
run(&sched)
|
|
.expect("the second build runs once the first is done")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
}
|
|
|
|
/// Every node whose first request goes to the forge, run against a forge
|
|
/// that accepts the connection and never answers: each ends `Failed`
|
|
/// with an error naming the node instead of holding its executor slot.
|
|
/// `MintAgentForgeToken` is not in the list: it logs in to the secret
|
|
/// store before its first forge request, and no test here has a store.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_forge_node_fails_when_the_forge_never_answers() {
|
|
// Bound but never accepted: the kernel completes the handshake and
|
|
// buffers the request, and no response ever comes.
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap();
|
|
let forge = super::Late::<std::sync::Arc<crate::forge::Client>>::default();
|
|
assert!(
|
|
forge
|
|
.set(std::sync::Arc::new(crate::forge::Client::for_test_forge(
|
|
url
|
|
)))
|
|
.is_ok()
|
|
);
|
|
let agent = || "atlas".to_owned();
|
|
let nodes = [
|
|
("create_repo", SwarmNodeKind::CreateRepo { agent: agent() }),
|
|
(
|
|
"create_forge_user",
|
|
SwarmNodeKind::CreateForgeUser { agent: agent() },
|
|
),
|
|
(
|
|
"add_repo_member",
|
|
SwarmNodeKind::AddRepoMember { agent: agent() },
|
|
),
|
|
(
|
|
"grant_docs_read",
|
|
SwarmNodeKind::GrantDocsRead { agent: agent() },
|
|
),
|
|
(
|
|
"init_agent_config_repo",
|
|
SwarmNodeKind::InitAgentConfigRepo { agent: agent() },
|
|
),
|
|
(
|
|
"mint_hive_forge_token",
|
|
SwarmNodeKind::MintHiveForgeToken {
|
|
hive: "pr1ma".to_owned(),
|
|
},
|
|
),
|
|
(
|
|
"mint_classifier_forge_token",
|
|
SwarmNodeKind::MintClassifierForgeToken,
|
|
),
|
|
(
|
|
"mint_builder_forge_token",
|
|
SwarmNodeKind::MintBuilderForgeToken,
|
|
),
|
|
(
|
|
"mint_ci_metrics_forge_token",
|
|
SwarmNodeKind::MintCiMetricsForgeToken,
|
|
),
|
|
];
|
|
for (label, kind) in nodes {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched.append(kind, Vec::new(), None).expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: forge.clone(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
tokio::time::timeout(std::time::Duration::from_mins(10), runner)
|
|
.await
|
|
.unwrap_or_else(|_| panic!("{label}: node still running after 10 minutes"))
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed, "{label}");
|
|
let error = node.error.as_deref().unwrap_or_default();
|
|
assert!(
|
|
error.contains(label) && error.contains("no answer from the forge"),
|
|
"{label}: expected a deadline error naming the node, got {error:?}"
|
|
);
|
|
}
|
|
drop(listener);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() {
|
|
unsafe {
|
|
std::env::remove_var("SWARM_CONTROLLER_FORGE_URL");
|
|
}
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::CreateRepo {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("forge"),
|
|
"expected a forge-not-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// Sibling of the `CreateRepo` test above, same shape: drives
|
|
/// `SwarmNodeKind::CreateForgeUser` through the real scheduler with no
|
|
/// forge configured, asserting the graceful-absence-is-failure branch —
|
|
/// this node's happy path needs a live forge, same caveat as
|
|
/// `CreateRepo`'s.
|
|
#[tokio::test]
|
|
async fn create_forge_user_node_runs_end_to_end_and_fails_without_forge_configured() {
|
|
unsafe {
|
|
std::env::remove_var("SWARM_CONTROLLER_FORGE_URL");
|
|
}
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::CreateForgeUser {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("forge"),
|
|
"expected a forge-not-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// Third sibling of the two above, and the same deliberate caveat: with
|
|
/// no agent PKI named this reaches only the graceful-absence-is-failure
|
|
/// branch, before any network call. The happy path is a live store and a
|
|
/// real login, which is exactly why it is `mint_and_verify`'s own job to
|
|
/// prove it at agent-creation time rather than a unit test's.
|
|
///
|
|
/// What this does pin is the degrade: a host whose environment does not
|
|
/// name the agent PKI fails this one node with a reason that names the
|
|
/// variable, and creates the agent anyway.
|
|
#[tokio::test]
|
|
async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_the_agent_pki_named() {
|
|
let pki = [
|
|
crate::agent_identity::ENV_AGENT_PKI_MOUNT,
|
|
crate::agent_identity::ENV_AGENT_PKI_ROLE,
|
|
];
|
|
assert!(
|
|
pki.iter().all(|var| std::env::var_os(var).is_none()),
|
|
"{} and {} must be unset: with them set this test would reach a live store",
|
|
pki[0],
|
|
pki[1]
|
|
);
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::MintAgentIdentity {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
let error = node.error.as_deref().unwrap_or_default();
|
|
assert!(
|
|
error.contains(crate::agent_identity::ENV_AGENT_PKI_MOUNT),
|
|
"the reason must name the variable an operator has to set, got {error:?}"
|
|
);
|
|
}
|
|
|
|
/// Fourth sibling, same shape and the same caveat once more: with no
|
|
/// queue configured this reaches only the graceful-absence-is-failure
|
|
/// branch. Writing the declaration for real needs a `JetStream` server,
|
|
/// which no test in this crate has — `wanted::apply`'s own unit tests
|
|
/// cover what gets written, and [`a_new_agent_is_declared_paused`] below
|
|
/// pins the state this node asks for.
|
|
#[tokio::test]
|
|
async fn set_agent_wanted_node_runs_end_to_end_and_fails_without_a_queue() {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::SetAgentWanted {
|
|
hive: "pr1ma".to_owned(),
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("queue"),
|
|
"expected a no-queue-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// The whole point of the node: a freshly created agent is declared
|
|
/// `Paused`, so it does not start driving turns the moment its hive
|
|
/// brings it up. Any other state here is the bug this fixes.
|
|
#[test]
|
|
fn a_new_agent_is_declared_paused() {
|
|
assert_eq!(
|
|
super::NEW_AGENT_WANTED_STATE,
|
|
swarm_wire::wanted::AgentState::Paused
|
|
);
|
|
}
|
|
|
|
/// Same reasoning as `a_mint_node_renders_both_the_agent_and_the_hive`:
|
|
/// this variant carries a hive too, and has to have joined `data`'s
|
|
/// two-field or-pattern rather than the agent-only one.
|
|
#[test]
|
|
fn a_set_wanted_node_renders_both_the_agent_and_the_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::SetAgentWanted {
|
|
hive: "pr1ma".to_owned(),
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "set_agent_wanted");
|
|
let data = kind.data(1);
|
|
assert_eq!(data["agent"], "atlas");
|
|
assert_eq!(data["hive"], "pr1ma");
|
|
}
|
|
|
|
/// The ordering the fix depends on: the pause has to be in the
|
|
/// wanted-state store *before* the hive is told to deploy, or a freshly
|
|
/// created agent runs undeclared for the window between the two.
|
|
///
|
|
/// Asserted on the graph `create_agent` builds, same as the mint edge's
|
|
/// test above and for the same reason — the edge is one line in a
|
|
/// builder closure whose absence changes nothing observable until a real
|
|
/// agent boots and starts taking turns nobody asked for.
|
|
#[tokio::test]
|
|
async fn the_deploy_waits_for_the_pause_to_be_declared() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let set_wanted = id_of("set_agent_wanted");
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
|
|
let when = deploy
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == set_wanted => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the declaration");
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a declaration that never landed must not be deployed past; this \
|
|
edge has to be `after_ok`, not `after_any`"
|
|
);
|
|
}
|
|
|
|
/// Whether the edge from the `from` node to the `on` node accepts a
|
|
/// failed `on`, in a graph built by `declare`.
|
|
fn edge_accepts_failure(
|
|
declare: impl FnOnce(&hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>),
|
|
from: &str,
|
|
on: &str,
|
|
) -> bool {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
let graph = sched.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let (from, on) = (id_of(from), id_of(on));
|
|
graph
|
|
.node(from)
|
|
.expect("just found")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == on => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the edge exists")
|
|
.accepts(hive_jobq::TerminalState::Failed)
|
|
}
|
|
|
|
type TestBuilder = hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>;
|
|
|
|
fn create_dag(b: &TestBuilder) {
|
|
let _ = super::declare_agent_job(b, "atlas", "pr1ma", false);
|
|
}
|
|
|
|
fn merged_dag(b: &TestBuilder) {
|
|
let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc", 7);
|
|
}
|
|
|
|
/// A deploy needs the meta update to succeed: its edge refuses a failed
|
|
/// update, in both DAGs that queue one.
|
|
#[test]
|
|
fn a_deploy_needs_the_meta_update_to_succeed() {
|
|
assert!(!edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"update_meta"
|
|
));
|
|
assert!(!edge_accepts_failure(
|
|
merged_dag,
|
|
"trigger_deploy",
|
|
"update_meta"
|
|
));
|
|
// Controls: the same lookup finds the edges that do accept a failure.
|
|
assert!(edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"mint_agent_identity"
|
|
));
|
|
assert!(!edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"set_agent_wanted"
|
|
));
|
|
}
|
|
|
|
/// A deploy needs the swarm's build of its agent to succeed: its edge
|
|
/// refuses a failed build, in both DAGs that declare an update. The mint
|
|
/// edge is the control that the lookup finds an edge that accepts one.
|
|
#[test]
|
|
fn a_deploy_needs_the_meta_build_to_succeed() {
|
|
for declare in [create_dag as fn(&TestBuilder), merged_dag] {
|
|
assert!(!edge_accepts_failure(
|
|
declare,
|
|
"trigger_deploy",
|
|
"build_meta"
|
|
));
|
|
assert!(!edge_accepts_failure(declare, "build_meta", "update_meta"));
|
|
}
|
|
assert!(edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"mint_agent_identity"
|
|
));
|
|
}
|
|
|
|
/// Every `TriggerDeploy` in `graph` waits, strictly, on a `BuildMeta` of the
|
|
/// same agent that needs the builder. Panics, naming `path`, when one does
|
|
/// not; returns how many deploys it checked.
|
|
fn assert_every_deploy_is_gated(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
path: &str,
|
|
) -> usize {
|
|
let mut checked = 0;
|
|
for deploy in graph.nodes() {
|
|
let SwarmNodeKind::TriggerDeploy { agent, .. } = &deploy.payload else {
|
|
continue;
|
|
};
|
|
let gate = deploy.deps.iter().find_map(|d| {
|
|
match d {
|
|
hive_jobq::Dep::Node { id, when } => graph
|
|
.node(*id)
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent)
|
|
})
|
|
.map(|n| (n, *when)),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
}
|
|
});
|
|
let (build, when) =
|
|
gate.unwrap_or_else(|| panic!("{path}: {agent}'s deploy has no build"));
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"{path}: {agent}'s deploy runs after a failed build"
|
|
);
|
|
assert!(
|
|
build.deps.iter().any(|d| matches!(
|
|
d,
|
|
hive_jobq::Dep::Resource {
|
|
name: SwarmResourceKind::Builder,
|
|
..
|
|
}
|
|
)),
|
|
"{path}: {agent}'s build does not need the builder"
|
|
);
|
|
checked += 1;
|
|
}
|
|
checked
|
|
}
|
|
|
|
/// The four paths that send a deploy all declare the same build and deploy
|
|
/// pair, so a path that forgets the gate fails here.
|
|
#[tokio::test]
|
|
async fn all_four_deploy_paths_declare_the_gated_pair() {
|
|
let graph_of = |declare: fn(&TestBuilder)| {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
sched
|
|
};
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(graph_of(merged_dag).graph(), "merge"),
|
|
1
|
|
);
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(graph_of(create_dag).graph(), "creation"),
|
|
1
|
|
);
|
|
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "fan-out"),
|
|
3
|
|
);
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
// bob's and atlas's first deploys and atlas's retry: all gated.
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "timer retry"),
|
|
3
|
|
);
|
|
}
|
|
|
|
/// With the swarm's build red, the deploy never runs, in both DAGs that
|
|
/// declare an update. The first run in each pair is the control, where the
|
|
/// build is green and the deploy does run.
|
|
#[tokio::test]
|
|
async fn a_red_build_means_no_deploy_runs() {
|
|
use hive_jobq::State;
|
|
|
|
for (name, declare) in [
|
|
("merged", merged_dag as fn(&TestBuilder)),
|
|
("create", create_dag as fn(&TestBuilder)),
|
|
] {
|
|
let green = run_dag_failing(declare, None).await;
|
|
assert_eq!(state_of(&green, "build_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&green, "trigger_deploy"), State::Done, "{name}");
|
|
|
|
let red = run_dag_failing(declare, Some("build_meta")).await;
|
|
assert_eq!(state_of(&red, "update_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&red, "build_meta"), State::Failed, "{name}");
|
|
assert_eq!(
|
|
state_of(&red, "trigger_deploy"),
|
|
State::Skipped,
|
|
"{name}: the deploy must not run after a red build"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Every node of the DAG `declare` builds, run to the end with the meta
|
|
/// update failing or succeeding and every other node succeeding: the
|
|
/// final state of each, by label.
|
|
async fn run_dag(
|
|
declare: impl FnOnce(&TestBuilder),
|
|
update_meta_fails: bool,
|
|
) -> Vec<(String, hive_jobq::State)> {
|
|
run_dag_failing(declare, update_meta_fails.then_some("update_meta")).await
|
|
}
|
|
|
|
/// [`run_dag`], with the nodes labelled `failing` failing.
|
|
async fn run_dag_failing(
|
|
declare: impl FnOnce(&TestBuilder),
|
|
failing: Option<&'static str>,
|
|
) -> Vec<(String, hive_jobq::State)> {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
while let Some(runner) = Scheduler::claim_next(&sched, |_id, kind, builder| async move {
|
|
let outcome = if failing == Some(kind.label().as_str()) {
|
|
Outcome::Failed("no builder".to_owned())
|
|
} else {
|
|
Outcome::Done
|
|
};
|
|
(builder, outcome)
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
let guard = sched.lock().unwrap();
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| (n.payload.label(), n.state))
|
|
.collect()
|
|
}
|
|
|
|
fn state_of(states: &[(String, hive_jobq::State)], label: &str) -> hive_jobq::State {
|
|
states
|
|
.iter()
|
|
.find(|(l, _)| l == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.1
|
|
}
|
|
|
|
/// The scheduler's side of the same edge, for both DAGs that queue an
|
|
/// update: with it failed, the deploy never runs. The graph shows the
|
|
/// update `Failed` and the deploy `Skipped`; the first run in each pair is
|
|
/// the control, where the deploy does run.
|
|
#[tokio::test]
|
|
async fn a_failed_meta_update_means_no_deploy_runs() {
|
|
use hive_jobq::State;
|
|
|
|
for (name, declare) in [
|
|
("merged", merged_dag as fn(&TestBuilder)),
|
|
("create", create_dag as fn(&TestBuilder)),
|
|
] {
|
|
let ok = run_dag(declare, false).await;
|
|
assert_eq!(state_of(&ok, "update_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&ok, "trigger_deploy"), State::Done, "{name}");
|
|
|
|
let failed = run_dag(declare, true).await;
|
|
assert_eq!(state_of(&failed, "update_meta"), State::Failed, "{name}");
|
|
assert_eq!(
|
|
state_of(&failed, "trigger_deploy"),
|
|
State::Skipped,
|
|
"{name}: the deploy must not run after a failed update"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A fan-out record held in memory; `unreadable` stands for a bucket that
|
|
/// cannot be opened.
|
|
#[derive(Default)]
|
|
struct FakePins {
|
|
pin: Mutex<Option<String>>,
|
|
unreadable: bool,
|
|
}
|
|
|
|
impl FakePins {
|
|
fn holding(pin: &str) -> Self {
|
|
Self {
|
|
pin: Mutex::new(Some(pin.to_owned())),
|
|
unreadable: false,
|
|
}
|
|
}
|
|
|
|
fn held(&self) -> Option<String> {
|
|
self.pin.lock().unwrap().clone()
|
|
}
|
|
}
|
|
|
|
impl meta_fanout::PinStore for FakePins {
|
|
async fn recorded(&self) -> Result<Option<String>> {
|
|
anyhow::ensure!(!self.unreadable, "the bucket did not open");
|
|
Ok(self.held())
|
|
}
|
|
|
|
async fn record(&self, pin: &str) -> Result<()> {
|
|
*self.pin.lock().unwrap() = Some(pin.to_owned());
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn report(pin: &str, moved: bool) -> meta_flake::Report {
|
|
meta_flake::Report {
|
|
updated: meta_flake::Updated::Committed,
|
|
pin: pin.to_owned(),
|
|
hyperhive_moved: moved,
|
|
}
|
|
}
|
|
|
|
type TestScheduler = hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>;
|
|
|
|
/// One `UpdateMeta` pass against `pins`, with every node it grows run: the
|
|
/// deploy to `failing_agent` fails, and `RecordFanout` dies before it
|
|
/// writes when `crash_before_record`. Returns what the pass decided and the
|
|
/// scheduler it ran in.
|
|
async fn pass(
|
|
pins: &Arc<FakePins>,
|
|
report: &meta_flake::Report,
|
|
placed: &[(&str, &str)],
|
|
failing_agent: Option<&'static str>,
|
|
crash_before_record: bool,
|
|
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
|
|
pass_with(
|
|
pins,
|
|
report,
|
|
placed,
|
|
failing_agent,
|
|
crash_before_record,
|
|
None,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// [`pass`], with the build of one agent failing with the given error when
|
|
/// `failing_build` names it.
|
|
async fn pass_with(
|
|
pins: &Arc<FakePins>,
|
|
report: &meta_flake::Report,
|
|
placed: &[(&str, &str)],
|
|
failing_agent: Option<&'static str>,
|
|
crash_before_record: bool,
|
|
failing_build: Option<(&'static str, String)>,
|
|
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
use meta_fanout::PinStore as _;
|
|
|
|
let mut sched = Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.append(SwarmNodeKind::UpdateMeta, Vec::new(), None)
|
|
.expect("insert");
|
|
let sched = Arc::new(Mutex::new(sched));
|
|
let placements: std::collections::BTreeSet<(String, String)> = placed
|
|
.iter()
|
|
.map(|(h, a)| ((*h).to_owned(), (*a).to_owned()))
|
|
.collect();
|
|
let fan_out = decide_fan_out(report, pins.recorded().await);
|
|
let decided = fan_out.clone();
|
|
let update = Scheduler::claim_next(&sched, move |_id, _kind, builder| async move {
|
|
declare_fan_out(&builder, &fan_out, &placements);
|
|
(builder, Outcome::Done)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
update.await.1.expect("the grown nodes insert");
|
|
while let Some(runner) = Scheduler::claim_next(&sched, {
|
|
let (pins, sched, failing_build) =
|
|
(Arc::clone(pins), Arc::clone(&sched), failing_build.clone());
|
|
move |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::TriggerDeploy { agent, .. }
|
|
if failing_agent == Some(agent.as_str()) =>
|
|
{
|
|
Outcome::Failed("the hive did not take it".to_owned())
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, .. }
|
|
if failing_build.as_ref().is_some_and(|(a, _)| *a == agent) =>
|
|
{
|
|
Outcome::Failed(failing_build.map(|(_, error)| error).expect("matched"))
|
|
}
|
|
SwarmNodeKind::RecordFanout { .. } if crash_before_record => {
|
|
Outcome::Failed("the controller died".to_owned())
|
|
}
|
|
SwarmNodeKind::RecordFanout { pin } => {
|
|
let deploys_done = sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
|
|
.all(|n| {
|
|
matches!(
|
|
n.state,
|
|
hive_jobq::State::Done | hive_jobq::State::Skipped
|
|
)
|
|
});
|
|
if deploys_done {
|
|
pins.record(&pin).await.expect("recorded");
|
|
Outcome::Done
|
|
} else {
|
|
Outcome::Failed("recorded before the deploys were settled".to_owned())
|
|
}
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
(decided, sched)
|
|
}
|
|
|
|
/// The states of the nodes labelled `label`.
|
|
fn states_of(sched: &Mutex<TestScheduler>, label: &str) -> Vec<hive_jobq::State> {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == label)
|
|
.map(|n| n.state)
|
|
.collect()
|
|
}
|
|
|
|
const PLACED: [(&str, &str); 3] = [("pr1ma", "atlas"), ("pr1ma", "argus"), ("sec0nd", "iris")];
|
|
|
|
/// A controller with no record sends every placed agent a deploy, whether
|
|
/// or not the pin moved, and records the pin once they are done.
|
|
#[tokio::test]
|
|
async fn a_pin_with_no_record_deploys_every_placed_agent_and_is_recorded() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (fan_out, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
assert_eq!(
|
|
states_of(&sched, "trigger_deploy"),
|
|
[hive_jobq::State::Done; 3]
|
|
);
|
|
assert_eq!(states_of(&sched, "record_fanout"), [hive_jobq::State::Done]);
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
}
|
|
|
|
/// The deploys declared are one per placement, to the hive that places it.
|
|
#[tokio::test]
|
|
async fn each_deploy_goes_to_the_hive_that_places_the_agent() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
let mut deploys: Vec<_> = sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
|
|
Some((hive.clone(), agent.clone(), rev.clone()))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
deploys.sort();
|
|
assert_eq!(
|
|
deploys,
|
|
vec![
|
|
("pr1ma".to_owned(), "argus".to_owned(), None),
|
|
("pr1ma".to_owned(), "atlas".to_owned(), None),
|
|
("sec0nd".to_owned(), "iris".to_owned(), None),
|
|
]
|
|
);
|
|
}
|
|
|
|
/// The record, not whether the pin moved, decides: a pin already sent
|
|
/// sends nothing even on a pass that moved it, and a different one does.
|
|
#[tokio::test]
|
|
async fn a_pin_that_is_recorded_deploys_nothing() {
|
|
let pins = Arc::new(FakePins::holding("p1"));
|
|
let (fan_out, sched) = pass(&pins, &report("p1", true), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
assert!(states_of(&sched, "record_fanout").is_empty());
|
|
|
|
let (fan_out, sched) = pass(&pins, &report("p2", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p2".into() });
|
|
assert_eq!(states_of(&sched, "trigger_deploy").len(), 3);
|
|
assert_eq!(pins.held().as_deref(), Some("p2"));
|
|
}
|
|
|
|
/// A controller that dies after the commit and before the record is
|
|
/// written finds the commit unchanged on its next pass, and still sends.
|
|
#[tokio::test]
|
|
async fn a_pass_that_dies_before_the_record_is_written_sends_again() {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass(&pins, &report("p1", true), &PLACED, None, true).await;
|
|
assert_eq!(states_of(&sched, "trigger_deploy").len(), 3);
|
|
assert_eq!(pins.held().as_deref(), Some("p0"), "nothing was recorded");
|
|
|
|
// The restart: the commit is the render now, so nothing moved.
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
}
|
|
|
|
/// The record waits on every deploy of its pass to be settled: done, or
|
|
/// skipped by its build gate. A failed deploy still rules it out.
|
|
#[tokio::test]
|
|
async fn the_record_waits_on_every_deploy_being_done_or_skipped() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
let guard = sched.lock().unwrap();
|
|
let graph = guard.graph();
|
|
let record = graph
|
|
.nodes()
|
|
.find(|n| matches!(n.payload, SwarmNodeKind::RecordFanout { .. }))
|
|
.expect("a record node");
|
|
let mut waits: Vec<_> = record
|
|
.deps
|
|
.iter()
|
|
.map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } => (*id, *when),
|
|
hive_jobq::Dep::Resource { .. } => panic!("the record needs no resource"),
|
|
})
|
|
.collect();
|
|
waits.sort_by_key(|(id, _)| id.get());
|
|
let mut deploys: Vec<_> = graph
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
|
|
.map(|n| {
|
|
(
|
|
n.id,
|
|
hive_jobq::DepWhen::of(&[
|
|
hive_jobq::TerminalState::Done,
|
|
hive_jobq::TerminalState::Skipped,
|
|
]),
|
|
)
|
|
})
|
|
.collect();
|
|
deploys.sort_by_key(|(id, _)| id.get());
|
|
assert_eq!(deploys.len(), 3);
|
|
assert_eq!(waits, deploys);
|
|
}
|
|
|
|
/// One deploy failing keeps the pin unrecorded: the record is ruled out,
|
|
/// the others still run, and the next pass sends again.
|
|
#[tokio::test]
|
|
async fn a_failed_deploy_leaves_the_pin_unrecorded() {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass(&pins, &report("p1", true), &PLACED, Some("argus"), false).await;
|
|
let mut deploys = states_of(&sched, "trigger_deploy");
|
|
deploys.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(
|
|
deploys,
|
|
[
|
|
hive_jobq::State::Done,
|
|
hive_jobq::State::Done,
|
|
hive_jobq::State::Failed
|
|
]
|
|
);
|
|
assert_eq!(
|
|
states_of(&sched, "record_fanout"),
|
|
[hive_jobq::State::Skipped]
|
|
);
|
|
assert_eq!(pins.held().as_deref(), Some("p0"));
|
|
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
}
|
|
|
|
/// A failed build for one agent skips that agent's deploy only, and the pin
|
|
/// is still recorded, so the healthy agents are not sent again. The timer
|
|
/// declares a build and deploy for the failed agent alone, whatever the
|
|
/// build failed with: `nix`'s error, a timeout, or no verdict.
|
|
#[tokio::test]
|
|
async fn a_failed_build_skips_its_deploy_and_the_pin_is_still_recorded() {
|
|
use hive_jobq::State;
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
for error in [
|
|
"error: builder for argus failed".to_owned(),
|
|
"timed out: nix did not finish".to_owned(),
|
|
format!(
|
|
"{}no swarm builder is listening",
|
|
meta_build::NO_VERDICT_PREFIX
|
|
),
|
|
] {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass_with(
|
|
&pins,
|
|
&report("p1", true),
|
|
&PLACED,
|
|
None,
|
|
false,
|
|
Some(("argus", error.clone())),
|
|
)
|
|
.await;
|
|
let mut builds = states_of(&sched, "build_meta");
|
|
builds.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(builds, [State::Done, State::Done, State::Failed], "{error}");
|
|
let mut deploys = states_of(&sched, "trigger_deploy");
|
|
deploys.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(
|
|
deploys,
|
|
[State::Done, State::Done, State::Skipped],
|
|
"{error}"
|
|
);
|
|
assert_eq!(states_of(&sched, "record_fanout"), [State::Done], "{error}");
|
|
assert_eq!(pins.held().as_deref(), Some("p1"), "{error}");
|
|
|
|
assert!(queue_meta_update(&sched).expect("tick"), "{error}");
|
|
let guard = sched.lock().unwrap();
|
|
let again = |label: &str| {
|
|
let mut found: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == label)
|
|
.filter(|n| n.state == State::Pending)
|
|
.map(|n| match &n.payload {
|
|
SwarmNodeKind::BuildMeta { agent, .. }
|
|
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
|
|
other => panic!("not a build or a deploy: {other:?}"),
|
|
})
|
|
.collect();
|
|
found.sort();
|
|
found
|
|
};
|
|
assert_eq!(again("build_meta"), ["argus"], "{error}");
|
|
assert_eq!(again("trigger_deploy"), ["argus"], "{error}");
|
|
}
|
|
}
|
|
|
|
/// With nothing placed there is nothing to wait on, and the pin is
|
|
/// recorded all the same.
|
|
#[tokio::test]
|
|
async fn a_pin_with_no_placed_agent_is_recorded() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &[], None, false).await;
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
assert_eq!(states_of(&sched, "record_fanout"), [hive_jobq::State::Done]);
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
}
|
|
|
|
/// While the record cannot be read, a moved pin deploys as it did before
|
|
/// the record existed, and there is nothing to record into.
|
|
#[tokio::test]
|
|
async fn an_unreadable_record_falls_back_to_the_moved_pin() {
|
|
let pins = Arc::new(FakePins {
|
|
unreadable: true,
|
|
..FakePins::default()
|
|
});
|
|
let (fan_out, sched) = pass(&pins, &report("p1", true), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Unrecorded);
|
|
assert_eq!(
|
|
states_of(&sched, "trigger_deploy"),
|
|
[hive_jobq::State::Done; 3]
|
|
);
|
|
assert!(states_of(&sched, "record_fanout").is_empty());
|
|
|
|
let (fan_out, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
}
|
|
|
|
/// Node labels of the graph [`super::declare_agent_job`] builds, and the
|
|
/// labels the deploy node waits on.
|
|
fn creation_graph(archetype: bool) -> (Vec<String>, Vec<String>) {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
super::declare_agent_job(b, "atlas", "pr1ma", archetype)
|
|
})
|
|
.expect("the graph inserts");
|
|
let graph = sched.graph();
|
|
let labels = graph.nodes().map(|n| n.payload.label()).collect();
|
|
let deploy = graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == "trigger_deploy")
|
|
.expect("every creation deploys");
|
|
let waits = deploy
|
|
.deps
|
|
.iter()
|
|
.filter_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, .. } => graph.node(*id).map(|n| n.payload.label()),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
})
|
|
.collect();
|
|
(labels, waits)
|
|
}
|
|
|
|
/// An archetype agent's graph has no repo nodes and its deploy waits on
|
|
/// no repo; a custom agent's has all three, and its deploy waits on the
|
|
/// seeded repo.
|
|
#[test]
|
|
fn an_archetype_agent_is_created_without_a_config_repo() {
|
|
const REPO_NODES: [&str; 3] = ["create_repo", "add_repo_member", "init_agent_config_repo"];
|
|
|
|
let (custom, custom_waits) = creation_graph(false);
|
|
for repo in REPO_NODES {
|
|
assert!(custom.iter().any(|l| l == repo), "custom lacks {repo}");
|
|
}
|
|
assert!(custom_waits.iter().any(|l| l == "init_agent_config_repo"));
|
|
|
|
let (archetype, archetype_waits) = creation_graph(true);
|
|
for repo in REPO_NODES {
|
|
assert!(!archetype.iter().any(|l| l == repo), "archetype has {repo}");
|
|
}
|
|
for kept in [
|
|
"create_identity",
|
|
"create_forge_user",
|
|
"grant_docs_read",
|
|
"mint_agent_identity",
|
|
"mint_agent_forge_token",
|
|
"mint_agent_matrix_account",
|
|
"set_agent_wanted",
|
|
] {
|
|
assert!(
|
|
archetype.iter().any(|l| l == kept),
|
|
"archetype lacks {kept}"
|
|
);
|
|
assert!(custom.iter().any(|l| l == kept), "custom lacks {kept}");
|
|
}
|
|
assert!(archetype_waits.iter().any(|l| l == "set_agent_wanted"));
|
|
assert_eq!(archetype.len() + REPO_NODES.len(), custom.len());
|
|
}
|
|
|
|
/// The ordering the operator's ruling requires: a hive cannot pass down
|
|
/// a certificate the swarm has not published, so the deploy message must
|
|
/// not leave before the mint is terminal.
|
|
///
|
|
/// Asserted on the graph `create_agent` builds, because the edge is one
|
|
/// line in a builder closure and its absence changes nothing observable
|
|
/// until a real agent boots without an identity.
|
|
#[tokio::test]
|
|
async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let mint = id_of("mint_agent_identity");
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
|
|
let when = deploy
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the mint");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a failed or unconfigured agent PKI issue must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// The forge-token mint sits between the forge user and the deploy: it
|
|
/// needs the user to exist, and the deploy must not overtake it — but a
|
|
/// host with no forge or no store must still deploy the agent.
|
|
#[tokio::test]
|
|
async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let edge = |from: hive_jobq::NodeId, to: &str| {
|
|
graph
|
|
.node(id_of(to))
|
|
.expect("just found")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == from => Some(*when),
|
|
_ => None,
|
|
})
|
|
.unwrap_or_else(|| panic!("{to} waits for node {}", from.get()))
|
|
};
|
|
|
|
let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token");
|
|
assert!(
|
|
!user_to_mint.accepts(hive_jobq::TerminalState::Failed),
|
|
"a token for a user that was never created cannot be minted; this \
|
|
edge has to be `after_ok`"
|
|
);
|
|
let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy");
|
|
assert!(
|
|
mint_to_deploy.accepts(hive_jobq::TerminalState::Failed),
|
|
"a host with no forge or store must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// Agent creation grants `internal/docs` read once the forge user exists,
|
|
/// and a failed grant does not hold up the deploy (the pass retries it).
|
|
#[tokio::test]
|
|
async fn the_docs_grant_follows_the_user_and_is_not_a_deploy_parent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let grant = graph.node(id_of("grant_docs_read")).expect("just found");
|
|
assert_eq!(grant.payload.data(grant.id.get())["agent"], "atlas");
|
|
let when = grant
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == id_of("create_forge_user") => {
|
|
Some(*when)
|
|
}
|
|
_ => None,
|
|
})
|
|
.expect("the grant waits for the forge user");
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"adding a user that was never created as a collaborator is a forge error; \
|
|
this edge has to be `after_ok`"
|
|
);
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
assert!(
|
|
!deploy.deps.iter().any(|d| matches!(
|
|
d,
|
|
hive_jobq::Dep::Node { id, .. } if *id == grant.id
|
|
)),
|
|
"the deploy must not wait on the docs grant"
|
|
);
|
|
}
|
|
|
|
/// The periodic pass queues the same pair of nodes agent creation does.
|
|
#[test]
|
|
fn a_queued_docs_grant_creates_the_forge_user_first() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_docs_grants(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2, "one returned id per agent: its grant node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let mut labels: Vec<String> = graph.nodes().map(|n| n.payload.label()).collect();
|
|
labels.sort();
|
|
assert_eq!(
|
|
labels,
|
|
[
|
|
"create_forge_user",
|
|
"create_forge_user",
|
|
"grant_docs_read",
|
|
"grant_docs_read"
|
|
]
|
|
);
|
|
for grant in graph
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == "grant_docs_read")
|
|
{
|
|
assert!(ids.contains(&grant.id));
|
|
assert!(
|
|
grant
|
|
.deps
|
|
.iter()
|
|
.any(|d| matches!(d, hive_jobq::Dep::Node { when, .. } if !when.accepts(hive_jobq::TerminalState::Failed))),
|
|
"each grant waits `after_ok` on its user"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_forge_token_node_renders_the_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::MintAgentForgeToken {
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "mint_agent_forge_token");
|
|
assert_eq!(kind.data(1)["agent"], "atlas");
|
|
}
|
|
|
|
#[test]
|
|
fn a_matrix_account_node_renders_the_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::MintAgentMatrixAccount {
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "mint_agent_matrix_account");
|
|
assert_eq!(kind.data(1)["agent"], "atlas");
|
|
}
|
|
|
|
/// The periodic pass's queueing: a certificate re-issue is the node agent
|
|
/// creation mints with, a secret re-mint its own node, and an agent due
|
|
/// both gets the secret `after_any` the certificate.
|
|
#[test]
|
|
fn a_queued_renewal_chains_the_secret_after_the_certificate() {
|
|
use crate::agent_renewal::Renewal;
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let r = |agent: &str, cert, secret| Renewal {
|
|
agent: agent.to_owned(),
|
|
cert,
|
|
secret,
|
|
};
|
|
let ids = super::queue_agent_renewals(
|
|
&sched,
|
|
vec![
|
|
r("both", true, true),
|
|
r("cert", true, false),
|
|
r("secret", false, true),
|
|
],
|
|
)
|
|
.expect("three jobs insert");
|
|
assert_eq!(ids.len(), 4, "one returned id per node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let agent_of = |n: &hive_jobq::Node<SwarmNodeKind, super::SwarmResourceKind>| {
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
};
|
|
let mut nodes: Vec<(String, String)> = graph
|
|
.nodes()
|
|
.map(|n| (agent_of(n), n.payload.label()))
|
|
.collect();
|
|
nodes.sort();
|
|
assert_eq!(
|
|
nodes,
|
|
[
|
|
("both".to_owned(), "mint_agent_identity".to_owned()),
|
|
("both".to_owned(), "renew_agent_queue_credential".to_owned()),
|
|
("cert".to_owned(), "mint_agent_identity".to_owned()),
|
|
(
|
|
"secret".to_owned(),
|
|
"renew_agent_queue_credential".to_owned()
|
|
),
|
|
]
|
|
);
|
|
let find = |agent: &str, label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label && agent_of(n) == agent)
|
|
.unwrap_or_else(|| panic!("{agent} has a {label} node"))
|
|
};
|
|
let cert = find("both", "mint_agent_identity").id;
|
|
let secret = find("both", "renew_agent_queue_credential");
|
|
let when = secret
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == cert => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the secret waits for the certificate");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a certificate that could not be re-issued must not hold up the secret"
|
|
);
|
|
assert!(
|
|
find("secret", "renew_agent_queue_credential")
|
|
.deps
|
|
.is_empty(),
|
|
"a secret on its own waits for nothing"
|
|
);
|
|
}
|
|
|
|
fn mint_nodes_for(
|
|
sched: &std::sync::Mutex<
|
|
hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
|
|
>,
|
|
agent: &str,
|
|
) -> Vec<hive_jobq::NodeId> {
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
|
|
})
|
|
.map(|n| n.id)
|
|
.collect()
|
|
}
|
|
|
|
/// A pass that finds a mint for the agent still queued or running (agent
|
|
/// creation's or its own earlier one) queues none; one that finds only
|
|
/// finished mints queues the next. The other agent is the control: the
|
|
/// check is per agent.
|
|
#[test]
|
|
fn a_mint_in_flight_blocks_another_for_that_agent_only() {
|
|
use crate::agent_renewal::Renewal;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let renewal = |agent: &str| Renewal {
|
|
agent: agent.to_owned(),
|
|
cert: true,
|
|
secret: false,
|
|
};
|
|
|
|
let first = super::queue_agent_renewals(&sched, vec![renewal("sock"), renewal("other")])
|
|
.expect("two jobs insert");
|
|
assert_eq!(first.len(), 2, "no mint in flight: both are queued");
|
|
|
|
let second = super::queue_agent_renewals(&sched, vec![renewal("sock")])
|
|
.expect("a skipped agent is not an error");
|
|
assert!(second.is_empty(), "the first pass's mint is still pending");
|
|
assert_eq!(mint_nodes_for(&sched, "sock").len(), 1);
|
|
|
|
// The next pass after that mint finished.
|
|
let pending = mint_nodes_for(&sched, "sock")[0];
|
|
assert!(
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.cancel_node(pending)
|
|
);
|
|
let third = super::queue_agent_renewals(&sched, vec![renewal("sock")])
|
|
.expect("a finished mint does not block");
|
|
assert_eq!(
|
|
third.len(),
|
|
1,
|
|
"a finished mint is retried by the next pass"
|
|
);
|
|
assert_eq!(mint_nodes_for(&sched, "sock").len(), 2);
|
|
assert_eq!(
|
|
mint_nodes_for(&sched, "other").len(),
|
|
1,
|
|
"the other agent's pending mint was never touched"
|
|
);
|
|
}
|
|
|
|
/// Agent creation mints the matrix account as a root of its own, and the
|
|
/// deploy waits for it without being cancelled by it: a host with no
|
|
/// homeserver configured must still deploy the agent.
|
|
#[tokio::test]
|
|
async fn the_matrix_account_mint_is_a_root_and_does_not_block_the_deploy() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let find = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
};
|
|
let mint = find("mint_agent_matrix_account");
|
|
assert!(mint.deps.is_empty(), "the mint waits for nothing");
|
|
let when = find("trigger_deploy")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == mint.id => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the matrix mint");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a host with no homeserver must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// The backfill's queueing: one mint node per agent, nothing else.
|
|
#[test]
|
|
fn a_queued_matrix_mint_is_one_node_per_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_matrix_account_mints(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut agents: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_agent_matrix_account");
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
})
|
|
.collect();
|
|
agents.sort();
|
|
assert_eq!(agents, ["a", "b"]);
|
|
}
|
|
|
|
/// The hive sender-token pass's queueing: one mint node per hive, and
|
|
/// the node names the hive rather than an agent.
|
|
#[test]
|
|
fn a_queued_hive_sender_mint_is_one_node_per_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids =
|
|
super::queue_hive_sender_mints(&sched, vec!["alpha".to_owned(), "beta".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut hives: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_hive_sender_token");
|
|
let data = n.payload.data(n.id.get());
|
|
assert!(data.get("agent").is_none(), "{data}");
|
|
data["hive"].as_str().expect("hive is a string").to_owned()
|
|
})
|
|
.collect();
|
|
hives.sort();
|
|
assert_eq!(hives, ["alpha", "beta"]);
|
|
}
|
|
|
|
/// The hive forge-token pass's queueing: one node per hive, naming the
|
|
/// hive rather than an agent.
|
|
#[test]
|
|
fn a_queued_hive_forge_token_mint_is_one_node_per_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_hive_forge_token_mints(
|
|
&sched,
|
|
vec!["alpha".to_owned(), "beta".to_owned()],
|
|
)
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut hives: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_hive_forge_token");
|
|
let data = n.payload.data(n.id.get());
|
|
assert!(data.get("agent").is_none(), "{data}");
|
|
data["hive"].as_str().expect("hive is a string").to_owned()
|
|
})
|
|
.collect();
|
|
hives.sort();
|
|
assert_eq!(hives, ["alpha", "beta"]);
|
|
}
|
|
|
|
/// The manual route and the periodic pass both go through
|
|
/// `queue_forge_token_mints`, so this is the assertion that a backfilled
|
|
/// mint is the same pair of nodes agent creation inserts: the forge user,
|
|
/// then the mint `after_ok` it, per agent.
|
|
#[test]
|
|
fn a_queued_mint_creates_the_forge_user_first() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2, "one returned id per agent: its mint node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let agent_of = |n: &hive_jobq::Node<SwarmNodeKind, super::SwarmResourceKind>| {
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
};
|
|
for agent in ["a", "b"] {
|
|
let find = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label && agent_of(n) == agent)
|
|
.unwrap_or_else(|| panic!("{agent} has a {label} node"))
|
|
};
|
|
let user = find("create_forge_user").id;
|
|
let mint = find("mint_agent_forge_token");
|
|
assert!(ids.contains(&mint.id), "the returned id is {agent}'s mint");
|
|
let when = mint
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == user => Some(*when),
|
|
_ => None,
|
|
})
|
|
.unwrap_or_else(|| panic!("{agent}'s mint waits for its own forge user"));
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a token cannot be minted for a user that was never created; this \
|
|
edge has to be `after_ok`"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
graph.nodes().count(),
|
|
4,
|
|
"two nodes per agent, nothing else"
|
|
);
|
|
}
|
|
|
|
/// The backfill end to end, minus the IO: an agent the pass observed with
|
|
/// no forge user is planned, and the job it gets creates that user before
|
|
/// it mints.
|
|
#[test]
|
|
fn an_agent_with_no_forge_user_gets_a_user_and_a_mint() {
|
|
use crate::forge::agent_token::{Observed, plan};
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let agents = plan(&[("ruth".to_owned(), Observed::NoForgeUser)]);
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
super::queue_forge_token_mints(&sched, agents).expect("one job inserts");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut labels: Vec<String> = guard.graph().nodes().map(|n| n.payload.label()).collect();
|
|
labels.sort();
|
|
assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]);
|
|
}
|
|
|
|
/// A scheduler holding one `UpdateMeta` that ran and ended `outcome`.
|
|
async fn after_one_update_meta(
|
|
outcome: hive_jobq::scheduler::Outcome,
|
|
) -> std::sync::Arc<
|
|
std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
> {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { (b, outcome) })
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
sched
|
|
}
|
|
|
|
fn update_metas(
|
|
sched: &std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> usize {
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
.count()
|
|
}
|
|
|
|
/// With none run yet the tick queues one, and queues no second while that
|
|
/// one is pending or running.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_queues_first_and_skips_while_one_is_in_flight() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "a pending one blocks the tick");
|
|
|
|
let running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Failed("late".to_owned()))
|
|
})
|
|
.expect("runnable");
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "a running one blocks the tick");
|
|
running.await.1.expect("no growth");
|
|
}
|
|
|
|
/// Once the latest run succeeded, ticks add no node.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_adds_nothing_after_a_success() {
|
|
let sched = after_one_update_meta(hive_jobq::scheduler::Outcome::Done).await;
|
|
for _ in 0..3 {
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
}
|
|
assert_eq!(update_metas(&sched), 1);
|
|
}
|
|
|
|
/// A failed latest run is retried; a success after it ends the retries.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_retries_after_a_failure_until_a_success() {
|
|
let sched =
|
|
after_one_update_meta(hive_jobq::scheduler::Outcome::Failed("no".to_owned())).await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 2);
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 2);
|
|
}
|
|
|
|
type SharedSched = std::sync::Arc<
|
|
std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
>;
|
|
|
|
/// Declare `agent`'s creation, as the endpoint does, and run every node to
|
|
/// the end: the meta update fails when `update_meta_fails`, the pause
|
|
/// declaration when `set_wanted_fails`, everything else succeeds.
|
|
async fn create_and_run(
|
|
sched: &SharedSched,
|
|
agent: &str,
|
|
update_meta_fails: bool,
|
|
set_wanted_fails: bool,
|
|
) {
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| super::declare_agent_job(b, agent, "pr1ma", false))
|
|
.expect("the graph inserts");
|
|
drain(sched, update_meta_fails, set_wanted_fails).await;
|
|
}
|
|
|
|
/// Run every runnable node; outcomes as for [`create_and_run`].
|
|
async fn drain(sched: &SharedSched, update_meta_fails: bool, set_wanted_fails: bool) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
|
|
while let Some(runner) = Scheduler::claim_next(sched, |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::UpdateMeta if update_meta_fails => {
|
|
Outcome::Failed("no builder".to_owned())
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { .. } if set_wanted_fails => {
|
|
Outcome::Failed("no queue".to_owned())
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
}
|
|
|
|
/// The states of `agent`'s `TriggerDeploy` nodes, oldest first.
|
|
fn deploys_of(sched: &SharedSched, agent: &str) -> Vec<hive_jobq::State> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(&n.payload, SwarmNodeKind::TriggerDeploy { agent: a, .. } if a == agent))
|
|
.collect();
|
|
nodes.sort_by_key(|n| n.id);
|
|
nodes.into_iter().map(|n| n.state).collect()
|
|
}
|
|
|
|
/// `bob` deployed under a working update; then `atlas` was created while
|
|
/// the update failed, so its deploy is `Skipped`.
|
|
async fn bob_deployed_and_atlas_skipped() -> SharedSched {
|
|
use hive_jobq::State;
|
|
|
|
let sched: SharedSched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
create_and_run(&sched, "bob", false, false).await;
|
|
create_and_run(&sched, "atlas", true, false).await;
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
sched
|
|
}
|
|
|
|
/// An agent created while the update was failing is deployed by the run
|
|
/// that succeeds, exactly once.
|
|
#[tokio::test]
|
|
async fn the_next_successful_update_deploys_the_agent_a_failed_one_skipped() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
// A retry whose update fails too skips the new deploy as well.
|
|
drain(&sched, true, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped]
|
|
);
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
// The success ends the retries: no tick adds a node, so no second deploy.
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
}
|
|
|
|
/// An agent whose latest deploy ran gets no extra one from the retry, even
|
|
/// when an older deploy of its was skipped.
|
|
#[tokio::test]
|
|
async fn an_agent_that_was_deployed_gets_no_extra_deploy_from_the_retry() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
|
|
// atlas now holds an old Skipped deploy and a Done one. Another
|
|
// creation fails its update; the retry deploys carol and only carol.
|
|
create_and_run(&sched, "carol", true, false).await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "carol"), [State::Skipped, State::Done]);
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped, State::Done]);
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
}
|
|
|
|
/// A deploy skipped because the pause declaration failed stays skipped: the
|
|
/// retry must not deploy past the declaration it waits for.
|
|
#[tokio::test]
|
|
async fn a_deploy_skipped_for_another_reason_is_not_redeclared() {
|
|
use hive_jobq::State;
|
|
|
|
let sched: SharedSched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
create_and_run(&sched, "atlas", false, true).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
}
|
|
|
|
/// The mint waits for the declaration its node re-reads: with the
|
|
/// declaration failed it is skipped, with it written it runs.
|
|
#[tokio::test]
|
|
async fn creation_mints_the_identity_only_after_the_declaration() {
|
|
use hive_jobq::State;
|
|
|
|
let mint_states = |sched: &SharedSched| -> Vec<State> {
|
|
let guard = sched.lock().unwrap();
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { .. }))
|
|
.map(|n| n.state)
|
|
.collect()
|
|
};
|
|
let sched = fresh_sched();
|
|
create_and_run(&sched, "atlas", false, true).await;
|
|
assert_eq!(mint_states(&sched), [State::Skipped]);
|
|
let sched = fresh_sched();
|
|
create_and_run(&sched, "atlas", false, false).await;
|
|
assert_eq!(mint_states(&sched), [State::Done]);
|
|
}
|
|
|
|
/// Run every runnable node; the build of `agent` fails with `error` and
|
|
/// everything else succeeds.
|
|
async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
|
|
while let Some(runner) = Scheduler::claim_next(sched, {
|
|
let error = error.clone();
|
|
move |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent => {
|
|
Outcome::Failed(error)
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
}
|
|
|
|
fn fresh_sched() -> SharedSched {
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)))
|
|
}
|
|
|
|
/// The `rev` of `agent`'s deploys, oldest first.
|
|
fn deploy_revs(sched: &SharedSched, agent: &str) -> Vec<Option<String>> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::TriggerDeploy { agent: a, rev, .. } if a == agent => {
|
|
Some((n.id, rev.clone()))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
nodes.sort_by_key(|(id, _)| *id);
|
|
nodes.into_iter().map(|(_, rev)| rev).collect()
|
|
}
|
|
|
|
/// The `pr` of `agent`'s builds, oldest first.
|
|
fn build_prs(sched: &SharedSched, agent: &str) -> Vec<Option<u64>> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::BuildMeta { agent: a, pr } if a == agent => Some((n.id, *pr)),
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
nodes.sort_by_key(|(id, _)| *id);
|
|
nodes.into_iter().map(|(_, pr)| pr).collect()
|
|
}
|
|
|
|
fn merged_atlas() -> SharedSched {
|
|
let sched = fresh_sched();
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| merged_deploy_job(b, "pr1ma", "atlas", "abc", 7))
|
|
.expect("the graph inserts");
|
|
sched
|
|
}
|
|
|
|
/// A merged config's build that fails, with the update `Done`, whatever it
|
|
/// fails with: the deploy is skipped, and the timer declares a build and
|
|
/// the same deploy again on every round, without an update, until a build
|
|
/// succeeds. Then it stops.
|
|
#[tokio::test]
|
|
async fn a_failed_build_is_built_again_every_round_until_it_succeeds() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = merged_atlas();
|
|
let errors = [
|
|
"error: attribute missing".to_owned(),
|
|
"timed out: nix did not finish".to_owned(),
|
|
format!(
|
|
"{}no swarm builder is listening",
|
|
meta_build::NO_VERDICT_PREFIX
|
|
),
|
|
];
|
|
drain_failing_build(&sched, "atlas", errors[0].clone()).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
assert_eq!(update_metas(&sched), 1);
|
|
|
|
for error in &errors[1..] {
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "the retry needs no update");
|
|
drain_failing_build(&sched, "atlas", error.clone()).await;
|
|
}
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Skipped]
|
|
);
|
|
|
|
// A success on a later round deploys, once.
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
// The merged deploy's rev survives every retry.
|
|
assert!(
|
|
deploy_revs(&sched, "atlas")
|
|
.iter()
|
|
.all(|rev| rev.as_deref() == Some("abc"))
|
|
);
|
|
// So does the PR it belongs to: every retried build reports on it too.
|
|
assert_eq!(build_prs(&sched, "atlas"), [Some(7); 4]);
|
|
assert!(
|
|
!queue_meta_update(&sched).expect("tick"),
|
|
"the success ends the retries"
|
|
);
|
|
}
|
|
|
|
/// Only a merged config PR's build carries a PR: the creation DAG's does
|
|
/// not, so its failure has nobody to comment on.
|
|
#[test]
|
|
fn only_a_merged_prs_build_carries_its_pr() {
|
|
let sched = merged_atlas();
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| declare_agent_job(b, "iris", "pr1ma", false))
|
|
.expect("the graph inserts");
|
|
assert_eq!(build_prs(&sched, "atlas"), [Some(7)]);
|
|
assert_eq!(build_prs(&sched, "iris"), [None]);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod mint_identity_tests {
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use hive_jobq::scheduler::Outcome;
|
|
use tracing_subscriber::layer::SubscriberExt as _;
|
|
|
|
use super::{SwarmNodeKind, WorkerDeps, mint_identity, mint_if_live, run_swarm_node};
|
|
|
|
/// One event's level and its fields, each by name.
|
|
type Event = (tracing::Level, Vec<(String, String)>);
|
|
|
|
/// Every event.
|
|
#[derive(Clone, Default)]
|
|
struct Events(Arc<Mutex<Vec<Event>>>);
|
|
|
|
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Events {
|
|
fn on_event(
|
|
&self,
|
|
event: &tracing::Event<'_>,
|
|
_: tracing_subscriber::layer::Context<'_, S>,
|
|
) {
|
|
let mut fields = Vec::new();
|
|
event.record(
|
|
&mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
|
|
fields.push((field.name().to_owned(), format!("{value:?}")));
|
|
},
|
|
);
|
|
self.0
|
|
.lock()
|
|
.expect("unpoisoned")
|
|
.push((*event.metadata().level(), fields));
|
|
}
|
|
}
|
|
|
|
/// An agent name the store would refuse fails the mint before any store
|
|
/// call, whatever this process's environment holds. The line names the
|
|
/// step it stopped at. The literal is the
|
|
/// `_msg` that `hh-agent-identity-mint-failed` in
|
|
/// `nix/host-modules/swarm-grafana.nix` matches.
|
|
#[tokio::test]
|
|
async fn a_failed_mint_logs_the_agent_and_the_error_on_the_line_the_alert_rule_matches() {
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
|
|
let Outcome::Failed(error) = mint_identity(None, "a/b").await else {
|
|
panic!("an illegal agent name cannot be minted");
|
|
};
|
|
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
let [(level, fields)] = got.as_slice() else {
|
|
panic!("one event, got {got:?}");
|
|
};
|
|
assert_eq!(*level, tracing::Level::WARN);
|
|
let field = |name: &str| {
|
|
fields
|
|
.iter()
|
|
.find(|(n, _)| n == name)
|
|
.map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
|
|
};
|
|
assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed");
|
|
assert_eq!(field("agent"), "a/b");
|
|
assert_eq!(field("step"), "setup");
|
|
assert_eq!(field("error"), error);
|
|
}
|
|
|
|
fn declared_as(
|
|
agents: &[(&str, swarm_wire::wanted::AgentState)],
|
|
) -> swarm_wire::wanted::HiveWanted {
|
|
let mut d = swarm_wire::wanted::HiveWanted::default();
|
|
for (agent, state) in agents {
|
|
d.agents.insert(
|
|
(*agent).to_owned(),
|
|
swarm_wire::wanted::AgentWanted { state: *state },
|
|
);
|
|
}
|
|
d
|
|
}
|
|
|
|
/// The mint runs for a declared agent and not for one declared `Destroyed`
|
|
/// or not at all: nothing is written for the second, and the node is done.
|
|
#[tokio::test]
|
|
async fn a_mint_for_an_agent_destroyed_since_it_was_queued_writes_nothing() {
|
|
use swarm_wire::wanted::AgentState;
|
|
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
let ran = std::sync::atomic::AtomicBool::new(false);
|
|
let mint = async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
};
|
|
let declarations =
|
|
declared_as(&[("live", AgentState::Up), ("gone", AgentState::Destroyed)]);
|
|
|
|
for agent in ["gone", "unknown"] {
|
|
let outcome = mint_if_live(agent, Some(Ok(vec![declarations.clone()])), async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
assert!(matches!(outcome, Outcome::Done), "{agent}");
|
|
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst), "{agent}");
|
|
}
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
assert!(
|
|
got.iter()
|
|
.any(|(level, fields)| *level == tracing::Level::INFO
|
|
&& fields
|
|
.iter()
|
|
.any(|(n, v)| n == "message"
|
|
&& v == "identity mint skipped: gone is destroyed")),
|
|
"{got:?}"
|
|
);
|
|
|
|
// Presence controls: a live agent mints, and so does one with no
|
|
// wanted-state writer to ask.
|
|
let outcome = mint_if_live("live", Some(Ok(vec![declarations])), mint).await;
|
|
assert!(matches!(outcome, Outcome::Done));
|
|
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
ran.store(false, std::sync::atomic::Ordering::SeqCst);
|
|
let outcome = mint_if_live("anyone", None, async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
assert!(matches!(outcome, Outcome::Done));
|
|
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
}
|
|
|
|
/// A declaration that cannot be read fails the node, naming the cause,
|
|
/// and mints nothing.
|
|
#[tokio::test]
|
|
async fn an_unreadable_declaration_fails_the_mint_without_minting() {
|
|
let ran = std::sync::atomic::AtomicBool::new(false);
|
|
let outcome = mint_if_live("atlas", Some(Err(anyhow::anyhow!("hive down"))), async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
let Outcome::Failed(error) = outcome else {
|
|
panic!("a failed read must fail the node");
|
|
};
|
|
assert!(error.contains("hive down"), "{error}");
|
|
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
}
|
|
/// A node of any kind whose work returns `Failed` logs one warn naming the
|
|
/// node's id, its kind and the error, from the one place that runs nodes.
|
|
#[tokio::test]
|
|
async fn a_failed_node_logs_its_kind_id_and_error() {
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
let sched = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
let ids = sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::SetAgentWanted {
|
|
hive: "h".to_owned(),
|
|
agent: "a".to_owned(),
|
|
})
|
|
.guid(),
|
|
]
|
|
})
|
|
.expect("inserted");
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: super::Late::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth declared");
|
|
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
let [(level, fields)] = got.as_slice() else {
|
|
panic!("one event, got {got:?}");
|
|
};
|
|
assert_eq!(*level, tracing::Level::WARN);
|
|
let field = |name: &str| {
|
|
fields
|
|
.iter()
|
|
.find(|(n, _)| n == name)
|
|
.map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
|
|
};
|
|
assert_eq!(field("kind"), "set_agent_wanted");
|
|
assert_eq!(field("node"), ids[0].get().to_string());
|
|
assert!(
|
|
field("error").contains("no swarm queue is configured"),
|
|
"{fields:?}"
|
|
);
|
|
}
|
|
}
|