4513 lines
177 KiB
Rust
4513 lines
177 KiB
Rust
//! The swarm-level job graph: its node and resource kinds, the worker that
|
|
//! runs claimed nodes, and the `queue_*` fns that insert them.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use anyhow::Result;
|
|
use swarm_authelia_bridge_sock::BridgeResponse;
|
|
|
|
use crate::routes::AppState;
|
|
use crate::routes::agents::{claiming_hives, read_declarations};
|
|
use crate::startup::PUBLIC_URL_ENV;
|
|
use crate::{
|
|
agent_identity, agent_renewal, auth, builder, config_pr, forge, matrix_account, meta_build,
|
|
meta_fanout, meta_flake, wanted,
|
|
};
|
|
|
|
/// Node payload for the swarm-level job graph. Named `Swarm*` rather than
|
|
/// the bare `NodeKind`/`Resource` `hive-c0re::job_queue::model` already
|
|
/// uses, so a grep for either doesn't land on both crates.
|
|
///
|
|
/// `CreateIdentity` was the first real variant — "the minimal shape for
|
|
/// agent creation is creating the identity and wiring that in" (the design
|
|
/// thread's own framing for why that landed before the forge-node work
|
|
/// here). `CreateRepo`/`AddRepoMember`/`InitAgentConfigRepo` are three
|
|
/// separate nodes rather than one combined "create the repo" step so each
|
|
/// is independently retryable/observable in the job graph, same as every
|
|
/// other multi-step provisioning flow in this codebase (`hive-c0re`'s own
|
|
/// `NodeKind` never folds unrelated forge calls into one node either).
|
|
#[derive(Clone, Debug)]
|
|
pub(crate) enum SwarmNodeKind {
|
|
/// Ensure `agent` exists as an authelia subject at the swarm level.
|
|
CreateIdentity { agent: String },
|
|
/// Create the agent's repo in `forge::CONFIG_ORG` with the operator
|
|
/// merge gate on its default branch. See `forge::Client::create_repo`.
|
|
CreateRepo { agent: String },
|
|
/// Ensure `agent` exists as a Forgejo user account. Independent of
|
|
/// `CreateIdentity`/`CreateRepo` (a forge user needs neither an
|
|
/// authelia subject nor an existing repo) but a prerequisite for
|
|
/// `AddRepoMember`, which fails with "user does not exist" against an
|
|
/// account this node hasn't created yet. See
|
|
/// `forge::Client::ensure_agent_user`.
|
|
CreateForgeUser { agent: String },
|
|
/// Add `agent` as a write collaborator on its own repo. See
|
|
/// `forge::Client::add_repo_member`.
|
|
AddRepoMember { agent: String },
|
|
/// Add `agent` as a read collaborator on `internal/docs`. See
|
|
/// `forge::docs_grant`.
|
|
///
|
|
/// Carries no hive: the grant is on a swarm-wide repo.
|
|
GrantDocsRead { agent: String },
|
|
/// Seed the agent's repo with `agent.nix` + `flake.nix`. See
|
|
/// `forge::Client::seed_agent_config`.
|
|
///
|
|
/// Deliberately carries no hive: seeding a config repo is the same
|
|
/// work whichever hive the agent is bound for, and an agent's config
|
|
/// states nothing about where it runs. The hive is an address the
|
|
/// swarm routes a deploy message to — it belongs on the node that
|
|
/// sends that message, not on this one.
|
|
InitAgentConfigRepo { agent: String },
|
|
/// Seed the agent's config repo with the archetype's module, for turn
|
|
/// into custom. See `agent_custom::seed`.
|
|
SeedArchetypeConfig {
|
|
agent: String,
|
|
archetype: swarm_wire::agent_archetype::ArchetypeRef,
|
|
},
|
|
/// Delete the agent's archetype key, which makes it custom. See
|
|
/// `agent_custom::drop_archetype`.
|
|
DropArchetype { agent: String },
|
|
/// Mint `agent`'s own client certificate for the swarm secret store,
|
|
/// publish it there, grant it, and log in with it. See
|
|
/// `agent_identity::mint_and_verify` — including why this node does not
|
|
/// report success on a write.
|
|
///
|
|
/// Carries no hive: neither the certificate, the queue credential nor
|
|
/// the agent's policy names one, so an agent keeps one store identity
|
|
/// whichever hive it runs on.
|
|
MintAgentIdentity { agent: String },
|
|
/// Make sure `agent` holds a live forge access token in the swarm secret
|
|
/// store, minting one with the forge's admin API when it does not. See
|
|
/// `forge::agent_token` — including why a rotation is a delete then a
|
|
/// create.
|
|
///
|
|
/// Carries no hive: the token's store path has no hive segment, and the
|
|
/// agent pulls it from wherever it runs.
|
|
MintAgentForgeToken { agent: String },
|
|
/// Make sure `agent` holds a live token for its own account on the swarm's
|
|
/// homeserver, creating the account with the swarm's appservice token
|
|
/// when it does not. See `matrix_account::agent_token` — including why a
|
|
/// stored token is read back before anything is minted.
|
|
///
|
|
/// Carries no hive: the path it writes has no hive segment, so an agent
|
|
/// that moves between hives keeps one matrix identity.
|
|
MintAgentMatrixAccount { agent: String },
|
|
/// Make sure `hive`'s sender account (`@hive-<hive>:`) holds a live token
|
|
/// in the swarm store, where that hive's matrix sweep reads it. See
|
|
/// `matrix_account::hive_sender`.
|
|
MintHiveSenderToken { hive: String },
|
|
/// Make sure `hive`'s read-only forge account (`hive-<hive>`) exists, is in
|
|
/// the `hives` team, and holds a live token in the swarm store, where
|
|
/// that hive's hive-c0re reads it. See `forge::hive_token`.
|
|
MintHiveForgeToken { hive: String },
|
|
/// Make sure the review classifier's forge account exists and holds a
|
|
/// live token in the swarm store, where this daemon reads it to post the
|
|
/// classifier's reviews. See `forge::classifier_account`.
|
|
MintClassifierForgeToken,
|
|
/// Make sure the swarm builder's read-only forge account exists, is in
|
|
/// the `hives` team, and holds a live token in the swarm store, where the
|
|
/// builder reads it to fetch config repos. See `forge::builder_account`.
|
|
MintBuilderForgeToken,
|
|
/// Make sure the CI metrics poller's read-only forge account exists, is in
|
|
/// its team in every organisation, and holds a live token in the swarm
|
|
/// store, where the poller reads it to list CI runs. See
|
|
/// `forge::ci_metrics_account`.
|
|
MintCiMetricsForgeToken,
|
|
/// Build `agent`'s config at `rev`, the head of its config PR `pr`, on
|
|
/// the swarm builder, and write the result as a commit status on `rev`.
|
|
/// See `builder`.
|
|
///
|
|
/// Carries no hive: the check builds on the builder's own agent base,
|
|
/// not on any hive.
|
|
BuildAgent { agent: String, rev: String, pr: u64 },
|
|
/// Build `agent` from `internal/meta`'s head on the swarm builder, and
|
|
/// record the closure in `agent-closures`. The `TriggerDeploy` behind it
|
|
/// runs only when the build is green. See `meta_build`.
|
|
///
|
|
/// Carries no rev: it reads the head when it runs, which in most DAGs is
|
|
/// after the `UpdateMeta` that commits it.
|
|
///
|
|
/// `pr` is the merged config PR the build belongs to, if any: a red `nix`
|
|
/// build is reported on it once.
|
|
BuildMeta { agent: String, pr: Option<u64> },
|
|
/// Re-mint `agent`'s queue secret if it is still stored and old enough.
|
|
/// See `agent_renewal`.
|
|
///
|
|
/// Carries no hive: the secret's store path has none.
|
|
RenewAgentQueueCredential { agent: String },
|
|
/// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state
|
|
/// store, so a freshly created agent does not start driving turns the
|
|
/// moment it's deployed — the operator has to explicitly flip it to `Up`.
|
|
/// Carries the hive because the wanted-state bucket is keyed per hive.
|
|
SetAgentWanted { hive: String, agent: String },
|
|
/// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy
|
|
/// subject. The one node kind whose effect leaves this host.
|
|
///
|
|
/// Carries the hive, unlike every variant above — this is the node
|
|
/// `InitAgentConfigRepo`'s doc points at when it says the hive belongs
|
|
/// on the node that sends the deploy message. It names the subject the
|
|
/// message goes to, one per hive, so no other hive is woken by it.
|
|
TriggerDeploy {
|
|
hive: String,
|
|
agent: String,
|
|
/// The config commit to deploy, from a merged config PR. `None` from
|
|
/// agent creation, where the hive deploys what it seeds.
|
|
rev: Option<String>,
|
|
},
|
|
/// Pin the swarm meta flake (`internal/meta`): render, lock on the swarm
|
|
/// builder, commit. See `meta_flake`. When the pin `internal/meta` holds is
|
|
/// not the one `meta_fanout` recorded, a `BuildMeta` and a `TriggerDeploy`
|
|
/// per placed agent follow, then a `RecordFanout`.
|
|
///
|
|
/// Carries nothing: it reads every head when it runs, so one pending node
|
|
/// covers every trigger queued behind it.
|
|
UpdateMeta,
|
|
/// Record `pin` in `meta_fanout` as sent to the hives. Runs once every
|
|
/// `TriggerDeploy` declared beside it is done or skipped by its build gate.
|
|
/// A deploy that failed or was lost leaves the record stale, and the next
|
|
/// `UpdateMeta` sends them again.
|
|
RecordFanout { pin: String },
|
|
}
|
|
|
|
impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|
fn label(&self) -> String {
|
|
match self {
|
|
SwarmNodeKind::CreateIdentity { .. } => "create_identity".to_owned(),
|
|
SwarmNodeKind::CreateRepo { .. } => "create_repo".to_owned(),
|
|
SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(),
|
|
SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
|
|
SwarmNodeKind::GrantDocsRead { .. } => "grant_docs_read".to_owned(),
|
|
SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
|
|
SwarmNodeKind::SeedArchetypeConfig { .. } => "seed_archetype_config".to_owned(),
|
|
SwarmNodeKind::DropArchetype { .. } => "drop_archetype".to_owned(),
|
|
SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
|
|
SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(),
|
|
SwarmNodeKind::MintAgentMatrixAccount { .. } => "mint_agent_matrix_account".to_owned(),
|
|
SwarmNodeKind::MintHiveSenderToken { .. } => "mint_hive_sender_token".to_owned(),
|
|
SwarmNodeKind::MintHiveForgeToken { .. } => "mint_hive_forge_token".to_owned(),
|
|
SwarmNodeKind::MintClassifierForgeToken => "mint_classifier_forge_token".to_owned(),
|
|
SwarmNodeKind::MintBuilderForgeToken => "mint_builder_forge_token".to_owned(),
|
|
SwarmNodeKind::MintCiMetricsForgeToken => "mint_ci_metrics_forge_token".to_owned(),
|
|
SwarmNodeKind::BuildAgent { .. } => "build_agent".to_owned(),
|
|
SwarmNodeKind::BuildMeta { .. } => "build_meta".to_owned(),
|
|
SwarmNodeKind::RenewAgentQueueCredential { .. } => {
|
|
"renew_agent_queue_credential".to_owned()
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
|
|
SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
|
|
SwarmNodeKind::UpdateMeta => "update_meta".to_owned(),
|
|
SwarmNodeKind::RecordFanout { .. } => "record_fanout".to_owned(),
|
|
}
|
|
}
|
|
|
|
fn data(&self, _id: hive_jobq_wire::WireId) -> serde_json::Value {
|
|
// Spelled out as an or-pattern rather than a catch-all on purpose: a
|
|
// new variant then fails to compile here instead of silently
|
|
// rendering as an agent name. That gate has now fired once —
|
|
// `TriggerDeploy` is the first node that is about an agent *and a
|
|
// hive*, and a catch-all would have dropped the hive from the
|
|
// viewer without anyone noticing.
|
|
match self {
|
|
SwarmNodeKind::CreateIdentity { agent }
|
|
| SwarmNodeKind::CreateRepo { agent }
|
|
| SwarmNodeKind::CreateForgeUser { agent }
|
|
| SwarmNodeKind::AddRepoMember { agent }
|
|
| SwarmNodeKind::GrantDocsRead { agent }
|
|
| SwarmNodeKind::InitAgentConfigRepo { agent }
|
|
| SwarmNodeKind::DropArchetype { agent }
|
|
| SwarmNodeKind::MintAgentIdentity { agent }
|
|
| SwarmNodeKind::MintAgentForgeToken { agent }
|
|
| SwarmNodeKind::MintAgentMatrixAccount { agent }
|
|
| SwarmNodeKind::RenewAgentQueueCredential { agent } => {
|
|
serde_json::json!({ "agent": agent })
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, pr } => {
|
|
serde_json::json!({ "agent": agent, "pr": pr })
|
|
}
|
|
SwarmNodeKind::SeedArchetypeConfig { agent, archetype } => {
|
|
serde_json::json!({ "agent": agent, "archetype": archetype.name })
|
|
}
|
|
SwarmNodeKind::MintHiveSenderToken { hive }
|
|
| SwarmNodeKind::MintHiveForgeToken { hive } => serde_json::json!({ "hive": hive }),
|
|
SwarmNodeKind::MintClassifierForgeToken
|
|
| SwarmNodeKind::MintBuilderForgeToken
|
|
| SwarmNodeKind::MintCiMetricsForgeToken
|
|
| SwarmNodeKind::UpdateMeta => serde_json::json!({}),
|
|
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
|
|
serde_json::json!({ "agent": agent, "rev": rev, "pr": pr })
|
|
}
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
|
|
serde_json::json!({ "agent": agent, "hive": hive, "rev": rev })
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { hive, agent } => {
|
|
serde_json::json!({ "agent": agent, "hive": hive })
|
|
}
|
|
SwarmNodeKind::RecordFanout { pin } => serde_json::json!({ "pin": pin }),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A named counting semaphore a node acquires before it starts. Every name
|
|
/// has capacity 1: the scheduler's default, which no caller overrides.
|
|
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
|
pub(crate) enum SwarmResourceKind {
|
|
/// The swarm builder: one `BuildAgent` or `BuildMeta` node runs at a time,
|
|
/// and the rest wait pending. A newer push replaces its PR's pending
|
|
/// `BuildAgent`.
|
|
Builder,
|
|
/// `internal/meta`: one `UpdateMeta` commits at a time, so two never race
|
|
/// on the same file's blob.
|
|
Meta,
|
|
}
|
|
|
|
impl hive_jobq_wire::WireResource for SwarmResourceKind {
|
|
fn name(&self) -> String {
|
|
match self {
|
|
SwarmResourceKind::Builder => "builder".to_owned(),
|
|
SwarmResourceKind::Meta => "meta".to_owned(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A dependency this daemon may only gain after it starts serving: empty
|
|
/// until then, filled once. Readers take what is there at the moment they ask.
|
|
pub(crate) type Late<T> = Arc<std::sync::OnceLock<T>>;
|
|
|
|
/// Everything a claimed node's executor arm might need to reach outside
|
|
/// this process — bundled into one `Clone` struct rather than growing
|
|
/// `run_swarm_node`'s parameter list per node kind (three forge-shaped
|
|
/// node kinds landed in one slice; a fourth parameter each would have made
|
|
/// the signature the least readable part of this file). Each field is
|
|
/// built once at startup (see `main`) and is `None` exactly when that
|
|
/// dependency isn't configured on this host — every arm below treats
|
|
/// absence as *this node's* failure, not a reason to skip silently.
|
|
#[derive(Clone)]
|
|
pub(crate) struct WorkerDeps {
|
|
pub(crate) auth: Option<std::sync::Arc<auth::AuthBridge>>,
|
|
/// Empty until the forge token has been read (`forge::enable`), so a node
|
|
/// run before then fails as "no forge access", and one run after succeeds.
|
|
pub(crate) forge: Late<Arc<forge::Client>>,
|
|
/// The swarm queue connection, for nodes whose effect is a published
|
|
/// event rather than an API call. A handle rather than the status
|
|
/// reader it is cloned from: a node that announces a deploy has no
|
|
/// business reading hive status, and `status.rs`'s own doc says the
|
|
/// connection living there is an accident of construction order, not a
|
|
/// claim that events are a kind of status.
|
|
pub(crate) queue: Option<async_nats::Client>,
|
|
/// The wanted-state writer, for nodes that declare what a hive should
|
|
/// converge an agent to. Shares the status reader's queue connection —
|
|
/// see `wanted_writer`. `None` exactly when no swarm queue is configured
|
|
/// on this host, same as `queue`.
|
|
pub(crate) wanted: Option<std::sync::Arc<wanted::WantedWriter>>,
|
|
/// Client-server API base of the swarm's homeserver, for the node that
|
|
/// mints agents' accounts on it. `None` when
|
|
/// `matrix_account::DEFAULT_HOMESERVER_ENV` is unset, which that node
|
|
/// **fails** on rather than skips, same as every other field here.
|
|
pub(crate) matrix_homeserver: Option<std::sync::Arc<str>>,
|
|
}
|
|
|
|
/// What `UpdateMeta` renders from, fixed for the process.
|
|
struct MetaDeps {
|
|
settings: meta_flake::Settings,
|
|
/// Every hive's name: each places agents, and each hive's forge account
|
|
/// gets read on the repo.
|
|
hives: Vec<String>,
|
|
}
|
|
|
|
/// What [`SwarmNodeKind::SetAgentWanted`] declares a brand-new agent to be.
|
|
///
|
|
/// A named constant rather than the literal inline, because this *is* the
|
|
/// decision the node exists to carry — writing a declaration at all is the
|
|
/// mechanism, `Paused` is the policy — and a test can pin a name where it
|
|
/// cannot pin a value written into a KV bucket no unit test has a server for.
|
|
const NEW_AGENT_WANTED_STATE: swarm_wire::wanted::AgentState =
|
|
swarm_wire::wanted::AgentState::Paused;
|
|
|
|
/// Run a claimed node and log one warn when it fails, whatever its kind.
|
|
pub(crate) async fn run_swarm_node(
|
|
id: hive_jobq::NodeId,
|
|
kind: SwarmNodeKind,
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
deps: WorkerDeps,
|
|
) -> (
|
|
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive_jobq::scheduler::Outcome,
|
|
) {
|
|
let node_kind = hive_jobq_wire::WireNode::label(&kind);
|
|
let (builder, outcome) = run_node_work(kind, builder, deps).await;
|
|
if let hive_jobq::scheduler::Outcome::Failed(error) = &outcome {
|
|
tracing::warn!(node = id.get(), kind = %node_kind, %error, "swarm jobq: node failed");
|
|
}
|
|
(builder, outcome)
|
|
}
|
|
|
|
/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/
|
|
/// exec.rs::run_node`'s role exactly — the one place a `SwarmNodeKind`
|
|
/// variant turns into a real effect.
|
|
async fn run_node_work(
|
|
kind: SwarmNodeKind,
|
|
builder: hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
deps: WorkerDeps,
|
|
) -> (
|
|
hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive_jobq::scheduler::Outcome,
|
|
) {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let forge = deps.forge.get().cloned();
|
|
let outcome = match kind {
|
|
SwarmNodeKind::CreateIdentity { agent } => match deps.auth {
|
|
None => {
|
|
Outcome::Failed("no swarm-authelia-bridge is configured on this host".to_owned())
|
|
}
|
|
Some(bridge) => match bridge.ensure_agent_identity(&agent).await {
|
|
// Matched rather than discarded with `Ok(_)`: a heal writes
|
|
// to a subject this job did not create, and that has to
|
|
// reach a human. `Outcome` has no success-carrying variant,
|
|
// so the controller's own log is the only channel a
|
|
// succeeding node has today.
|
|
Ok(BridgeResponse::Healed) => {
|
|
tracing::warn!(
|
|
%agent,
|
|
"swarm jobq: create_identity marked an EXISTING identity as an \
|
|
agent — verify this name did not belong to a person"
|
|
);
|
|
Outcome::Done
|
|
}
|
|
Ok(_) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
},
|
|
},
|
|
SwarmNodeKind::CreateRepo { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => match forge_node("create_repo", client.create_repo(&agent)).await {
|
|
Ok(full_name) => {
|
|
tracing::info!(%full_name, "swarm jobq: create_repo done");
|
|
Outcome::Done
|
|
}
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
},
|
|
},
|
|
SwarmNodeKind::CreateForgeUser { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => {
|
|
match forge_node("create_forge_user", client.ensure_agent_user(&agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
},
|
|
SwarmNodeKind::AddRepoMember { agent } => match forge {
|
|
None => Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => {
|
|
match forge_node("add_repo_member", client.add_repo_member(&agent, &agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
},
|
|
SwarmNodeKind::GrantDocsRead { agent } => grant_docs_read(forge, &agent).await,
|
|
SwarmNodeKind::InitAgentConfigRepo { agent } => init_agent_config_repo(forge, &agent).await,
|
|
SwarmNodeKind::SeedArchetypeConfig { .. } | SwarmNodeKind::DropArchetype { .. } => {
|
|
convert_node(&kind, forge, deps.queue.as_ref()).await
|
|
}
|
|
SwarmNodeKind::MintAgentIdentity { agent } => {
|
|
mint_identity(deps.wanted.as_deref(), &agent).await
|
|
}
|
|
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(forge, &agent).await,
|
|
SwarmNodeKind::MintAgentMatrixAccount { agent } => {
|
|
mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await
|
|
}
|
|
SwarmNodeKind::MintHiveSenderToken { hive } => {
|
|
mint_hive_sender(deps.matrix_homeserver.as_deref(), &hive).await
|
|
}
|
|
SwarmNodeKind::MintHiveForgeToken { hive } => mint_hive_forge_token(forge, &hive).await,
|
|
SwarmNodeKind::MintClassifierForgeToken => mint_classifier_forge_token(forge).await,
|
|
SwarmNodeKind::MintBuilderForgeToken => mint_builder_forge_token(forge).await,
|
|
SwarmNodeKind::MintCiMetricsForgeToken => mint_ci_metrics_forge_token(forge).await,
|
|
SwarmNodeKind::BuildAgent { agent, rev, pr } => {
|
|
build_agent(&deps, forge, agent, rev, pr).await
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, pr } => build_meta(&deps, forge, agent, pr).await,
|
|
SwarmNodeKind::RenewAgentQueueCredential { agent } => renew_queue_credential(&agent).await,
|
|
SwarmNodeKind::SetAgentWanted { hive, agent } => {
|
|
set_agent_wanted(deps.wanted.as_deref(), &hive, &agent).await
|
|
}
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => match deps.queue {
|
|
None => Outcome::Failed(
|
|
"no swarm queue is configured on this host, so no hive can be told to deploy"
|
|
.to_owned(),
|
|
),
|
|
Some(client) => publish_deploy(&client, &hive, &agent, rev).await,
|
|
},
|
|
SwarmNodeKind::UpdateMeta => finish_update_meta(update_meta(&deps, forge).await, &builder),
|
|
SwarmNodeKind::RecordFanout { pin } => record_fanout(deps.queue.as_ref(), &pin).await,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
|
|
/// A forge-calling node's work, run under [`forge::FORGE_DEADLINE`]. Elapsing
|
|
/// is the node's own error, naming `node`, so it fails like any other refused
|
|
/// step instead of holding its executor slot.
|
|
pub(crate) async fn forge_node<T>(
|
|
node: &str,
|
|
work: impl std::future::Future<Output = Result<T>>,
|
|
) -> Result<T> {
|
|
forge::within_forge_deadline(node, work)
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
Err(anyhow::anyhow!(
|
|
"{node}: no answer from the forge within {}s",
|
|
forge::FORGE_DEADLINE.as_secs()
|
|
))
|
|
})
|
|
}
|
|
|
|
/// Logged at `warn`, with the agent, the step and the error, when
|
|
/// [`mint_identity`] fails. The Grafana rule `hh-agent-identity-mint-failed`
|
|
/// (`nix/host-modules/swarm-grafana.nix`) matches this exact text: change both
|
|
/// together.
|
|
const MINT_IDENTITY_FAILED: &str = "swarm jobq: mint_agent_identity failed";
|
|
|
|
/// The `MintAgentIdentity` arm, lifted out so `run_node_work` stays under
|
|
/// `clippy::too_many_lines`.
|
|
///
|
|
/// Re-reads the wanted-state declarations first (see [`mint_if_live`]); with
|
|
/// no wanted-state writer there is nothing to read and the mint runs.
|
|
pub(crate) async fn mint_identity(
|
|
wanted: Option<&wanted::WantedWriter>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let declarations = match wanted {
|
|
Some(writer) => {
|
|
let hives: Vec<String> = crate::startup::load_hives()
|
|
.into_iter()
|
|
.map(|h| h.name)
|
|
.collect();
|
|
Some(agent_renewal::read_declarations(writer, &hives).await)
|
|
}
|
|
None => None,
|
|
};
|
|
mint_if_live(agent, declarations, agent_identity::mint_and_verify(agent)).await
|
|
}
|
|
|
|
/// Run `mint` unless `declarations` (every hive's, read when the node runs)
|
|
/// leave `agent` not live: declared `Destroyed` everywhere, or nowhere. A mint
|
|
/// queued before a destroy then writes no credential for the destroyed name,
|
|
/// and the node is `Done`. Creation orders its mint after the declaration, so a
|
|
/// created agent is always declared. A failed read fails the node rather than
|
|
/// minting blind.
|
|
async fn mint_if_live(
|
|
agent: &str,
|
|
declarations: Option<anyhow::Result<Vec<swarm_wire::wanted::HiveWanted>>>,
|
|
mint: impl std::future::Future<Output = anyhow::Result<()>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match declarations {
|
|
Some(Ok(declarations)) if !agent_renewal::live_agents(&declarations).contains(agent) => {
|
|
tracing::info!(agent, "identity mint skipped: {agent} is destroyed");
|
|
return Outcome::Done;
|
|
}
|
|
Some(Err(e)) => {
|
|
let error = format!("{e:#}");
|
|
tracing::warn!(%agent, step = "declaration", %error, "{MINT_IDENTITY_FAILED}");
|
|
return Outcome::Failed(error);
|
|
}
|
|
Some(Ok(_)) | None => {}
|
|
}
|
|
match mint.await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => {
|
|
let error = format!("{e:#}");
|
|
let step =
|
|
agent_identity::failed_step(&e).map_or("unknown", agent_identity::MintStep::as_str);
|
|
tracing::warn!(%agent, %step, %error, "{MINT_IDENTITY_FAILED}");
|
|
Outcome::Failed(error)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The `MintAgentForgeToken` arm, lifted out so `run_node_work` stays under
|
|
/// `clippy::too_many_lines`.
|
|
async fn mint_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no agent forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_agent_forge_token",
|
|
client.ensure_agent_forge_token(agent),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `GrantDocsRead` arm, lifted out for the same reason.
|
|
pub(crate) async fn grant_docs_read(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no docs grant can be made"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node("grant_docs_read", client.grant_docs_read(agent)).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `InitAgentConfigRepo` arm, lifted out for the same reason.
|
|
async fn init_agent_config_repo(
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"init_agent_config_repo",
|
|
client.seed_agent_config(agent, agent),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `SeedArchetypeConfig` and `DropArchetype` arms, lifted out for the
|
|
/// same reason. Any other kind is not this function's to run.
|
|
async fn convert_node(
|
|
kind: &SwarmNodeKind,
|
|
forge: Option<Arc<forge::Client>>,
|
|
queue: Option<&async_nats::Client>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let done = |result: Result<()>| match result {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
};
|
|
match kind {
|
|
SwarmNodeKind::SeedArchetypeConfig { agent, archetype } => {
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store)"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
done(
|
|
forge_node(
|
|
"seed_archetype_config",
|
|
crate::agent_custom::seed(client.as_ref(), agent, archetype),
|
|
)
|
|
.await,
|
|
)
|
|
}
|
|
SwarmNodeKind::DropArchetype { agent } => match queue {
|
|
Some(client) => done(crate::agent_custom::drop_archetype(client, agent).await),
|
|
None => Outcome::Failed("no swarm queue is configured on this host".to_owned()),
|
|
},
|
|
other => unreachable!("{other:?} is not a conversion node"),
|
|
}
|
|
}
|
|
|
|
/// The `RenewAgentQueueCredential` arm, lifted out for the same reason.
|
|
async fn renew_queue_credential(agent: &str) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match agent_renewal::renew(agent).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintHiveForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_hive_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
hive: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no hive forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_hive_forge_token",
|
|
client.ensure_hive_forge_token(hive),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `BuildAgent` arm, lifted out for `clippy::too_many_lines`.
|
|
pub(crate) async fn build_agent(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: String,
|
|
rev: String,
|
|
pr: u64,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let job = swarm_wire::builder::BuildAgent { agent, rev, pr };
|
|
let builds = builder::runs::Builds::Live(deps.queue.clone());
|
|
let link_base = std::env::var(PUBLIC_URL_ENV).ok();
|
|
builder::run(
|
|
deps.queue.as_ref(),
|
|
forge.as_deref(),
|
|
&builds,
|
|
link_base.as_deref(),
|
|
job,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// The `BuildMeta` arm, lifted out for `clippy::too_many_lines`.
|
|
async fn build_meta(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
agent: String,
|
|
pr: Option<u64>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
let builds = builder::runs::Builds::Live(deps.queue.clone());
|
|
meta_build::run(forge.as_deref(), deps.queue.as_ref(), &builds, agent, pr).await
|
|
}
|
|
|
|
/// What an `UpdateMeta` pass decided: the deploys to send, and the placements
|
|
/// to send them to.
|
|
struct MetaPass {
|
|
fan_out: FanOut,
|
|
placements: std::collections::BTreeSet<(String, String)>,
|
|
}
|
|
|
|
/// What a pass sends the placed agents.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
enum FanOut {
|
|
/// Nothing: the pin is the one recorded.
|
|
None,
|
|
/// A deploy each, then record `pin`.
|
|
Record { pin: String },
|
|
/// A deploy each, with nothing to record into: the record cannot be read.
|
|
Unrecorded,
|
|
}
|
|
|
|
/// The fan-out for the pin `report` leaves, given the pin `meta_fanout` has
|
|
/// recorded. With no readable record, only a move of the pin sends deploys,
|
|
/// and nothing can be recorded.
|
|
fn decide_fan_out(report: &meta_flake::Report, recorded: Result<Option<String>>) -> FanOut {
|
|
match recorded {
|
|
Ok(Some(pin)) if pin == report.pin => FanOut::None,
|
|
Ok(_) => FanOut::Record {
|
|
pin: report.pin.clone(),
|
|
},
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = %format!("{e:#}"),
|
|
moved = report.hyperhive_moved,
|
|
"update_meta: the recorded fan-out pin is unreadable; deploying only on a moved pin"
|
|
);
|
|
if report.hyperhive_moved {
|
|
FanOut::Unrecorded
|
|
} else {
|
|
FanOut::None
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The `UpdateMeta` arm, lifted out for the same reason: what the update
|
|
/// decided to send, and the placements it rendered for. `Err` is the node's
|
|
/// failure.
|
|
async fn update_meta(
|
|
deps: &WorkerDeps,
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> Result<MetaPass, String> {
|
|
let Some(client) = forge else {
|
|
return Err(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so internal/meta cannot be written"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
let Some(meta) = meta_deps() else {
|
|
return Err(format!(
|
|
"{} is unset, so there are no swarm values to render internal/meta from",
|
|
meta_flake::SETTINGS_ENV
|
|
));
|
|
};
|
|
let (Some(queue), Some(wanted)) = (&deps.queue, &deps.wanted) else {
|
|
return Err(
|
|
"no swarm queue is configured on this host, so neither the placed agents \
|
|
nor a lock can be had"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
let placements = placements(wanted, &meta.hives)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
let agents = placed_agents(queue, &placements)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
let report = forge_node(
|
|
"update_meta",
|
|
meta_flake::update(client.as_ref(), queue, &meta.settings, &agents, &meta.hives),
|
|
)
|
|
.await
|
|
.map_err(|e| format!("{e:#}"))?;
|
|
tracing::info!(?report, agents = agents.len(), "update_meta: done");
|
|
let fan_out = decide_fan_out(&report, meta_fanout::recorded(queue).await);
|
|
Ok(MetaPass {
|
|
fan_out,
|
|
placements,
|
|
})
|
|
}
|
|
|
|
/// The `UpdateMeta` node's outcome from what [`update_meta`] decided, with the
|
|
/// deploys declared into `builder` (the scheduler inserts them under the node
|
|
/// when it completes). Synchronous: the builder is not `Sync`, so it cannot be
|
|
/// held across an await.
|
|
fn finish_update_meta(
|
|
result: Result<MetaPass, String>,
|
|
builder: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
match result {
|
|
Ok(pass) => {
|
|
declare_fan_out(builder, &pass.fan_out, &pass.placements);
|
|
Outcome::Done
|
|
}
|
|
Err(e) => Outcome::Failed(e),
|
|
}
|
|
}
|
|
|
|
/// The `RecordFanout` arm, lifted out so `run_node_work` stays under the line limit.
|
|
async fn record_fanout(
|
|
queue: Option<&async_nats::Client>,
|
|
pin: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = queue else {
|
|
return Outcome::Failed(
|
|
"no swarm queue is configured on this host, so the fan-out cannot be recorded"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match meta_fanout::record(client, pin).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The build and the deploy every path declares for one agent: a `BuildMeta`
|
|
/// of `agent`, then the `TriggerDeploy`, which runs only when the build is
|
|
/// green (`after_ok`), so a red build leaves it `Skipped`.
|
|
///
|
|
/// `update` is the `UpdateMeta` the pair follows when it is not this pair's
|
|
/// parent. The build waits on it, since the build reads the head that update
|
|
/// commits. The deploy keeps its own edge to it too: `skipped_deploys` finds
|
|
/// the deploys a failed update skipped by that edge.
|
|
fn gated_deploy<'a>(
|
|
b: &'a hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: Option<String>,
|
|
pr: Option<u64>,
|
|
update: Option<hive_jobq::builder::NodeGuid>,
|
|
) -> hive_jobq::builder::NodeRef<'a, SwarmNodeKind, SwarmResourceKind> {
|
|
let mut build = b
|
|
.node(SwarmNodeKind::BuildMeta {
|
|
agent: agent.to_owned(),
|
|
pr,
|
|
})
|
|
.needs(SwarmResourceKind::Builder);
|
|
if let Some(update) = update {
|
|
build = build.after_ok(update);
|
|
}
|
|
let mut deploy = b
|
|
.node(SwarmNodeKind::TriggerDeploy {
|
|
hive: hive.to_owned(),
|
|
agent: agent.to_owned(),
|
|
rev,
|
|
})
|
|
.after_ok(build);
|
|
if let Some(update) = update {
|
|
deploy = deploy.after_ok(update);
|
|
}
|
|
deploy
|
|
}
|
|
|
|
/// Declare one build and rebuild of each `(hive, agent)` placement, which
|
|
/// nothing else tells to build again, and for a [`FanOut::Record`] the
|
|
/// `RecordFanout` that waits on every one of them. A deploy the build gate
|
|
/// skipped counts as sent: the pin is recorded, so one failing agent does
|
|
/// not resend every healthy one on each retry. A deploy skipped by a failed
|
|
/// build is re-declared by `queue_meta_update` on every round.
|
|
fn declare_fan_out(
|
|
grown: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
fan_out: &FanOut,
|
|
placements: &std::collections::BTreeSet<(String, String)>,
|
|
) {
|
|
if *fan_out == FanOut::None {
|
|
return;
|
|
}
|
|
let deploys: Vec<hive_jobq::builder::NodeGuid> = placements
|
|
.iter()
|
|
.map(|(hive, agent)| gated_deploy(grown, hive, agent, None, None, None).guid())
|
|
.collect();
|
|
if let FanOut::Record { pin } = fan_out {
|
|
let mut record = grown.node(SwarmNodeKind::RecordFanout { pin: pin.clone() });
|
|
for deploy in deploys {
|
|
record = record.on_outcome(
|
|
deploy,
|
|
&[
|
|
hive_jobq::TerminalState::Done,
|
|
hive_jobq::TerminalState::Skipped,
|
|
],
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Every `(hive, agent)` a hive in `hives` places.
|
|
pub(crate) async fn placements(
|
|
wanted: &wanted::WantedWriter,
|
|
hives: &[String],
|
|
) -> Result<std::collections::BTreeSet<(String, String)>> {
|
|
let mut placed = std::collections::BTreeSet::new();
|
|
for hive in hives {
|
|
let Some(declared) = wanted
|
|
.view(hive)
|
|
.await
|
|
.map_err(|e| e.context(format!("read where hive {hive} places agents")))?
|
|
else {
|
|
continue;
|
|
};
|
|
placed.extend(
|
|
declared
|
|
.agents
|
|
.into_iter()
|
|
.filter(|(_, w)| crate::routes::agents::places_agent(w.state))
|
|
.map(|(name, _)| (hive.clone(), name)),
|
|
);
|
|
}
|
|
Ok(placed)
|
|
}
|
|
|
|
/// Every agent in `placements`, with its archetype.
|
|
async fn placed_agents(
|
|
queue: &async_nats::Client,
|
|
placements: &std::collections::BTreeSet<(String, String)>,
|
|
) -> Result<Vec<meta_flake::Placed>> {
|
|
let names: std::collections::BTreeSet<&String> =
|
|
placements.iter().map(|(_, agent)| agent).collect();
|
|
let archetypes = swarm_queue_client::agent_archetype::open_or_create(queue)
|
|
.await
|
|
.map_err(|e| anyhow::anyhow!("open the agent-archetype bucket: {e}"))?;
|
|
let mut placed = Vec::with_capacity(names.len());
|
|
for name in names {
|
|
let (archetype, _, _) = crate::schedules::read_or_default::<
|
|
Option<swarm_wire::agent_archetype::ArchetypeRef>,
|
|
>(&archetypes, name)
|
|
.await
|
|
.map_err(|e| e.context(format!("read the archetype of {name}")))?;
|
|
placed.push(meta_flake::Placed {
|
|
name: name.clone(),
|
|
archetype,
|
|
});
|
|
}
|
|
Ok(placed)
|
|
}
|
|
|
|
/// The `MintBuilderForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_builder_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no builder forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_builder_forge_token",
|
|
client.ensure_builder_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintCiMetricsForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_ci_metrics_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no ci metrics forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_ci_metrics_forge_token",
|
|
client.ensure_ci_metrics_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintClassifierForgeToken` arm, lifted out for the same reason.
|
|
async fn mint_classifier_forge_token(
|
|
forge: Option<Arc<forge::Client>>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(client) = forge else {
|
|
return Outcome::Failed(
|
|
"no forge access here (SWARM_CONTROLLER_FORGE_URL unset, or its token \
|
|
not in the swarm secret store), so no review classifier forge token can be minted"
|
|
.to_owned(),
|
|
);
|
|
};
|
|
match forge_node(
|
|
"mint_classifier_forge_token",
|
|
client.ensure_classifier_forge_token(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintAgentMatrixAccount` arm, lifted out for the same reason.
|
|
async fn mint_matrix_account(
|
|
homeserver: Option<&str>,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
// Loud on purpose: an agent quietly created without a matrix account is
|
|
// an agent nothing can talk to, and no hive mints one any more.
|
|
let Some(base) = homeserver else {
|
|
return Outcome::Failed(format!(
|
|
"no matrix homeserver configured on this host ({} unset), so no agent \
|
|
matrix account can be minted",
|
|
matrix_account::DEFAULT_HOMESERVER_ENV
|
|
));
|
|
};
|
|
match matrix_account::agent_token::ensure_agent_matrix_account(base, agent).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `MintHiveSenderToken` arm, lifted out for the same reason.
|
|
async fn mint_hive_sender(homeserver: Option<&str>, hive: &str) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
let Some(base) = homeserver else {
|
|
return Outcome::Failed(format!(
|
|
"no matrix homeserver configured on this host ({} unset), so no hive \
|
|
sender token can be minted",
|
|
matrix_account::DEFAULT_HOMESERVER_ENV
|
|
));
|
|
};
|
|
match matrix_account::hive_sender::ensure_hive_sender_token(base, hive).await {
|
|
Ok(()) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// The `SetAgentWanted` arm, lifted out so `run_node_work` stays under the line
|
|
/// limit: it fails without a wanted-state writer.
|
|
async fn set_agent_wanted(
|
|
wanted: Option<&wanted::WantedWriter>,
|
|
hive: &str,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
match wanted {
|
|
None => hive_jobq::scheduler::Outcome::Failed(
|
|
"no swarm queue is configured on this host, so no wanted-state \
|
|
declaration can be written"
|
|
.to_owned(),
|
|
),
|
|
Some(writer) => declare_new_agent(writer, hive, agent).await,
|
|
}
|
|
}
|
|
|
|
/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns
|
|
/// out not to be new: an agent that already has a declaration (other than
|
|
/// `Destroyed`, which this treats as reusable) is left alone, so a retried
|
|
/// or migrating create is idempotent rather than pausing an already-running
|
|
/// agent. `wanted.rs` itself has no such carve-out, so this node makes the
|
|
/// call.
|
|
async fn declare_new_agent(
|
|
writer: &wanted::WantedWriter,
|
|
hive: &str,
|
|
agent: &str,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
use swarm_wire::wanted::AgentState;
|
|
|
|
let existing = match writer.view(hive).await {
|
|
Ok(declaration) => declaration.and_then(|d| d.agents.get(agent).map(|a| a.state)),
|
|
Err(e) => return Outcome::Failed(format!("{e:#}")),
|
|
};
|
|
if let Some(state) = existing
|
|
&& state != AgentState::Destroyed
|
|
{
|
|
tracing::debug!(
|
|
hive,
|
|
agent,
|
|
?state,
|
|
"agent already declared, leaving its wanted state alone"
|
|
);
|
|
return Outcome::Done;
|
|
}
|
|
match writer.set(hive, agent, NEW_AGENT_WANTED_STATE).await {
|
|
Ok(_) => Outcome::Done,
|
|
Err(e) => Outcome::Failed(format!("{e:#}")),
|
|
}
|
|
}
|
|
|
|
/// Publish one deploy request and report whether it left this process.
|
|
///
|
|
/// The flush is not belt-and-braces: `publish` hands the message to the
|
|
/// client's write buffer and returns, so a node that reported `Done` on
|
|
/// that alone would be claiming a delivery it has no evidence for — the
|
|
/// same ordering `webhook::announce_knowledge_change` documents.
|
|
async fn publish_deploy(
|
|
client: &async_nats::Client,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: Option<String>,
|
|
) -> hive_jobq::scheduler::Outcome {
|
|
use hive_jobq::scheduler::Outcome;
|
|
|
|
// One subject per hive, so the other hives are never woken by this.
|
|
let subject = swarm_queue_client::deploy_subject(hive);
|
|
let request = swarm_queue_client::DeployRequest {
|
|
agent: agent.to_owned(),
|
|
rev,
|
|
};
|
|
let payload = match serde_json::to_vec(&request) {
|
|
Ok(payload) => payload,
|
|
Err(e) => return Outcome::Failed(format!("encoding the deploy request failed: {e}")),
|
|
};
|
|
if let Err(e) = client.publish(subject.clone(), payload.into()).await {
|
|
return Outcome::Failed(format!("publishing to {subject} failed: {e}"));
|
|
}
|
|
if let Err(e) = client.flush().await {
|
|
return Outcome::Failed(format!(
|
|
"flushing the deploy event to {subject} failed: {e}"
|
|
));
|
|
}
|
|
tracing::info!(%subject, %hive, %agent, rev = ?request.rev, "swarm jobq: deploy requested");
|
|
Outcome::Done
|
|
}
|
|
|
|
/// Spawn the swarm-level job-graph scheduler loop. Mirrors `hive-c0re/src/
|
|
/// job_queue/scheduler.rs::run_worker`'s shape: claim one runnable node,
|
|
/// spawn the future that runs + completes it, loop again immediately if
|
|
/// something started (more may now be runnable), otherwise back off
|
|
/// briefly before re-polling.
|
|
///
|
|
/// No shutdown signal to wire in — unlike `hive-c0re`'s `coord.shutdown_rx()`,
|
|
/// this daemon has no graceful-shutdown machinery at all yet (`main`'s
|
|
/// `axum::serve` runs unconditionally to process exit), so this loop
|
|
/// matches that: it rides the runtime down with the process, same as
|
|
/// every in-flight HTTP request does.
|
|
///
|
|
/// Cheap to run with an empty graph: `claim_next` on a graph nothing was
|
|
/// ever inserted into just returns `None` every poll.
|
|
///
|
|
/// `deps` is cloned per iteration (its fields are `Arc` clones, not
|
|
/// reconnects) and moved into the closure `claim_next` takes ownership
|
|
/// of — `run_swarm_node` needs its own owned copy since the claimed
|
|
/// future may outlive this loop iteration.
|
|
pub(crate) fn spawn_jobq_worker(
|
|
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
|
deps: WorkerDeps,
|
|
) {
|
|
tokio::spawn(async move {
|
|
loop {
|
|
let deps = deps.clone();
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
});
|
|
match runner {
|
|
Some(runner) => {
|
|
tokio::spawn(async move {
|
|
let (id, grew) = runner.await;
|
|
if let Err(e) = grew {
|
|
tracing::warn!(
|
|
node = id.get(),
|
|
error = %e,
|
|
"swarm jobq: grown job rejected"
|
|
);
|
|
}
|
|
});
|
|
// Something just started — more may be runnable right
|
|
// now, so loop again immediately rather than sleeping.
|
|
}
|
|
None => {
|
|
// Nothing runnable. Bounded poll rather than an event
|
|
// wake (unlike hive-c0re's `notify.notify_one()`,
|
|
// there is no completion-signal channel here yet) —
|
|
// fine at this daemon's scale (one graph, no
|
|
// submitters yet); revisit if/when that stops holding.
|
|
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Queue a build check of a config PR's new head. Every head is built, on
|
|
/// whichever hive the agent runs: the check builds on the builder's own
|
|
/// agent base.
|
|
pub(crate) fn queue_config_build(state: &AppState, changed: config_pr::ChangedConfigPr) {
|
|
let (agent, rev, pr) = (changed.agent.clone(), changed.rev.clone(), changed.pr);
|
|
let queued = queue_build(
|
|
&mut state
|
|
.jobq
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
|
changed,
|
|
);
|
|
match queued {
|
|
Ok(superseded) => {
|
|
tracing::info!(%agent, %rev, pr, superseded, "config-pr head: build queued");
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%agent, %rev, pr, error = %e, "config-pr head: queueing the build failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Cancel every pending build of the same PR, then insert the new one behind
|
|
/// [`SwarmResourceKind::Builder`]. A build already running is left to finish:
|
|
/// its status lands on its own commit, not on the new head. Returns how many
|
|
/// pending builds were cancelled.
|
|
pub(crate) fn queue_build(
|
|
sched: &mut hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>,
|
|
changed: config_pr::ChangedConfigPr,
|
|
) -> Result<usize> {
|
|
let config_pr::ChangedConfigPr { agent, rev, pr } = changed;
|
|
let stale: Vec<hive_jobq::NodeId> = sched
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.state == hive_jobq::State::Pending)
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::BuildAgent { agent: a, pr: p, .. } if *a == agent && *p == pr)
|
|
})
|
|
.map(|n| n.id)
|
|
.collect();
|
|
let superseded = stale
|
|
.into_iter()
|
|
.filter(|&id| sched.cancel_node(id))
|
|
.count();
|
|
sched
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::BuildAgent { agent, rev, pr })
|
|
.needs(SwarmResourceKind::Builder)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
Ok(superseded)
|
|
}
|
|
|
|
/// Queue the deploy of a merged config PR on the one hive that runs its agent.
|
|
///
|
|
/// The hive is found by scanning every hive's wanted state, since nothing
|
|
/// records an agent's hive (see `forge::initial_agent_nix`). Zero claimants or
|
|
/// several deploy nothing: there is no hive to send it to, or no telling which.
|
|
pub(crate) async fn queue_merged_config_deploy(
|
|
state: &AppState,
|
|
merged: config_pr::MergedConfigPr,
|
|
) {
|
|
let config_pr::MergedConfigPr { agent, rev, pr } = merged;
|
|
let declared = match read_declarations(state, state.hives.iter()).await {
|
|
Ok(declared) => declared,
|
|
Err((hive, e)) => {
|
|
tracing::warn!(
|
|
%agent, %rev, %hive, error = %format!("{e:#}"),
|
|
"config-pr merge: cannot read where the agent is placed; nothing deployed"
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
let claimants = claiming_hives(&agent, &declared);
|
|
let [hive] = claimants.as_slice() else {
|
|
tracing::warn!(
|
|
%agent, %rev, ?claimants,
|
|
"config-pr merge: not exactly one hive places this agent; nothing deployed"
|
|
);
|
|
return;
|
|
};
|
|
insert_merged_deploy(&state.jobq, hive, &agent, &rev, Some(pr));
|
|
}
|
|
|
|
/// Insert the [`merged_deploy_job`] of `agent`'s config at `rev` on `hive`.
|
|
/// `pr` is the merged PR a failed build reports on, when there is one. Whether
|
|
/// the job was inserted.
|
|
pub(crate) fn insert_merged_deploy(
|
|
jobq: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: &str,
|
|
pr: Option<u64>,
|
|
) -> bool {
|
|
let inserted = jobq
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| merged_deploy_job(b, hive, agent, rev, pr));
|
|
match inserted {
|
|
Ok(_) => {
|
|
tracing::info!(%agent, %rev, %hive, "config deploy queued");
|
|
true
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(%agent, %rev, %hive, error = %e, "queueing the config deploy failed");
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The sub-DAG a merged config PR is: the meta update, the build of the agent
|
|
/// from the updated flake, then the deploy of `rev` on `hive`. `pr` is the
|
|
/// merged PR, `None` for a head no PR delivery reported.
|
|
///
|
|
/// `after_ok`: a hive builds only from `internal/meta`, so a deploy asked for
|
|
/// after a failed update (no builder, an unreadable config repo) could only
|
|
/// fail its build, and one asked for after a red build would fail the same
|
|
/// way. The failed node is `Failed` in the graph and the deploy `Skipped`.
|
|
fn merged_deploy_job(
|
|
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
hive: &str,
|
|
agent: &str,
|
|
rev: &str,
|
|
pr: Option<u64>,
|
|
) -> Vec<hive_jobq::builder::NodeGuid> {
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta);
|
|
let root = update_meta.guid();
|
|
let _deploy = gated_deploy(b, hive, agent, Some(rev.to_owned()), pr, Some(root));
|
|
vec![root]
|
|
}
|
|
|
|
/// The sub-DAG one agent creation is: the nodes, and the edges between them.
|
|
///
|
|
/// A function rather than a closure inside [`create_agent`](crate::routes::agents::create_agent) so the endpoint's
|
|
/// validation and the graph's shape can each be read without scrolling past
|
|
/// the other — and so the one handle the response reports is returned from
|
|
/// the place that decides which node it is.
|
|
pub(crate) fn declare_agent_job(
|
|
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
agent: &str,
|
|
hive: &str,
|
|
archetype: bool,
|
|
) -> Vec<hive_jobq::builder::NodeGuid> {
|
|
let create_identity = b.node(SwarmNodeKind::CreateIdentity {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// A second, independent root: a forge user needs neither an authelia
|
|
// subject nor an existing repo, so it does not chain off
|
|
// `create_identity` (see the doc comment above).
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// An archetype agent has no config repo: its hive builds it from the
|
|
// archetype's module, so there is no repo to create, share or seed.
|
|
let init_config = (!archetype).then(|| {
|
|
let create_repo = b
|
|
.node(SwarmNodeKind::CreateRepo {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_identity);
|
|
// `AddRepoMember` needs both parents: the repo to add a collaborator
|
|
// to, and the forge user to add as one — adding a nonexistent user is
|
|
// a Forgejo validation error, not an idempotent no-op.
|
|
// `InitAgentConfigRepo` needs only the repo — see the doc comment
|
|
// above for why.
|
|
let _add_repo_member = b
|
|
.node(SwarmNodeKind::AddRepoMember {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_repo)
|
|
.after_ok(create_forge_user);
|
|
b.node(SwarmNodeKind::InitAgentConfigRepo {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_repo)
|
|
.guid()
|
|
});
|
|
// Read on `internal/docs`, a repo that exists whatever this agent is, so
|
|
// it needs only the user. Not a parent of the deploy: the agent reads docs
|
|
// once running, and a failed grant is retried by `forge::docs_grant::spawn`.
|
|
let _grant_docs = b
|
|
.node(SwarmNodeKind::GrantDocsRead {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_forge_user);
|
|
// Declared before the deploy trigger so the pause is visible in the
|
|
// wanted-state store before the hive brings the container up — see the
|
|
// node's own doc comment for why "before", not just "eventually". Needs
|
|
// no parent of its own: the declaration is written from the hive/agent
|
|
// names known at request time, same as `create_forge_user`.
|
|
let set_wanted = b.node(SwarmNodeKind::SetAgentWanted {
|
|
hive: hive.to_owned(),
|
|
agent: agent.to_owned(),
|
|
});
|
|
// The agent's own identity at the swarm secret store, minted and
|
|
// published at swarm level so no hive ever needs the capability to mint
|
|
// one. After `create_identity` because the certificate names a principal
|
|
// the swarm has agreed exists — not because anything in the store reads
|
|
// authelia. After `set_wanted` because the node re-reads the declaration
|
|
// and writes nothing for an agent declared nowhere or `Destroyed`.
|
|
let mint_identity = b
|
|
.node(SwarmNodeKind::MintAgentIdentity {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_identity)
|
|
.after_ok(set_wanted);
|
|
// The agent's own forge token, in the store before the hive deploys
|
|
// the container that pulls it. After the forge user, because the token
|
|
// is minted for that user.
|
|
let mint_forge_token = b
|
|
.node(SwarmNodeKind::MintAgentForgeToken {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_forge_user);
|
|
// The agent's account on the swarm's homeserver. A root of its own:
|
|
// creating it needs neither an authelia subject nor a forge user nor a
|
|
// repo, and chaining it behind one of those would make an unrelated
|
|
// failure look like a matrix failure.
|
|
let mint_matrix = b.node(SwarmNodeKind::MintAgentMatrixAccount {
|
|
agent: agent.to_owned(),
|
|
});
|
|
// The hive builds the agent from `internal/meta` only, so the deploy needs
|
|
// the update and the swarm's build from it to succeed (`after_ok`): after a
|
|
// failed update (no builder, an unreadable config repo) or a red build, the
|
|
// hive's build could only fail.
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta)
|
|
.after_ok(set_wanted);
|
|
// Last, and specifically after the config repo is seeded: the hive
|
|
// deploys by reading that repo, so a deploy asked for any earlier would
|
|
// find nothing to build. This is the edge that makes creating an agent at
|
|
// swarm level actually put it on a hive, rather than leaving a
|
|
// provisioned name nobody runs. An archetype agent has no repo, so its
|
|
// deploy waits on the mints and the declaration alone.
|
|
//
|
|
// `after_any` on the mint, not `after_ok`: a hive cannot pass down a
|
|
// certificate the swarm has not published, so the deploy must not
|
|
// overtake the mint — but a host whose store or agent PKI is not set up
|
|
// must still create agents. `after_ok` there would turn a store outage
|
|
// into an agent nobody runs.
|
|
//
|
|
// The forge-token mint gets `after_any` for the same reason: a host with
|
|
// no forge or no store must still create agents; only that node fails,
|
|
// by name, and the agent's container fetches the token on its next
|
|
// timer tick once one is minted.
|
|
//
|
|
// `after_ok` on the declaration, though: an agent deployed without its
|
|
// pause landing first is the race this node exists to close, so a
|
|
// declaration that did not land must not be deployed past. A host with no
|
|
// queue at all fails this node — but `TriggerDeploy` has nothing to
|
|
// publish to on such a host either, so the strict edge cancels a node
|
|
// that could not have succeeded anyway.
|
|
let trigger_deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()))
|
|
.after_any(mint_identity)
|
|
.after_any(mint_forge_token)
|
|
// `after_any` for the same reason: the container reads this token
|
|
// rather than producing it, so the deploy must not overtake the mint —
|
|
// but a host with no homeserver configured must still create agents,
|
|
// and only this node fails, by name.
|
|
.after_any(mint_matrix)
|
|
.after_ok(set_wanted);
|
|
if let Some(init_config) = init_config {
|
|
let _ = trigger_deploy.after_ok(init_config);
|
|
let _ = update_meta.after_ok(init_config);
|
|
}
|
|
vec![create_identity.guid()]
|
|
}
|
|
|
|
/// The sub-DAG turning the archetype agent `agent` into a custom one: its
|
|
/// config repo is created and seeded, and only then is its archetype key
|
|
/// deleted, so a failed step leaves the agent archetype-built. The meta
|
|
/// update and the deploy follow the delete (`after_ok`), as in
|
|
/// [`merged_deploy_job`]. Returns the repo node, the handle the response
|
|
/// reports.
|
|
///
|
|
/// `CreateRepo`, `CreateForgeUser` and `AddRepoMember` are no-ops for what
|
|
/// already exists, so the same graph can be queued again after a failure.
|
|
///
|
|
/// With `archetype` `None` the agent is already custom: its key was deleted and
|
|
/// a later step failed, so the graph is the meta update and the deploy alone,
|
|
/// and the handle is the update. A build that fails after the key delete leaves
|
|
/// the agent custom, which is how that failure is retried.
|
|
pub(crate) fn declare_custom_job(
|
|
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
|
agent: &str,
|
|
hive: &str,
|
|
archetype: Option<&swarm_wire::agent_archetype::ArchetypeRef>,
|
|
) -> Vec<hive_jobq::builder::NodeGuid> {
|
|
let Some(archetype) = archetype else {
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta);
|
|
let _deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()));
|
|
return vec![update_meta.guid()];
|
|
};
|
|
let create_repo = b.node(SwarmNodeKind::CreateRepo {
|
|
agent: agent.to_owned(),
|
|
});
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.to_owned(),
|
|
});
|
|
let add_repo_member = b
|
|
.node(SwarmNodeKind::AddRepoMember {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(create_repo)
|
|
.after_ok(create_forge_user);
|
|
let seed = b
|
|
.node(SwarmNodeKind::SeedArchetypeConfig {
|
|
agent: agent.to_owned(),
|
|
archetype: archetype.clone(),
|
|
})
|
|
.after_ok(create_repo);
|
|
let drop_archetype = b
|
|
.node(SwarmNodeKind::DropArchetype {
|
|
agent: agent.to_owned(),
|
|
})
|
|
.after_ok(seed)
|
|
.after_ok(add_repo_member);
|
|
let update_meta = b
|
|
.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta)
|
|
.after_ok(drop_archetype);
|
|
let _deploy = gated_deploy(b, hive, agent, None, None, Some(update_meta.guid()));
|
|
vec![create_repo.guid()]
|
|
}
|
|
|
|
/// Insert one job per agent: `CreateForgeUser`, then `MintAgentForgeToken`
|
|
/// `after_ok` it — the same edge [`declare_agent_job`] uses. Returns the mint
|
|
/// nodes' ids.
|
|
///
|
|
/// The one place that node is queued outside agent creation: both the manual
|
|
/// route below and `forge::agent_token::spawn`'s periodic pass come through
|
|
/// here, so a backfilled mint is the same node a new agent gets. The user
|
|
/// node is what lets the pass reach an agent that has a store identity but
|
|
/// no forge account; for one that has an account it is a no-op, because
|
|
/// `forge::Client::ensure_agent_user` folds "already exists" into success.
|
|
pub(crate) fn queue_forge_token_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.clone(),
|
|
});
|
|
vec![
|
|
b.node(SwarmNodeKind::MintAgentForgeToken { agent })
|
|
.after_ok(create_forge_user)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one job per agent: `CreateForgeUser`, then `GrantDocsRead` `after_ok`
|
|
/// it — the same edge [`declare_agent_job`] uses. Returns the grant nodes' ids.
|
|
/// `forge::docs_grant::spawn`'s periodic pass comes through here, so a
|
|
/// backfilled grant is the same node a new agent gets; the user node is a
|
|
/// no-op for an agent that has an account.
|
|
pub(crate) fn queue_docs_grants(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
|
agent: agent.clone(),
|
|
});
|
|
vec![
|
|
b.node(SwarmNodeKind::GrantDocsRead { agent })
|
|
.after_ok(create_forge_user)
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintAgentMatrixAccount` job per agent and return the nodes'
|
|
/// ids. `matrix_account::agent_token::spawn`'s periodic pass comes through
|
|
/// here, so a backfilled mint is the same node a new agent gets.
|
|
pub(crate) fn queue_matrix_account_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
agents: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(agents.len());
|
|
for agent in agents {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::MintAgentMatrixAccount { agent })
|
|
.guid(),
|
|
]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintHiveSenderToken` job per hive and return the nodes' ids.
|
|
/// `matrix_account::hive_sender::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_hive_sender_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
hives: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(hives.len());
|
|
for hive in hives {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintHiveSenderToken { hive }).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintHiveForgeToken` job per hive and return the nodes' ids.
|
|
/// `forge::hive_token::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_hive_forge_token_mints(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
hives: Vec<String>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(hives.len());
|
|
for hive in hives {
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintHiveForgeToken { hive }).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
/// Insert one `MintClassifierForgeToken` job and return its node's id.
|
|
/// `forge::classifier_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_classifier_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintClassifierForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// Insert one `MintBuilderForgeToken` job and return its node's id.
|
|
/// `forge::builder_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_builder_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintBuilderForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// How often [`spawn_meta_update_timer`] looks at the last `UpdateMeta`.
|
|
const META_UPDATE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(10);
|
|
|
|
/// Look at the last `UpdateMeta` now and every [`META_UPDATE_INTERVAL`] after,
|
|
/// and queue one when none has run yet or the latest did not succeed, so a
|
|
/// boot run that failed or hung is retried. The first tick is the boot run.
|
|
/// Once a run has succeeded a tick adds nothing: the graph keeps every node,
|
|
/// so a clock-driven node per tick would grow it without bound.
|
|
pub(crate) fn spawn_meta_update_timer(
|
|
sched: Arc<Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>>,
|
|
) {
|
|
tokio::spawn(async move {
|
|
let mut ticker = tokio::time::interval(META_UPDATE_INTERVAL);
|
|
loop {
|
|
ticker.tick().await;
|
|
if let Err(e) = queue_meta_update(&sched) {
|
|
tracing::warn!(error = %format!("{e:#}"), "update_meta: queueing failed");
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Why a deploy is `Skipped` and the timer should declare it again.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum SkipCause {
|
|
/// It waited on an `UpdateMeta` that `Failed`.
|
|
FailedUpdate,
|
|
/// It waited on a `BuildMeta` that `Failed`, whatever the reason.
|
|
FailedBuild,
|
|
}
|
|
|
|
/// A deploy [`skipped_deploys`] found.
|
|
struct SkippedDeploy {
|
|
hive: String,
|
|
agent: String,
|
|
rev: Option<String>,
|
|
/// The merged config PR the failed build belonged to, so the build
|
|
/// declared again reports on it too.
|
|
pr: Option<u64>,
|
|
cause: SkipCause,
|
|
}
|
|
|
|
/// The deploys the timer declares again: for each agent, its latest
|
|
/// `TriggerDeploy` when the scheduler skipped it because an `UpdateMeta` or a
|
|
/// `BuildMeta` it waited on `Failed`. A failed build is built again on every
|
|
/// round until one succeeds. A deploy skipped for another reason is left alone:
|
|
/// the pause declaration or the config seeding failing means the update that
|
|
/// would unblock the deploy was skipped too, and deploying past those is the
|
|
/// race those edges close.
|
|
fn skipped_deploys(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
) -> Vec<SkippedDeploy> {
|
|
let mut latest = std::collections::BTreeMap::new();
|
|
for n in graph.nodes() {
|
|
if let SwarmNodeKind::TriggerDeploy { hive, agent, .. } = &n.payload {
|
|
let key = (hive.clone(), agent.clone());
|
|
if latest
|
|
.get(&key)
|
|
.is_none_or(|l: &&hive_jobq::Node<_, _>| l.id < n.id)
|
|
{
|
|
latest.insert(key, n);
|
|
}
|
|
}
|
|
}
|
|
latest
|
|
.into_values()
|
|
.filter_map(|n| {
|
|
let SwarmNodeKind::TriggerDeploy { hive, agent, rev } = &n.payload else {
|
|
return None;
|
|
};
|
|
let dep_nodes = || {
|
|
n.deps.iter().filter_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, .. } => graph.node(*id),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
})
|
|
};
|
|
let failed = |is: fn(&SwarmNodeKind) -> bool| {
|
|
dep_nodes().any(|dep| is(&dep.payload) && dep.state == hive_jobq::State::Failed)
|
|
};
|
|
let cause = if failed(|p| matches!(p, SwarmNodeKind::UpdateMeta)) {
|
|
SkipCause::FailedUpdate
|
|
} else if failed(|p| matches!(p, SwarmNodeKind::BuildMeta { .. })) {
|
|
SkipCause::FailedBuild
|
|
} else {
|
|
return None;
|
|
};
|
|
let pr = dep_nodes().find_map(|dep| match &dep.payload {
|
|
SwarmNodeKind::BuildMeta { pr, .. } => *pr,
|
|
_ => None,
|
|
});
|
|
Some(SkippedDeploy {
|
|
hive: hive.clone(),
|
|
agent: agent.clone(),
|
|
rev: rev.clone(),
|
|
pr,
|
|
cause,
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Insert one job when the latest `UpdateMeta`, by node id, is not `Done` (or
|
|
/// none exists) and none is in flight: an `UpdateMeta`, with a fresh build and
|
|
/// `TriggerDeploy` behind it ([`gated_deploy`]) for every deploy the failed
|
|
/// run skipped ([`skipped_deploys`]), so an agent created or merged while the
|
|
/// update was failing is deployed by the run that succeeds.
|
|
///
|
|
/// When the latest `UpdateMeta` is `Done`, a deploy skipped by a failed build
|
|
/// still gets a fresh build and deploy, with no update, so a build that failed
|
|
/// is tried again on every round until it succeeds. That adds two nodes per
|
|
/// failing agent on every tick. Returns whether a job was inserted.
|
|
fn queue_meta_update(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<bool> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut update_metas = sched
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
.collect::<Vec<_>>();
|
|
if update_metas.iter().any(|n| {
|
|
matches!(
|
|
n.state,
|
|
hive_jobq::State::Pending | hive_jobq::State::Running | hive_jobq::State::Finishing
|
|
)
|
|
}) {
|
|
return Ok(false);
|
|
}
|
|
update_metas.sort_by_key(|n| n.id);
|
|
let update_due = update_metas
|
|
.last()
|
|
.is_none_or(|n| n.state != hive_jobq::State::Done);
|
|
let skipped: Vec<SkippedDeploy> = skipped_deploys(sched.graph())
|
|
.into_iter()
|
|
.filter(|s| update_due || s.cause == SkipCause::FailedBuild)
|
|
.collect();
|
|
if !update_due && skipped.is_empty() {
|
|
return Ok(false);
|
|
}
|
|
sched
|
|
.insert_job(None, |b| {
|
|
let update = update_due.then(|| {
|
|
b.node(SwarmNodeKind::UpdateMeta)
|
|
.needs(SwarmResourceKind::Meta)
|
|
.guid()
|
|
});
|
|
let deploys: Vec<_> = skipped
|
|
.into_iter()
|
|
.map(|s| gated_deploy(b, &s.hive, &s.agent, s.rev, s.pr, update).guid())
|
|
.collect();
|
|
update.map_or(deploys, |update| vec![update])
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
Ok(true)
|
|
}
|
|
|
|
/// [`MetaDeps`] from the environment, read on first use; `None`, logged, when
|
|
/// the settings are unset or do not parse, which `UpdateMeta` then fails on
|
|
/// by name.
|
|
fn meta_deps() -> Option<&'static MetaDeps> {
|
|
static DEPS: std::sync::OnceLock<Option<MetaDeps>> = std::sync::OnceLock::new();
|
|
DEPS.get_or_init(read_meta_deps).as_ref()
|
|
}
|
|
|
|
fn read_meta_deps() -> Option<MetaDeps> {
|
|
let settings = match meta_flake::Settings::from_env() {
|
|
Ok(Some(s)) => s,
|
|
Ok(None) => {
|
|
tracing::warn!(
|
|
"{} unset; internal/meta will not be written",
|
|
meta_flake::SETTINGS_ENV
|
|
);
|
|
return None;
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(error = %format!("{e:#}"), "internal/meta will not be written");
|
|
return None;
|
|
}
|
|
};
|
|
Some(MetaDeps {
|
|
settings,
|
|
hives: crate::startup::load_hives()
|
|
.into_iter()
|
|
.map(|h| h.name)
|
|
.collect(),
|
|
})
|
|
}
|
|
|
|
/// Insert one `MintCiMetricsForgeToken` job and return its node's id.
|
|
/// `forge::ci_metrics_account::spawn`'s periodic pass is the one caller.
|
|
pub(crate) fn queue_ci_metrics_forge_token_mint(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.insert_job(None, |b| {
|
|
vec![b.node(SwarmNodeKind::MintCiMetricsForgeToken).guid()]
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))
|
|
}
|
|
|
|
/// Whether a `MintAgentIdentity` for `agent` is queued or running.
|
|
fn identity_mint_in_flight(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
agent: &str,
|
|
) -> bool {
|
|
graph.nodes().any(|n| {
|
|
!n.state.is_terminal()
|
|
&& matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
|
|
})
|
|
}
|
|
|
|
/// Insert one job per renewal and return the ids of its nodes:
|
|
/// `MintAgentIdentity` for a certificate, `RenewAgentQueueCredential` for a
|
|
/// queue secret, the second `after_any` the first when an agent needs both.
|
|
/// `agent_renewal::spawn`'s periodic pass comes through here.
|
|
///
|
|
/// An agent whose `MintAgentIdentity` has not finished (agent creation's, or an
|
|
/// earlier pass's) is skipped whole until the next pass: a mint issues a new
|
|
/// leaf each run, and a pass comes every five minutes whether or not the last
|
|
/// mint has run yet. Its secret renewal waits too, since the chain below
|
|
/// cannot reach a mint it did not queue.
|
|
///
|
|
/// Chained rather than parallel because `mint_and_verify` reads the queue
|
|
/// secret back and compares it with the one it read first; a re-mint landing
|
|
/// in between fails that node. `after_any`, so a certificate that could not be
|
|
/// re-issued does not hold up the secret.
|
|
pub(crate) fn queue_agent_renewals(
|
|
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
renewals: Vec<agent_renewal::Renewal>,
|
|
) -> Result<Vec<hive_jobq::NodeId>> {
|
|
let mut sched = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut ids = Vec::with_capacity(renewals.len());
|
|
for renewal in renewals {
|
|
if renewal.cert && identity_mint_in_flight(sched.graph(), &renewal.agent) {
|
|
tracing::info!(
|
|
agent = %renewal.agent,
|
|
"agent identity mint already in flight; not queueing another"
|
|
);
|
|
continue;
|
|
}
|
|
let queued = sched
|
|
.insert_job(None, |b| {
|
|
let mut asked = Vec::new();
|
|
let cert = renewal.cert.then(|| {
|
|
b.node(SwarmNodeKind::MintAgentIdentity {
|
|
agent: renewal.agent.clone(),
|
|
})
|
|
.guid()
|
|
});
|
|
asked.extend(cert);
|
|
if renewal.secret {
|
|
let secret = b.node(SwarmNodeKind::RenewAgentQueueCredential {
|
|
agent: renewal.agent.clone(),
|
|
});
|
|
let secret = match cert {
|
|
Some(cert) => secret.after_any(cert),
|
|
None => secret,
|
|
};
|
|
asked.push(secret.guid());
|
|
}
|
|
asked
|
|
})
|
|
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
|
ids.extend(queued);
|
|
}
|
|
Ok(ids)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::routes::tests::state_with_roster_queued;
|
|
|
|
pub(crate) fn changed(agent: &str, rev: &str, pr: u64) -> crate::config_pr::ChangedConfigPr {
|
|
crate::config_pr::ChangedConfigPr {
|
|
agent: agent.to_owned(),
|
|
rev: rev.to_owned(),
|
|
pr,
|
|
}
|
|
}
|
|
|
|
fn build_state(
|
|
sched: &hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
|
|
rev: &str,
|
|
) -> hive_jobq::State {
|
|
sched
|
|
.graph()
|
|
.nodes()
|
|
.find(|n| matches!(&n.payload, SwarmNodeKind::BuildAgent { rev: r, .. } if r == rev))
|
|
.map(|n| n.state)
|
|
.expect("a build of that rev")
|
|
}
|
|
|
|
/// A newer head of the same PR replaces its pending build; another PR's
|
|
/// build, and the same PR's running build, are left alone.
|
|
#[tokio::test]
|
|
async fn a_newer_push_replaces_the_prs_pending_build_only() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
let queue = |c| super::queue_build(&mut sched.lock().unwrap(), c).expect("queued");
|
|
assert_eq!(queue(changed("atlas", "r1", 3)), 0);
|
|
// r1 takes the builder and stays running: its future is never awaited.
|
|
let _running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
.expect("r1 is runnable");
|
|
assert_eq!(
|
|
queue(changed("atlas", "r2", 3)),
|
|
0,
|
|
"r1 is running, not pending"
|
|
);
|
|
assert_eq!(queue(changed("argus", "o1", 4)), 0);
|
|
assert_eq!(queue(changed("atlas", "r3", 3)), 1, "r2 was pending");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
assert_eq!(build_state(&guard, "r1"), hive_jobq::State::Running);
|
|
assert_eq!(build_state(&guard, "r2"), hive_jobq::State::Cancelled);
|
|
assert_eq!(build_state(&guard, "r3"), hive_jobq::State::Pending);
|
|
assert_eq!(build_state(&guard, "o1"), hive_jobq::State::Pending);
|
|
}
|
|
|
|
/// Builds share one builder slot: a second build is not claimable while
|
|
/// the first runs, and is once it has finished.
|
|
#[tokio::test]
|
|
async fn one_build_runs_at_a_time() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
for c in [changed("atlas", "a1", 1), changed("argus", "b1", 2)] {
|
|
super::queue_build(&mut sched.lock().unwrap(), c).expect("queued");
|
|
}
|
|
let run = |sched| {
|
|
hive_jobq::scheduler::Scheduler::claim_next(sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
};
|
|
let first = run(&sched).expect("one build is runnable");
|
|
assert!(
|
|
run(&sched).is_none(),
|
|
"a second build started beside the first"
|
|
);
|
|
first.await.1.expect("no growth");
|
|
run(&sched)
|
|
.expect("the second build runs once the first is done")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
}
|
|
|
|
/// Every node whose first request goes to the forge, run against a forge
|
|
/// that accepts the connection and never answers: each ends `Failed`
|
|
/// with an error naming the node instead of holding its executor slot.
|
|
/// `MintAgentForgeToken` is not in the list: it logs in to the secret
|
|
/// store before its first forge request, and no test here has a store.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_forge_node_fails_when_the_forge_never_answers() {
|
|
// Bound but never accepted: the kernel completes the handshake and
|
|
// buffers the request, and no response ever comes.
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap();
|
|
let forge = super::Late::<std::sync::Arc<crate::forge::Client>>::default();
|
|
assert!(
|
|
forge
|
|
.set(std::sync::Arc::new(crate::forge::Client::for_test_forge(
|
|
url
|
|
)))
|
|
.is_ok()
|
|
);
|
|
let agent = || "atlas".to_owned();
|
|
let nodes = [
|
|
("create_repo", SwarmNodeKind::CreateRepo { agent: agent() }),
|
|
(
|
|
"create_forge_user",
|
|
SwarmNodeKind::CreateForgeUser { agent: agent() },
|
|
),
|
|
(
|
|
"add_repo_member",
|
|
SwarmNodeKind::AddRepoMember { agent: agent() },
|
|
),
|
|
(
|
|
"grant_docs_read",
|
|
SwarmNodeKind::GrantDocsRead { agent: agent() },
|
|
),
|
|
(
|
|
"init_agent_config_repo",
|
|
SwarmNodeKind::InitAgentConfigRepo { agent: agent() },
|
|
),
|
|
(
|
|
"mint_hive_forge_token",
|
|
SwarmNodeKind::MintHiveForgeToken {
|
|
hive: "pr1ma".to_owned(),
|
|
},
|
|
),
|
|
(
|
|
"mint_classifier_forge_token",
|
|
SwarmNodeKind::MintClassifierForgeToken,
|
|
),
|
|
(
|
|
"mint_builder_forge_token",
|
|
SwarmNodeKind::MintBuilderForgeToken,
|
|
),
|
|
(
|
|
"mint_ci_metrics_forge_token",
|
|
SwarmNodeKind::MintCiMetricsForgeToken,
|
|
),
|
|
];
|
|
for (label, kind) in nodes {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched.append(kind, Vec::new(), None).expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: forge.clone(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
tokio::time::timeout(std::time::Duration::from_mins(10), runner)
|
|
.await
|
|
.unwrap_or_else(|_| panic!("{label}: node still running after 10 minutes"))
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed, "{label}");
|
|
let error = node.error.as_deref().unwrap_or_default();
|
|
assert!(
|
|
error.contains(label) && error.contains("no answer from the forge"),
|
|
"{label}: expected a deadline error naming the node, got {error:?}"
|
|
);
|
|
}
|
|
drop(listener);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() {
|
|
unsafe {
|
|
std::env::remove_var("SWARM_CONTROLLER_FORGE_URL");
|
|
}
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::CreateRepo {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("forge"),
|
|
"expected a forge-not-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// Sibling of the `CreateRepo` test above, same shape: drives
|
|
/// `SwarmNodeKind::CreateForgeUser` through the real scheduler with no
|
|
/// forge configured, asserting the graceful-absence-is-failure branch —
|
|
/// this node's happy path needs a live forge, same caveat as
|
|
/// `CreateRepo`'s.
|
|
#[tokio::test]
|
|
async fn create_forge_user_node_runs_end_to_end_and_fails_without_forge_configured() {
|
|
unsafe {
|
|
std::env::remove_var("SWARM_CONTROLLER_FORGE_URL");
|
|
}
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::CreateForgeUser {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("forge"),
|
|
"expected a forge-not-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// Third sibling of the two above, and the same deliberate caveat: with
|
|
/// no agent PKI named this reaches only the graceful-absence-is-failure
|
|
/// branch, before any network call. The happy path is a live store and a
|
|
/// real login, which is exactly why it is `mint_and_verify`'s own job to
|
|
/// prove it at agent-creation time rather than a unit test's.
|
|
///
|
|
/// What this does pin is the degrade: a host whose environment does not
|
|
/// name the agent PKI fails this one node with a reason that names the
|
|
/// variable, and creates the agent anyway.
|
|
#[tokio::test]
|
|
async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_the_agent_pki_named() {
|
|
let pki = [
|
|
crate::agent_identity::ENV_AGENT_PKI_MOUNT,
|
|
crate::agent_identity::ENV_AGENT_PKI_ROLE,
|
|
];
|
|
assert!(
|
|
pki.iter().all(|var| std::env::var_os(var).is_none()),
|
|
"{} and {} must be unset: with them set this test would reach a live store",
|
|
pki[0],
|
|
pki[1]
|
|
);
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::MintAgentIdentity {
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
let error = node.error.as_deref().unwrap_or_default();
|
|
assert!(
|
|
error.contains(crate::agent_identity::ENV_AGENT_PKI_MOUNT),
|
|
"the reason must name the variable an operator has to set, got {error:?}"
|
|
);
|
|
}
|
|
|
|
/// Fourth sibling, same shape and the same caveat once more: with no
|
|
/// queue configured this reaches only the graceful-absence-is-failure
|
|
/// branch. Writing the declaration for real needs a `JetStream` server,
|
|
/// which no test in this crate has — `wanted::apply`'s own unit tests
|
|
/// cover what gets written, and [`a_new_agent_is_declared_paused`] below
|
|
/// pins the state this node asks for.
|
|
#[tokio::test]
|
|
async fn set_agent_wanted_node_runs_end_to_end_and_fails_without_a_queue() {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
let id = sched
|
|
.append(
|
|
SwarmNodeKind::SetAgentWanted {
|
|
hive: "pr1ma".to_owned(),
|
|
agent: "atlas".to_owned(),
|
|
},
|
|
Vec::new(),
|
|
None,
|
|
)
|
|
.expect("insert");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: std::sync::Arc::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
let runner =
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
runner
|
|
.await
|
|
.1
|
|
.expect("no growth declared, nothing to reject");
|
|
|
|
let guard = sched.lock().unwrap();
|
|
let node = guard.graph().node(id).expect("node still present");
|
|
assert_eq!(node.state, hive_jobq::State::Failed);
|
|
assert!(
|
|
node.error.as_deref().unwrap_or_default().contains("queue"),
|
|
"expected a no-queue-configured error, got {:?}",
|
|
node.error
|
|
);
|
|
}
|
|
|
|
/// The whole point of the node: a freshly created agent is declared
|
|
/// `Paused`, so it does not start driving turns the moment its hive
|
|
/// brings it up. Any other state here is the bug this fixes.
|
|
#[test]
|
|
fn a_new_agent_is_declared_paused() {
|
|
assert_eq!(
|
|
super::NEW_AGENT_WANTED_STATE,
|
|
swarm_wire::wanted::AgentState::Paused
|
|
);
|
|
}
|
|
|
|
/// Same reasoning as `a_mint_node_renders_both_the_agent_and_the_hive`:
|
|
/// this variant carries a hive too, and has to have joined `data`'s
|
|
/// two-field or-pattern rather than the agent-only one.
|
|
#[test]
|
|
fn a_set_wanted_node_renders_both_the_agent_and_the_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::SetAgentWanted {
|
|
hive: "pr1ma".to_owned(),
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "set_agent_wanted");
|
|
let data = kind.data(1);
|
|
assert_eq!(data["agent"], "atlas");
|
|
assert_eq!(data["hive"], "pr1ma");
|
|
}
|
|
|
|
/// The ordering the fix depends on: the pause has to be in the
|
|
/// wanted-state store *before* the hive is told to deploy, or a freshly
|
|
/// created agent runs undeclared for the window between the two.
|
|
///
|
|
/// Asserted on the graph `create_agent` builds, same as the mint edge's
|
|
/// test above and for the same reason — the edge is one line in a
|
|
/// builder closure whose absence changes nothing observable until a real
|
|
/// agent boots and starts taking turns nobody asked for.
|
|
#[tokio::test]
|
|
async fn the_deploy_waits_for_the_pause_to_be_declared() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let set_wanted = id_of("set_agent_wanted");
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
|
|
let when = deploy
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == set_wanted => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the declaration");
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a declaration that never landed must not be deployed past; this \
|
|
edge has to be `after_ok`, not `after_any`"
|
|
);
|
|
}
|
|
|
|
/// Whether the edge from the `from` node to the `on` node accepts a
|
|
/// failed `on`, in a graph built by `declare`.
|
|
fn edge_accepts_failure(
|
|
declare: impl FnOnce(&hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>),
|
|
from: &str,
|
|
on: &str,
|
|
) -> bool {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
let graph = sched.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let (from, on) = (id_of(from), id_of(on));
|
|
graph
|
|
.node(from)
|
|
.expect("just found")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == on => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the edge exists")
|
|
.accepts(hive_jobq::TerminalState::Failed)
|
|
}
|
|
|
|
type TestBuilder = hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>;
|
|
|
|
fn create_dag(b: &TestBuilder) {
|
|
let _ = super::declare_agent_job(b, "atlas", "pr1ma", false);
|
|
}
|
|
|
|
fn merged_dag(b: &TestBuilder) {
|
|
let _ = super::merged_deploy_job(b, "pr1ma", "atlas", "abc", Some(7));
|
|
}
|
|
|
|
fn custom_dag(b: &TestBuilder) {
|
|
let archetype = swarm_wire::agent_archetype::ArchetypeRef {
|
|
source: "hyperhive".to_owned(),
|
|
name: "reviewer".to_owned(),
|
|
};
|
|
let _ = super::declare_custom_job(b, "atlas", "pr1ma", Some(&archetype));
|
|
}
|
|
|
|
/// Turn into custom: with every node succeeding the agent is seeded, loses
|
|
/// its key, and is deployed.
|
|
#[tokio::test]
|
|
async fn a_conversion_seeds_then_drops_the_key_then_deploys() {
|
|
use hive_jobq::State;
|
|
|
|
let done = run_dag_failing(custom_dag, None).await;
|
|
for label in [
|
|
"create_repo",
|
|
"seed_archetype_config",
|
|
"drop_archetype",
|
|
"update_meta",
|
|
"trigger_deploy",
|
|
] {
|
|
assert_eq!(state_of(&done, label), State::Done, "{label}");
|
|
}
|
|
// The key is deleted only after the seed and the member grant, and
|
|
// each edge refuses a failure. The deploy edge is the control that
|
|
// the lookup finds an `after_ok` edge in this graph.
|
|
for (from, on) in [
|
|
("drop_archetype", "seed_archetype_config"),
|
|
("drop_archetype", "add_repo_member"),
|
|
("update_meta", "drop_archetype"),
|
|
("trigger_deploy", "update_meta"),
|
|
] {
|
|
assert!(
|
|
!edge_accepts_failure(custom_dag, from, on),
|
|
"{from} -> {on}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// The retry of a conversion whose build failed after the key was deleted:
|
|
/// with no archetype the graph is the meta update and the deploy, with no
|
|
/// repo, seed or key delete. The conversion is the control that those
|
|
/// nodes exist when there is one.
|
|
#[tokio::test]
|
|
async fn a_retry_of_a_custom_agent_is_only_the_update_and_the_deploy() {
|
|
use hive_jobq::State;
|
|
|
|
fn retry_dag(b: &TestBuilder) {
|
|
let _ = super::declare_custom_job(b, "atlas", "pr1ma", None);
|
|
}
|
|
let done = run_dag_failing(retry_dag, None).await;
|
|
let labels: Vec<&str> = done.iter().map(|(l, _)| l.as_str()).collect();
|
|
for absent in ["create_repo", "seed_archetype_config", "drop_archetype"] {
|
|
assert!(!labels.contains(&absent), "{absent} in {labels:?}");
|
|
}
|
|
assert_eq!(state_of(&done, "update_meta"), State::Done);
|
|
assert_eq!(state_of(&done, "trigger_deploy"), State::Done);
|
|
let converting = run_dag_failing(custom_dag, None).await;
|
|
assert!(converting.iter().any(|(l, _)| l == "drop_archetype"));
|
|
// A failed build leaves the deploy skipped, so a retry has work to do.
|
|
let red = run_dag_failing(retry_dag, Some("build_meta")).await;
|
|
assert_eq!(state_of(&red, "trigger_deploy"), State::Skipped);
|
|
}
|
|
|
|
/// A seed that fails leaves the key in place and nothing deployed; the
|
|
/// same graph, queued again with the seed succeeding, completes.
|
|
#[tokio::test]
|
|
async fn a_failed_seed_keeps_the_key_and_a_retry_completes() {
|
|
use hive_jobq::State;
|
|
|
|
let failed = run_dag_failing(custom_dag, Some("seed_archetype_config")).await;
|
|
assert_eq!(state_of(&failed, "seed_archetype_config"), State::Failed);
|
|
for label in ["drop_archetype", "update_meta", "trigger_deploy"] {
|
|
assert_eq!(state_of(&failed, label), State::Skipped, "{label}");
|
|
}
|
|
let retried = run_dag_failing(custom_dag, None).await;
|
|
assert_eq!(state_of(&retried, "drop_archetype"), State::Done);
|
|
assert_eq!(state_of(&retried, "trigger_deploy"), State::Done);
|
|
}
|
|
|
|
/// A deploy needs the meta update to succeed: its edge refuses a failed
|
|
/// update, in both DAGs that queue one.
|
|
#[test]
|
|
fn a_deploy_needs_the_meta_update_to_succeed() {
|
|
assert!(!edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"update_meta"
|
|
));
|
|
assert!(!edge_accepts_failure(
|
|
merged_dag,
|
|
"trigger_deploy",
|
|
"update_meta"
|
|
));
|
|
// Controls: the same lookup finds the edges that do accept a failure.
|
|
assert!(edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"mint_agent_identity"
|
|
));
|
|
assert!(!edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"set_agent_wanted"
|
|
));
|
|
}
|
|
|
|
/// A deploy needs the swarm's build of its agent to succeed: its edge
|
|
/// refuses a failed build, in both DAGs that declare an update. The mint
|
|
/// edge is the control that the lookup finds an edge that accepts one.
|
|
#[test]
|
|
fn a_deploy_needs_the_meta_build_to_succeed() {
|
|
for declare in [create_dag as fn(&TestBuilder), merged_dag] {
|
|
assert!(!edge_accepts_failure(
|
|
declare,
|
|
"trigger_deploy",
|
|
"build_meta"
|
|
));
|
|
assert!(!edge_accepts_failure(declare, "build_meta", "update_meta"));
|
|
}
|
|
assert!(edge_accepts_failure(
|
|
create_dag,
|
|
"trigger_deploy",
|
|
"mint_agent_identity"
|
|
));
|
|
}
|
|
|
|
/// Every `TriggerDeploy` in `graph` waits, strictly, on a `BuildMeta` of the
|
|
/// same agent that needs the builder. Panics, naming `path`, when one does
|
|
/// not; returns how many deploys it checked.
|
|
fn assert_every_deploy_is_gated(
|
|
graph: &hive_jobq::Graph<SwarmNodeKind, SwarmResourceKind>,
|
|
path: &str,
|
|
) -> usize {
|
|
let mut checked = 0;
|
|
for deploy in graph.nodes() {
|
|
let SwarmNodeKind::TriggerDeploy { agent, .. } = &deploy.payload else {
|
|
continue;
|
|
};
|
|
let gate = deploy.deps.iter().find_map(|d| {
|
|
match d {
|
|
hive_jobq::Dep::Node { id, when } => graph
|
|
.node(*id)
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent)
|
|
})
|
|
.map(|n| (n, *when)),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
}
|
|
});
|
|
let (build, when) =
|
|
gate.unwrap_or_else(|| panic!("{path}: {agent}'s deploy has no build"));
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"{path}: {agent}'s deploy runs after a failed build"
|
|
);
|
|
assert!(
|
|
build.deps.iter().any(|d| matches!(
|
|
d,
|
|
hive_jobq::Dep::Resource {
|
|
name: SwarmResourceKind::Builder,
|
|
..
|
|
}
|
|
)),
|
|
"{path}: {agent}'s build does not need the builder"
|
|
);
|
|
checked += 1;
|
|
}
|
|
checked
|
|
}
|
|
|
|
/// The four paths that send a deploy all declare the same build and deploy
|
|
/// pair, so a path that forgets the gate fails here.
|
|
#[tokio::test]
|
|
async fn all_four_deploy_paths_declare_the_gated_pair() {
|
|
let graph_of = |declare: fn(&TestBuilder)| {
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
sched
|
|
};
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(graph_of(merged_dag).graph(), "merge"),
|
|
1
|
|
);
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(graph_of(create_dag).graph(), "creation"),
|
|
1
|
|
);
|
|
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "fan-out"),
|
|
3
|
|
);
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
// bob's and atlas's first deploys and atlas's retry: all gated.
|
|
assert_eq!(
|
|
assert_every_deploy_is_gated(sched.lock().unwrap().graph(), "timer retry"),
|
|
3
|
|
);
|
|
}
|
|
|
|
/// With the swarm's build red, the deploy never runs, in both DAGs that
|
|
/// declare an update. The first run in each pair is the control, where the
|
|
/// build is green and the deploy does run.
|
|
#[tokio::test]
|
|
async fn a_red_build_means_no_deploy_runs() {
|
|
use hive_jobq::State;
|
|
|
|
for (name, declare) in [
|
|
("merged", merged_dag as fn(&TestBuilder)),
|
|
("create", create_dag as fn(&TestBuilder)),
|
|
] {
|
|
let green = run_dag_failing(declare, None).await;
|
|
assert_eq!(state_of(&green, "build_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&green, "trigger_deploy"), State::Done, "{name}");
|
|
|
|
let red = run_dag_failing(declare, Some("build_meta")).await;
|
|
assert_eq!(state_of(&red, "update_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&red, "build_meta"), State::Failed, "{name}");
|
|
assert_eq!(
|
|
state_of(&red, "trigger_deploy"),
|
|
State::Skipped,
|
|
"{name}: the deploy must not run after a red build"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Every node of the DAG `declare` builds, run to the end with the meta
|
|
/// update failing or succeeding and every other node succeeding: the
|
|
/// final state of each, by label.
|
|
async fn run_dag(
|
|
declare: impl FnOnce(&TestBuilder),
|
|
update_meta_fails: bool,
|
|
) -> Vec<(String, hive_jobq::State)> {
|
|
run_dag_failing(declare, update_meta_fails.then_some("update_meta")).await
|
|
}
|
|
|
|
/// [`run_dag`], with the nodes labelled `failing` failing.
|
|
async fn run_dag_failing(
|
|
declare: impl FnOnce(&TestBuilder),
|
|
failing: Option<&'static str>,
|
|
) -> Vec<(String, hive_jobq::State)> {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
declare(b);
|
|
Vec::new()
|
|
})
|
|
.expect("the graph inserts");
|
|
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
|
while let Some(runner) = Scheduler::claim_next(&sched, |_id, kind, builder| async move {
|
|
let outcome = if failing == Some(kind.label().as_str()) {
|
|
Outcome::Failed("no builder".to_owned())
|
|
} else {
|
|
Outcome::Done
|
|
};
|
|
(builder, outcome)
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
let guard = sched.lock().unwrap();
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| (n.payload.label(), n.state))
|
|
.collect()
|
|
}
|
|
|
|
fn state_of(states: &[(String, hive_jobq::State)], label: &str) -> hive_jobq::State {
|
|
states
|
|
.iter()
|
|
.find(|(l, _)| l == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.1
|
|
}
|
|
|
|
/// The scheduler's side of the same edge, for both DAGs that queue an
|
|
/// update: with it failed, the deploy never runs. The graph shows the
|
|
/// update `Failed` and the deploy `Skipped`; the first run in each pair is
|
|
/// the control, where the deploy does run.
|
|
#[tokio::test]
|
|
async fn a_failed_meta_update_means_no_deploy_runs() {
|
|
use hive_jobq::State;
|
|
|
|
for (name, declare) in [
|
|
("merged", merged_dag as fn(&TestBuilder)),
|
|
("create", create_dag as fn(&TestBuilder)),
|
|
] {
|
|
let ok = run_dag(declare, false).await;
|
|
assert_eq!(state_of(&ok, "update_meta"), State::Done, "{name}");
|
|
assert_eq!(state_of(&ok, "trigger_deploy"), State::Done, "{name}");
|
|
|
|
let failed = run_dag(declare, true).await;
|
|
assert_eq!(state_of(&failed, "update_meta"), State::Failed, "{name}");
|
|
assert_eq!(
|
|
state_of(&failed, "trigger_deploy"),
|
|
State::Skipped,
|
|
"{name}: the deploy must not run after a failed update"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A fan-out record held in memory; `unreadable` stands for a bucket that
|
|
/// cannot be opened.
|
|
#[derive(Default)]
|
|
struct FakePins {
|
|
pin: Mutex<Option<String>>,
|
|
unreadable: bool,
|
|
}
|
|
|
|
impl FakePins {
|
|
fn holding(pin: &str) -> Self {
|
|
Self {
|
|
pin: Mutex::new(Some(pin.to_owned())),
|
|
unreadable: false,
|
|
}
|
|
}
|
|
|
|
fn held(&self) -> Option<String> {
|
|
self.pin.lock().unwrap().clone()
|
|
}
|
|
}
|
|
|
|
impl meta_fanout::PinStore for FakePins {
|
|
async fn recorded(&self) -> Result<Option<String>> {
|
|
anyhow::ensure!(!self.unreadable, "the bucket did not open");
|
|
Ok(self.held())
|
|
}
|
|
|
|
async fn record(&self, pin: &str) -> Result<()> {
|
|
*self.pin.lock().unwrap() = Some(pin.to_owned());
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn report(pin: &str, moved: bool) -> meta_flake::Report {
|
|
meta_flake::Report {
|
|
updated: meta_flake::Updated::Committed,
|
|
pin: pin.to_owned(),
|
|
hyperhive_moved: moved,
|
|
}
|
|
}
|
|
|
|
type TestScheduler = hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>;
|
|
|
|
/// One `UpdateMeta` pass against `pins`, with every node it grows run: the
|
|
/// deploy to `failing_agent` fails, and `RecordFanout` dies before it
|
|
/// writes when `crash_before_record`. Returns what the pass decided and the
|
|
/// scheduler it ran in.
|
|
async fn pass(
|
|
pins: &Arc<FakePins>,
|
|
report: &meta_flake::Report,
|
|
placed: &[(&str, &str)],
|
|
failing_agent: Option<&'static str>,
|
|
crash_before_record: bool,
|
|
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
|
|
pass_with(
|
|
pins,
|
|
report,
|
|
placed,
|
|
failing_agent,
|
|
crash_before_record,
|
|
None,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// [`pass`], with the build of one agent failing with the given error when
|
|
/// `failing_build` names it.
|
|
async fn pass_with(
|
|
pins: &Arc<FakePins>,
|
|
report: &meta_flake::Report,
|
|
placed: &[(&str, &str)],
|
|
failing_agent: Option<&'static str>,
|
|
crash_before_record: bool,
|
|
failing_build: Option<(&'static str, String)>,
|
|
) -> (FanOut, Arc<Mutex<TestScheduler>>) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
use meta_fanout::PinStore as _;
|
|
|
|
let mut sched = Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.append(SwarmNodeKind::UpdateMeta, Vec::new(), None)
|
|
.expect("insert");
|
|
let sched = Arc::new(Mutex::new(sched));
|
|
let placements: std::collections::BTreeSet<(String, String)> = placed
|
|
.iter()
|
|
.map(|(h, a)| ((*h).to_owned(), (*a).to_owned()))
|
|
.collect();
|
|
let fan_out = decide_fan_out(report, pins.recorded().await);
|
|
let decided = fan_out.clone();
|
|
let update = Scheduler::claim_next(&sched, move |_id, _kind, builder| async move {
|
|
declare_fan_out(&builder, &fan_out, &placements);
|
|
(builder, Outcome::Done)
|
|
})
|
|
.expect("the node just inserted is runnable");
|
|
update.await.1.expect("the grown nodes insert");
|
|
while let Some(runner) = Scheduler::claim_next(&sched, {
|
|
let (pins, sched, failing_build) =
|
|
(Arc::clone(pins), Arc::clone(&sched), failing_build.clone());
|
|
move |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::TriggerDeploy { agent, .. }
|
|
if failing_agent == Some(agent.as_str()) =>
|
|
{
|
|
Outcome::Failed("the hive did not take it".to_owned())
|
|
}
|
|
SwarmNodeKind::BuildMeta { agent, .. }
|
|
if failing_build.as_ref().is_some_and(|(a, _)| *a == agent) =>
|
|
{
|
|
Outcome::Failed(failing_build.map(|(_, error)| error).expect("matched"))
|
|
}
|
|
SwarmNodeKind::RecordFanout { .. } if crash_before_record => {
|
|
Outcome::Failed("the controller died".to_owned())
|
|
}
|
|
SwarmNodeKind::RecordFanout { pin } => {
|
|
let deploys_done = sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
|
|
.all(|n| {
|
|
matches!(
|
|
n.state,
|
|
hive_jobq::State::Done | hive_jobq::State::Skipped
|
|
)
|
|
});
|
|
if deploys_done {
|
|
pins.record(&pin).await.expect("recorded");
|
|
Outcome::Done
|
|
} else {
|
|
Outcome::Failed("recorded before the deploys were settled".to_owned())
|
|
}
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
(decided, sched)
|
|
}
|
|
|
|
/// The states of the nodes labelled `label`.
|
|
fn states_of(sched: &Mutex<TestScheduler>, label: &str) -> Vec<hive_jobq::State> {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == label)
|
|
.map(|n| n.state)
|
|
.collect()
|
|
}
|
|
|
|
const PLACED: [(&str, &str); 3] = [("pr1ma", "atlas"), ("pr1ma", "argus"), ("sec0nd", "iris")];
|
|
|
|
/// A controller with no record sends every placed agent a deploy, whether
|
|
/// or not the pin moved, and records the pin once they are done.
|
|
#[tokio::test]
|
|
async fn a_pin_with_no_record_deploys_every_placed_agent_and_is_recorded() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (fan_out, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
assert_eq!(
|
|
states_of(&sched, "trigger_deploy"),
|
|
[hive_jobq::State::Done; 3]
|
|
);
|
|
assert_eq!(states_of(&sched, "record_fanout"), [hive_jobq::State::Done]);
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
}
|
|
|
|
/// The deploys declared are one per placement, to the hive that places it.
|
|
#[tokio::test]
|
|
async fn each_deploy_goes_to_the_hive_that_places_the_agent() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
let mut deploys: Vec<_> = sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::TriggerDeploy { hive, agent, rev } => {
|
|
Some((hive.clone(), agent.clone(), rev.clone()))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
deploys.sort();
|
|
assert_eq!(
|
|
deploys,
|
|
vec![
|
|
("pr1ma".to_owned(), "argus".to_owned(), None),
|
|
("pr1ma".to_owned(), "atlas".to_owned(), None),
|
|
("sec0nd".to_owned(), "iris".to_owned(), None),
|
|
]
|
|
);
|
|
}
|
|
|
|
/// The record, not whether the pin moved, decides: a pin already sent
|
|
/// sends nothing even on a pass that moved it, and a different one does.
|
|
#[tokio::test]
|
|
async fn a_pin_that_is_recorded_deploys_nothing() {
|
|
let pins = Arc::new(FakePins::holding("p1"));
|
|
let (fan_out, sched) = pass(&pins, &report("p1", true), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
assert!(states_of(&sched, "record_fanout").is_empty());
|
|
|
|
let (fan_out, sched) = pass(&pins, &report("p2", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p2".into() });
|
|
assert_eq!(states_of(&sched, "trigger_deploy").len(), 3);
|
|
assert_eq!(pins.held().as_deref(), Some("p2"));
|
|
}
|
|
|
|
/// A controller that dies after the commit and before the record is
|
|
/// written finds the commit unchanged on its next pass, and still sends.
|
|
#[tokio::test]
|
|
async fn a_pass_that_dies_before_the_record_is_written_sends_again() {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass(&pins, &report("p1", true), &PLACED, None, true).await;
|
|
assert_eq!(states_of(&sched, "trigger_deploy").len(), 3);
|
|
assert_eq!(pins.held().as_deref(), Some("p0"), "nothing was recorded");
|
|
|
|
// The restart: the commit is the render now, so nothing moved.
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
}
|
|
|
|
/// The record waits on every deploy of its pass to be settled: done, or
|
|
/// skipped by its build gate. A failed deploy still rules it out.
|
|
#[tokio::test]
|
|
async fn the_record_waits_on_every_deploy_being_done_or_skipped() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
let guard = sched.lock().unwrap();
|
|
let graph = guard.graph();
|
|
let record = graph
|
|
.nodes()
|
|
.find(|n| matches!(n.payload, SwarmNodeKind::RecordFanout { .. }))
|
|
.expect("a record node");
|
|
let mut waits: Vec<_> = record
|
|
.deps
|
|
.iter()
|
|
.map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } => (*id, *when),
|
|
hive_jobq::Dep::Resource { .. } => panic!("the record needs no resource"),
|
|
})
|
|
.collect();
|
|
waits.sort_by_key(|(id, _)| id.get());
|
|
let mut deploys: Vec<_> = graph
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::TriggerDeploy { .. }))
|
|
.map(|n| {
|
|
(
|
|
n.id,
|
|
hive_jobq::DepWhen::of(&[
|
|
hive_jobq::TerminalState::Done,
|
|
hive_jobq::TerminalState::Skipped,
|
|
]),
|
|
)
|
|
})
|
|
.collect();
|
|
deploys.sort_by_key(|(id, _)| id.get());
|
|
assert_eq!(deploys.len(), 3);
|
|
assert_eq!(waits, deploys);
|
|
}
|
|
|
|
/// One deploy failing keeps the pin unrecorded: the record is ruled out,
|
|
/// the others still run, and the next pass sends again.
|
|
#[tokio::test]
|
|
async fn a_failed_deploy_leaves_the_pin_unrecorded() {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass(&pins, &report("p1", true), &PLACED, Some("argus"), false).await;
|
|
let mut deploys = states_of(&sched, "trigger_deploy");
|
|
deploys.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(
|
|
deploys,
|
|
[
|
|
hive_jobq::State::Done,
|
|
hive_jobq::State::Done,
|
|
hive_jobq::State::Failed
|
|
]
|
|
);
|
|
assert_eq!(
|
|
states_of(&sched, "record_fanout"),
|
|
[hive_jobq::State::Skipped]
|
|
);
|
|
assert_eq!(pins.held().as_deref(), Some("p0"));
|
|
|
|
let (fan_out, _) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Record { pin: "p1".into() });
|
|
}
|
|
|
|
/// A failed build for one agent skips that agent's deploy only, and the pin
|
|
/// is still recorded, so the healthy agents are not sent again. The timer
|
|
/// declares a build and deploy for the failed agent alone, whatever the
|
|
/// build failed with: `nix`'s error, a timeout, or no verdict.
|
|
#[tokio::test]
|
|
async fn a_failed_build_skips_its_deploy_and_the_pin_is_still_recorded() {
|
|
use hive_jobq::State;
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
for error in [
|
|
"error: builder for argus failed".to_owned(),
|
|
"timed out: nix did not finish".to_owned(),
|
|
format!(
|
|
"{}no swarm builder is listening",
|
|
meta_build::NO_VERDICT_PREFIX
|
|
),
|
|
] {
|
|
let pins = Arc::new(FakePins::holding("p0"));
|
|
let (_, sched) = pass_with(
|
|
&pins,
|
|
&report("p1", true),
|
|
&PLACED,
|
|
None,
|
|
false,
|
|
Some(("argus", error.clone())),
|
|
)
|
|
.await;
|
|
let mut builds = states_of(&sched, "build_meta");
|
|
builds.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(builds, [State::Done, State::Done, State::Failed], "{error}");
|
|
let mut deploys = states_of(&sched, "trigger_deploy");
|
|
deploys.sort_by_key(|s| format!("{s:?}"));
|
|
assert_eq!(
|
|
deploys,
|
|
[State::Done, State::Done, State::Skipped],
|
|
"{error}"
|
|
);
|
|
assert_eq!(states_of(&sched, "record_fanout"), [State::Done], "{error}");
|
|
assert_eq!(pins.held().as_deref(), Some("p1"), "{error}");
|
|
|
|
assert!(queue_meta_update(&sched).expect("tick"), "{error}");
|
|
let guard = sched.lock().unwrap();
|
|
let again = |label: &str| {
|
|
let mut found: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == label)
|
|
.filter(|n| n.state == State::Pending)
|
|
.map(|n| match &n.payload {
|
|
SwarmNodeKind::BuildMeta { agent, .. }
|
|
| SwarmNodeKind::TriggerDeploy { agent, .. } => agent.clone(),
|
|
other => panic!("not a build or a deploy: {other:?}"),
|
|
})
|
|
.collect();
|
|
found.sort();
|
|
found
|
|
};
|
|
assert_eq!(again("build_meta"), ["argus"], "{error}");
|
|
assert_eq!(again("trigger_deploy"), ["argus"], "{error}");
|
|
}
|
|
}
|
|
|
|
/// With nothing placed there is nothing to wait on, and the pin is
|
|
/// recorded all the same.
|
|
#[tokio::test]
|
|
async fn a_pin_with_no_placed_agent_is_recorded() {
|
|
let pins = Arc::new(FakePins::default());
|
|
let (_, sched) = pass(&pins, &report("p1", false), &[], None, false).await;
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
assert_eq!(states_of(&sched, "record_fanout"), [hive_jobq::State::Done]);
|
|
assert_eq!(pins.held().as_deref(), Some("p1"));
|
|
}
|
|
|
|
/// While the record cannot be read, a moved pin deploys as it did before
|
|
/// the record existed, and there is nothing to record into.
|
|
#[tokio::test]
|
|
async fn an_unreadable_record_falls_back_to_the_moved_pin() {
|
|
let pins = Arc::new(FakePins {
|
|
unreadable: true,
|
|
..FakePins::default()
|
|
});
|
|
let (fan_out, sched) = pass(&pins, &report("p1", true), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::Unrecorded);
|
|
assert_eq!(
|
|
states_of(&sched, "trigger_deploy"),
|
|
[hive_jobq::State::Done; 3]
|
|
);
|
|
assert!(states_of(&sched, "record_fanout").is_empty());
|
|
|
|
let (fan_out, sched) = pass(&pins, &report("p1", false), &PLACED, None, false).await;
|
|
assert_eq!(fan_out, FanOut::None);
|
|
assert!(states_of(&sched, "trigger_deploy").is_empty());
|
|
}
|
|
|
|
/// Node labels of the graph [`super::declare_agent_job`] builds, and the
|
|
/// labels the deploy node waits on.
|
|
fn creation_graph(archetype: bool) -> (Vec<String>, Vec<String>) {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
);
|
|
sched
|
|
.insert_job(None, |b| {
|
|
super::declare_agent_job(b, "atlas", "pr1ma", archetype)
|
|
})
|
|
.expect("the graph inserts");
|
|
let graph = sched.graph();
|
|
let labels = graph.nodes().map(|n| n.payload.label()).collect();
|
|
let deploy = graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == "trigger_deploy")
|
|
.expect("every creation deploys");
|
|
let waits = deploy
|
|
.deps
|
|
.iter()
|
|
.filter_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, .. } => graph.node(*id).map(|n| n.payload.label()),
|
|
hive_jobq::Dep::Resource { .. } => None,
|
|
})
|
|
.collect();
|
|
(labels, waits)
|
|
}
|
|
|
|
/// An archetype agent's graph has no repo nodes and its deploy waits on
|
|
/// no repo; a custom agent's has all three, and its deploy waits on the
|
|
/// seeded repo.
|
|
#[test]
|
|
fn an_archetype_agent_is_created_without_a_config_repo() {
|
|
const REPO_NODES: [&str; 3] = ["create_repo", "add_repo_member", "init_agent_config_repo"];
|
|
|
|
let (custom, custom_waits) = creation_graph(false);
|
|
for repo in REPO_NODES {
|
|
assert!(custom.iter().any(|l| l == repo), "custom lacks {repo}");
|
|
}
|
|
assert!(custom_waits.iter().any(|l| l == "init_agent_config_repo"));
|
|
|
|
let (archetype, archetype_waits) = creation_graph(true);
|
|
for repo in REPO_NODES {
|
|
assert!(!archetype.iter().any(|l| l == repo), "archetype has {repo}");
|
|
}
|
|
for kept in [
|
|
"create_identity",
|
|
"create_forge_user",
|
|
"grant_docs_read",
|
|
"mint_agent_identity",
|
|
"mint_agent_forge_token",
|
|
"mint_agent_matrix_account",
|
|
"set_agent_wanted",
|
|
] {
|
|
assert!(
|
|
archetype.iter().any(|l| l == kept),
|
|
"archetype lacks {kept}"
|
|
);
|
|
assert!(custom.iter().any(|l| l == kept), "custom lacks {kept}");
|
|
}
|
|
assert!(archetype_waits.iter().any(|l| l == "set_agent_wanted"));
|
|
assert_eq!(archetype.len() + REPO_NODES.len(), custom.len());
|
|
}
|
|
|
|
/// The ordering the operator's ruling requires: a hive cannot pass down
|
|
/// a certificate the swarm has not published, so the deploy message must
|
|
/// not leave before the mint is terminal.
|
|
///
|
|
/// Asserted on the graph `create_agent` builds, because the edge is one
|
|
/// line in a builder closure and its absence changes nothing observable
|
|
/// until a real agent boots without an identity.
|
|
#[tokio::test]
|
|
async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let mint = id_of("mint_agent_identity");
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
|
|
let when = deploy
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the mint");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a failed or unconfigured agent PKI issue must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// The forge-token mint sits between the forge user and the deploy: it
|
|
/// needs the user to exist, and the deploy must not overtake it — but a
|
|
/// host with no forge or no store must still deploy the agent.
|
|
#[tokio::test]
|
|
async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let edge = |from: hive_jobq::NodeId, to: &str| {
|
|
graph
|
|
.node(id_of(to))
|
|
.expect("just found")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == from => Some(*when),
|
|
_ => None,
|
|
})
|
|
.unwrap_or_else(|| panic!("{to} waits for node {}", from.get()))
|
|
};
|
|
|
|
let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token");
|
|
assert!(
|
|
!user_to_mint.accepts(hive_jobq::TerminalState::Failed),
|
|
"a token for a user that was never created cannot be minted; this \
|
|
edge has to be `after_ok`"
|
|
);
|
|
let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy");
|
|
assert!(
|
|
mint_to_deploy.accepts(hive_jobq::TerminalState::Failed),
|
|
"a host with no forge or store must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// Agent creation grants `internal/docs` read once the forge user exists,
|
|
/// and a failed grant does not hold up the deploy (the pass retries it).
|
|
#[tokio::test]
|
|
async fn the_docs_grant_follows_the_user_and_is_not_a_deploy_parent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let id_of = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
.id
|
|
};
|
|
let grant = graph.node(id_of("grant_docs_read")).expect("just found");
|
|
assert_eq!(grant.payload.data(grant.id.get())["agent"], "atlas");
|
|
let when = grant
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == id_of("create_forge_user") => {
|
|
Some(*when)
|
|
}
|
|
_ => None,
|
|
})
|
|
.expect("the grant waits for the forge user");
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"adding a user that was never created as a collaborator is a forge error; \
|
|
this edge has to be `after_ok`"
|
|
);
|
|
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
|
assert!(
|
|
!deploy.deps.iter().any(|d| matches!(
|
|
d,
|
|
hive_jobq::Dep::Node { id, .. } if *id == grant.id
|
|
)),
|
|
"the deploy must not wait on the docs grant"
|
|
);
|
|
}
|
|
|
|
/// The periodic pass queues the same pair of nodes agent creation does.
|
|
#[test]
|
|
fn a_queued_docs_grant_creates_the_forge_user_first() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_docs_grants(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2, "one returned id per agent: its grant node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let mut labels: Vec<String> = graph.nodes().map(|n| n.payload.label()).collect();
|
|
labels.sort();
|
|
assert_eq!(
|
|
labels,
|
|
[
|
|
"create_forge_user",
|
|
"create_forge_user",
|
|
"grant_docs_read",
|
|
"grant_docs_read"
|
|
]
|
|
);
|
|
for grant in graph
|
|
.nodes()
|
|
.filter(|n| n.payload.label() == "grant_docs_read")
|
|
{
|
|
assert!(ids.contains(&grant.id));
|
|
assert!(
|
|
grant
|
|
.deps
|
|
.iter()
|
|
.any(|d| matches!(d, hive_jobq::Dep::Node { when, .. } if !when.accepts(hive_jobq::TerminalState::Failed))),
|
|
"each grant waits `after_ok` on its user"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_forge_token_node_renders_the_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::MintAgentForgeToken {
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "mint_agent_forge_token");
|
|
assert_eq!(kind.data(1)["agent"], "atlas");
|
|
}
|
|
|
|
#[test]
|
|
fn a_matrix_account_node_renders_the_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let kind = SwarmNodeKind::MintAgentMatrixAccount {
|
|
agent: "atlas".to_owned(),
|
|
};
|
|
assert_eq!(kind.label(), "mint_agent_matrix_account");
|
|
assert_eq!(kind.data(1)["agent"], "atlas");
|
|
}
|
|
|
|
/// The periodic pass's queueing: a certificate re-issue is the node agent
|
|
/// creation mints with, a secret re-mint its own node, and an agent due
|
|
/// both gets the secret `after_any` the certificate.
|
|
#[test]
|
|
fn a_queued_renewal_chains_the_secret_after_the_certificate() {
|
|
use crate::agent_renewal::Renewal;
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let r = |agent: &str, cert, secret| Renewal {
|
|
agent: agent.to_owned(),
|
|
cert,
|
|
secret,
|
|
};
|
|
let ids = super::queue_agent_renewals(
|
|
&sched,
|
|
vec![
|
|
r("both", true, true),
|
|
r("cert", true, false),
|
|
r("secret", false, true),
|
|
],
|
|
)
|
|
.expect("three jobs insert");
|
|
assert_eq!(ids.len(), 4, "one returned id per node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let agent_of = |n: &hive_jobq::Node<SwarmNodeKind, super::SwarmResourceKind>| {
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
};
|
|
let mut nodes: Vec<(String, String)> = graph
|
|
.nodes()
|
|
.map(|n| (agent_of(n), n.payload.label()))
|
|
.collect();
|
|
nodes.sort();
|
|
assert_eq!(
|
|
nodes,
|
|
[
|
|
("both".to_owned(), "mint_agent_identity".to_owned()),
|
|
("both".to_owned(), "renew_agent_queue_credential".to_owned()),
|
|
("cert".to_owned(), "mint_agent_identity".to_owned()),
|
|
(
|
|
"secret".to_owned(),
|
|
"renew_agent_queue_credential".to_owned()
|
|
),
|
|
]
|
|
);
|
|
let find = |agent: &str, label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label && agent_of(n) == agent)
|
|
.unwrap_or_else(|| panic!("{agent} has a {label} node"))
|
|
};
|
|
let cert = find("both", "mint_agent_identity").id;
|
|
let secret = find("both", "renew_agent_queue_credential");
|
|
let when = secret
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == cert => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the secret waits for the certificate");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a certificate that could not be re-issued must not hold up the secret"
|
|
);
|
|
assert!(
|
|
find("secret", "renew_agent_queue_credential")
|
|
.deps
|
|
.is_empty(),
|
|
"a secret on its own waits for nothing"
|
|
);
|
|
}
|
|
|
|
fn mint_nodes_for(
|
|
sched: &std::sync::Mutex<
|
|
hive_jobq::scheduler::Scheduler<SwarmNodeKind, super::SwarmResourceKind>,
|
|
>,
|
|
agent: &str,
|
|
) -> Vec<hive_jobq::NodeId> {
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| {
|
|
matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { agent: a } if a == agent)
|
|
})
|
|
.map(|n| n.id)
|
|
.collect()
|
|
}
|
|
|
|
/// A pass that finds a mint for the agent still queued or running (agent
|
|
/// creation's or its own earlier one) queues none; one that finds only
|
|
/// finished mints queues the next. The other agent is the control: the
|
|
/// check is per agent.
|
|
#[test]
|
|
fn a_mint_in_flight_blocks_another_for_that_agent_only() {
|
|
use crate::agent_renewal::Renewal;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let renewal = |agent: &str| Renewal {
|
|
agent: agent.to_owned(),
|
|
cert: true,
|
|
secret: false,
|
|
};
|
|
|
|
let first = super::queue_agent_renewals(&sched, vec![renewal("sock"), renewal("other")])
|
|
.expect("two jobs insert");
|
|
assert_eq!(first.len(), 2, "no mint in flight: both are queued");
|
|
|
|
let second = super::queue_agent_renewals(&sched, vec![renewal("sock")])
|
|
.expect("a skipped agent is not an error");
|
|
assert!(second.is_empty(), "the first pass's mint is still pending");
|
|
assert_eq!(mint_nodes_for(&sched, "sock").len(), 1);
|
|
|
|
// The next pass after that mint finished.
|
|
let pending = mint_nodes_for(&sched, "sock")[0];
|
|
assert!(
|
|
sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.cancel_node(pending)
|
|
);
|
|
let third = super::queue_agent_renewals(&sched, vec![renewal("sock")])
|
|
.expect("a finished mint does not block");
|
|
assert_eq!(
|
|
third.len(),
|
|
1,
|
|
"a finished mint is retried by the next pass"
|
|
);
|
|
assert_eq!(mint_nodes_for(&sched, "sock").len(), 2);
|
|
assert_eq!(
|
|
mint_nodes_for(&sched, "other").len(),
|
|
1,
|
|
"the other agent's pending mint was never touched"
|
|
);
|
|
}
|
|
|
|
/// Agent creation mints the matrix account as a root of its own, and the
|
|
/// deploy waits for it without being cancelled by it: a host with no
|
|
/// homeserver configured must still deploy the agent.
|
|
#[tokio::test]
|
|
async fn the_matrix_account_mint_is_a_root_and_does_not_block_the_deploy() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let (_nats, state, sched) = state_with_roster_queued().await;
|
|
let _queued = crate::routes::agents::create_agent(
|
|
axum::extract::State(state),
|
|
axum::Json(swarm_controller_api::agents::CreateAgentRequest {
|
|
name: "atlas".to_owned(),
|
|
hive: "pr1ma".to_owned(),
|
|
limits: None,
|
|
inference: None,
|
|
permissions: None,
|
|
unrestricted_internet: None,
|
|
archetype: None,
|
|
agents_md: None,
|
|
}),
|
|
)
|
|
.await
|
|
.expect("a hive in the roster must be accepted");
|
|
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let find = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label)
|
|
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
|
};
|
|
let mint = find("mint_agent_matrix_account");
|
|
assert!(mint.deps.is_empty(), "the mint waits for nothing");
|
|
let when = find("trigger_deploy")
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == mint.id => Some(*when),
|
|
_ => None,
|
|
})
|
|
.expect("the deploy waits for the matrix mint");
|
|
assert!(
|
|
when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a host with no homeserver must not cancel the deploy; this edge \
|
|
has to be `after_any`, not `after_ok`"
|
|
);
|
|
}
|
|
|
|
/// The backfill's queueing: one mint node per agent, nothing else.
|
|
#[test]
|
|
fn a_queued_matrix_mint_is_one_node_per_agent() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_matrix_account_mints(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut agents: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_agent_matrix_account");
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
})
|
|
.collect();
|
|
agents.sort();
|
|
assert_eq!(agents, ["a", "b"]);
|
|
}
|
|
|
|
/// The hive sender-token pass's queueing: one mint node per hive, and
|
|
/// the node names the hive rather than an agent.
|
|
#[test]
|
|
fn a_queued_hive_sender_mint_is_one_node_per_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids =
|
|
super::queue_hive_sender_mints(&sched, vec!["alpha".to_owned(), "beta".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut hives: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_hive_sender_token");
|
|
let data = n.payload.data(n.id.get());
|
|
assert!(data.get("agent").is_none(), "{data}");
|
|
data["hive"].as_str().expect("hive is a string").to_owned()
|
|
})
|
|
.collect();
|
|
hives.sort();
|
|
assert_eq!(hives, ["alpha", "beta"]);
|
|
}
|
|
|
|
/// The hive forge-token pass's queueing: one node per hive, naming the
|
|
/// hive rather than an agent.
|
|
#[test]
|
|
fn a_queued_hive_forge_token_mint_is_one_node_per_hive() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_hive_forge_token_mints(
|
|
&sched,
|
|
vec!["alpha".to_owned(), "beta".to_owned()],
|
|
)
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2);
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut hives: Vec<String> = guard
|
|
.graph()
|
|
.nodes()
|
|
.map(|n| {
|
|
assert_eq!(n.payload.label(), "mint_hive_forge_token");
|
|
let data = n.payload.data(n.id.get());
|
|
assert!(data.get("agent").is_none(), "{data}");
|
|
data["hive"].as_str().expect("hive is a string").to_owned()
|
|
})
|
|
.collect();
|
|
hives.sort();
|
|
assert_eq!(hives, ["alpha", "beta"]);
|
|
}
|
|
|
|
/// The manual route and the periodic pass both go through
|
|
/// `queue_forge_token_mints`, so this is the assertion that a backfilled
|
|
/// mint is the same pair of nodes agent creation inserts: the forge user,
|
|
/// then the mint `after_ok` it, per agent.
|
|
#[test]
|
|
fn a_queued_mint_creates_the_forge_user_first() {
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()])
|
|
.expect("two jobs insert");
|
|
assert_eq!(ids.len(), 2, "one returned id per agent: its mint node");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let graph = guard.graph();
|
|
let agent_of = |n: &hive_jobq::Node<SwarmNodeKind, super::SwarmResourceKind>| {
|
|
n.payload.data(n.id.get())["agent"]
|
|
.as_str()
|
|
.expect("agent is a string")
|
|
.to_owned()
|
|
};
|
|
for agent in ["a", "b"] {
|
|
let find = |label: &str| {
|
|
graph
|
|
.nodes()
|
|
.find(|n| n.payload.label() == label && agent_of(n) == agent)
|
|
.unwrap_or_else(|| panic!("{agent} has a {label} node"))
|
|
};
|
|
let user = find("create_forge_user").id;
|
|
let mint = find("mint_agent_forge_token");
|
|
assert!(ids.contains(&mint.id), "the returned id is {agent}'s mint");
|
|
let when = mint
|
|
.deps
|
|
.iter()
|
|
.find_map(|d| match d {
|
|
hive_jobq::Dep::Node { id, when } if *id == user => Some(*when),
|
|
_ => None,
|
|
})
|
|
.unwrap_or_else(|| panic!("{agent}'s mint waits for its own forge user"));
|
|
assert!(
|
|
!when.accepts(hive_jobq::TerminalState::Failed),
|
|
"a token cannot be minted for a user that was never created; this \
|
|
edge has to be `after_ok`"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
graph.nodes().count(),
|
|
4,
|
|
"two nodes per agent, nothing else"
|
|
);
|
|
}
|
|
|
|
/// The backfill end to end, minus the IO: an agent the pass observed with
|
|
/// no forge user is planned, and the job it gets creates that user before
|
|
/// it mints.
|
|
#[test]
|
|
fn an_agent_with_no_forge_user_gets_a_user_and_a_mint() {
|
|
use crate::forge::agent_token::{Observed, plan};
|
|
use hive_jobq_wire::WireNode as _;
|
|
|
|
let agents = plan(&[("ruth".to_owned(), Observed::NoForgeUser)]);
|
|
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
));
|
|
super::queue_forge_token_mints(&sched, agents).expect("one job inserts");
|
|
let guard = sched
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
let mut labels: Vec<String> = guard.graph().nodes().map(|n| n.payload.label()).collect();
|
|
labels.sort();
|
|
assert_eq!(labels, ["create_forge_user", "mint_agent_forge_token"]);
|
|
}
|
|
|
|
/// A scheduler holding one `UpdateMeta` that ran and ended `outcome`.
|
|
async fn after_one_update_meta(
|
|
outcome: hive_jobq::scheduler::Outcome,
|
|
) -> std::sync::Arc<
|
|
std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
> {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move { (b, outcome) })
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
sched
|
|
}
|
|
|
|
fn update_metas(
|
|
sched: &std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
) -> usize {
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(n.payload, SwarmNodeKind::UpdateMeta))
|
|
.count()
|
|
}
|
|
|
|
/// With none run yet the tick queues one, and queues no second while that
|
|
/// one is pending or running.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_queues_first_and_skips_while_one_is_in_flight() {
|
|
let sched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "a pending one blocks the tick");
|
|
|
|
let running = hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Failed("late".to_owned()))
|
|
})
|
|
.expect("runnable");
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "a running one blocks the tick");
|
|
running.await.1.expect("no growth");
|
|
}
|
|
|
|
/// Once the latest run succeeded, ticks add no node.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_adds_nothing_after_a_success() {
|
|
let sched = after_one_update_meta(hive_jobq::scheduler::Outcome::Done).await;
|
|
for _ in 0..3 {
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
}
|
|
assert_eq!(update_metas(&sched), 1);
|
|
}
|
|
|
|
/// A failed latest run is retried; a success after it ends the retries.
|
|
#[tokio::test]
|
|
async fn the_meta_timer_retries_after_a_failure_until_a_success() {
|
|
let sched =
|
|
after_one_update_meta(hive_jobq::scheduler::Outcome::Failed("no".to_owned())).await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 2);
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, |_, _, b| async move {
|
|
(b, hive_jobq::scheduler::Outcome::Done)
|
|
})
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth");
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 2);
|
|
}
|
|
|
|
type SharedSched = std::sync::Arc<
|
|
std::sync::Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
|
>;
|
|
|
|
/// Declare `agent`'s creation, as the endpoint does, and run every node to
|
|
/// the end: the meta update fails when `update_meta_fails`, the pause
|
|
/// declaration when `set_wanted_fails`, everything else succeeds.
|
|
async fn create_and_run(
|
|
sched: &SharedSched,
|
|
agent: &str,
|
|
update_meta_fails: bool,
|
|
set_wanted_fails: bool,
|
|
) {
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| super::declare_agent_job(b, agent, "pr1ma", false))
|
|
.expect("the graph inserts");
|
|
drain(sched, update_meta_fails, set_wanted_fails).await;
|
|
}
|
|
|
|
/// Run every runnable node; outcomes as for [`create_and_run`].
|
|
async fn drain(sched: &SharedSched, update_meta_fails: bool, set_wanted_fails: bool) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
|
|
while let Some(runner) = Scheduler::claim_next(sched, |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::UpdateMeta if update_meta_fails => {
|
|
Outcome::Failed("no builder".to_owned())
|
|
}
|
|
SwarmNodeKind::SetAgentWanted { .. } if set_wanted_fails => {
|
|
Outcome::Failed("no queue".to_owned())
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
}
|
|
|
|
/// The states of `agent`'s `TriggerDeploy` nodes, oldest first.
|
|
fn deploys_of(sched: &SharedSched, agent: &str) -> Vec<hive_jobq::State> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(&n.payload, SwarmNodeKind::TriggerDeploy { agent: a, .. } if a == agent))
|
|
.collect();
|
|
nodes.sort_by_key(|n| n.id);
|
|
nodes.into_iter().map(|n| n.state).collect()
|
|
}
|
|
|
|
/// `bob` deployed under a working update; then `atlas` was created while
|
|
/// the update failed, so its deploy is `Skipped`.
|
|
async fn bob_deployed_and_atlas_skipped() -> SharedSched {
|
|
use hive_jobq::State;
|
|
|
|
let sched: SharedSched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
create_and_run(&sched, "bob", false, false).await;
|
|
create_and_run(&sched, "atlas", true, false).await;
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
sched
|
|
}
|
|
|
|
/// An agent created while the update was failing is deployed by the run
|
|
/// that succeeds, exactly once.
|
|
#[tokio::test]
|
|
async fn the_next_successful_update_deploys_the_agent_a_failed_one_skipped() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
// A retry whose update fails too skips the new deploy as well.
|
|
drain(&sched, true, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped]
|
|
);
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
// The success ends the retries: no tick adds a node, so no second deploy.
|
|
assert!(!super::queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
}
|
|
|
|
/// An agent whose latest deploy ran gets no extra one from the retry, even
|
|
/// when an older deploy of its was skipped.
|
|
#[tokio::test]
|
|
async fn an_agent_that_was_deployed_gets_no_extra_deploy_from_the_retry() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = bob_deployed_and_atlas_skipped().await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
|
|
// atlas now holds an old Skipped deploy and a Done one. Another
|
|
// creation fails its update; the retry deploys carol and only carol.
|
|
create_and_run(&sched, "carol", true, false).await;
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "carol"), [State::Skipped, State::Done]);
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped, State::Done]);
|
|
assert_eq!(deploys_of(&sched, "bob"), [State::Done]);
|
|
}
|
|
|
|
/// A deploy skipped because the pause declaration failed stays skipped: the
|
|
/// retry must not deploy past the declaration it waits for.
|
|
#[tokio::test]
|
|
async fn a_deploy_skipped_for_another_reason_is_not_redeclared() {
|
|
use hive_jobq::State;
|
|
|
|
let sched: SharedSched =
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
create_and_run(&sched, "atlas", false, true).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
assert!(super::queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
}
|
|
|
|
/// The mint waits for the declaration its node re-reads: with the
|
|
/// declaration failed it is skipped, with it written it runs.
|
|
#[tokio::test]
|
|
async fn creation_mints_the_identity_only_after_the_declaration() {
|
|
use hive_jobq::State;
|
|
|
|
let mint_states = |sched: &SharedSched| -> Vec<State> {
|
|
let guard = sched.lock().unwrap();
|
|
guard
|
|
.graph()
|
|
.nodes()
|
|
.filter(|n| matches!(&n.payload, SwarmNodeKind::MintAgentIdentity { .. }))
|
|
.map(|n| n.state)
|
|
.collect()
|
|
};
|
|
let sched = fresh_sched();
|
|
create_and_run(&sched, "atlas", false, true).await;
|
|
assert_eq!(mint_states(&sched), [State::Skipped]);
|
|
let sched = fresh_sched();
|
|
create_and_run(&sched, "atlas", false, false).await;
|
|
assert_eq!(mint_states(&sched), [State::Done]);
|
|
}
|
|
|
|
/// Run every runnable node; the build of `agent` fails with `error` and
|
|
/// everything else succeeds.
|
|
async fn drain_failing_build(sched: &SharedSched, agent: &'static str, error: String) {
|
|
use hive_jobq::scheduler::{Outcome, Scheduler};
|
|
|
|
while let Some(runner) = Scheduler::claim_next(sched, {
|
|
let error = error.clone();
|
|
move |_id, kind, builder| async move {
|
|
let outcome = match kind {
|
|
SwarmNodeKind::BuildMeta { agent: a, .. } if a == agent => {
|
|
Outcome::Failed(error)
|
|
}
|
|
_ => Outcome::Done,
|
|
};
|
|
(builder, outcome)
|
|
}
|
|
}) {
|
|
runner.await.1.expect("no growth declared");
|
|
}
|
|
}
|
|
|
|
fn fresh_sched() -> SharedSched {
|
|
std::sync::Arc::new(std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)))
|
|
}
|
|
|
|
/// The `rev` of `agent`'s deploys, oldest first.
|
|
fn deploy_revs(sched: &SharedSched, agent: &str) -> Vec<Option<String>> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::TriggerDeploy { agent: a, rev, .. } if a == agent => {
|
|
Some((n.id, rev.clone()))
|
|
}
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
nodes.sort_by_key(|(id, _)| *id);
|
|
nodes.into_iter().map(|(_, rev)| rev).collect()
|
|
}
|
|
|
|
/// The `pr` of `agent`'s builds, oldest first.
|
|
fn build_prs(sched: &SharedSched, agent: &str) -> Vec<Option<u64>> {
|
|
let guard = sched.lock().unwrap();
|
|
let mut nodes: Vec<_> = guard
|
|
.graph()
|
|
.nodes()
|
|
.filter_map(|n| match &n.payload {
|
|
SwarmNodeKind::BuildMeta { agent: a, pr } if a == agent => Some((n.id, *pr)),
|
|
_ => None,
|
|
})
|
|
.collect();
|
|
nodes.sort_by_key(|(id, _)| *id);
|
|
nodes.into_iter().map(|(_, pr)| pr).collect()
|
|
}
|
|
|
|
fn merged_atlas() -> SharedSched {
|
|
let sched = fresh_sched();
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| {
|
|
merged_deploy_job(b, "pr1ma", "atlas", "abc", Some(7))
|
|
})
|
|
.expect("the graph inserts");
|
|
sched
|
|
}
|
|
|
|
/// A merged config's build that fails, with the update `Done`, whatever it
|
|
/// fails with: the deploy is skipped, and the timer declares a build and
|
|
/// the same deploy again on every round, without an update, until a build
|
|
/// succeeds. Then it stops.
|
|
#[tokio::test]
|
|
async fn a_failed_build_is_built_again_every_round_until_it_succeeds() {
|
|
use hive_jobq::State;
|
|
|
|
let sched = merged_atlas();
|
|
let errors = [
|
|
"error: attribute missing".to_owned(),
|
|
"timed out: nix did not finish".to_owned(),
|
|
format!(
|
|
"{}no swarm builder is listening",
|
|
meta_build::NO_VERDICT_PREFIX
|
|
),
|
|
];
|
|
drain_failing_build(&sched, "atlas", errors[0].clone()).await;
|
|
assert_eq!(deploys_of(&sched, "atlas"), [State::Skipped]);
|
|
assert_eq!(update_metas(&sched), 1);
|
|
|
|
for error in &errors[1..] {
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
assert_eq!(update_metas(&sched), 1, "the retry needs no update");
|
|
drain_failing_build(&sched, "atlas", error.clone()).await;
|
|
}
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Skipped]
|
|
);
|
|
|
|
// A success on a later round deploys, once.
|
|
assert!(queue_meta_update(&sched).expect("tick"));
|
|
drain(&sched, false, false).await;
|
|
assert_eq!(
|
|
deploys_of(&sched, "atlas"),
|
|
[State::Skipped, State::Skipped, State::Skipped, State::Done]
|
|
);
|
|
// The merged deploy's rev survives every retry.
|
|
assert!(
|
|
deploy_revs(&sched, "atlas")
|
|
.iter()
|
|
.all(|rev| rev.as_deref() == Some("abc"))
|
|
);
|
|
// So does the PR it belongs to: every retried build reports on it too.
|
|
assert_eq!(build_prs(&sched, "atlas"), [Some(7); 4]);
|
|
assert!(
|
|
!queue_meta_update(&sched).expect("tick"),
|
|
"the success ends the retries"
|
|
);
|
|
}
|
|
|
|
/// Only a merged config PR's build carries a PR: the creation DAG's does
|
|
/// not, so its failure has nobody to comment on.
|
|
#[test]
|
|
fn only_a_merged_prs_build_carries_its_pr() {
|
|
let sched = merged_atlas();
|
|
sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| declare_agent_job(b, "iris", "pr1ma", false))
|
|
.expect("the graph inserts");
|
|
assert_eq!(build_prs(&sched, "atlas"), [Some(7)]);
|
|
assert_eq!(build_prs(&sched, "iris"), [None]);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod mint_identity_tests {
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use hive_jobq::scheduler::Outcome;
|
|
use tracing_subscriber::layer::SubscriberExt as _;
|
|
|
|
use super::{SwarmNodeKind, WorkerDeps, mint_identity, mint_if_live, run_swarm_node};
|
|
|
|
/// One event's level and its fields, each by name.
|
|
type Event = (tracing::Level, Vec<(String, String)>);
|
|
|
|
/// Every event.
|
|
#[derive(Clone, Default)]
|
|
struct Events(Arc<Mutex<Vec<Event>>>);
|
|
|
|
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Events {
|
|
fn on_event(
|
|
&self,
|
|
event: &tracing::Event<'_>,
|
|
_: tracing_subscriber::layer::Context<'_, S>,
|
|
) {
|
|
let mut fields = Vec::new();
|
|
event.record(
|
|
&mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
|
|
fields.push((field.name().to_owned(), format!("{value:?}")));
|
|
},
|
|
);
|
|
self.0
|
|
.lock()
|
|
.expect("unpoisoned")
|
|
.push((*event.metadata().level(), fields));
|
|
}
|
|
}
|
|
|
|
/// An agent name the store would refuse fails the mint before any store
|
|
/// call, whatever this process's environment holds. The line names the
|
|
/// step it stopped at. The literal is the
|
|
/// `_msg` that `hh-agent-identity-mint-failed` in
|
|
/// `nix/host-modules/swarm-grafana.nix` matches.
|
|
#[tokio::test]
|
|
async fn a_failed_mint_logs_the_agent_and_the_error_on_the_line_the_alert_rule_matches() {
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
|
|
let Outcome::Failed(error) = mint_identity(None, "a/b").await else {
|
|
panic!("an illegal agent name cannot be minted");
|
|
};
|
|
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
let [(level, fields)] = got.as_slice() else {
|
|
panic!("one event, got {got:?}");
|
|
};
|
|
assert_eq!(*level, tracing::Level::WARN);
|
|
let field = |name: &str| {
|
|
fields
|
|
.iter()
|
|
.find(|(n, _)| n == name)
|
|
.map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
|
|
};
|
|
assert_eq!(field("message"), "swarm jobq: mint_agent_identity failed");
|
|
assert_eq!(field("agent"), "a/b");
|
|
assert_eq!(field("step"), "setup");
|
|
assert_eq!(field("error"), error);
|
|
}
|
|
|
|
fn declared_as(
|
|
agents: &[(&str, swarm_wire::wanted::AgentState)],
|
|
) -> swarm_wire::wanted::HiveWanted {
|
|
let mut d = swarm_wire::wanted::HiveWanted::default();
|
|
for (agent, state) in agents {
|
|
d.agents.insert(
|
|
(*agent).to_owned(),
|
|
swarm_wire::wanted::AgentWanted { state: *state },
|
|
);
|
|
}
|
|
d
|
|
}
|
|
|
|
/// The mint runs for a declared agent and not for one declared `Destroyed`
|
|
/// or not at all: nothing is written for the second, and the node is done.
|
|
#[tokio::test]
|
|
async fn a_mint_for_an_agent_destroyed_since_it_was_queued_writes_nothing() {
|
|
use swarm_wire::wanted::AgentState;
|
|
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
let ran = std::sync::atomic::AtomicBool::new(false);
|
|
let mint = async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
};
|
|
let declarations =
|
|
declared_as(&[("live", AgentState::Up), ("gone", AgentState::Destroyed)]);
|
|
|
|
for agent in ["gone", "unknown"] {
|
|
let outcome = mint_if_live(agent, Some(Ok(vec![declarations.clone()])), async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
assert!(matches!(outcome, Outcome::Done), "{agent}");
|
|
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst), "{agent}");
|
|
}
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
assert!(
|
|
got.iter()
|
|
.any(|(level, fields)| *level == tracing::Level::INFO
|
|
&& fields
|
|
.iter()
|
|
.any(|(n, v)| n == "message"
|
|
&& v == "identity mint skipped: gone is destroyed")),
|
|
"{got:?}"
|
|
);
|
|
|
|
// Presence controls: a live agent mints, and so does one with no
|
|
// wanted-state writer to ask.
|
|
let outcome = mint_if_live("live", Some(Ok(vec![declarations])), mint).await;
|
|
assert!(matches!(outcome, Outcome::Done));
|
|
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
ran.store(false, std::sync::atomic::Ordering::SeqCst);
|
|
let outcome = mint_if_live("anyone", None, async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
assert!(matches!(outcome, Outcome::Done));
|
|
assert!(ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
}
|
|
|
|
/// A declaration that cannot be read fails the node, naming the cause,
|
|
/// and mints nothing.
|
|
#[tokio::test]
|
|
async fn an_unreadable_declaration_fails_the_mint_without_minting() {
|
|
let ran = std::sync::atomic::AtomicBool::new(false);
|
|
let outcome = mint_if_live("atlas", Some(Err(anyhow::anyhow!("hive down"))), async {
|
|
ran.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(())
|
|
})
|
|
.await;
|
|
let Outcome::Failed(error) = outcome else {
|
|
panic!("a failed read must fail the node");
|
|
};
|
|
assert!(error.contains("hive down"), "{error}");
|
|
assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
}
|
|
/// A node of any kind whose work returns `Failed` logs one warn naming the
|
|
/// node's id, its kind and the error, from the one place that runs nodes.
|
|
#[tokio::test]
|
|
async fn a_failed_node_logs_its_kind_id_and_error() {
|
|
let events = Events::default();
|
|
let _guard =
|
|
tracing::subscriber::set_default(tracing_subscriber::registry().with(events.clone()));
|
|
let sched = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
|
hive_jobq::Graph::new(),
|
|
hive_jobq::resources::ResourceTable::new(),
|
|
)));
|
|
let ids = sched
|
|
.lock()
|
|
.unwrap()
|
|
.insert_job(None, |b| {
|
|
vec![
|
|
b.node(SwarmNodeKind::SetAgentWanted {
|
|
hive: "h".to_owned(),
|
|
agent: "a".to_owned(),
|
|
})
|
|
.guid(),
|
|
]
|
|
})
|
|
.expect("inserted");
|
|
let deps = WorkerDeps {
|
|
auth: None,
|
|
forge: super::Late::default(),
|
|
queue: None,
|
|
wanted: None,
|
|
matrix_homeserver: None,
|
|
};
|
|
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
|
run_swarm_node(id, kind, builder, deps)
|
|
})
|
|
.expect("runnable")
|
|
.await
|
|
.1
|
|
.expect("no growth declared");
|
|
|
|
let got = events.0.lock().expect("unpoisoned").clone();
|
|
let [(level, fields)] = got.as_slice() else {
|
|
panic!("one event, got {got:?}");
|
|
};
|
|
assert_eq!(*level, tracing::Level::WARN);
|
|
let field = |name: &str| {
|
|
fields
|
|
.iter()
|
|
.find(|(n, _)| n == name)
|
|
.map_or_else(|| panic!("no {name} in {fields:?}"), |(_, v)| v.clone())
|
|
};
|
|
assert_eq!(field("kind"), "set_agent_wanted");
|
|
assert_eq!(field("node"), ids[0].get().to_string());
|
|
assert!(
|
|
field("error").contains("no swarm queue is configured"),
|
|
"{fields:?}"
|
|
);
|
|
}
|
|
}
|