swarm-controller: mint each agent's forge token and store it in bao
A MintAgentForgeToken node mints a fixed-name swarm-agent token with the admin API, keeps it when the stored value's last eight and the normalised scopes match the forge's list, and otherwise deletes and re-creates it. The token is stored at swarm/agents/<agent>/forge-token. Agent creation inserts the node, and a pass at start and every five minutes inserts it for every agent holding a store identity whose token is missing or stale. Refs #3782
This commit is contained in:
atlas
mara
parent
2fda529ca8
commit
52c8c0b0de
7 changed files with 977 additions and 4 deletions
|
|
@ -105,6 +105,14 @@ enum SwarmNodeKind {
|
|||
/// its hive's shared queue credential, so the document cannot be rendered
|
||||
/// without knowing which hive the agent belongs to.
|
||||
MintAgentIdentity { hive: String, agent: String },
|
||||
/// Make sure `agent` holds a live forge access token in the swarm secret
|
||||
/// store, minting one with the forge's admin API when it does not. See
|
||||
/// `forge::agent_token` — including why a rotation is a delete then a
|
||||
/// create.
|
||||
///
|
||||
/// Carries no hive: the token's store path has no hive segment, and the
|
||||
/// agent pulls it from wherever it runs.
|
||||
MintAgentForgeToken { agent: String },
|
||||
/// Declare `agent` on `hive` as `Paused` in the swarm's wanted-state
|
||||
/// store, so a freshly created agent does not start driving turns the
|
||||
/// moment it's deployed — the operator has to explicitly flip it to `Up`.
|
||||
|
|
@ -130,6 +138,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|||
SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
|
||||
SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
|
||||
SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
|
||||
SwarmNodeKind::MintAgentForgeToken { .. } => "mint_agent_forge_token".to_owned(),
|
||||
SwarmNodeKind::SetAgentWanted { .. } => "set_agent_wanted".to_owned(),
|
||||
SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
|
||||
}
|
||||
|
|
@ -147,7 +156,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|||
| SwarmNodeKind::CreateRepo { agent }
|
||||
| SwarmNodeKind::CreateForgeUser { agent }
|
||||
| SwarmNodeKind::AddRepoMember { agent }
|
||||
| SwarmNodeKind::InitAgentConfigRepo { agent } => {
|
||||
| SwarmNodeKind::InitAgentConfigRepo { agent }
|
||||
| SwarmNodeKind::MintAgentForgeToken { agent } => {
|
||||
serde_json::json!({ "agent": agent })
|
||||
}
|
||||
SwarmNodeKind::TriggerDeploy { hive, agent }
|
||||
|
|
@ -308,6 +318,7 @@ async fn run_swarm_node(
|
|||
}
|
||||
}
|
||||
},
|
||||
SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(deps.forge, &agent).await,
|
||||
SwarmNodeKind::SetAgentWanted { hive, agent } => match deps.wanted {
|
||||
None => Outcome::Failed(
|
||||
"no swarm queue is configured on this host, so no wanted-state \
|
||||
|
|
@ -327,6 +338,27 @@ async fn run_swarm_node(
|
|||
(builder, outcome)
|
||||
}
|
||||
|
||||
/// The `MintAgentForgeToken` arm, lifted out so `run_swarm_node` stays under
|
||||
/// `clippy::too_many_lines`.
|
||||
async fn mint_forge_token(
|
||||
forge: Option<Arc<forge::Client>>,
|
||||
agent: &str,
|
||||
) -> hive_jobq::scheduler::Outcome {
|
||||
use hive_jobq::scheduler::Outcome;
|
||||
|
||||
let Some(client) = forge else {
|
||||
return Outcome::Failed(
|
||||
"no forge configured on this host (SWARM_CONTROLLER_FORGE_URL / \
|
||||
SWARM_CONTROLLER_FORGE_TOKEN_FILE unset), so no agent forge token can be minted"
|
||||
.to_owned(),
|
||||
);
|
||||
};
|
||||
match client.ensure_agent_forge_token(agent).await {
|
||||
Ok(()) => Outcome::Done,
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Declare a brand-new agent at [`NEW_AGENT_WANTED_STATE`] — unless it turns
|
||||
/// out not to be new: an agent that already has a declaration (other than
|
||||
/// `Destroyed`, which this treats as reusable) is left alone, so a retried
|
||||
|
|
@ -1470,6 +1502,14 @@ fn declare_agent_job(
|
|||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_identity);
|
||||
// The agent's own forge token, in the store before the hive deploys
|
||||
// the container that pulls it. After the forge user, because the token
|
||||
// is minted for that user.
|
||||
let mint_forge_token = b
|
||||
.node(SwarmNodeKind::MintAgentForgeToken {
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_forge_user);
|
||||
// Declared before the deploy trigger so the pause is visible in the
|
||||
// wanted-state store before the hive brings the container up — see the
|
||||
// node's own doc comment for why "before", not just "eventually". Needs
|
||||
|
|
@ -1491,6 +1531,11 @@ fn declare_agent_job(
|
|||
// create agents exactly as it does today. `after_ok` there would turn an
|
||||
// unconfigured option into an agent nobody runs.
|
||||
//
|
||||
// The forge-token mint gets `after_any` for the same reason: a host with
|
||||
// no forge or no store must still create agents; only that node fails,
|
||||
// by name, and the agent's container fetches the token on its next
|
||||
// timer tick once one is minted.
|
||||
//
|
||||
// `after_ok` on the declaration, though: an agent deployed without its
|
||||
// pause landing first is the race this node exists to close, so a
|
||||
// declaration that did not land must not be deployed past. A host with no
|
||||
|
|
@ -1504,6 +1549,7 @@ fn declare_agent_job(
|
|||
})
|
||||
.after_ok(init_config)
|
||||
.after_any(mint_identity)
|
||||
.after_any(mint_forge_token)
|
||||
.after_ok(set_wanted);
|
||||
vec![create_identity.guid()]
|
||||
}
|
||||
|
|
@ -1703,6 +1749,73 @@ async fn mint_agent_identity(
|
|||
Ok(Json(MintAgentIdentityResponse { node_id: id.get() }))
|
||||
}
|
||||
|
||||
/// Success body of `POST /api/agents/{name}/forge-token`.
|
||||
#[derive(Clone, Debug, Serialize, ToSchema)]
|
||||
struct MintAgentForgeTokenResponse {
|
||||
/// The queued node, so a caller can follow it in the job view.
|
||||
node_id: u64,
|
||||
}
|
||||
|
||||
/// Insert one `MintAgentForgeToken` node per agent, each its own job.
|
||||
///
|
||||
/// The one place that node is queued outside agent creation: both the manual
|
||||
/// route below and `forge::agent_token::spawn`'s periodic pass come through
|
||||
/// here, so a backfilled mint is the same node a new agent gets.
|
||||
fn queue_forge_token_mints(
|
||||
sched: &Mutex<hive_jobq::scheduler::Scheduler<SwarmNodeKind, SwarmResourceKind>>,
|
||||
agents: Vec<String>,
|
||||
) -> Result<Vec<hive_jobq::NodeId>> {
|
||||
let mut sched = sched
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let mut ids = Vec::with_capacity(agents.len());
|
||||
for agent in agents {
|
||||
let queued = sched
|
||||
.insert_job(None, |b| {
|
||||
vec![b.node(SwarmNodeKind::MintAgentForgeToken { agent }).guid()]
|
||||
})
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
ids.extend(queued);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// Check an agent's forge token now, and mint one if it is missing or stale.
|
||||
///
|
||||
/// The periodic pass (`forge::agent_token::spawn`) does the same every five
|
||||
/// minutes for every agent with a store identity; this is for an operator who
|
||||
/// does not want to wait, or for an agent the pass skipped. Queues the same
|
||||
/// node agent creation does. Idempotent: a current token is left alone.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/agents/{name}/forge-token",
|
||||
params(("name" = String, Path, description = "agent name")),
|
||||
responses(
|
||||
(status = 200, description = "mint queued", body = MintAgentForgeTokenResponse),
|
||||
(status = 400, description = "`name` is not a valid identifier (problem+json)", body = String),
|
||||
(status = 500, description = "the job could not be queued (problem+json)", body = String),
|
||||
),
|
||||
tag = "agents"
|
||||
)]
|
||||
async fn mint_agent_forge_token(
|
||||
State(state): State<AppState>,
|
||||
Path(name): Path<String>,
|
||||
) -> Result<Json<MintAgentForgeTokenResponse>, problem_details::ProblemDetails> {
|
||||
let agent = hive_types::Ident::parse(&name)
|
||||
.map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))?
|
||||
.into_string();
|
||||
let ids = queue_forge_token_mints(&state.jobq, vec![agent]).map_err(|e| {
|
||||
error_problem(
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&e.to_string(),
|
||||
)
|
||||
})?;
|
||||
let [id] = ids[..] else {
|
||||
unreachable!("exactly one handle was asked for");
|
||||
};
|
||||
Ok(Json(MintAgentForgeTokenResponse { node_id: id.get() }))
|
||||
}
|
||||
|
||||
/// Every agent with an open config PR, in one response — the bulk
|
||||
/// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs
|
||||
/// every agent's status to render, and fetching them one at a time doesn't
|
||||
|
|
@ -1990,6 +2103,14 @@ async fn main() -> Result<()> {
|
|||
);
|
||||
}
|
||||
|
||||
if let Some(client) = forge_client.clone() {
|
||||
let sched = Arc::clone(&jobq);
|
||||
forge::agent_token::spawn(client, move |agents| {
|
||||
if let Err(e) = queue_forge_token_mints(&sched, agents) {
|
||||
tracing::warn!(error = %format!("{e:#}"), "agent forge tokens: queueing failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
let config_prs = forge_client.clone().map(config_pr::spawn);
|
||||
let state_forge = keep_forge_for_state(forge_client, webhook_secret.clone());
|
||||
|
||||
|
|
@ -2041,6 +2162,7 @@ fn build_app(state: AppState) -> axum::Router {
|
|||
.routes(routes!(get_config_prs))
|
||||
.routes(routes!(create_agent))
|
||||
.routes(routes!(mint_agent_identity))
|
||||
.routes(routes!(mint_agent_forge_token))
|
||||
.routes(routes!(get_agents))
|
||||
.routes(routes!(get_agents_status))
|
||||
.routes(routes!(set_agent_state))
|
||||
|
|
@ -2842,6 +2964,105 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// The forge-token mint sits between the forge user and the deploy: it
|
||||
/// needs the user to exist, and the deploy must not overtake it — but a
|
||||
/// host with no forge or no store must still deploy the agent.
|
||||
#[tokio::test]
|
||||
async fn the_forge_token_mint_follows_the_user_and_does_not_block_the_deploy() {
|
||||
use hive_jobq_wire::WireNode as _;
|
||||
|
||||
let (state, sched) = state_with_roster();
|
||||
let _queued = super::create_agent(
|
||||
axum::extract::State(state),
|
||||
axum::Json(super::CreateAgentRequest {
|
||||
name: "atlas".to_owned(),
|
||||
hive: "pr1ma".to_owned(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("a hive in the roster must be accepted");
|
||||
|
||||
let guard = sched
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let graph = guard.graph();
|
||||
let id_of = |label: &str| {
|
||||
graph
|
||||
.nodes()
|
||||
.find(|n| n.payload.label() == label)
|
||||
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
||||
.id
|
||||
};
|
||||
let edge = |from: hive_jobq::NodeId, to: &str| {
|
||||
graph
|
||||
.node(id_of(to))
|
||||
.expect("just found")
|
||||
.deps
|
||||
.iter()
|
||||
.find_map(|d| match d {
|
||||
hive_jobq::Dep::Node { id, when } if *id == from => Some(*when),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap_or_else(|| panic!("{to} waits for node {}", from.get()))
|
||||
};
|
||||
|
||||
let user_to_mint = edge(id_of("create_forge_user"), "mint_agent_forge_token");
|
||||
assert!(
|
||||
!user_to_mint.accepts(hive_jobq::TerminalState::Failed),
|
||||
"a token for a user that was never created cannot be minted; this \
|
||||
edge has to be `after_ok`"
|
||||
);
|
||||
let mint_to_deploy = edge(id_of("mint_agent_forge_token"), "trigger_deploy");
|
||||
assert!(
|
||||
mint_to_deploy.accepts(hive_jobq::TerminalState::Failed),
|
||||
"a host with no forge or store must not cancel the deploy; this edge \
|
||||
has to be `after_any`, not `after_ok`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forge_token_node_renders_the_agent() {
|
||||
use hive_jobq_wire::WireNode as _;
|
||||
|
||||
let kind = SwarmNodeKind::MintAgentForgeToken {
|
||||
agent: "atlas".to_owned(),
|
||||
};
|
||||
assert_eq!(kind.label(), "mint_agent_forge_token");
|
||||
assert_eq!(kind.data(1)["agent"], "atlas");
|
||||
}
|
||||
|
||||
/// The manual route and the periodic pass both go through
|
||||
/// `queue_forge_token_mints`, so this is the assertion that a backfilled
|
||||
/// mint is the same node agent creation inserts.
|
||||
#[test]
|
||||
fn a_queued_mint_is_one_forge_token_node_per_agent() {
|
||||
use hive_jobq_wire::WireNode as _;
|
||||
|
||||
let sched = std::sync::Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
||||
hive_jobq::Graph::new(),
|
||||
hive_jobq::resources::ResourceTable::new(),
|
||||
));
|
||||
let ids = super::queue_forge_token_mints(&sched, vec!["a".to_owned(), "b".to_owned()])
|
||||
.expect("two single-node jobs insert");
|
||||
assert_eq!(ids.len(), 2);
|
||||
let guard = sched
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let mut agents: Vec<String> = guard
|
||||
.graph()
|
||||
.nodes()
|
||||
.map(|n| {
|
||||
assert_eq!(n.payload.label(), "mint_agent_forge_token");
|
||||
n.payload.data(n.id.get())["agent"]
|
||||
.as_str()
|
||||
.expect("agent is a string")
|
||||
.to_owned()
|
||||
})
|
||||
.collect();
|
||||
agents.sort();
|
||||
assert_eq!(agents, ["a", "b"]);
|
||||
}
|
||||
|
||||
/// The socket must not share a directory with anything else, because
|
||||
/// the socket is `0666` and the directory is therefore the only access
|
||||
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
|
||||
|
|
|
|||
Loading…
Reference in a new issue