From 14421687151397f9b9dc12de3421bb96d91d6b3b Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 21 Sep 2026 18:36:57 +0200 Subject: [PATCH] swarmctl: re-mint an existing agent's store identity Agent creation at swarm level is event-driven and nothing sweeps for agents missing a credential, so an agent created before a credential joined the mint never receives one -- nothing comes back around to it. Without a way to re-run the mint by hand, the only route to giving an existing agent its queue credential would be to delete and recreate the agent. POST /api/agents/{name}/identity enqueues the same MintAgentIdentity node POST /api/agents declares, rather than writing inline: a second code path that mints an identity is a second place for the four strings that have to agree to disagree. swarmctl agent mint-identity is the operator end, the same POST-and-print-the-node-id shape agent create already has. --hive is required on both ends. Neither the CLI nor the controller keeps a roster of which agent runs where, and the credentials this mints name a hive, so a default would be a guess that hands an agent subjects on a hive it does not run on. Documents the backfill as a runbook step, and fills in the renewal cell the credential matrix requires for the new row. --- docs/swarm/credentials.md | 42 +++++++--- docs/tools/swarmctl-cli.md | 29 +++++++ swarm-controller/src/agent_identity.rs | 26 +++--- swarm-controller/src/main.rs | 106 +++++++++++++++++++++++++ swarmctl/src/agent.rs | 89 ++++++++++++++++++--- swarmctl/src/main.rs | 77 ++++++++++++++++++ 6 files changed, 332 insertions(+), 37 deletions(-) diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 0143621a..706f7b18 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -54,15 +54,16 @@ strategy for every credential, including the mTLS leaf. -| store path | minter | reader — pulls at runtime, holds in memory | renewal | -| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------- | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | -| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | -| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | -| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | -| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | -| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | -| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | +| store path | minter | reader — pulls at runtime, holds in memory | renewal | +| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | +| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | +| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | none: the secret is fixed for the life of the agent and is revoked by deleting the path. A rotation mechanism is tracked as separate work, because rotating this credential needs a reconnect path — a queue client holding a revoked secret does not find out until it reconnects | +| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | +| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | +| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | +| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | +| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | @@ -76,6 +77,29 @@ carries it, and no hive ever needs the capability to mint an identity. `swarm-controller` proves the leaf it publishes before the creation job reports success, by logging in with it and reading the row back. +**Backfilling an agent that predates a credential.** Agent creation at swarm +level is purely event-driven — `swarm-controller` mints an agent's store +identity on the job graph `POST /api/agents` inserts, and nothing sweeps for +agents that already exist. So an agent created before a credential joined that +mint never receives one, and nothing will ever come back around to it. Re-run +the mint for one agent with: + +```sh +swarmctl agent mint-identity --hive +``` + +`--hive` is required: neither the CLI nor the controller keeps a roster of +which agent runs where, and the credentials this mints name a hive. The queue +secret half is idempotent — an agent that already has one keeps exactly the +value it has, so running this against an already-migrated agent does not drop +its queue connection. The certificate half is not: the agent gets a fresh leaf +and picks it up on its next boot. + +⚠️ **Run this for every existing agent before deploying a hive-side change +that makes a container require a credential it may not have.** A container +whose credential is absent does not start — that is deliberate, and it is what +makes the backfill a step rather than a suggestion. + **Who reads that row, and what happens to it.** `hive-c0re` reads it every time it writes an agent's container configuration (`lifecycle::agent_identity`), stages the certificate and its key `0600` diff --git a/docs/tools/swarmctl-cli.md b/docs/tools/swarmctl-cli.md index 4cc0af2e..67d90610 100644 --- a/docs/tools/swarmctl-cli.md +++ b/docs/tools/swarmctl-cli.md @@ -7,6 +7,7 @@ This document contains the help content for the `swarmctl` command-line program. * [`swarmctl`↴](#swarmctl) * [`swarmctl agent`↴](#swarmctl-agent) * [`swarmctl agent create`↴](#swarmctl-agent-create) +* [`swarmctl agent mint-identity`↴](#swarmctl-agent-mint-identity) * [`swarmctl user`↴](#swarmctl-user) * [`swarmctl user add`↴](#swarmctl-user-add) * [`swarmctl user update`↴](#swarmctl-user-update) @@ -43,6 +44,7 @@ Manage agents across the swarm ###### **Subcommands:** * `create` — Queue creation of a new agent on a hive in this swarm +* `mint-identity` — Queue a re-mint of an existing agent's identity at the swarm's secret store @@ -75,6 +77,33 @@ No approval gate guards this: running this binary already means being root on th +## `swarmctl agent mint-identity` + +Queue a re-mint of an existing agent's identity at the swarm's secret store. + +**The backfill verb.** Agent creation is event-driven and nothing at swarm level sweeps for agents that are missing a credential, so an agent created before a credential joined the mint never receives one. This re-runs the mint for one agent that already exists. + +⚠️ **It re-mints the agent's store certificate**, which that agent picks up the next time its container boots. The agent's queue secret is left exactly as it is if it already has one, so running this against an already-migrated agent does not disturb its queue connection. + +Queues and returns, the same way `agent create` does — watch the swarm UI's job view for the outcome. + +**Usage:** `swarmctl agent mint-identity [OPTIONS] --hive ` + +###### **Arguments:** + +* `` — Name of an agent that already exists + +###### **Options:** + +* `--hive ` — The hive that agent runs on. + + Required, and deliberately not defaulted: the credentials this mints name a hive, and neither this CLI nor the controller keeps a roster of which agent is on which hive. Naming the wrong one gives the agent an identity scoped to a hive it does not run on. The controller checks the value against the swarm's hive roster and names the known hives if it misses. +* `--controller-socket ` — swarm-controller's unix socket. + + Supplied by the nix module that installs this binary, from the same `socketPath` option the daemon binds; falls back to `SWARM_CONTROLLER_SOCKET`. + + + ## `swarmctl user` Manage subjects in the swarm's SSO provider diff --git a/swarm-controller/src/agent_identity.rs b/swarm-controller/src/agent_identity.rs index 262ec77c..b3205f73 100644 --- a/swarm-controller/src/agent_identity.rs +++ b/swarm-controller/src/agent_identity.rs @@ -8,14 +8,12 @@ //! store, and its grant already covers exactly the objects written here //! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on //! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on -//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere — the -//! agent's queue secret lives under the same `swarm/agents/` prefix the -//! certificate does, which is why adding it costs no grant on either side. +//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere. //! -//! **Two credentials, deliberately unrelated.** The certificate is how the -//! agent reaches the store; the queue secret is how it identifies itself to the -//! swarm queue. The second is not derived from the first, so renewing either is -//! a question that can be answered without reference to the other. +//! **Two credentials, deliberately unrelated.** The certificate reaches the +//! store; the queue secret identifies the agent to the swarm queue. Both sit +//! under `swarm/agents/`, so neither costs a grant — but the second is +//! not derived from the first, so either renews without reference to the other. //! //! Four separate strings have to agree before an agent can authenticate: the //! policy's name, the cert-auth role's name, the certificate's common name, @@ -247,14 +245,12 @@ fn generate_queue_secret() -> Result { /// names the policy, so the other order leaves a window in which it points at /// nothing. /// -/// ⚠️ **Step 3 is idempotent and step 2 is not.** Re-running this function -/// re-mints the agent's certificate — a fresh leaf the agent picks up on its -/// next boot — but leaves an existing queue secret exactly as it is. The -/// asymmetry is deliberate: an agent holds its queue secret in a live -/// connection, so replacing it would drop that agent off the queue until it -/// reconnected, and this function is re-run deliberately (by the backfill -/// route) against agents that are already running. Nothing here rotates a -/// queue secret; revoking one means deleting the path. +/// ⚠️ **Step 3 is idempotent and step 2 is not.** Re-running re-mints the +/// certificate — a fresh leaf the agent picks up on its next boot — but leaves +/// an existing queue secret alone. An agent holds that secret in a live +/// connection, and this function is re-run deliberately against agents that +/// are already running, so replacing it would drop them off the queue. +/// Nothing here rotates one; revoking means deleting the path. /// /// # Errors /// Anything that stops one of those five steps, with the step named. A diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 69fe1cad..3621ad39 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1500,6 +1500,111 @@ async fn get_agent_config_pr( Ok(Json(cache.get(&name))) } +/// Body of `POST /api/agents/{name}/identity`. +#[derive(Deserialize, ToSchema)] +struct MintAgentIdentityRequest { + /// The hive this agent belongs to. + /// + /// Required, for the same reason [`CreateAgentRequest`]'s is: the + /// credentials this mints name a hive, and the controller has nowhere to + /// look one up — agents are created on hives at runtime and this daemon + /// keeps no roster of which agent is where. An operator naming the wrong + /// one would hand the agent subjects on a hive it does not run on, so it + /// is asked for rather than guessed at. + hive: String, +} + +/// Success body of `POST /api/agents/{name}/identity`. +#[derive(Serialize, ToSchema)] +struct MintAgentIdentityResponse { + /// The queued node, so a caller can follow it in the job view. + node_id: u64, +} + +/// Re-run the store-identity mint for an agent that already exists. +/// +/// **This route exists for backfill.** Agent creation is event-driven and +/// there is no reconcile sweep at swarm level, so an agent created before a +/// credential was part of the mint never gets one — nothing would ever come +/// back around to it. Without a way to re-run the node by hand, the only way +/// to give an existing agent its queue credential would be to delete and +/// recreate the agent. +/// +/// A job node rather than an inline write, and specifically the *same* node +/// `POST /api/agents` declares: a second code path that mints an identity is a +/// second place for the four strings in [`agent_identity`] to disagree. +/// +/// ⚠️ **Not idempotent in one respect**: the node re-mints the agent's mTLS +/// leaf, which the agent picks up on its next boot. Its queue secret is left +/// alone — see [`agent_identity::mint_and_verify`] for which half is which and +/// why. +#[utoipa::path( + post, + path = "/api/agents/{name}/identity", + params(("name" = String, Path, description = "agent name")), + request_body = MintAgentIdentityRequest, + responses( + (status = 200, description = "mint queued", body = MintAgentIdentityResponse), + (status = 400, description = "`name` or `hive` is not a valid identifier, or `hive` is not in this swarm (problem+json)", body = String), + (status = 500, description = "the job could not be queued (problem+json)", body = String), + ), + tag = "agents" +)] +async fn mint_agent_identity( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, problem_details::ProblemDetails> { + // Both names are interpolated into store paths and policy documents + // downstream, so both are validated here as well as there. + let agent = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + let hive = hive_types::Ident::parse(&req.hive) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + if !state.hives.iter().any(|h| h.name == hive) { + let known: Vec<&str> = state.hives.iter().map(|h| h.name.as_str()).collect(); + let known = if known.is_empty() { + "(none configured)".to_owned() + } else { + known.join(", ") + }; + return Err(error_problem( + axum::http::StatusCode::BAD_REQUEST, + &format!("hive {hive:?} is not in this swarm — known hives: {known}"), + )); + } + + // No reserved-name or collision warnings here, unlike `create_agent`: + // those answer "is this name available", and this route is only ever + // pointed at a name that was already taken. + let mut sched = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let ids = sched + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::MintAgentIdentity { + hive: hive.clone(), + agent: agent.clone(), + }) + .guid(), + ] + }) + .map_err(|e| { + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &e.to_string(), + ) + })?; + let [id] = ids[..] else { + unreachable!("exactly one handle was asked for"); + }; + Ok(Json(MintAgentIdentityResponse { node_id: id.get() })) +} + /// Every agent with an open config PR, in one response — the bulk /// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs /// every agent's status to render, and fetching them one at a time doesn't @@ -1837,6 +1942,7 @@ fn build_app(state: AppState) -> axum::Router { .routes(routes!(get_agent_config_pr)) .routes(routes!(get_config_prs)) .routes(routes!(create_agent)) + .routes(routes!(mint_agent_identity)) .routes(routes!(get_agents)) .routes(routes!(get_agents_status)) .routes(routes!(set_agent_state)) diff --git a/swarmctl/src/agent.rs b/swarmctl/src/agent.rs index ad228c9c..7b6811d1 100644 --- a/swarmctl/src/agent.rs +++ b/swarmctl/src/agent.rs @@ -1,11 +1,11 @@ -//! `swarmctl agent create` — queue the swarm-controller's agent-creation -//! job graph. +//! `swarmctl agent create` and `swarmctl agent mint-identity` — queue work +//! on the swarm-controller's job graph. //! -//! `POST /api/agents` inserts a DAG and returns as soon as it is -//! *inserted*. This verb prints the queued node id and stops: the graph's -//! last node only *publishes* a deploy, after which the hive converges on -//! its own clock, so even a settled graph would not mean the agent is up -//! and there is nothing here that could be waited on honestly. +//! Each POSTs and returns as soon as the work is *inserted*. Both verbs +//! print the queued node id and stop: creation's last node only *publishes* +//! a deploy, after which the hive converges on its own clock, so even a +//! settled graph would not mean the agent is up and there is nothing here +//! that could be waited on honestly. //! //! Transport is a bare `hyper` HTTP/1.1 client handshaked onto a //! [`tokio::net::UnixStream`] via [`hyper_util::rt::TokioIo`] — the @@ -51,6 +51,19 @@ struct CreateAgentResponse { warnings: Vec, } +/// Body of `POST /api/agents/{name}/identity`. Mirrors the controller's own +/// `MintAgentIdentityRequest` — see this module's doc comment. +#[derive(Serialize)] +struct MintIdentityRequest<'a> { + hive: &'a str, +} + +/// Success body of `POST /api/agents/{name}/identity`. +#[derive(Deserialize)] +struct MintIdentityResponse { + node_id: u64, +} + /// Run `swarmctl agent create`. /// /// Synchronous on purpose: every other verb in this crate is, and this is @@ -94,10 +107,60 @@ fn parse_ident(value: &str, what: &str) -> Result { .map_err(|reason| anyhow::anyhow!("invalid {what} {value:?}: {reason}")) } +/// Run `swarmctl agent mint-identity`. +/// +/// Synchronous for the same reason [`create`] is, and built on the same +/// round trip. +pub(crate) fn mint_identity(socket: &Path, name: &str, hive: &str) -> Result<()> { + // Client-side first, so a typo is a local error rather than a 400 the + // operator waits for. The controller validates both again. + let name = parse_ident(name, "agent name")?; + let hive = parse_ident(hive, "hive")?; + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_io() + .build() + .context("starting a tokio runtime for the controller request")?; + let resp: MintIdentityResponse = rt.block_on(post( + socket, + &format!("/api/agents/{name}/identity"), + &MintIdentityRequest { hive: &hive }, + "mint-identity", + ))?; + + println!("queued: job node {}", resp.node_id); + println!( + "agent {name:?} will have its identity re-minted on hive {hive:?} once the job graph \ + runs; `swarmctl` does not wait for it. An existing queue secret is kept as it is; the \ + store certificate is re-minted and the agent picks the new one up on its next boot" + ); + Ok(()) +} + /// One `POST /api/agents` round trip over the controller's unix socket. async fn post_create(socket: &Path, name: &str, hive: &str) -> Result { - let body = serde_json::to_vec(&CreateAgentRequest { name, hive }) - .context("serialising the create-agent request")?; + post( + socket, + "/api/agents", + &CreateAgentRequest { name, hive }, + "create-agent", + ) + .await +} + +/// One JSON `POST` to `uri` over the controller's unix socket. +/// +/// `what` names the request in error messages — the operator needs to know +/// which call failed, and every other part of this function is identical +/// between the two verbs. +async fn post( + socket: &Path, + uri: &str, + request: &Req, + what: &str, +) -> Result { + let body = + serde_json::to_vec(request).with_context(|| format!("serialising the {what} request"))?; let stream = UnixStream::connect(socket) .await @@ -112,18 +175,18 @@ async fn post_create(socket: &Path, name: &str, hive: &str) -> Result Result, } +#[derive(Args)] +struct AgentMintIdentityArgs { + /// Name of an agent that already exists. + name: String, + /// The hive that agent runs on. + /// + /// Required, and deliberately not defaulted: the credentials this mints + /// name a hive, and neither this CLI nor the controller keeps a roster of + /// which agent is on which hive. Naming the wrong one gives the agent an + /// identity scoped to a hive it does not run on. The controller checks + /// the value against the swarm's hive roster and names the known hives if + /// it misses. + #[arg(long, value_name = "HIVE")] + hive: String, + /// swarm-controller's unix socket. + /// + /// Supplied by the nix module that installs this binary, from the same + /// `socketPath` option the daemon binds; falls back to + /// `SWARM_CONTROLLER_SOCKET`. + #[arg(long, value_name = "PATH")] + controller_socket: Option, +} + #[derive(Subcommand)] enum UserVerb { /// Add a user, generating a password for them. @@ -262,6 +302,13 @@ fn main() -> Result<()> { let socket = path_from(args.controller_socket, "SWARM_CONTROLLER_SOCKET")?; agent::create(&socket, &args.name, &args.hive) } + // Same socket-resolution reasoning as `Create` above. + Verb::Agent { + command: AgentVerb::MintIdentity(args), + } => { + let socket = path_from(args.controller_socket, "SWARM_CONTROLLER_SOCKET")?; + agent::mint_identity(&socket, &args.name, &args.hive) + } // Resolved lazily, inside the one arm that actually touches the // deployment env vars — see the `MarkdownDocs` doc comment above // for why an unconditional resolve up front would be wrong. @@ -621,6 +668,36 @@ mod tests { assert!(args.controller_socket.is_none()); } + /// The backfill verb takes the same two names as `create`, and `--hive` + /// is required on it for the same reason: it is an address nobody can + /// infer. + #[test] + fn the_backfill_verb_takes_an_agent_and_a_hive() { + let cli = Cli::try_parse_from([ + "swarmctl", + "agent", + "mint-identity", + "scribe", + "--hive", + "alpha", + ]) + .expect("the minimal form parses"); + let Verb::Agent { + command: AgentVerb::MintIdentity(args), + } = cli.command + else { + panic!("expected `agent mint-identity`"); + }; + assert_eq!(args.name, "scribe"); + assert_eq!(args.hive, "alpha"); + assert!(args.controller_socket.is_none()); + + assert!( + Cli::try_parse_from(["swarmctl", "agent", "mint-identity", "scribe"]).is_err(), + "an omitted hive must not be defaulted" + ); + } + #[test] fn parses_authelia_hash_output() { let out = "Random Password: hunter2\nDigest: $argon2id$v=19$m=65536$abc\n";