diff --git a/Cargo.lock b/Cargo.lock index e7bd31a4..097329b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4893,6 +4893,7 @@ dependencies = [ "bytes", "forgejo-api", "futures-util", + "getrandom 0.3.4", "hive-jobq", "hive-jobq-metrics", "hive-jobq-wire", diff --git a/Cargo.toml b/Cargo.toml index 28c22610..edae4e54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,12 @@ chrono = { version = "0.4", default-features = false, features = [ clap = { version = "4", features = ["derive"] } clap_complete = "4" enumflags2 = { version = "0.7.12", features = ["serde"] } +# The OS CSPRNG, for the one thing in this tree that generates a secret rather +# than receiving one (`swarm-controller::agent_identity`). `getrandom` rather +# than `rand`: the whole need is "fill these bytes from the kernel", and `rand` +# would add `rand_core` + `rand_chacha` to do it through a userspace generator +# this has no use for. +getrandom = "0.3" indicatif = "0.18" hive-sh4re = { path = "hive-sh4re" } hive-agent-sock = { path = "hive-agent-sock" } diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 0143621a..3a05fb70 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -54,15 +54,16 @@ strategy for every credential, including the mTLS leaf. -| store path | minter | reader — pulls at runtime, holds in memory | renewal | -| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------- | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | -| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | -| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | -| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | -| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | -| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | -| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | +| store path | minter | reader — pulls at runtime, holds in memory | renewal | +| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | +| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | +| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | none: the secret is fixed for the life of the agent and is revoked by deleting the path. A rotation mechanism is tracked as separate work, because rotating this credential needs a reconnect path — a queue client holding a revoked secret doesn't find out until it reconnects | +| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | +| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | +| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | +| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | +| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | @@ -76,6 +77,29 @@ carries it, and no hive ever needs the capability to mint an identity. `swarm-controller` proves the leaf it publishes before the creation job reports success, by logging in with it and reading the row back. +**Backfilling an agent that predates a credential.** Agent creation at swarm +level is purely event-driven — `swarm-controller` mints an agent's store +identity on the job graph `POST /api/agents` inserts, and nothing sweeps for +agents that already exist. An agent created before a credential joined that +mint therefore never receives one, and nothing will ever come back around to +it. Re-run the mint for one agent with: + +```sh +swarmctl agent mint-identity --hive +``` + +`--hive` has no default: neither the CLI nor the controller keeps a roster of +which agent runs where, and the credentials this mints name a hive. The queue +secret half is idempotent — an agent that already has one keeps exactly the +value it holds, so running this against an already-migrated agent doesn't drop +its queue connection. The certificate half isn't: the agent gets a fresh leaf +and picks it up on its next boot. + +⚠️ **Run this for every existing agent before deploying a hive-side change +that makes a container require a credential it may not have.** A container +whose credential is absent doesn't start — that's deliberate, and it's what +makes the backfill a step rather than a suggestion. + **Who reads that row, and what happens to it.** `hive-c0re` reads it every time it writes an agent's container configuration (`lifecycle::agent_identity`), stages the certificate and its key `0600` diff --git a/docs/tools/swarmctl-cli.md b/docs/tools/swarmctl-cli.md index 4cc0af2e..96d0eb20 100644 --- a/docs/tools/swarmctl-cli.md +++ b/docs/tools/swarmctl-cli.md @@ -7,6 +7,7 @@ This document contains the help content for the `swarmctl` command-line program. * [`swarmctl`↴](#swarmctl) * [`swarmctl agent`↴](#swarmctl-agent) * [`swarmctl agent create`↴](#swarmctl-agent-create) +* [`swarmctl agent mint-identity`↴](#swarmctl-agent-mint-identity) * [`swarmctl user`↴](#swarmctl-user) * [`swarmctl user add`↴](#swarmctl-user-add) * [`swarmctl user update`↴](#swarmctl-user-update) @@ -43,6 +44,7 @@ Manage agents across the swarm ###### **Subcommands:** * `create` — Queue creation of a new agent on a hive in this swarm +* `mint-identity` — Queue a re-mint of an existing agent's identity at the swarm's secret store @@ -75,6 +77,33 @@ No approval gate guards this: running this binary already means being root on th +## `swarmctl agent mint-identity` + +Queue a re-mint of an existing agent's identity at the swarm's secret store. + +**The backfill verb.** Agent creation is event-driven and nothing at swarm level sweeps for agents that are missing a credential, so an agent created before a credential joined the mint never receives one. This re-runs the mint for one agent that already exists. + +⚠️ **It re-mints the agent's store certificate**, which that agent picks up the next time its container boots. It leaves an existing queue secret exactly as it stands, so running this against an already-migrated agent doesn't disturb its queue connection. + +Queues and returns, the same way `agent create` does — watch the swarm UI's job view for the outcome. + +**Usage:** `swarmctl agent mint-identity [OPTIONS] --hive ` + +###### **Arguments:** + +* `` — Name of an agent that already exists + +###### **Options:** + +* `--hive ` — The hive that agent runs on. + + Required, and deliberately not defaulted: the credentials this mints name a hive, and neither this CLI nor the controller keeps a roster of which agent is on which hive. Naming the wrong one gives the agent an identity scoped to a hive it doesn't run on. The controller checks the value against the swarm's hive roster and names the known hives if it misses. +* `--controller-socket ` — swarm-controller's unix socket. + + Supplied by the nix module that installs this binary, from the same `socketPath` option the daemon binds; falls back to `SWARM_CONTROLLER_SOCKET`. + + + ## `swarmctl user` Manage subjects in the swarm's SSO provider diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index c64eea31..412c9fa5 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -26,6 +26,10 @@ forgejo-api.workspace = true # `repo_change_files` calls — forgejo's content API takes base64, never # raw bytes. base64.workspace = true +# `agent_identity.rs` mints the per-agent queue secret, the one value in this +# tree this daemon invents rather than receives. Straight from the kernel's +# CSPRNG — see the workspace entry for why this and not `rand`. +getrandom.workspace = true futures-util.workspace = true # RFC 9457 `application/problem+json` error bodies. Same version + `axum` # feature as hive-c0re: the two daemons answer the same operator UIs, so a diff --git a/swarm-controller/src/agent_identity.rs b/swarm-controller/src/agent_identity.rs index 8475222c..b3205f73 100644 --- a/swarm-controller/src/agent_identity.rs +++ b/swarm-controller/src/agent_identity.rs @@ -5,11 +5,16 @@ //! The swarm mints the agent's certificate so that no hive ever needs the //! capability to mint one; the hive only carries it down. The controller is //! the swarm-level service that does it because it already logs in to the -//! store, and its grant already covers exactly the three objects written here +//! store, and its grant already covers exactly the objects written here //! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on //! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on //! `auth/cert/certs/hive-*`). No new authority is asked for anywhere. //! +//! **Two credentials, deliberately unrelated.** The certificate reaches the +//! store; the queue secret identifies the agent to the swarm queue. Both sit +//! under `swarm/agents/`, so neither costs a grant — but the second is +//! not derived from the first, so either renews without reference to the other. +//! //! Four separate strings have to agree before an agent can authenticate: the //! policy's name, the cert-auth role's name, the certificate's common name, //! and the authority the role pins. That is the kind of agreement that holds @@ -30,7 +35,7 @@ use anyhow::{Context, Result, bail}; use swarm_secret_client::{ SecretStore, client::{DEFAULT_CERT_MOUNT, Settings}, - mtls, policy, + mtls, policy, queue, }; use time::OffsetDateTime; @@ -191,22 +196,62 @@ fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, Offs Ok((not_before, not_after)) } +/// How many bytes of kernel randomness a queue secret is before encoding. +/// +/// Thirty-two because the secret is a bearer token compared for equality and +/// nothing else — there is no work factor and no rate limit behind it, so the +/// only defence is that guessing is not worth attempting. Encoded it is 43 +/// characters. +const QUEUE_SECRET_BYTES: usize = 32; + +/// Generate a queue secret: [`QUEUE_SECRET_BYTES`] from the kernel's CSPRNG, +/// base64url without padding. +/// +/// The alphabet matters and is the reason for `URL_SAFE_NO_PAD` rather than +/// the standard engine: the secret is destined to be carried in a token the +/// verifying end splits on `.`, so it must not be able to contain one. This +/// alphabet is `[A-Za-z0-9_-]`, and `=` padding is dropped as well so the +/// value survives anything that treats it as a word. +/// +/// # Errors +/// When the kernel will not supply randomness. Bubbled rather than panicked +/// on: the caller is a job node that reports a named failure, and a secret +/// from a degraded source is worse than no secret. +fn generate_queue_secret() -> Result { + let mut bytes = [0u8; QUEUE_SECRET_BYTES]; + getrandom::fill(&mut bytes).context("drawing a queue secret from the kernel's CSPRNG")?; + Ok(base64::Engine::encode( + &base64::engine::general_purpose::URL_SAFE_NO_PAD, + bytes, + )) +} + /// Give `agent` an identity at the store, and prove it works. /// -/// Four store writes' worth of agreement, then the login that checks it: +/// Five store writes' worth of agreement, then the login that checks it: /// /// 1. mint a leaf whose common name is [`policy::agent_object_name`]; /// 2. publish it at [`mtls::identity_path`], where the agent's hive collects /// it under the hive's own certificate; -/// 3. write the ACL document [`policy::render_agent_with_queue`] renders — +/// 3. publish a queue secret at [`queue::agent_queue_path`] — the agent's own +/// identity at the swarm queue, minted here so that the credential an agent +/// presents names *it* rather than its hive; +/// 4. write the ACL document [`policy::render_agent_with_queue`] renders — /// read on this one agent's paths, plus the hive-shared queue credential /// every agent container on `hive` already receives out of band; -/// 4. write the cert-auth role that ties the three together. +/// 5. write the cert-auth role that ties the three together. /// /// Policy before role, for the reason `read_policy::provision` gives: the role /// names the policy, so the other order leaves a window in which it points at /// nothing. /// +/// ⚠️ **Step 3 is idempotent and step 2 is not.** Re-running re-mints the +/// certificate — a fresh leaf the agent picks up on its next boot — but leaves +/// an existing queue secret alone. An agent holds that secret in a live +/// connection, and this function is re-run deliberately against agents that +/// are already running, so replacing it would drop them off the queue. +/// Nothing here rotates one; revoking means deleting the path. +/// /// # Errors /// Anything that stops one of those five steps, with the step named. A /// failure here fails the job node and nothing else — the agent is still @@ -214,6 +259,7 @@ fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, Offs pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> { let name = policy::agent_object_name(agent)?; let path = mtls::identity_path(agent)?; + let queue_path = queue::agent_queue_path(agent)?; let (cert, key) = authority.mint_leaf(&name)?; let credential = mtls::Credential { @@ -229,6 +275,50 @@ pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> .write(&path, &credential) .await .with_context(|| format!("publishing the agent identity at {path}"))?; + + // Read before write, and `read_optional` rather than `read`, so that only + // a genuine 404 leads to a new secret — see that method for why every + // other failure has to stay a failure here. + let existing: Option = store + .read_optional(&queue_path) + .await + .with_context(|| format!("checking whether {queue_path} already holds a credential"))?; + // The secret survives a re-run; the principal it names does not get to. + // An object whose `hive` disagrees with the hive this node was invoked + // with would grant its holder subjects on the wrong hive, so it is + // corrected — but by rewriting the two name fields around the *same* + // `value`, which is a correction no live connection notices. + let wanted = queue::AgentCredential { + value: match &existing { + Some(existing) => existing.value.clone(), + None => generate_queue_secret()?, + }, + agent: agent.to_owned(), + hive: hive.to_owned(), + }; + if existing.as_ref() == Some(&wanted) { + tracing::info!( + agent, + %queue_path, + "agent queue credential already published; left as it is" + ); + } else { + store + .write(&queue_path, &wanted) + .await + .with_context(|| format!("publishing the agent queue credential at {queue_path}"))?; + tracing::info!( + agent, + hive, + %queue_path, + // Never "rotated": the secret is the same one, only the names + // around it moved. + corrected = existing.is_some(), + "agent queue credential published" + ); + } + let queue_credential = wanted; + store .write_policy(&name, &policy::render_agent_with_queue(agent, hive)?) .await @@ -239,29 +329,42 @@ pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> .with_context(|| format!("writing the cert-auth role {name}"))?; tracing::info!(agent, %path, role = %name, "agent store identity published"); - read_back_as_agent(&credential, &name, &path).await?; + read_back_as_agent(&credential, &name, &path, &queue_path, &queue_credential).await?; tracing::info!( agent, role = %name, - "agent store identity verified: the minted leaf logged in and read its own path" + "agent store identity verified: the minted leaf logged in and read both its own paths" ); Ok(()) } /// The consumer of everything [`mint_and_verify`] wrote: log in **as the -/// agent**, with the leaf just minted, and read back the path just published. +/// agent**, with the leaf just minted, and read back both paths just +/// published. /// /// The address and the store's CA come from this process's own `BAO_*` /// environment; the *identity* deliberately does not — see /// [`SecretStore::connect_with_identity`]. The freshly minted private key /// never touches a filesystem. /// +/// Both paths, not just the certificate's, for the reason this function +/// exists at all: the queue credential is readable only because it sits inside +/// the stanza [`policy::render_agent`] already grants, and a policy that +/// drifted from that path would otherwise fail at the agent's first connection +/// — far from here, as a denial naming neither the policy nor the path. +/// /// # Errors /// When the store refuses the login (the role, the authority or the common -/// name disagree), when the read is denied (the policy does not cover the +/// name disagree), when either read is denied (the policy does not cover the /// path, or the role attached the wrong policy), or when what comes back is /// not what went in. -async fn read_back_as_agent(credential: &mtls::Credential, role: &str, path: &str) -> Result<()> { +async fn read_back_as_agent( + credential: &mtls::Credential, + role: &str, + path: &str, + queue_path: &str, + queue_credential: &queue::AgentCredential, +) -> Result<()> { let settings = Settings::from_env() .context("reading this daemon's own store settings for the read-back")?; @@ -287,12 +390,25 @@ async fn read_back_as_agent(credential: &mtls::Credential, role: &str, path: &st if read_back != *credential { bail!("the store returned a different object at {path} than the one just published"); } + + let read_back: queue::AgentCredential = as_agent + .read(queue_path) + .await + .with_context(|| format!("reading {queue_path} back under {role}'s own token"))?; + // The two name fields are compared as well as the secret: they are what + // the verifying end will grant subjects from, so a mismatch here is the + // same class of fault as an unreadable path. + if read_back != *queue_credential { + bail!("the store returned a different object at {queue_path} than the one just published"); + } Ok(()) } #[cfg(test)] mod tests { - use super::{Authority, CLOCK_SKEW, LEAF_LIFETIME, validity}; + use super::{ + Authority, CLOCK_SKEW, LEAF_LIFETIME, QUEUE_SECRET_BYTES, generate_queue_secret, validity, + }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; /// A throwaway CA, minted in-process so no test needs a fixture file. @@ -378,6 +494,28 @@ mod tests { reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity"); } + /// The alphabet claim the token format rests on: the secret is carried in + /// a composite the verifying end splits on `.`, so a secret that could + /// contain one would make that split ambiguous. Also the entropy claim — + /// a generator that silently returned a short or constant value would pass + /// every other test in this file. + #[test] + fn a_queue_secret_is_high_entropy_and_carries_no_separator() { + let a = generate_queue_secret().expect("the kernel supplies randomness"); + let b = generate_queue_secret().expect("twice"); + assert_ne!(a, b, "two draws must not agree"); + + // base64 without padding: one character per six bits, rounded up. + assert_eq!(a.len(), (QUEUE_SECRET_BYTES * 8).div_ceil(6)); + assert!( + a.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'), + "{a}" + ); + assert!(!a.contains('.'), "{a}"); + assert!(!a.contains('='), "{a}"); + } + /// A misconfiguration that would otherwise look like "not configured": /// half an authority has to be an error, not a silent `None`. /// diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 69fe1cad..04f69945 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1500,6 +1500,111 @@ async fn get_agent_config_pr( Ok(Json(cache.get(&name))) } +/// Body of `POST /api/agents/{name}/identity`. +#[derive(Deserialize, ToSchema)] +struct MintAgentIdentityRequest { + /// The hive this agent belongs to. + /// + /// Required, for the same reason [`CreateAgentRequest`]'s is: the + /// credentials this mints name a hive, and the controller has nowhere to + /// look one up — agents are created on hives at runtime and this daemon + /// keeps no roster of which agent is where. An operator naming the wrong + /// one would hand the agent subjects on a hive it does not run on, so it + /// is asked for rather than guessed at. + hive: String, +} + +/// Success body of `POST /api/agents/{name}/identity`. +#[derive(Clone, Debug, Serialize, ToSchema)] +struct MintAgentIdentityResponse { + /// The queued node, so a caller can follow it in the job view. + node_id: u64, +} + +/// Re-run the store-identity mint for an agent that already exists. +/// +/// **This route exists for backfill.** Agent creation is event-driven and +/// there is no reconcile sweep at swarm level, so an agent created before a +/// credential was part of the mint never gets one — nothing would ever come +/// back around to it. Without a way to re-run the node by hand, the only way +/// to give an existing agent its queue credential would be to delete and +/// recreate the agent. +/// +/// A job node rather than an inline write, and specifically the *same* node +/// `POST /api/agents` declares: a second code path that mints an identity is a +/// second place for the four strings in [`agent_identity`] to disagree. +/// +/// ⚠️ **Not idempotent in one respect**: the node re-mints the agent's mTLS +/// leaf, which the agent picks up on its next boot. Its queue secret is left +/// alone — see [`agent_identity::mint_and_verify`] for which half is which and +/// why. +#[utoipa::path( + post, + path = "/api/agents/{name}/identity", + params(("name" = String, Path, description = "agent name")), + request_body = MintAgentIdentityRequest, + responses( + (status = 200, description = "mint queued", body = MintAgentIdentityResponse), + (status = 400, description = "`name` or `hive` is not a valid identifier, or `hive` is not in this swarm (problem+json)", body = String), + (status = 500, description = "the job could not be queued (problem+json)", body = String), + ), + tag = "agents" +)] +async fn mint_agent_identity( + State(state): State, + Path(name): Path, + Json(req): Json, +) -> Result, problem_details::ProblemDetails> { + // Both names are interpolated into store paths and policy documents + // downstream, so both are validated here as well as there. + let agent = hive_types::Ident::parse(&name) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + let hive = hive_types::Ident::parse(&req.hive) + .map_err(|reason| error_problem(axum::http::StatusCode::BAD_REQUEST, reason))? + .into_string(); + if !state.hives.iter().any(|h| h.name == hive) { + let known: Vec<&str> = state.hives.iter().map(|h| h.name.as_str()).collect(); + let known = if known.is_empty() { + "(none configured)".to_owned() + } else { + known.join(", ") + }; + return Err(error_problem( + axum::http::StatusCode::BAD_REQUEST, + &format!("hive {hive:?} is not in this swarm — known hives: {known}"), + )); + } + + // No reserved-name or collision warnings here, unlike `create_agent`: + // those answer "is this name available", and this route is only ever + // pointed at a name that was already taken. + let mut sched = state + .jobq + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let ids = sched + .insert_job(None, |b| { + vec![ + b.node(SwarmNodeKind::MintAgentIdentity { + hive: hive.clone(), + agent: agent.clone(), + }) + .guid(), + ] + }) + .map_err(|e| { + error_problem( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + &e.to_string(), + ) + })?; + let [id] = ids[..] else { + unreachable!("exactly one handle was asked for"); + }; + Ok(Json(MintAgentIdentityResponse { node_id: id.get() })) +} + /// Every agent with an open config PR, in one response — the bulk /// counterpart to [`get_agent_config_pr`]. swarm-ui's config-PR table needs /// every agent's status to render, and fetching them one at a time doesn't @@ -1837,6 +1942,7 @@ fn build_app(state: AppState) -> axum::Router { .routes(routes!(get_agent_config_pr)) .routes(routes!(get_config_prs)) .routes(routes!(create_agent)) + .routes(routes!(mint_agent_identity)) .routes(routes!(get_agents)) .routes(routes!(get_agents_status)) .routes(routes!(set_agent_state)) @@ -2070,6 +2176,107 @@ mod tests { assert!(queued > 0, "an accepted creation must queue work"); } + /// The backfill route's roster check, asserted by effect for the same + /// reason its sibling above is: a refusal that queued first would still + /// re-mint the agent's certificate, which every running agent on the + /// named hive picks up on its next boot. + #[tokio::test] + async fn a_backfill_for_a_hive_outside_the_roster_queues_nothing() { + let (state, sched) = state_with_roster(); + + let err = super::mint_agent_identity( + axum::extract::State(state), + axum::extract::Path("atlas".to_owned()), + axum::Json(super::MintAgentIdentityRequest { + hive: "pr1maa".to_owned(), + }), + ) + .await + .expect_err("a hive outside the roster must be refused"); + + let rendered = format!("{err:?}"); + assert!( + rendered.contains("pr1ma"), + "the refusal should name the known hives, got: {rendered}" + ); + + let queued = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count(); + assert_eq!(queued, 0, "a refused backfill must queue no work"); + } + + /// A name that is not an identifier is refused before it can reach a + /// store path. `create_agent` gets this for free from the roster check + /// on the hive; the agent name has no roster to check against, so this + /// is the only thing standing between `../beta` and a path built out of + /// it. + #[tokio::test] + async fn a_backfill_for_a_name_that_is_not_an_identifier_queues_nothing() { + let (state, sched) = state_with_roster(); + + super::mint_agent_identity( + axum::extract::State(state), + axum::extract::Path("../beta".to_owned()), + axum::Json(super::MintAgentIdentityRequest { + hive: "pr1ma".to_owned(), + }), + ) + .await + .expect_err("a traversal in the agent name must be refused"); + + let queued = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .graph() + .nodes() + .count(); + assert_eq!(queued, 0, "a refused backfill must queue no work"); + } + + /// The accept arm, and the property that makes this a *backfill* route + /// rather than a second creation route: it queues the mint node **and + /// nothing else**. `create_agent`'s graph also scaffolds a config repo + /// and publishes a deploy; running either of those against an agent that + /// already exists is the failure this assertion exists to catch. + #[tokio::test] + async fn a_backfill_queues_the_mint_node_and_only_the_mint_node() { + let (state, sched) = state_with_roster(); + + let queued = super::mint_agent_identity( + axum::extract::State(state), + axum::extract::Path("atlas".to_owned()), + axum::Json(super::MintAgentIdentityRequest { + hive: "pr1ma".to_owned(), + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let nodes: Vec<_> = guard.graph().nodes().collect(); + let [node] = nodes[..] else { + panic!("a backfill queues exactly one node, got {}", nodes.len()); + }; + assert!( + matches!( + &node.payload, + SwarmNodeKind::MintAgentIdentity { hive, agent } + if hive == "pr1ma" && agent == "atlas" + ), + "the one node must be the mint, carrying both names: {:?}", + node.payload + ); + // The id the operator is told to watch has to be the node that was + // actually inserted, or `swarmctl` prints a handle to nothing. + assert_eq!(queued.node_id, node.id.get()); + } + #[tokio::test] async fn create_repo_node_runs_end_to_end_and_fails_without_forge_configured() { unsafe { diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index ad9deef3..0a8232f3 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -174,6 +174,31 @@ impl SecretStore { Ok(vaultrs::kv2::read(&self.inner, MOUNT, path).await?) } + /// Read the object stored at `path`, or `None` when nothing is stored + /// there. + /// + /// The verb for a caller whose write must be a no-op if the path is + /// already populated. [`read`][Self::read] cannot serve that: it collapses + /// *absent*, *denied* and *undecodable* into one `Err`, so a caller + /// treating every failure as absence would overwrite a live credential + /// whenever the store was merely unreachable. + /// + /// **Only a 404 is absence.** A denial is a 403 and stays an error — a + /// principal whose grant does not cover the path must not conclude the + /// path is empty, which is how a write ends up clobbering something the + /// caller was never allowed to see. + /// + /// # Errors + /// [`Error::Vault`] for anything that is not a 404: a denial, an + /// unreachable store, or an object that does not decode as `T`. + pub async fn read_optional(&self, path: &str) -> Result, Error> { + match vaultrs::kv2::read(&self.inner, MOUNT, path).await { + Ok(value) => Ok(Some(value)), + Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(e.into()), + } + } + /// Write `value` at `path`, creating a new version. /// /// # Errors diff --git a/swarm-secret-client/src/queue.rs b/swarm-secret-client/src/queue.rs index ee424e42..3bd5a781 100644 --- a/swarm-secret-client/src/queue.rs +++ b/swarm-secret-client/src/queue.rs @@ -1,11 +1,27 @@ -//! The queue agreement: where a hive's agent-container credential lives in the -//! store, and what the object at that path holds. +//! The queue agreements: where a queue credential lives in the store, and what +//! the object at that path holds. //! -//! The sibling of [`crate::matrix`], and it differs from it in one way worth -//! reading before using either: a matrix credential is keyed per **agent**, -//! this one per **hive**. Agents are created at runtime, so the queue -//! identity they present is minted once per hive at deploy time and says which -//! hive an agent belongs to, never which agent. +//! **Two kinds live here, and which is which is the thing to get right.** +//! +//! - [`agent_client_path`] / [`Credential`] are keyed per **hive**: one OIDC +//! client minted per hive at deploy time, shared by every agent container on +//! it. It says which hive a caller belongs to, never which agent. +//! - [`agent_queue_path`] / [`AgentCredential`] are keyed per **agent**: a +//! secret the swarm mints for one agent, at swarm level, with no hive in the +//! chain. It is what makes one agent distinguishable from its co-hived +//! neighbours. +//! +//! They are separate objects rather than one with a nullable field because they +//! are minted by different principals on different events — the hive-scoped one +//! by authelia at deploy time, the per-agent one by `swarm-controller` when an +//! agent is created — and an object whose shape depends on who wrote it is a +//! reader that has to guess. +//! +//! The per-agent secret is deliberately **not** derived from the agent's mTLS +//! identity (the leaf at [`crate::mtls::identity_path`]). That leaf is for +//! reaching the store and nothing else; deriving a queue identity from it would +//! couple the two credentials' lifetimes, so that renewing one would mean +//! renewing the other. use serde::{Deserialize, Serialize}; @@ -25,6 +41,48 @@ pub fn agent_client_path(hive: &str) -> Result { Ok(format!("{prefix}/queue/agent")) } +/// The path holding the secret **one agent** presents to the swarm queue. +/// +/// Under the agent's own principal prefix, and that is the whole reason for +/// this spelling rather than a new top-level one: every agent's ACL document +/// already grants read on `swarm/agents//*` +/// ([`crate::policy::render_agent`]), so a credential here needs no new grant, +/// no policy re-render, and no rewrite of any existing agent's document. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another agent's secret. +pub fn agent_queue_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/queue")) +} + +/// What [`agent_queue_path`] holds: the secret, and the principal it proves. +/// +/// Both names ride **in the object** rather than being parsed back out of a +/// composite principal string. Hive and agent names draw from the same +/// alphabet (`hive_types::Ident`, `[a-z0-9-]`), so a principal spelled +/// `hive--agent-` parses two ways for a name containing `-agent-` +/// — and an ambiguous principal parse in an authorisation path is a caller that +/// authenticates fine and is handed somebody else's grant. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentCredential { + /// The secret itself. Named to match [`Credential::value`] and + /// [`crate::matrix::Credential::value`] so a nix-side reader spells + /// `bao kv get -field=value` for every kind. + pub value: String, + + /// The agent this secret authenticates. + pub agent: String, + + /// The hive that agent belongs to. + /// + /// Here because the verifying end has no roster to look it up in, and + /// because the subjects an agent is granted are hive-templated — without + /// this field the verifier would know *who* is connecting and not *where*. + pub hive: String, +} + /// What the path holds: the client secret, plus the client id it belongs to. /// /// The id rides with the secret for the same reason the homeserver rides with @@ -97,4 +155,108 @@ mod tests { assert_eq!(json["value"], "s3cr3t"); assert_eq!(json["client_id"], "hive-alpha-agent"); } + + #[test] + fn an_agent_name_lands_under_its_own_principal_prefix() { + assert_eq!( + agent_queue_path("atlas").expect("a plain name is legal"), + "swarm/agents/atlas/queue" + ); + } + + #[test] + fn a_traversal_in_the_agent_name_is_refused() { + let e = agent_queue_path("../beta").expect_err("a traversal is not"); + assert!(matches!(e, Error::PathSegment { kind: "agent", .. }), "{e}"); + } + + #[test] + fn two_agents_never_share_a_path() { + assert_ne!( + agent_queue_path("atlas").expect("legal"), + agent_queue_path("argus").expect("legal") + ); + } + + /// The reason this path was chosen over a new top-level prefix: it is + /// already inside the stanza every agent's own ACL document grants, so no + /// policy has to change for the credential to be readable by the one agent + /// it belongs to. + #[test] + fn the_agents_existing_policy_already_covers_its_queue_path() { + let document = crate::policy::render_agent("atlas").expect("legal"); + let path = agent_queue_path("atlas").expect("legal"); + let stanza = path + .rsplit_once('/') + .map(|(prefix, _)| format!("secret/data/{prefix}/*")) + .expect("the path has a parent"); + assert!(document.contains(&stanza), "{document}"); + } + + #[test] + fn the_agent_object_round_trips_through_the_store_representation() { + let c = AgentCredential { + value: "s3cr3t".to_owned(), + agent: "atlas".to_owned(), + hive: "alpha".to_owned(), + }; + let json = serde_json::to_string(&c).expect("serialises"); + assert_eq!( + serde_json::from_str::(&json).expect("deserialises"), + c + ); + } + + #[test] + fn the_agent_objects_field_names_are_the_ones_a_nix_reader_asks_for() { + let json = serde_json::to_value(AgentCredential { + value: "s3cr3t".to_owned(), + agent: "atlas".to_owned(), + hive: "alpha".to_owned(), + }) + .expect("serialises"); + assert_eq!(json["value"], "s3cr3t"); + assert_eq!(json["agent"], "atlas"); + assert_eq!(json["hive"], "alpha"); + } + + /// Neither name is optional. An object missing one is not a usable + /// credential — a verifier holding `None` for the hive can only guess at + /// the subjects to grant, and guessing is the failure this shape exists to + /// prevent. + #[test] + fn an_agent_object_missing_a_principal_does_not_decode() { + assert!( + serde_json::from_str::(r#"{"value":"s","agent":"atlas"}"#).is_err() + ); + assert!( + serde_json::from_str::(r#"{"value":"s","hive":"alpha"}"#).is_err() + ); + } + + /// The two kinds are different objects at different paths, and neither + /// decodes as the other — the property that keeps a reader from picking up + /// a hive-shared credential where a per-agent one was meant. + #[test] + fn the_hive_credential_and_the_agent_credential_are_not_interchangeable() { + let hive_json = serde_json::to_string(&Credential { + value: "s".to_owned(), + client_id: "hive-alpha-agent".to_owned(), + }) + .expect("serialises"); + assert!(serde_json::from_str::(&hive_json).is_err()); + + let agent_json = serde_json::to_string(&AgentCredential { + value: "s".to_owned(), + agent: "atlas".to_owned(), + hive: "alpha".to_owned(), + }) + .expect("serialises"); + assert!(serde_json::from_str::(&agent_json).is_err()); + + assert_ne!( + agent_queue_path("atlas").expect("legal"), + agent_client_path("atlas").expect("legal"), + ); + } } diff --git a/swarmctl/src/agent.rs b/swarmctl/src/agent.rs index ad228c9c..7b6811d1 100644 --- a/swarmctl/src/agent.rs +++ b/swarmctl/src/agent.rs @@ -1,11 +1,11 @@ -//! `swarmctl agent create` — queue the swarm-controller's agent-creation -//! job graph. +//! `swarmctl agent create` and `swarmctl agent mint-identity` — queue work +//! on the swarm-controller's job graph. //! -//! `POST /api/agents` inserts a DAG and returns as soon as it is -//! *inserted*. This verb prints the queued node id and stops: the graph's -//! last node only *publishes* a deploy, after which the hive converges on -//! its own clock, so even a settled graph would not mean the agent is up -//! and there is nothing here that could be waited on honestly. +//! Each POSTs and returns as soon as the work is *inserted*. Both verbs +//! print the queued node id and stop: creation's last node only *publishes* +//! a deploy, after which the hive converges on its own clock, so even a +//! settled graph would not mean the agent is up and there is nothing here +//! that could be waited on honestly. //! //! Transport is a bare `hyper` HTTP/1.1 client handshaked onto a //! [`tokio::net::UnixStream`] via [`hyper_util::rt::TokioIo`] — the @@ -51,6 +51,19 @@ struct CreateAgentResponse { warnings: Vec, } +/// Body of `POST /api/agents/{name}/identity`. Mirrors the controller's own +/// `MintAgentIdentityRequest` — see this module's doc comment. +#[derive(Serialize)] +struct MintIdentityRequest<'a> { + hive: &'a str, +} + +/// Success body of `POST /api/agents/{name}/identity`. +#[derive(Deserialize)] +struct MintIdentityResponse { + node_id: u64, +} + /// Run `swarmctl agent create`. /// /// Synchronous on purpose: every other verb in this crate is, and this is @@ -94,10 +107,60 @@ fn parse_ident(value: &str, what: &str) -> Result { .map_err(|reason| anyhow::anyhow!("invalid {what} {value:?}: {reason}")) } +/// Run `swarmctl agent mint-identity`. +/// +/// Synchronous for the same reason [`create`] is, and built on the same +/// round trip. +pub(crate) fn mint_identity(socket: &Path, name: &str, hive: &str) -> Result<()> { + // Client-side first, so a typo is a local error rather than a 400 the + // operator waits for. The controller validates both again. + let name = parse_ident(name, "agent name")?; + let hive = parse_ident(hive, "hive")?; + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_io() + .build() + .context("starting a tokio runtime for the controller request")?; + let resp: MintIdentityResponse = rt.block_on(post( + socket, + &format!("/api/agents/{name}/identity"), + &MintIdentityRequest { hive: &hive }, + "mint-identity", + ))?; + + println!("queued: job node {}", resp.node_id); + println!( + "agent {name:?} will have its identity re-minted on hive {hive:?} once the job graph \ + runs; `swarmctl` does not wait for it. An existing queue secret is kept as it is; the \ + store certificate is re-minted and the agent picks the new one up on its next boot" + ); + Ok(()) +} + /// One `POST /api/agents` round trip over the controller's unix socket. async fn post_create(socket: &Path, name: &str, hive: &str) -> Result { - let body = serde_json::to_vec(&CreateAgentRequest { name, hive }) - .context("serialising the create-agent request")?; + post( + socket, + "/api/agents", + &CreateAgentRequest { name, hive }, + "create-agent", + ) + .await +} + +/// One JSON `POST` to `uri` over the controller's unix socket. +/// +/// `what` names the request in error messages — the operator needs to know +/// which call failed, and every other part of this function is identical +/// between the two verbs. +async fn post( + socket: &Path, + uri: &str, + request: &Req, + what: &str, +) -> Result { + let body = + serde_json::to_vec(request).with_context(|| format!("serialising the {what} request"))?; let stream = UnixStream::connect(socket) .await @@ -112,18 +175,18 @@ async fn post_create(socket: &Path, name: &str, hive: &str) -> Result Result, } +#[derive(Args)] +struct AgentMintIdentityArgs { + /// Name of an agent that already exists. + name: String, + /// The hive that agent runs on. + /// + /// Required, and deliberately not defaulted: the credentials this mints + /// name a hive, and neither this CLI nor the controller keeps a roster of + /// which agent is on which hive. Naming the wrong one gives the agent an + /// identity scoped to a hive it doesn't run on. The controller checks + /// the value against the swarm's hive roster and names the known hives if + /// it misses. + #[arg(long, value_name = "HIVE")] + hive: String, + /// swarm-controller's unix socket. + /// + /// Supplied by the nix module that installs this binary, from the same + /// `socketPath` option the daemon binds; falls back to + /// `SWARM_CONTROLLER_SOCKET`. + #[arg(long, value_name = "PATH")] + controller_socket: Option, +} + #[derive(Subcommand)] enum UserVerb { /// Add a user, generating a password for them. @@ -262,6 +301,13 @@ fn main() -> Result<()> { let socket = path_from(args.controller_socket, "SWARM_CONTROLLER_SOCKET")?; agent::create(&socket, &args.name, &args.hive) } + // Same socket-resolution reasoning as `Create` above. + Verb::Agent { + command: AgentVerb::MintIdentity(args), + } => { + let socket = path_from(args.controller_socket, "SWARM_CONTROLLER_SOCKET")?; + agent::mint_identity(&socket, &args.name, &args.hive) + } // Resolved lazily, inside the one arm that actually touches the // deployment env vars — see the `MarkdownDocs` doc comment above // for why an unconditional resolve up front would be wrong. @@ -621,6 +667,36 @@ mod tests { assert!(args.controller_socket.is_none()); } + /// The backfill verb takes the same two names as `create`, and `--hive` + /// is required on it for the same reason: it is an address nobody can + /// infer. + #[test] + fn the_backfill_verb_takes_an_agent_and_a_hive() { + let cli = Cli::try_parse_from([ + "swarmctl", + "agent", + "mint-identity", + "scribe", + "--hive", + "alpha", + ]) + .expect("the minimal form parses"); + let Verb::Agent { + command: AgentVerb::MintIdentity(args), + } = cli.command + else { + panic!("expected `agent mint-identity`"); + }; + assert_eq!(args.name, "scribe"); + assert_eq!(args.hive, "alpha"); + assert!(args.controller_socket.is_none()); + + assert!( + Cli::try_parse_from(["swarmctl", "agent", "mint-identity", "scribe"]).is_err(), + "an omitted hive must not be defaulted" + ); + } + #[test] fn parses_authelia_hash_output() { let out = "Random Password: hunter2\nDigest: $argon2id$v=19$m=65536$abc\n";