//! One agent's own identity at the swarm's secret store: minted here, //! published here, granted here — and, before the job node reports success, //! **used** here. //! //! The swarm mints the agent's certificate so that no hive ever needs the //! capability to mint one; the hive only carries it down. The controller is //! the swarm-level service that does it because it already logs in to the //! store, and its grant already covers exactly the objects written here //! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on //! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on //! `auth/cert/certs/hive-*`). No new authority is asked for anywhere. //! //! **Two credentials, deliberately unrelated.** The certificate reaches the //! store; the queue secret identifies the agent to the swarm queue. Both sit //! under `swarm/agents/`, so neither costs a grant — but the second is //! not derived from the first, so either renews without reference to the other. //! //! Four separate strings have to agree before an agent can authenticate: the //! policy's name, the cert-auth role's name, the certificate's common name, //! and the authority the role pins. That is the kind of agreement that holds //! in review and fails in production — a mismatch is a 403 naming none of the //! four — so [`mint_and_verify`] does not finish on a write. See //! [`read_back_as_agent`]. //! //! ⚠️ The authority is **not** `/var/lib/swarm-bao-pki/ca-key.pem`. A //! cert-auth role pins its authority by value, per role (see //! [`SecretStore::write_cert_role`][swarm_secret_client::SecretStore::write_cert_role]), //! so a role this daemon writes carries whatever authority this daemon hands //! it — which is what lets the controller mint from its own CA on its own //! host, with nothing co-located and no existing role changed. use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use swarm_secret_client::{ SecretStore, client::{DEFAULT_CERT_MOUNT, Settings}, mtls, policy, queue, }; use time::OffsetDateTime; /// File holding the authority agent leaves are issued from, as /// `swarm-controller.nix` names it. Public material. pub const ENV_AGENT_CA: &str = "SWARM_CONTROLLER_AGENT_CA_FILE"; /// File holding the private key for [`ENV_AGENT_CA`]. 🩸 A path, never a /// value — the key's bytes must not reach a unit file or the nix store. pub const ENV_AGENT_CA_KEY: &str = "SWARM_CONTROLLER_AGENT_CA_KEY_FILE"; /// How long a minted leaf is good for. /// /// Short enough that a leaked key is not permanent, long enough that the /// absence of a renewal path is not immediately fatal. Nothing re-mints a leaf /// today, so until a renewal path lands this is the interval after which an /// operator re-runs agent creation. It is deliberately far shorter than the ten /// years /// `glue-bao-tls.nix` gives the store's own CA: that one is an authority /// whose reissue invalidates every leaf under it, this one is a leaf. /// Spelled in hours because `Duration::from_days` is not yet a stable `const /// fn`; `read_policy`'s `RETRY_WINDOW` is the same workaround. const LEAF_LIFETIME: Duration = Duration::from_hours(90 * 24); /// How far a leaf is backdated. /// /// The verifier is the store, on another machine: a certificate whose /// `notBefore` is this exact instant is refused outright by a clock a second /// behind ours, and the resulting error names a validity window rather than a /// clock. const CLOCK_SKEW: Duration = Duration::from_mins(5); /// The authority this daemon issues agent leaves from, loaded once at startup. /// /// ⚠️ **No `Debug` derive**, and the key field is private: this struct is /// reachable from `WorkerDeps`, which is formatted nowhere today and is one /// `#[derive(Debug)]` away from being formatted everywhere. pub struct Authority { /// The authority's certificate, PEM. Public material — it is also what /// goes into each agent's cert-auth role and into each agent's published /// credential. ca_pem: String, /// The authority's private key, PEM. Never logged, never published, /// never leaves this struct. key_pem: String, } impl Authority { /// Load the authority from the files [`ENV_AGENT_CA`] and /// [`ENV_AGENT_CA_KEY`] name, or `None` when this host was given neither. /// /// `None` is a supported deployment, not a failure: it is the state every /// controller is in before an operator has turned agent identities on, and /// `run_swarm_node` reports it as that one node's named failure rather /// than refusing to start the daemon. /// /// # Errors /// When exactly one of the two variables is set — half an authority signs /// nothing, and silently doing nothing about it is how a host ends up /// looking configured — or when a named file cannot be read. pub fn from_env() -> Result> { match ( std::env::var_os(ENV_AGENT_CA), std::env::var_os(ENV_AGENT_CA_KEY), ) { (None, None) => Ok(None), (Some(_), None) => bail!( "{ENV_AGENT_CA} is set but {ENV_AGENT_CA_KEY} is not — an authority with no key signs nothing" ), (None, Some(_)) => bail!( "{ENV_AGENT_CA_KEY} is set but {ENV_AGENT_CA} is not — a key with no certificate is not an authority" ), (Some(ca), Some(key)) => { let ca_path = ca.to_string_lossy().into_owned(); let key_path = key.to_string_lossy().into_owned(); Ok(Some(Self { ca_pem: std::fs::read_to_string(&ca_path).with_context(|| { format!("reading the agent authority {ca_path} (from {ENV_AGENT_CA})") })?, key_pem: std::fs::read_to_string(&key_path).with_context(|| { format!( "reading the agent authority's key {key_path} (from {ENV_AGENT_CA_KEY})" ) })?, })) } } } /// Build one from PEM already in hand — the constructor a test uses, and /// the one that keeps [`Authority::from_env`] the only place this process /// reads a key off disk. #[cfg(test)] fn from_pem(ca_pem: String, key_pem: String) -> Self { Self { ca_pem, key_pem } } /// Issue a client leaf carrying `common_name`, returning `(certificate, /// private key)` as PEM. /// /// `clientAuth` and nothing else: this certificate authenticates a /// principal to the store and must not be usable to *serve* anything. /// /// # Errors /// When the authority's own PEM will not parse, or the leaf will not sign. fn mint_leaf(&self, common_name: &str) -> Result<(String, String)> { let issuer_key = rcgen::KeyPair::from_pem(&self.key_pem) .context("the agent authority's key is not a PEM key that can sign")?; let issuer = rcgen::Issuer::from_ca_cert_pem(&self.ca_pem, issuer_key) .context("the agent authority is not a PEM certificate that can issue")?; let (not_before, not_after) = validity(SystemTime::now(), LEAF_LIFETIME)?; let mut params = rcgen::CertificateParams::default(); params.distinguished_name = rcgen::DistinguishedName::new(); params .distinguished_name .push(rcgen::DnType::CommonName, common_name); params.is_ca = rcgen::IsCa::NoCa; params.use_authority_key_identifier_extension = true; params.key_usages = vec![ rcgen::KeyUsagePurpose::DigitalSignature, rcgen::KeyUsagePurpose::KeyEncipherment, ]; params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; params.not_before = not_before; params.not_after = not_after; let leaf_key = rcgen::KeyPair::generate().context("generating the leaf's key")?; let cert = params .signed_by(&leaf_key, &issuer) .with_context(|| format!("signing a leaf for {common_name}"))?; Ok((cert.pem(), leaf_key.serialize_pem())) } } /// The validity window of a leaf minted at `now`, backdated by [`CLOCK_SKEW`]. /// /// A free function taking `now` rather than reading the clock itself, so the /// arithmetic — the part that can be wrong by a factor of sixty — is testable /// without waiting ninety days. /// /// # Errors /// When the system clock is before the unix epoch, or so far past it that the /// window will not fit a timestamp. fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, OffsetDateTime)> { let secs = now .duration_since(UNIX_EPOCH) .context("the system clock is before the unix epoch")? .as_secs(); let secs = i64::try_from(secs).context("the system clock is past what a timestamp holds")?; let skew = i64::try_from(CLOCK_SKEW.as_secs()).expect("a five-minute constant fits an i64"); let life = i64::try_from(lifetime.as_secs()).context("the leaf lifetime does not fit")?; let not_before = OffsetDateTime::from_unix_timestamp(secs - skew) .context("the backdated start is not a representable time")?; let not_after = OffsetDateTime::from_unix_timestamp(secs + life) .context("the expiry is not a representable time")?; Ok((not_before, not_after)) } /// How many bytes of kernel randomness a queue secret is before encoding. /// /// Thirty-two because the secret is a bearer token compared for equality and /// nothing else — there is no work factor and no rate limit behind it, so the /// only defence is that guessing is not worth attempting. Encoded it is 43 /// characters. const QUEUE_SECRET_BYTES: usize = 32; /// Generate a queue secret: [`QUEUE_SECRET_BYTES`] from the kernel's CSPRNG, /// base64url without padding. /// /// The alphabet matters and is the reason for `URL_SAFE_NO_PAD` rather than /// the standard engine: the secret is destined to be carried in a token the /// verifying end splits on `.`, so it must not be able to contain one. This /// alphabet is `[A-Za-z0-9_-]`, and `=` padding is dropped as well so the /// value survives anything that treats it as a word. /// /// # Errors /// When the kernel will not supply randomness. Bubbled rather than panicked /// on: the caller is a job node that reports a named failure, and a secret /// from a degraded source is worse than no secret. fn generate_queue_secret() -> Result { let mut bytes = [0u8; QUEUE_SECRET_BYTES]; getrandom::fill(&mut bytes).context("drawing a queue secret from the kernel's CSPRNG")?; Ok(base64::Engine::encode( &base64::engine::general_purpose::URL_SAFE_NO_PAD, bytes, )) } /// Give `agent` an identity at the store, and prove it works. /// /// Five store writes' worth of agreement, then the login that checks it: /// /// 1. mint a leaf whose common name is [`policy::agent_object_name`]; /// 2. publish it at [`mtls::identity_path`], where the agent's hive collects /// it under the hive's own certificate; /// 3. publish a queue secret at [`queue::agent_queue_path`] — the agent's own /// identity at the swarm queue, minted here so that the credential an agent /// presents names *it* rather than its hive; /// 4. write the ACL document [`policy::render_agent_with_queue`] renders — /// read on this one agent's paths, plus the hive-shared queue credential /// every agent container on `hive` already receives out of band; /// 5. write the cert-auth role that ties the three together. /// /// Policy before role, for the reason `read_policy::provision` gives: the role /// names the policy, so the other order leaves a window in which it points at /// nothing. /// /// ⚠️ **Step 3 is idempotent and step 2 is not.** Re-running re-mints the /// certificate — a fresh leaf the agent picks up on its next boot — but leaves /// an existing queue secret alone. An agent holds that secret in a live /// connection, and this function is re-run deliberately against agents that /// are already running, so replacing it would drop them off the queue. /// Nothing here rotates one; revoking means deleting the path. /// /// # Errors /// Anything that stops one of those five steps, with the step named. A /// failure here fails the job node and nothing else — the agent is still /// created, exactly as capable as every agent is today. pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> { let name = policy::agent_object_name(agent)?; let path = mtls::identity_path(agent)?; let queue_path = queue::agent_queue_path(agent)?; let (cert, key) = authority.mint_leaf(&name)?; let credential = mtls::Credential { cert, key, ca: authority.ca_pem.clone(), }; let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; store .write(&path, &credential) .await .with_context(|| format!("publishing the agent identity at {path}"))?; // Read before write, and `read_optional` rather than `read`, so that only // a genuine 404 leads to a new secret — see that method for why every // other failure has to stay a failure here. let existing: Option = store .read_optional(&queue_path) .await .with_context(|| format!("checking whether {queue_path} already holds a credential"))?; // The secret survives a re-run; the principal it names does not get to. // An object whose `hive` disagrees with the hive this node was invoked // with would grant its holder subjects on the wrong hive, so it is // corrected — but by rewriting the two name fields around the *same* // `value`, which is a correction no live connection notices. let wanted = queue::AgentCredential { value: match &existing { Some(existing) => existing.value.clone(), None => generate_queue_secret()?, }, agent: agent.to_owned(), hive: hive.to_owned(), }; if existing.as_ref() == Some(&wanted) { tracing::info!( agent, %queue_path, "agent queue credential already published; left as it is" ); } else { store .write(&queue_path, &wanted) .await .with_context(|| format!("publishing the agent queue credential at {queue_path}"))?; tracing::info!( agent, hive, %queue_path, // Never "rotated": the secret is the same one, only the names // around it moved. corrected = existing.is_some(), "agent queue credential published" ); } let queue_credential = wanted; store .write_policy(&name, &policy::render_agent_with_queue(agent, hive)?) .await .with_context(|| format!("writing the read policy {name}"))?; store .write_cert_role(DEFAULT_CERT_MOUNT, &name, &authority.ca_pem, &name, &name) .await .with_context(|| format!("writing the cert-auth role {name}"))?; tracing::info!(agent, %path, role = %name, "agent store identity published"); read_back_as_agent(&credential, &name, &path, &queue_path, &queue_credential).await?; tracing::info!( agent, role = %name, "agent store identity verified: the minted leaf logged in and read both its own paths" ); Ok(()) } /// The consumer of everything [`mint_and_verify`] wrote: log in **as the /// agent**, with the leaf just minted, and read back both paths just /// published. /// /// The address and the store's CA come from this process's own `BAO_*` /// environment; the *identity* deliberately does not — see /// [`SecretStore::connect_with_identity`]. The freshly minted private key /// never touches a filesystem. /// /// Both paths, not just the certificate's, for the reason this function /// exists at all: the queue credential is readable only because it sits inside /// the stanza [`policy::render_agent`] already grants, and a policy that /// drifted from that path would otherwise fail at the agent's first connection /// — far from here, as a denial naming neither the policy nor the path. /// /// # Errors /// When the store refuses the login (the role, the authority or the common /// name disagree), when either read is denied (the policy does not cover the /// path, or the role attached the wrong policy), or when what comes back is /// not what went in. async fn read_back_as_agent( credential: &mtls::Credential, role: &str, path: &str, queue_path: &str, queue_credential: &queue::AgentCredential, ) -> Result<()> { let settings = Settings::from_env() .context("reading this daemon's own store settings for the read-back")?; // The concatenated blob `connect_with_identity` takes, built in memory. let mut identity = credential.cert.clone().into_bytes(); identity.push(b'\n'); identity.extend_from_slice(credential.key.as_bytes()); let as_agent = SecretStore::connect_with_identity(&settings, &identity, role, DEFAULT_CERT_MOUNT) .await .with_context(|| { format!("logging in to the store as {role} with the leaf just minted") })?; let read_back: mtls::Credential = as_agent .read(path) .await .with_context(|| format!("reading {path} back under {role}'s own token"))?; // Compared, not merely decoded: a successful read of an object written by // some earlier run would otherwise pass this check while this run's leaf // was the one nobody could use. Nothing about either value is printed. if read_back != *credential { bail!("the store returned a different object at {path} than the one just published"); } let read_back: queue::AgentCredential = as_agent .read(queue_path) .await .with_context(|| format!("reading {queue_path} back under {role}'s own token"))?; // The two name fields are compared as well as the secret: they are what // the verifying end will grant subjects from, so a mismatch here is the // same class of fault as an unreadable path. if read_back != *queue_credential { bail!("the store returned a different object at {queue_path} than the one just published"); } Ok(()) } #[cfg(test)] mod tests { use super::{ Authority, CLOCK_SKEW, LEAF_LIFETIME, QUEUE_SECRET_BYTES, generate_queue_secret, validity, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; /// A throwaway CA, minted in-process so no test needs a fixture file. fn test_authority() -> Authority { let key = rcgen::KeyPair::generate().expect("a key generates"); let mut params = rcgen::CertificateParams::default(); params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Constrained(0)); params .distinguished_name .push(rcgen::DnType::CommonName, "swarm-agent-ca"); let ca = params.self_signed(&key).expect("the CA self-signs"); Authority::from_pem(ca.pem(), key.serialize_pem()) } #[test] fn the_window_is_backdated_by_the_skew_and_as_long_as_the_lifetime() { // The arithmetic that is wrong by a factor of sixty if a unit slips. let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); let (before, after) = validity(now, LEAF_LIFETIME).expect("a plain instant is fine"); assert_eq!(before.unix_timestamp(), 1_700_000_000 - 300); assert_eq!(after.unix_timestamp(), 1_700_000_000 + 90 * 24 * 60 * 60); assert_eq!(CLOCK_SKEW, Duration::from_mins(5)); } #[test] fn a_minted_leaf_starts_valid_and_expires() { let now = i64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) .expect("the test host's clock is after 1970") .as_secs(), ) .expect("and before the end of time"); let (before, after) = validity(SystemTime::now(), LEAF_LIFETIME).expect("now is fine"); assert!( before.unix_timestamp() < now, "a leaf usable only in the future is unusable" ); assert!( after.unix_timestamp() > now, "a leaf that has already expired authenticates nothing" ); // The rule `docs/swarm/credentials.md` states: no credential's // renewal strategy may read NONE, which starts with it having an end. // A decade-long leaf is that rule broken in a way review would miss. assert!(after.unix_timestamp() - now < 3653 * 24 * 60 * 60); } /// The four-strings agreement `mint_and_verify` rests on, checked on the /// one of the four this process controls directly: the certificate really /// does carry the common name the cert-auth role will be told to match. #[test] fn the_leaf_carries_the_agents_object_name_as_its_common_name() { let name = swarm_secret_client::policy::agent_object_name("atlas").expect("legal"); assert_eq!(name, "hive-agent-atlas"); let (cert, key) = test_authority().mint_leaf(&name).expect("the leaf signs"); assert!(cert.contains("BEGIN CERTIFICATE"), "a PEM certificate"); assert!(key.contains("PRIVATE KEY"), "a PEM key"); // Searched in the SIGNED DER rather than asserted on the params we // built: the claim is that the name reached the bytes a verifier // reads. A byte search rather than an X.509 parse because the whole // crate would otherwise gain a parser dependency for one assertion — // the name is a UTF8String in the subject DN, so it appears verbatim. let body: String = cert .lines() .filter(|l| !l.starts_with("-----")) .collect::>() .join(""); let der = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, body) .expect("the PEM body is base64"); assert!( der.windows(name.len()).any(|w| w == name.as_bytes()), "the common name must be inside the signed certificate" ); // And the pair is a usable client identity — the first thing // `read_back_as_agent` does with it, in exactly this shape. let mut identity = cert.into_bytes(); identity.push(b'\n'); identity.extend_from_slice(key.as_bytes()); reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity"); } /// The alphabet claim the token format rests on: the secret is carried in /// a composite the verifying end splits on `.`, so a secret that could /// contain one would make that split ambiguous. Also the entropy claim — /// a generator that silently returned a short or constant value would pass /// every other test in this file. #[test] fn a_queue_secret_is_high_entropy_and_carries_no_separator() { let a = generate_queue_secret().expect("the kernel supplies randomness"); let b = generate_queue_secret().expect("twice"); assert_ne!(a, b, "two draws must not agree"); // base64 without padding: one character per six bits, rounded up. assert_eq!(a.len(), (QUEUE_SECRET_BYTES * 8).div_ceil(6)); assert!( a.chars() .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'), "{a}" ); assert!(!a.contains('.'), "{a}"); assert!(!a.contains('='), "{a}"); } /// A misconfiguration that would otherwise look like "not configured": /// half an authority has to be an error, not a silent `None`. /// /// SAFETY: single-threaded mutation of two env vars no other test in this /// crate reads, removed again before returning. #[test] fn half_an_authority_is_an_error_and_neither_half_is_absence() { unsafe { std::env::remove_var(super::ENV_AGENT_CA); std::env::remove_var(super::ENV_AGENT_CA_KEY); } assert!( Authority::from_env() .expect("neither set is a supported shape") .is_none(), "a controller with no authority configured is not an error" ); unsafe { std::env::set_var(super::ENV_AGENT_CA, "/nonexistent/ca.pem") } // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an // `Authority`, which deliberately has no `Debug` (it holds a key). let e = Authority::from_env() .err() .expect("a certificate with no key is half an authority"); assert!(format!("{e:#}").contains(super::ENV_AGENT_CA_KEY), "{e:#}"); unsafe { std::env::remove_var(super::ENV_AGENT_CA); std::env::set_var(super::ENV_AGENT_CA_KEY, "/nonexistent/ca-key.pem"); } // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an // `Authority`, which deliberately has no `Debug` (it holds a key). let e = Authority::from_env() .err() .expect("a key with no certificate is the other half"); assert!(format!("{e:#}").contains(super::ENV_AGENT_CA), "{e:#}"); unsafe { std::env::remove_var(super::ENV_AGENT_CA_KEY) } } }