hyperhive/swarm-controller/src/agent_identity.rs

//! One agent's own identity at the swarm's secret store: minted here,
//! published here, granted here — and, before the job node reports success,
//! **used** here.
//!
//! The swarm mints the agent's certificate so that no hive ever needs the
//! capability to mint one; the hive only carries it down. The controller is
//! the swarm-level service that does it because it already logs in to the
//! store, and its grant already covers exactly the three objects written here
//! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on
//! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on
//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere.
//!
//! Four separate strings have to agree before an agent can authenticate: the
//! policy's name, the cert-auth role's name, the certificate's common name,
//! and the authority the role pins. That is the kind of agreement that holds
//! in review and fails in production — a mismatch is a 403 naming none of the
//! four — so [`mint_and_verify`] does not finish on a write. See
//! [`read_back_as_agent`].
//!
//! ⚠️ The authority is **not** `/var/lib/swarm-bao-pki/ca-key.pem`. A
//! cert-auth role pins its authority by value, per role (see
//! [`SecretStore::write_cert_role`][swarm_secret_client::SecretStore::write_cert_role]),
//! so a role this daemon writes carries whatever authority this daemon hands
//! it — which is what lets the controller mint from its own CA on its own
//! host, with nothing co-located and no existing role changed.

use std::time::{Duration, SystemTime, UNIX_EPOCH};

use anyhow::{Context, Result, bail};
use swarm_secret_client::{
    SecretStore,
    client::{DEFAULT_CERT_MOUNT, Settings},
    mtls, policy,
};
use time::OffsetDateTime;

/// File holding the authority agent leaves are issued from, as
/// `swarm-controller.nix` names it. Public material.
pub const ENV_AGENT_CA: &str = "SWARM_CONTROLLER_AGENT_CA_FILE";

/// File holding the private key for [`ENV_AGENT_CA`]. 🩸 A path, never a
/// value — the key's bytes must not reach a unit file or the nix store.
pub const ENV_AGENT_CA_KEY: &str = "SWARM_CONTROLLER_AGENT_CA_KEY_FILE";

/// How long a minted leaf is good for.
///
/// Short enough that a leaked key is not permanent, long enough that the
/// absence of a renewal path is not immediately fatal. Nothing re-mints a leaf
/// today, so until a renewal path lands this is the interval after which an
/// operator re-runs agent creation. It is deliberately far shorter than the ten
/// years
/// `glue-bao-tls.nix` gives the store's own CA: that one is an authority
/// whose reissue invalidates every leaf under it, this one is a leaf.
/// Spelled in hours because `Duration::from_days` is not yet a stable `const
/// fn`; `read_policy`'s `RETRY_WINDOW` is the same workaround.
const LEAF_LIFETIME: Duration = Duration::from_hours(90 * 24);

/// How far a leaf is backdated.
///
/// The verifier is the store, on another machine: a certificate whose
/// `notBefore` is this exact instant is refused outright by a clock a second
/// behind ours, and the resulting error names a validity window rather than a
/// clock.
const CLOCK_SKEW: Duration = Duration::from_mins(5);

/// The authority this daemon issues agent leaves from, loaded once at startup.
///
/// ⚠️ **No `Debug` derive**, and the key field is private: this struct is
/// reachable from `WorkerDeps`, which is formatted nowhere today and is one
/// `#[derive(Debug)]` away from being formatted everywhere.
pub struct Authority {
    /// The authority's certificate, PEM. Public material — it is also what
    /// goes into each agent's cert-auth role and into each agent's published
    /// credential.
    ca_pem: String,
    /// The authority's private key, PEM. Never logged, never published,
    /// never leaves this struct.
    key_pem: String,
}

impl Authority {
    /// Load the authority from the files [`ENV_AGENT_CA`] and
    /// [`ENV_AGENT_CA_KEY`] name, or `None` when this host was given neither.
    ///
    /// `None` is a supported deployment, not a failure: it is the state every
    /// controller is in before an operator has turned agent identities on, and
    /// `run_swarm_node` reports it as that one node's named failure rather
    /// than refusing to start the daemon.
    ///
    /// # Errors
    /// When exactly one of the two variables is set — half an authority signs
    /// nothing, and silently doing nothing about it is how a host ends up
    /// looking configured — or when a named file cannot be read.
    pub fn from_env() -> Result<Option<Self>> {
        match (
            std::env::var_os(ENV_AGENT_CA),
            std::env::var_os(ENV_AGENT_CA_KEY),
        ) {
            (None, None) => Ok(None),
            (Some(_), None) => bail!(
                "{ENV_AGENT_CA} is set but {ENV_AGENT_CA_KEY} is not — an authority with no key signs nothing"
            ),
            (None, Some(_)) => bail!(
                "{ENV_AGENT_CA_KEY} is set but {ENV_AGENT_CA} is not — a key with no certificate is not an authority"
            ),
            (Some(ca), Some(key)) => {
                let ca_path = ca.to_string_lossy().into_owned();
                let key_path = key.to_string_lossy().into_owned();
                Ok(Some(Self {
                    ca_pem: std::fs::read_to_string(&ca_path).with_context(|| {
                        format!("reading the agent authority {ca_path} (from {ENV_AGENT_CA})")
                    })?,
                    key_pem: std::fs::read_to_string(&key_path).with_context(|| {
                        format!(
                            "reading the agent authority's key {key_path} (from {ENV_AGENT_CA_KEY})"
                        )
                    })?,
                }))
            }
        }
    }

    /// Build one from PEM already in hand — the constructor a test uses, and
    /// the one that keeps [`Authority::from_env`] the only place this process
    /// reads a key off disk.
    #[cfg(test)]
    fn from_pem(ca_pem: String, key_pem: String) -> Self {
        Self { ca_pem, key_pem }
    }

    /// Issue a client leaf carrying `common_name`, returning `(certificate,
    /// private key)` as PEM.
    ///
    /// `clientAuth` and nothing else: this certificate authenticates a
    /// principal to the store and must not be usable to *serve* anything.
    ///
    /// # Errors
    /// When the authority's own PEM will not parse, or the leaf will not sign.
    fn mint_leaf(&self, common_name: &str) -> Result<(String, String)> {
        let issuer_key = rcgen::KeyPair::from_pem(&self.key_pem)
            .context("the agent authority's key is not a PEM key that can sign")?;
        let issuer = rcgen::Issuer::from_ca_cert_pem(&self.ca_pem, issuer_key)
            .context("the agent authority is not a PEM certificate that can issue")?;

        let (not_before, not_after) = validity(SystemTime::now(), LEAF_LIFETIME)?;
        let mut params = rcgen::CertificateParams::default();
        params.distinguished_name = rcgen::DistinguishedName::new();
        params
            .distinguished_name
            .push(rcgen::DnType::CommonName, common_name);
        params.is_ca = rcgen::IsCa::NoCa;
        params.use_authority_key_identifier_extension = true;
        params.key_usages = vec![
            rcgen::KeyUsagePurpose::DigitalSignature,
            rcgen::KeyUsagePurpose::KeyEncipherment,
        ];
        params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth];
        params.not_before = not_before;
        params.not_after = not_after;

        let leaf_key = rcgen::KeyPair::generate().context("generating the leaf's key")?;
        let cert = params
            .signed_by(&leaf_key, &issuer)
            .with_context(|| format!("signing a leaf for {common_name}"))?;
        Ok((cert.pem(), leaf_key.serialize_pem()))
    }
}

/// The validity window of a leaf minted at `now`, backdated by [`CLOCK_SKEW`].
///
/// A free function taking `now` rather than reading the clock itself, so the
/// arithmetic — the part that can be wrong by a factor of sixty — is testable
/// without waiting ninety days.
///
/// # Errors
/// When the system clock is before the unix epoch, or so far past it that the
/// window will not fit a timestamp.
fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, OffsetDateTime)> {
    let secs = now
        .duration_since(UNIX_EPOCH)
        .context("the system clock is before the unix epoch")?
        .as_secs();
    let secs = i64::try_from(secs).context("the system clock is past what a timestamp holds")?;
    let skew = i64::try_from(CLOCK_SKEW.as_secs()).expect("a five-minute constant fits an i64");
    let life = i64::try_from(lifetime.as_secs()).context("the leaf lifetime does not fit")?;

    let not_before = OffsetDateTime::from_unix_timestamp(secs - skew)
        .context("the backdated start is not a representable time")?;
    let not_after = OffsetDateTime::from_unix_timestamp(secs + life)
        .context("the expiry is not a representable time")?;
    Ok((not_before, not_after))
}

/// Give `agent` an identity at the store, and prove it works.
///
/// Four store writes' worth of agreement, then the login that checks it:
///
/// 1. mint a leaf whose common name is [`policy::agent_object_name`];
/// 2. publish it at [`mtls::identity_path`], where the agent's hive collects
///    it under the hive's own certificate;
/// 3. write the ACL document [`policy::render_agent_with_queue`] renders —
///    read on this one agent's paths, plus the hive-shared queue credential
///    every agent container on `hive` already receives out of band;
/// 4. write the cert-auth role that ties the three together.
///
/// Policy before role, for the reason `read_policy::provision` gives: the role
/// names the policy, so the other order leaves a window in which it points at
/// nothing.
///
/// # Errors
/// Anything that stops one of those five steps, with the step named. A
/// failure here fails the job node and nothing else — the agent is still
/// created, exactly as capable as every agent is today.
pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> {
    let name = policy::agent_object_name(agent)?;
    let path = mtls::identity_path(agent)?;

    let (cert, key) = authority.mint_leaf(&name)?;
    let credential = mtls::Credential {
        cert,
        key,
        ca: authority.ca_pem.clone(),
    };

    let store = crate::store::connect()
        .await
        .context("logging in to the swarm secret store")?;
    store
        .write(&path, &credential)
        .await
        .with_context(|| format!("publishing the agent identity at {path}"))?;
    store
        .write_policy(&name, &policy::render_agent_with_queue(agent, hive)?)
        .await
        .with_context(|| format!("writing the read policy {name}"))?;
    store
        .write_cert_role(DEFAULT_CERT_MOUNT, &name, &authority.ca_pem, &name, &name)
        .await
        .with_context(|| format!("writing the cert-auth role {name}"))?;
    tracing::info!(agent, %path, role = %name, "agent store identity published");

    read_back_as_agent(&credential, &name, &path).await?;
    tracing::info!(
        agent,
        role = %name,
        "agent store identity verified: the minted leaf logged in and read its own path"
    );
    Ok(())
}

/// The consumer of everything [`mint_and_verify`] wrote: log in **as the
/// agent**, with the leaf just minted, and read back the path just published.
///
/// The address and the store's CA come from this process's own `BAO_*`
/// environment; the *identity* deliberately does not — see
/// [`SecretStore::connect_with_identity`]. The freshly minted private key
/// never touches a filesystem.
///
/// # Errors
/// When the store refuses the login (the role, the authority or the common
/// name disagree), when the read is denied (the policy does not cover the
/// path, or the role attached the wrong policy), or when what comes back is
/// not what went in.
async fn read_back_as_agent(credential: &mtls::Credential, role: &str, path: &str) -> Result<()> {
    let settings = Settings::from_env()
        .context("reading this daemon's own store settings for the read-back")?;

    // The concatenated blob `connect_with_identity` takes, built in memory.
    let mut identity = credential.cert.clone().into_bytes();
    identity.push(b'\n');
    identity.extend_from_slice(credential.key.as_bytes());

    let as_agent =
        SecretStore::connect_with_identity(&settings, &identity, role, DEFAULT_CERT_MOUNT)
            .await
            .with_context(|| {
                format!("logging in to the store as {role} with the leaf just minted")
            })?;
    let read_back: mtls::Credential = as_agent
        .read(path)
        .await
        .with_context(|| format!("reading {path} back under {role}'s own token"))?;

    // Compared, not merely decoded: a successful read of an object written by
    // some earlier run would otherwise pass this check while this run's leaf
    // was the one nobody could use. Nothing about either value is printed.
    if read_back != *credential {
        bail!("the store returned a different object at {path} than the one just published");
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::{Authority, CLOCK_SKEW, LEAF_LIFETIME, validity};
    use std::time::{Duration, SystemTime, UNIX_EPOCH};

    /// A throwaway CA, minted in-process so no test needs a fixture file.
    fn test_authority() -> Authority {
        let key = rcgen::KeyPair::generate().expect("a key generates");
        let mut params = rcgen::CertificateParams::default();
        params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Constrained(0));
        params
            .distinguished_name
            .push(rcgen::DnType::CommonName, "swarm-agent-ca");
        let ca = params.self_signed(&key).expect("the CA self-signs");
        Authority::from_pem(ca.pem(), key.serialize_pem())
    }

    #[test]
    fn the_window_is_backdated_by_the_skew_and_as_long_as_the_lifetime() {
        // The arithmetic that is wrong by a factor of sixty if a unit slips.
        let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000);
        let (before, after) = validity(now, LEAF_LIFETIME).expect("a plain instant is fine");
        assert_eq!(before.unix_timestamp(), 1_700_000_000 - 300);
        assert_eq!(after.unix_timestamp(), 1_700_000_000 + 90 * 24 * 60 * 60);
        assert_eq!(CLOCK_SKEW, Duration::from_mins(5));
    }

    #[test]
    fn a_minted_leaf_starts_valid_and_expires() {
        let now = i64::try_from(
            SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .expect("the test host's clock is after 1970")
                .as_secs(),
        )
        .expect("and before the end of time");
        let (before, after) = validity(SystemTime::now(), LEAF_LIFETIME).expect("now is fine");
        assert!(
            before.unix_timestamp() < now,
            "a leaf usable only in the future is unusable"
        );
        assert!(
            after.unix_timestamp() > now,
            "a leaf that has already expired authenticates nothing"
        );
        // The rule `docs/swarm/credentials.md` states: no credential's
        // renewal strategy may read NONE, which starts with it having an end.
        // A decade-long leaf is that rule broken in a way review would miss.
        assert!(after.unix_timestamp() - now < 3653 * 24 * 60 * 60);
    }

    /// The four-strings agreement `mint_and_verify` rests on, checked on the
    /// one of the four this process controls directly: the certificate really
    /// does carry the common name the cert-auth role will be told to match.
    #[test]
    fn the_leaf_carries_the_agents_object_name_as_its_common_name() {
        let name = swarm_secret_client::policy::agent_object_name("atlas").expect("legal");
        assert_eq!(name, "hive-agent-atlas");

        let (cert, key) = test_authority().mint_leaf(&name).expect("the leaf signs");
        assert!(cert.contains("BEGIN CERTIFICATE"), "a PEM certificate");
        assert!(key.contains("PRIVATE KEY"), "a PEM key");

        // Searched in the SIGNED DER rather than asserted on the params we
        // built: the claim is that the name reached the bytes a verifier
        // reads. A byte search rather than an X.509 parse because the whole
        // crate would otherwise gain a parser dependency for one assertion —
        // the name is a UTF8String in the subject DN, so it appears verbatim.
        let body: String = cert
            .lines()
            .filter(|l| !l.starts_with("-----"))
            .collect::<Vec<_>>()
            .join("");
        let der = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, body)
            .expect("the PEM body is base64");
        assert!(
            der.windows(name.len()).any(|w| w == name.as_bytes()),
            "the common name must be inside the signed certificate"
        );

        // And the pair is a usable client identity — the first thing
        // `read_back_as_agent` does with it, in exactly this shape.
        let mut identity = cert.into_bytes();
        identity.push(b'\n');
        identity.extend_from_slice(key.as_bytes());
        reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity");
    }

    /// A misconfiguration that would otherwise look like "not configured":
    /// half an authority has to be an error, not a silent `None`.
    ///
    /// SAFETY: single-threaded mutation of two env vars no other test in this
    /// crate reads, removed again before returning.
    #[test]
    fn half_an_authority_is_an_error_and_neither_half_is_absence() {
        unsafe {
            std::env::remove_var(super::ENV_AGENT_CA);
            std::env::remove_var(super::ENV_AGENT_CA_KEY);
        }
        assert!(
            Authority::from_env()
                .expect("neither set is a supported shape")
                .is_none(),
            "a controller with no authority configured is not an error"
        );

        unsafe { std::env::set_var(super::ENV_AGENT_CA, "/nonexistent/ca.pem") }
        // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an
        // `Authority`, which deliberately has no `Debug` (it holds a key).
        let e = Authority::from_env()
            .err()
            .expect("a certificate with no key is half an authority");
        assert!(format!("{e:#}").contains(super::ENV_AGENT_CA_KEY), "{e:#}");

        unsafe {
            std::env::remove_var(super::ENV_AGENT_CA);
            std::env::set_var(super::ENV_AGENT_CA_KEY, "/nonexistent/ca-key.pem");
        }
        // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an
        // `Authority`, which deliberately has no `Debug` (it holds a key).
        let e = Authority::from_env()
            .err()
            .expect("a key with no certificate is the other half");
        assert!(format!("{e:#}").contains(super::ENV_AGENT_CA), "{e:#}");

        unsafe { std::env::remove_var(super::ENV_AGENT_CA_KEY) }
    }
}