hyperhive/swarm-controller/src/read_policy.rs

//! Provisioning a hive's access to the swarm's secret store: the policy its
//! certificate carries, and the cert-auth role that hands it that policy.
//!
//! The controller writes a credential; the hive fetches it with its own
//! certificate. Without the role the certificate authenticates to nothing, and
//! without the policy the token it gets may read nothing — either way the read
//! half of a delivery answers 403.
//!
//! A pass runs at startup, and repeats until it reaches the store. The two
//! start together, so the store being the slower one is the ordinary case
//! rather than an edge one, and a pass that gave up there provisions nobody
//! until something restarts this daemon. A shared policy the store refuses is
//! the same kind of failure: the controller's own policy is rewritten by a unit
//! that is not ordered against this daemon, so the refusal can be a moment too.
//! Per-hive role failures are different: they are reported and skipped — one
//! hive that cannot be provisioned should not take down a daemon that serves
//! everything else, and a store that answers is answering the same way next
//! time, so the next start is the retry for those.
//!
//! The rule is written once for all hives, [`policy::SHARED_HIVE_POLICY`]: one
//! stanza is the same everywhere, the other is scoped to the reader's own name
//! ([`swarm_secret_client::policy`] explains the asymmetry), taken from its
//! login's alias on the cert mount instead of written into the text. A hive's
//! role attaches that policy and no other. The hive list is loaded once because
//! a config change means a redeploy.
//!
//! The shared agent policy is written by the same kind of pass
//! ([`ensure_agent_policies`]), together with each agent's identity entity and
//! the token lifetimes of its existing role ([`crate::agent_role_ttl`]).
//! Its roster is the store's `hive-agent-*` cert-auth roles.

use std::{future::Future, time::Duration};

use anyhow::{Context, Result};
use backon::{BackoffBuilder as _, ConstantBuilder};
use swarm_secret_client::{Error, client::DEFAULT_CERT_MOUNT, policy};

use crate::store::AccessStore;

/// File holding the authority hives are issued from, as
/// `swarm-controller.nix` names it.
pub const ENV_HIVE_CLIENT_CA: &str = "SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE";

/// How long to wait before re-attempting a pass that did not complete.
///
/// `nix/host-modules/swarm-bao.nix`'s bootstrap unit retries the other side of
/// this same race on the same cadence: the two halves of one race should not
/// disagree about how long a wait is worth.
const RETRY_INTERVAL: Duration = Duration::from_secs(30);

/// How many times [`RETRY_INTERVAL`] is waited before giving up. A store that
/// has not appeared within [`RETRY_WINDOW`] is absent rather than slow, and a
/// daemon logging into the void forever tells an operator nothing.
const RETRY_ATTEMPTS: u32 = 2_880;

/// What [`RETRY_ATTEMPTS`] at [`RETRY_INTERVAL`] come to. A named third
/// constant because it is the claim the other two make together, and either
/// one of them can be edited alone.
const RETRY_WINDOW: Duration = Duration::from_hours(24);

/// A pass that could not complete for a reason a later attempt may resolve: the
/// store was not reached, or it answered and refused (an accessor read or a
/// shared-policy write answered 403). Carries what to say about it.
///
/// Returned rather than logged where it happens: the first attempt deserves a
/// warning and the two-hundredth is noise, and only the caller knows which one
/// this is.
struct Unreachable(String);

/// Give every hive in `hives` the access its certificate needs.
///
/// Returns once the first pass is done, so a store that is already up leaves
/// nothing deferred. When that pass cannot complete, it repeats in the
/// background for up to [`RETRY_WINDOW`].
pub async fn ensure_hive_access(hives: Vec<String>) {
    if let Err(Unreachable(reason)) = provision(&hives).await {
        tracing::warn!(
            reason,
            retry_in = ?RETRY_INTERVAL,
            "hive provisioning could not complete (the swarm secret store was unreachable or refused); retrying in the background"
        );
        spawn_retry(hives);
    }
}

/// Repeat [`provision`] until it reaches the store, or [`RETRY_WINDOW`] passes.
fn spawn_retry(hives: Vec<String>) {
    tokio::spawn(async move {
        let reached = retry_until_ok(RETRY_INTERVAL, RETRY_ATTEMPTS, move || {
            let hives = hives.clone();
            async move {
                let outcome = provision(&hives).await;
                if let Err(Unreachable(reason)) = &outcome {
                    tracing::debug!(reason, "hive provisioning: still not complete");
                }
                outcome
            }
        })
        .await;
        if let Some(attempt) = reached {
            tracing::info!(attempt, "hive provisioning reached the store");
        } else {
            tracing::warn!(
                attempts = RETRY_ATTEMPTS,
                window = ?RETRY_WINDOW,
                "giving up on hive provisioning; no hive can log in to the store until this daemon is restarted"
            );
        }
    });
}

/// Run `attempt` every `interval` until it succeeds, at most `attempts` times.
/// Answers with the attempt that succeeded, or `None` if none did.
///
/// The cadence and the bound are parameters rather than the constants above so
/// that a test can drive this loop without waiting out a real day.
async fn retry_until_ok<F, Fut>(interval: Duration, attempts: u32, mut attempt: F) -> Option<u32>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = Result<(), Unreachable>>,
{
    // One delay per attempt, waited before it: the pass the caller already ran
    // before deciding to retry at all is the attempt that needs no wait.
    let delays = ConstantBuilder::default()
        .with_delay(interval)
        .with_max_times(attempts as usize)
        .build();
    for (n, delay) in (1..).zip(delays) {
        tokio::time::sleep(delay).await;
        if attempt().await.is_ok() {
            return Some(n);
        }
    }
    None
}

/// One pass: log in once, write the shared hive policy, then every hive's role.
///
/// Policy first, because the role names the policy — the other order leaves a
/// window in which the role points at nothing. Without the policy the roles are
/// left as they are: a role written to name a policy that does not exist logs
/// its hive in able to read nothing.
///
/// # Errors
/// [`Unreachable`] for the states a later attempt can resolve: a store that is
/// not up or that refuses the shared hive policy, and an authority file that is
/// not placed yet. A deployment that is configured for neither is `Ok` —
/// there is nothing to wait for.
async fn provision(hives: &[String]) -> Result<(), Unreachable> {
    if hives.is_empty() {
        return Ok(());
    }
    let store = match crate::store::connect().await {
        Ok(store) => store,
        // An unset `BAO_*` variable is a deployment that runs no store, not
        // one whose store is late: asking again changes nothing.
        Err(Error::MissingEnv(var)) => {
            tracing::info!(
                var,
                "no secret store configured; hive access is not managed here"
            );
            return Ok(());
        }
        Err(e) => return Err(Unreachable(e.to_string())),
    };
    provision_with(&store, hives, read_hive_authority).await
}

/// The hive authority from the file `ENV_HIVE_CLIENT_CA` names, or `None` when
/// none is configured.
fn read_hive_authority() -> Result<Option<String>, Unreachable> {
    let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else {
        return Ok(None);
    };
    let path = path.to_string_lossy().into_owned();
    // Retryable rather than fatal: the unit that places this file and this
    // daemon start together, so "not there yet" is a moment, not a verdict.
    std::fs::read_to_string(&path)
        .map(Some)
        .map_err(|e| Unreachable(format!("reading the hive authority {path}: {e}")))
}

/// [`provision`] against `store`, with the authority read by `read_authority`.
async fn provision_with(
    store: &impl AccessStore,
    hives: &[String],
    read_authority: impl FnOnce() -> Result<Option<String>, Unreachable>,
) -> Result<(), Unreachable> {
    write_shared_policy(store).await?;
    let Some(ca) = read_authority()? else {
        tracing::info!("no hive certificate authority configured; hive roles are not managed here");
        return Ok(());
    };
    for hive in hives {
        if let Err(e) = write_role(store, hive, &ca).await {
            tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed");
        }
    }
    Ok(())
}

/// Write [`policy::SHARED_HIVE_POLICY`], rendered with the cert mount's
/// accessor. Hive roles are written only once it is in place.
///
/// # Errors
/// [`Unreachable`] when the store refuses the accessor read (the controller's
/// policy may not carry it yet) or the write, or when the accessor is not
/// usable in a policy: the roles already in the store are left as they are.
async fn write_shared_policy(store: &impl AccessStore) -> Result<(), Unreachable> {
    let accessor = store
        .auth_mount_accessor(DEFAULT_CERT_MOUNT)
        .await
        .map_err(|e| {
            Unreachable(format!(
                "the cert auth mount's accessor could not be read: {e}"
            ))
        })?;
    let document = policy::render_shared_hives(&accessor).map_err(|e| {
        Unreachable(format!(
            "the cert auth mount's accessor is not usable in a policy: {e}"
        ))
    })?;
    store
        .write_policy(policy::SHARED_HIVE_POLICY, &document)
        .await
        .map_err(|e| {
            Unreachable(format!(
                "writing the {} policy failed: {e}",
                policy::SHARED_HIVE_POLICY
            ))
        })?;
    tracing::info!(
        policy = policy::SHARED_HIVE_POLICY,
        "shared hive policy in place"
    );
    Ok(())
}

/// The policies a hive's role attaches: the shared one, and no policy named for
/// the hive.
fn attached_policies() -> Vec<String> {
    vec![policy::SHARED_HIVE_POLICY.to_owned()]
}

/// The role for one hive: its own certificate's subject and the shared policy.
async fn write_role(store: &impl AccessStore, hive: &str, ca: &str) -> Result<()> {
    let name = policy::hive_object_name(hive)?;
    let attached = attached_policies();
    store
        // The subject is the hive's name: `glue-bao-tls.nix` mints a hive's
        // client leaf with its name as the CN, and cert auth matches on that.
        .write_cert_role(
            DEFAULT_CERT_MOUNT,
            &name,
            ca,
            hive,
            &attached.iter().map(String::as_str).collect::<Vec<_>>(),
        )
        .await
        .with_context(|| format!("writing the cert-auth role {name}"))?;
    tracing::info!(hive, role = %name, "hive store role in place");
    Ok(())
}

/// Write the shared agent policy and give every agent its identity entity, with
/// the retry [`ensure_hive_access`] has and for the same reason.
///
/// Returns once the first pass is done. When that pass cannot complete,
/// it repeats in the background for up to [`RETRY_WINDOW`].
pub async fn ensure_agent_policies() {
    if let Err(Unreachable(reason)) = rewrite_agent_policies().await {
        tracing::warn!(
            reason,
            retry_in = ?RETRY_INTERVAL,
            "agent policy rewrite could not complete (the swarm secret store was unreachable or refused); retrying in the background"
        );
        tokio::spawn(async {
            let reached = retry_until_ok(RETRY_INTERVAL, RETRY_ATTEMPTS, || async {
                let outcome = rewrite_agent_policies().await;
                if let Err(Unreachable(reason)) = &outcome {
                    tracing::debug!(reason, "agent policy rewrite: still not complete");
                }
                outcome
            })
            .await;
            if let Some(attempt) = reached {
                tracing::info!(attempt, "agent policy rewrite reached the store");
            } else {
                tracing::warn!(
                    attempts = RETRY_ATTEMPTS,
                    window = ?RETRY_WINDOW,
                    "giving up on the agent policy rewrite; the shared agent policy and the agents' identity entities are not checked until this daemon is restarted"
                );
            }
        });
    }
}

/// One pass: log in once, list the agents, write the shared agent policy, give
/// each agent its identity entity and bring each existing role's token
/// lifetimes current.
///
/// The roles' certificate, names and policies are not touched. They and the
/// certificate are `agent_identity::mint_and_verify`'s, and that reissues the
/// certificate, so it is not what a pass on every start may call.
///
/// # Errors
/// [`Unreachable`] when the store cannot be reached or will not list its
/// roles: either way no agent was written, and a later attempt may get
/// further. Also when it refuses the shared agent policy, after the agents
/// have still been given their entities and token lifetimes. A deployment
/// with no store configured is `Ok`.
async fn rewrite_agent_policies() -> Result<(), Unreachable> {
    let store = match crate::store::connect().await {
        Ok(store) => store,
        Err(Error::MissingEnv(var)) => {
            tracing::info!(
                var,
                "no secret store configured; agent policies are not managed here"
            );
            return Ok(());
        }
        Err(e) => return Err(Unreachable(e.to_string())),
    };
    let roles = store
        .list_cert_roles(DEFAULT_CERT_MOUNT)
        .await
        .map_err(|e| Unreachable(format!("listing the store's cert-auth roles: {e}")))?;
    write_agent_policy_and_entities(&store, &roles).await
}

/// The part of [`rewrite_agent_policies`] after the roles are listed.
async fn write_agent_policy_and_entities(
    store: &impl AccessStore,
    roles: &[String],
) -> Result<(), Unreachable> {
    // First, so the cert-auth role that names it never points at nothing.
    let written = store
        .write_policy(policy::SHARED_AGENT_POLICY, &policy::render_shared_agent())
        .await;
    if written.is_ok() {
        tracing::info!(
            policy = policy::SHARED_AGENT_POLICY,
            "shared agent policy in place"
        );
    }
    crate::agent_entity::ensure_agents(store, &policy::agents_from_role_names(roles)).await;
    crate::agent_role_ttl::bring_current(store, roles).await;
    written.map_err(|e| {
        Unreachable(format!(
            "writing the {} policy failed: {e}",
            policy::SHARED_AGENT_POLICY
        ))
    })
}

#[cfg(test)]
mod tests {
    use std::{cell::Cell, time::Duration};

    use crate::store::fake::FakeAccess;

    use super::{
        RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, attached_policies, policy,
        provision_with, retry_until_ok, write_agent_policy_and_entities,
    };

    #[test]
    fn the_retry_bound_is_the_window_it_claims() {
        assert_eq!(RETRY_INTERVAL * RETRY_ATTEMPTS, RETRY_WINDOW);
    }

    /// The property this issue is about: a pass that fails because the store is
    /// not up yet happens again.
    #[tokio::test]
    async fn a_pass_is_repeated_until_it_reaches_the_store() {
        let calls = Cell::new(0_u32);
        let reached = retry_until_ok(Duration::from_millis(1), 10, || {
            calls.set(calls.get() + 1);
            let still_down = calls.get() < 3;
            async move {
                if still_down {
                    Err(Unreachable("store not up".to_owned()))
                } else {
                    Ok(())
                }
            }
        })
        .await;
        assert_eq!(reached, Some(3));
        assert_eq!(calls.get(), 3, "the loop stops at the first success");
    }

    #[tokio::test]
    async fn the_loop_is_bounded_rather_than_endless() {
        let calls = Cell::new(0_u32);
        let reached = retry_until_ok(Duration::from_millis(1), 5, || {
            calls.set(calls.get() + 1);
            async { Err(Unreachable("store not up".to_owned())) }
        })
        .await;
        assert_eq!(reached, None, "a store that never appears is given up on");
        assert_eq!(calls.get(), 5, "one attempt per interval, and no more");
    }

    #[test]
    fn a_hives_role_attaches_the_shared_policy_and_none_named_for_the_hive() {
        let attached = attached_policies();
        assert_eq!(attached, [policy::SHARED_HIVE_POLICY]);
        let own = policy::hive_object_name("pr1ma").expect("legal");
        assert!(
            !attached.contains(&own),
            "the role must not attach a policy named for the hive"
        );
        // The control: the same lookup finds a policy that is attached.
        assert!(attached.contains(&policy::SHARED_HIVE_POLICY.to_owned()));
    }

    fn hives() -> Vec<String> {
        ["pr1ma", "beta"].map(str::to_owned).to_vec()
    }

    #[tokio::test]
    async fn every_hives_role_is_written_after_the_shared_policy_and_attaches_it_alone() {
        let store = FakeAccess::default();
        provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned())))
            .await
            .unwrap_or_else(|_| panic!("a store that accepts everything"));
        assert_eq!(
            store.roles_written(),
            [
                "write_cert_role hive-pr1ma hive-hives",
                "write_cert_role hive-beta hive-hives"
            ]
        );
        let calls = store.calls();
        let policy_at = calls
            .iter()
            .position(|c| c == "write_policy hive-hives")
            .expect("the shared policy was written");
        let first_role_at = calls
            .iter()
            .position(|c| c.starts_with("write_cert_role "))
            .expect("a role was written");
        assert!(policy_at < first_role_at, "{calls:?}");
    }

    #[tokio::test]
    async fn an_unreadable_accessor_leaves_every_hive_role_as_it_was() {
        let store = FakeAccess::without_accessor();
        let outcome = provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned()))).await;
        assert!(
            outcome.is_err(),
            "a policy that cannot be written is retried"
        );
        assert!(store.roles_written().is_empty(), "{:?}", store.calls());
        // The control: the accessor read was attempted, and the same fake
        // writes roles when it works (the test above).
        assert_eq!(store.calls(), ["auth_mount_accessor"]);
    }

    #[tokio::test]
    async fn a_refused_shared_policy_leaves_every_hive_role_as_it_was() {
        let store = FakeAccess::refusing_policies();
        let outcome = provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned()))).await;
        assert!(outcome.is_err(), "a refused write is retried");
        assert!(store.roles_written().is_empty(), "{:?}", store.calls());
        assert!(
            store
                .calls()
                .contains(&"write_policy hive-hives".to_owned())
        );
    }

    #[tokio::test]
    async fn without_an_authority_the_policy_is_written_and_no_role_is() {
        let store = FakeAccess::default();
        provision_with(&store, &hives(), || Ok(None))
            .await
            .unwrap_or_else(|_| panic!("no authority configured is not a retry"));
        assert!(store.roles_written().is_empty(), "{:?}", store.calls());
        assert!(
            store
                .calls()
                .contains(&"write_policy hive-hives".to_owned())
        );
    }

    #[tokio::test]
    async fn an_authority_that_cannot_be_read_is_retried_and_writes_no_role() {
        let store = FakeAccess::default();
        let outcome = provision_with(&store, &hives(), || {
            Err(Unreachable("not placed yet".to_owned()))
        })
        .await;
        assert!(outcome.is_err());
        assert!(store.roles_written().is_empty(), "{:?}", store.calls());
    }

    /// The property this change is about: a pass that is refused the shared
    /// policy is repeated, and the roles are written once the store accepts it.
    #[tokio::test]
    async fn a_refused_accessor_read_is_repeated_and_the_roles_follow() {
        let store = FakeAccess::with_accessor_refused(1);
        let hives = hives();
        let pass = || provision_with(&store, &hives, || Ok(Some("HIVE-CA".to_owned())));
        assert!(pass().await.is_err(), "the first pass is refused");
        assert!(store.roles_written().is_empty(), "{:?}", store.calls());
        let reached = retry_until_ok(Duration::from_millis(1), 10, pass).await;
        assert_eq!(reached, Some(1), "the first retry succeeds");
        assert_eq!(
            store.roles_written(),
            [
                "write_cert_role hive-pr1ma hive-hives",
                "write_cert_role hive-beta hive-hives"
            ],
            "{:?}",
            store.calls()
        );
    }

    #[tokio::test]
    async fn a_refused_shared_agent_policy_is_retried_and_a_written_one_is_not() {
        let refused = FakeAccess::refusing_policies();
        assert!(
            write_agent_policy_and_entities(&refused, &[])
                .await
                .is_err()
        );
        // The control: the same pass against a store that accepts the write.
        let accepting = FakeAccess::default();
        assert!(
            write_agent_policy_and_entities(&accepting, &[])
                .await
                .is_ok()
        );
        assert!(
            accepting
                .calls()
                .contains(&format!("write_policy {}", policy::SHARED_AGENT_POLICY))
        );
    }
}