489 lines
19 KiB
Rust
489 lines
19 KiB
Rust
//! Provisioning a hive's access to the swarm's secret store: the policy its
|
|
//! certificate carries, and the cert-auth role that hands it that policy.
|
|
//!
|
|
//! The controller writes a credential; the hive fetches it with its own
|
|
//! certificate. Without the role the certificate authenticates to nothing, and
|
|
//! without the policy the token it gets may read nothing — either way the read
|
|
//! half of a delivery answers 403.
|
|
//!
|
|
//! A pass runs at startup, and repeats until it reaches the store. The two
|
|
//! start together, so the store being the slower one is the ordinary case
|
|
//! rather than an edge one, and a pass that gave up there provisions nobody
|
|
//! until something restarts this daemon. Per-hive failures are different: they
|
|
//! are reported and skipped — one hive that cannot be provisioned should not
|
|
//! take down a daemon that serves everything else, and a store that answers is
|
|
//! answering the same way next time, so the next start is the retry for those.
|
|
//!
|
|
//! The rule is written once for all hives, [`policy::SHARED_HIVE_POLICY`]: one
|
|
//! stanza is the same everywhere, the other is scoped to the reader's own name
|
|
//! ([`swarm_secret_client::policy`] explains the asymmetry), taken from its
|
|
//! login's alias on the cert mount instead of written into the text. A hive's
|
|
//! role attaches that policy and no other. The hive list is loaded once because
|
|
//! a config change means a redeploy.
|
|
//!
|
|
//! The shared agent policy is written by the same kind of pass
|
|
//! ([`ensure_agent_policies`]), together with each agent's identity entity.
|
|
//! Its roster is the store's `hive-agent-*` cert-auth roles.
|
|
|
|
use std::{future::Future, time::Duration};
|
|
|
|
use anyhow::{Context, Result};
|
|
use backon::{BackoffBuilder as _, ConstantBuilder};
|
|
use swarm_secret_client::{Error, client::DEFAULT_CERT_MOUNT, policy};
|
|
|
|
use crate::store::AccessStore;
|
|
|
|
/// File holding the authority hives are issued from, as
|
|
/// `swarm-controller.nix` names it.
|
|
pub const ENV_HIVE_CLIENT_CA: &str = "SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE";
|
|
|
|
/// How long to wait before re-attempting a pass that never reached the store.
|
|
///
|
|
/// `nix/host-modules/swarm-bao.nix`'s bootstrap unit retries the other side of
|
|
/// this same race on the same cadence: the two halves of one race should not
|
|
/// disagree about how long a wait is worth.
|
|
const RETRY_INTERVAL: Duration = Duration::from_secs(30);
|
|
|
|
/// How many times [`RETRY_INTERVAL`] is waited before giving up. A store that
|
|
/// has not appeared within [`RETRY_WINDOW`] is absent rather than slow, and a
|
|
/// daemon logging into the void forever tells an operator nothing.
|
|
const RETRY_ATTEMPTS: u32 = 2_880;
|
|
|
|
/// What [`RETRY_ATTEMPTS`] at [`RETRY_INTERVAL`] come to. A named third
|
|
/// constant because it is the claim the other two make together, and either
|
|
/// one of them can be edited alone.
|
|
const RETRY_WINDOW: Duration = Duration::from_hours(24);
|
|
|
|
/// A pass that never got as far as the store, carrying what to say about it.
|
|
///
|
|
/// Returned rather than logged where it happens: the first attempt deserves a
|
|
/// warning and the two-hundredth is noise, and only the caller knows which one
|
|
/// this is.
|
|
struct Unreachable(String);
|
|
|
|
/// Give every hive in `hives` the access its certificate needs.
|
|
///
|
|
/// Returns once the first pass is done, so a store that is already up leaves
|
|
/// nothing deferred. When that pass cannot reach the store, it repeats in the
|
|
/// background for up to [`RETRY_WINDOW`].
|
|
pub async fn ensure_hive_access(hives: Vec<String>) {
|
|
if let Err(Unreachable(reason)) = provision(&hives).await {
|
|
tracing::warn!(
|
|
reason,
|
|
retry_in = ?RETRY_INTERVAL,
|
|
"hive provisioning could not reach the swarm secret store; retrying in the background"
|
|
);
|
|
spawn_retry(hives);
|
|
}
|
|
}
|
|
|
|
/// Repeat [`provision`] until it reaches the store, or [`RETRY_WINDOW`] passes.
|
|
fn spawn_retry(hives: Vec<String>) {
|
|
tokio::spawn(async move {
|
|
let reached = retry_until_ok(RETRY_INTERVAL, RETRY_ATTEMPTS, move || {
|
|
let hives = hives.clone();
|
|
async move {
|
|
let outcome = provision(&hives).await;
|
|
if let Err(Unreachable(reason)) = &outcome {
|
|
tracing::debug!(reason, "hive provisioning: store still unreachable");
|
|
}
|
|
outcome
|
|
}
|
|
})
|
|
.await;
|
|
if let Some(attempt) = reached {
|
|
tracing::info!(attempt, "hive provisioning reached the store");
|
|
} else {
|
|
tracing::warn!(
|
|
attempts = RETRY_ATTEMPTS,
|
|
window = ?RETRY_WINDOW,
|
|
"giving up on hive provisioning; no hive can log in to the store until this daemon is restarted"
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Run `attempt` every `interval` until it succeeds, at most `attempts` times.
|
|
/// Answers with the attempt that succeeded, or `None` if none did.
|
|
///
|
|
/// The cadence and the bound are parameters rather than the constants above so
|
|
/// that a test can drive this loop without waiting out a real day.
|
|
async fn retry_until_ok<F, Fut>(interval: Duration, attempts: u32, mut attempt: F) -> Option<u32>
|
|
where
|
|
F: FnMut() -> Fut,
|
|
Fut: Future<Output = Result<(), Unreachable>>,
|
|
{
|
|
// One delay per attempt, waited before it: the pass the caller already ran
|
|
// before deciding to retry at all is the attempt that needs no wait.
|
|
let delays = ConstantBuilder::default()
|
|
.with_delay(interval)
|
|
.with_max_times(attempts as usize)
|
|
.build();
|
|
for (n, delay) in (1..).zip(delays) {
|
|
tokio::time::sleep(delay).await;
|
|
if attempt().await.is_ok() {
|
|
return Some(n);
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
/// One pass: log in once, write the shared hive policy, then every hive's role.
|
|
///
|
|
/// Policy first, because the role names the policy — the other order leaves a
|
|
/// window in which the role points at nothing. Without the policy the roles are
|
|
/// left as they are: a role written to name a policy that does not exist logs
|
|
/// its hive in able to read nothing.
|
|
///
|
|
/// # Errors
|
|
/// [`Unreachable`] for the states a later attempt can resolve: a store that is
|
|
/// not up, and an authority file that is not placed yet. A deployment that is
|
|
/// configured for neither is `Ok` — there is nothing to wait for.
|
|
async fn provision(hives: &[String]) -> Result<(), Unreachable> {
|
|
if hives.is_empty() {
|
|
return Ok(());
|
|
}
|
|
let store = match crate::store::connect().await {
|
|
Ok(store) => store,
|
|
// An unset `BAO_*` variable is a deployment that runs no store, not
|
|
// one whose store is late: asking again changes nothing.
|
|
Err(Error::MissingEnv(var)) => {
|
|
tracing::info!(
|
|
var,
|
|
"no secret store configured; hive access is not managed here"
|
|
);
|
|
return Ok(());
|
|
}
|
|
Err(e) => return Err(Unreachable(e.to_string())),
|
|
};
|
|
provision_with(&store, hives, read_hive_authority).await
|
|
}
|
|
|
|
/// The hive authority from the file `ENV_HIVE_CLIENT_CA` names, or `None` when
|
|
/// none is configured.
|
|
fn read_hive_authority() -> Result<Option<String>, Unreachable> {
|
|
let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else {
|
|
return Ok(None);
|
|
};
|
|
let path = path.to_string_lossy().into_owned();
|
|
// Retryable rather than fatal: the unit that places this file and this
|
|
// daemon start together, so "not there yet" is a moment, not a verdict.
|
|
std::fs::read_to_string(&path)
|
|
.map(Some)
|
|
.map_err(|e| Unreachable(format!("reading the hive authority {path}: {e}")))
|
|
}
|
|
|
|
/// [`provision`] against `store`, with the authority read by `read_authority`.
|
|
async fn provision_with(
|
|
store: &impl AccessStore,
|
|
hives: &[String],
|
|
read_authority: impl FnOnce() -> Result<Option<String>, Unreachable>,
|
|
) -> Result<(), Unreachable> {
|
|
if !write_shared_policy(store).await {
|
|
return Ok(());
|
|
}
|
|
let Some(ca) = read_authority()? else {
|
|
tracing::info!("no hive certificate authority configured; hive roles are not managed here");
|
|
return Ok(());
|
|
};
|
|
for hive in hives {
|
|
if let Err(e) = write_role(store, hive, &ca).await {
|
|
tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Write [`policy::SHARED_HIVE_POLICY`], rendered with the cert mount's
|
|
/// accessor. Answers whether it is in place: hive roles are written only then.
|
|
///
|
|
/// A store that refuses the accessor read (the controller's policy may not
|
|
/// carry it yet) or the write leaves the roles already in the store as they are.
|
|
async fn write_shared_policy(store: &impl AccessStore) -> bool {
|
|
let accessor = match store.auth_mount_accessor(DEFAULT_CERT_MOUNT).await {
|
|
Ok(accessor) => accessor,
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
error = %e,
|
|
"the shared hive policy is not managed: the cert auth mount's accessor could not be read; hive roles are left as they are"
|
|
);
|
|
return false;
|
|
}
|
|
};
|
|
let document = match policy::render_shared_hives(&accessor) {
|
|
Ok(document) => document,
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "the shared hive policy is not managed: the cert auth mount's accessor is not usable in a policy; hive roles are left as they are");
|
|
return false;
|
|
}
|
|
};
|
|
match store
|
|
.write_policy(policy::SHARED_HIVE_POLICY, &document)
|
|
.await
|
|
{
|
|
Ok(()) => {
|
|
tracing::info!(
|
|
policy = policy::SHARED_HIVE_POLICY,
|
|
"shared hive policy in place"
|
|
);
|
|
true
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
policy = policy::SHARED_HIVE_POLICY,
|
|
error = %e,
|
|
"writing the shared hive policy failed; hive roles are left as they are"
|
|
);
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The policies a hive's role attaches: the shared one, and no policy named for
|
|
/// the hive.
|
|
fn attached_policies() -> Vec<String> {
|
|
vec![policy::SHARED_HIVE_POLICY.to_owned()]
|
|
}
|
|
|
|
/// The role for one hive: its own certificate's subject and the shared policy.
|
|
async fn write_role(store: &impl AccessStore, hive: &str, ca: &str) -> Result<()> {
|
|
let name = policy::hive_object_name(hive)?;
|
|
let attached = attached_policies();
|
|
store
|
|
// The subject is the hive's name: `glue-bao-tls.nix` mints a hive's
|
|
// client leaf with its name as the CN, and cert auth matches on that.
|
|
.write_cert_role(
|
|
DEFAULT_CERT_MOUNT,
|
|
&name,
|
|
ca,
|
|
hive,
|
|
&attached.iter().map(String::as_str).collect::<Vec<_>>(),
|
|
)
|
|
.await
|
|
.with_context(|| format!("writing the cert-auth role {name}"))?;
|
|
tracing::info!(hive, role = %name, "hive store role in place");
|
|
Ok(())
|
|
}
|
|
|
|
/// Write the shared agent policy and give every agent its identity entity, with
|
|
/// the retry [`ensure_hive_access`] has and for the same reason.
|
|
///
|
|
/// Returns once the first pass is done. When that pass cannot reach the store,
|
|
/// it repeats in the background for up to [`RETRY_WINDOW`].
|
|
pub async fn ensure_agent_policies() {
|
|
if let Err(Unreachable(reason)) = rewrite_agent_policies().await {
|
|
tracing::warn!(
|
|
reason,
|
|
retry_in = ?RETRY_INTERVAL,
|
|
"agent policy rewrite could not reach the swarm secret store; retrying in the background"
|
|
);
|
|
tokio::spawn(async {
|
|
let reached = retry_until_ok(RETRY_INTERVAL, RETRY_ATTEMPTS, || async {
|
|
let outcome = rewrite_agent_policies().await;
|
|
if let Err(Unreachable(reason)) = &outcome {
|
|
tracing::debug!(reason, "agent policy rewrite: store still unreachable");
|
|
}
|
|
outcome
|
|
})
|
|
.await;
|
|
if let Some(attempt) = reached {
|
|
tracing::info!(attempt, "agent policy rewrite reached the store");
|
|
} else {
|
|
tracing::warn!(
|
|
attempts = RETRY_ATTEMPTS,
|
|
window = ?RETRY_WINDOW,
|
|
"giving up on the agent policy rewrite; the shared agent policy and the agents' identity entities are not checked until this daemon is restarted"
|
|
);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
/// One pass: log in once, list the agents, write the shared agent policy and
|
|
/// give each agent its identity entity.
|
|
///
|
|
/// The policy and entities only. An agent's cert-auth role and certificate are
|
|
/// `agent_identity::mint_and_verify`'s, and that reissues the certificate, so
|
|
/// it is not what a pass on every start may call.
|
|
///
|
|
/// # Errors
|
|
/// [`Unreachable`] when the store cannot be reached or will not list its
|
|
/// roles: either way no agent was written, and a later attempt may get
|
|
/// further. A deployment with no store configured is `Ok`.
|
|
async fn rewrite_agent_policies() -> Result<(), Unreachable> {
|
|
let store = match crate::store::connect().await {
|
|
Ok(store) => store,
|
|
Err(Error::MissingEnv(var)) => {
|
|
tracing::info!(
|
|
var,
|
|
"no secret store configured; agent policies are not managed here"
|
|
);
|
|
return Ok(());
|
|
}
|
|
Err(e) => return Err(Unreachable(e.to_string())),
|
|
};
|
|
let roles = store
|
|
.list_cert_roles(DEFAULT_CERT_MOUNT)
|
|
.await
|
|
.map_err(|e| Unreachable(format!("listing the store's cert-auth roles: {e}")))?;
|
|
// First, so the cert-auth role that names it never points at nothing.
|
|
match store
|
|
.write_policy(policy::SHARED_AGENT_POLICY, &policy::render_shared_agent())
|
|
.await
|
|
{
|
|
Ok(()) => tracing::info!(
|
|
policy = policy::SHARED_AGENT_POLICY,
|
|
"shared agent policy in place"
|
|
),
|
|
Err(e) => tracing::warn!(
|
|
policy = policy::SHARED_AGENT_POLICY,
|
|
error = %e,
|
|
"writing the shared agent policy failed"
|
|
),
|
|
}
|
|
crate::agent_entity::ensure_agents(&store, &policy::agents_from_role_names(&roles)).await;
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::{cell::Cell, time::Duration};
|
|
|
|
use crate::store::fake::FakeAccess;
|
|
|
|
use super::{
|
|
RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, attached_policies, policy,
|
|
provision_with, retry_until_ok,
|
|
};
|
|
|
|
#[test]
|
|
fn the_retry_bound_is_the_window_it_claims() {
|
|
assert_eq!(RETRY_INTERVAL * RETRY_ATTEMPTS, RETRY_WINDOW);
|
|
}
|
|
|
|
/// The property this issue is about: a pass that fails because the store is
|
|
/// not up yet happens again.
|
|
#[tokio::test]
|
|
async fn a_pass_is_repeated_until_it_reaches_the_store() {
|
|
let calls = Cell::new(0_u32);
|
|
let reached = retry_until_ok(Duration::from_millis(1), 10, || {
|
|
calls.set(calls.get() + 1);
|
|
let still_down = calls.get() < 3;
|
|
async move {
|
|
if still_down {
|
|
Err(Unreachable("store not up".to_owned()))
|
|
} else {
|
|
Ok(())
|
|
}
|
|
}
|
|
})
|
|
.await;
|
|
assert_eq!(reached, Some(3));
|
|
assert_eq!(calls.get(), 3, "the loop stops at the first success");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_loop_is_bounded_rather_than_endless() {
|
|
let calls = Cell::new(0_u32);
|
|
let reached = retry_until_ok(Duration::from_millis(1), 5, || {
|
|
calls.set(calls.get() + 1);
|
|
async { Err(Unreachable("store not up".to_owned())) }
|
|
})
|
|
.await;
|
|
assert_eq!(reached, None, "a store that never appears is given up on");
|
|
assert_eq!(calls.get(), 5, "one attempt per interval, and no more");
|
|
}
|
|
|
|
#[test]
|
|
fn a_hives_role_attaches_the_shared_policy_and_none_named_for_the_hive() {
|
|
let attached = attached_policies();
|
|
assert_eq!(attached, [policy::SHARED_HIVE_POLICY]);
|
|
let own = policy::hive_object_name("pr1ma").expect("legal");
|
|
assert!(
|
|
!attached.contains(&own),
|
|
"the role must not attach a policy named for the hive"
|
|
);
|
|
// The control: the same lookup finds a policy that is attached.
|
|
assert!(attached.contains(&policy::SHARED_HIVE_POLICY.to_owned()));
|
|
}
|
|
|
|
fn hives() -> Vec<String> {
|
|
["pr1ma", "beta"].map(str::to_owned).to_vec()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn every_hives_role_is_written_after_the_shared_policy_and_attaches_it_alone() {
|
|
let store = FakeAccess::default();
|
|
provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned())))
|
|
.await
|
|
.unwrap_or_else(|_| panic!("a store that accepts everything"));
|
|
assert_eq!(
|
|
store.roles_written(),
|
|
[
|
|
"write_cert_role hive-pr1ma hive-hives",
|
|
"write_cert_role hive-beta hive-hives"
|
|
]
|
|
);
|
|
let calls = store.calls();
|
|
let policy_at = calls
|
|
.iter()
|
|
.position(|c| c == "write_policy hive-hives")
|
|
.expect("the shared policy was written");
|
|
let first_role_at = calls
|
|
.iter()
|
|
.position(|c| c.starts_with("write_cert_role "))
|
|
.expect("a role was written");
|
|
assert!(policy_at < first_role_at, "{calls:?}");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn an_unreadable_accessor_leaves_every_hive_role_as_it_was() {
|
|
let store = FakeAccess::without_accessor();
|
|
provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned())))
|
|
.await
|
|
.unwrap_or_else(|_| panic!("a policy that cannot be written is not a retry"));
|
|
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
|
|
// The control: the accessor read was attempted, and the same fake
|
|
// writes roles when it works (the test above).
|
|
assert_eq!(store.calls(), ["auth_mount_accessor"]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_refused_shared_policy_leaves_every_hive_role_as_it_was() {
|
|
let store = FakeAccess::refusing_policies();
|
|
provision_with(&store, &hives(), || Ok(Some("HIVE-CA".to_owned())))
|
|
.await
|
|
.unwrap_or_else(|_| panic!("a refused write is not a retry"));
|
|
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
|
|
assert!(
|
|
store
|
|
.calls()
|
|
.contains(&"write_policy hive-hives".to_owned())
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn without_an_authority_the_policy_is_written_and_no_role_is() {
|
|
let store = FakeAccess::default();
|
|
provision_with(&store, &hives(), || Ok(None))
|
|
.await
|
|
.unwrap_or_else(|_| panic!("no authority configured is not a retry"));
|
|
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
|
|
assert!(
|
|
store
|
|
.calls()
|
|
.contains(&"write_policy hive-hives".to_owned())
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn an_authority_that_cannot_be_read_is_retried_and_writes_no_role() {
|
|
let store = FakeAccess::default();
|
|
let outcome = provision_with(&store, &hives(), || {
|
|
Err(Unreachable("not placed yet".to_owned()))
|
|
})
|
|
.await;
|
|
assert!(outcome.is_err());
|
|
assert!(store.roles_written().is_empty(), "{:?}", store.calls());
|
|
}
|
|
}
|