3b007b1ccb
The boot run of the UpdateMeta node read hive placements before the queue client had connected, failed on the Pending client, and was not declared again until the next 10 minute timer tick. It now waits for the connection, bounded to 60s, and fails with a message naming the wait when the queue never connects. agent_renewal's wait_for_queue takes the bound as a parameter so both callers share it.
736 lines
27 KiB
Rust
736 lines
27 KiB
Rust
//! Renewing each live agent's two store credentials by age: its mTLS
|
|
//! certificate (`swarm/agents/<agent>/bao-mtls`) once it is past half its
|
|
//! validity, and its queue secret (`swarm/agents/<agent>/queue`) once it is
|
|
//! [`SECRET_RENEW_AFTER`] old.
|
|
//!
|
|
//! The certificate's age is its own `notBefore`/`notAfter`. The queue secret
|
|
//! has no expiry and `swarm-nats-auth` checks nothing about time, so its age is
|
|
//! the controller's own record, [`queue::AgentCredential::minted_at`]; a secret
|
|
//! without one is stamped, value unchanged ([`SecretStep::Backfill`]).
|
|
//!
|
|
//! A running agent re-reads its certificate every ten minutes under the one it
|
|
//! holds (`nix/agent-modules/bao.nix`); the old one stays valid until it
|
|
//! expires. The old queue secret stops working at once, but the queue checks it
|
|
//! only at `CONNECT`, so an open connection is unaffected. The agent reads its
|
|
//! secret from the store on every connection attempt after its first
|
|
//! (`hive-agent/src/swarm_queue.rs`), so a reconnect presents the new one.
|
|
//!
|
|
//! [`spawn`]'s pass, once the queue is up and every [`RECONCILE_INTERVAL`],
|
|
//! inserts a job per agent with anything due: `MintAgentIdentity` for the
|
|
//! certificate ([`crate::agent_identity::mint_and_verify`], which keeps the
|
|
//! queue secret as it is) and `RenewAgentQueueCredential` ([`renew`]) for the
|
|
//! secret, the second after the first when both are due. The decisions are
|
|
//! pure ([`live_agents`], [`cert_is_due`], [`secret_step`], [`plan`]) so the
|
|
//! tests pin them; the IO on either side only reads or acts.
|
|
//!
|
|
//! **Only agents some hive is declared to run are renewed** ([`live_agents`]):
|
|
//! a destroy is recorded in the wanted-state declarations, and an agent declared
|
|
//! nowhere, or only as `Destroyed`, is skipped. A live agent with no stored
|
|
//! certificate, or a current one with no listed cert-auth role, gets one minted
|
|
//! unless its mint is in flight. A missing queue secret stays missing.
|
|
|
|
use std::collections::BTreeSet;
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context, Result, bail};
|
|
use swarm_secret_client::{SecretStore, cert, client::DEFAULT_CERT_MOUNT, mtls, policy, queue};
|
|
use swarm_wire::wanted::{AgentState, HiveWanted};
|
|
|
|
use crate::wanted::WantedWriter;
|
|
|
|
/// Age at which a queue secret is re-minted. Independent of the agent
|
|
/// certificate's life (`agentPkiLeafTtl` in `swarm-bao.nix`), which renews at
|
|
/// half its own validity.
|
|
pub const SECRET_RENEW_AFTER: std::time::Duration = std::time::Duration::from_hours(45 * 24);
|
|
|
|
/// How often [`spawn`] re-checks every agent. Also the longest it waits for
|
|
/// the queue connection before its first pass.
|
|
const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5);
|
|
|
|
/// How often [`spawn`] checks the queue connection before its first pass.
|
|
const CONNECT_POLL: std::time::Duration = std::time::Duration::from_secs(5);
|
|
|
|
/// Now, in the unix seconds [`queue::AgentCredential::minted_at`] holds.
|
|
pub fn unix_now() -> i64 {
|
|
std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX))
|
|
}
|
|
|
|
/// What a stored queue secret needs at `now`.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum SecretStep {
|
|
/// Younger than [`SECRET_RENEW_AFTER`].
|
|
Keep,
|
|
/// No mint time recorded: stamp `now` beside the unchanged value, so its
|
|
/// clock starts here and no holder of it is cut off.
|
|
Backfill,
|
|
/// At least [`SECRET_RENEW_AFTER`] old: replace the value.
|
|
Remint,
|
|
}
|
|
|
|
/// Decide [`SecretStep`] for `stored` at `now`.
|
|
pub fn secret_step(stored: &queue::AgentCredential, now: i64) -> SecretStep {
|
|
let renew_after = i64::try_from(SECRET_RENEW_AFTER.as_secs()).unwrap_or(i64::MAX);
|
|
match stored.minted_at {
|
|
None => SecretStep::Backfill,
|
|
Some(at) if now.saturating_sub(at) >= renew_after => SecretStep::Remint,
|
|
Some(_) => SecretStep::Keep,
|
|
}
|
|
}
|
|
|
|
/// The object `step` writes for `agent` at `now`, or `None` for
|
|
/// [`SecretStep::Keep`].
|
|
///
|
|
/// # Errors
|
|
/// When a re-mint cannot draw randomness from the kernel.
|
|
pub fn apply_secret_step(
|
|
step: SecretStep,
|
|
stored: &queue::AgentCredential,
|
|
agent: &str,
|
|
now: i64,
|
|
) -> Result<Option<queue::AgentCredential>> {
|
|
Ok(match step {
|
|
SecretStep::Keep => None,
|
|
SecretStep::Backfill => Some(queue::AgentCredential {
|
|
minted_at: Some(now),
|
|
..stored.clone()
|
|
}),
|
|
SecretStep::Remint => Some(fresh(agent, now)?),
|
|
})
|
|
}
|
|
|
|
/// Whether a certificate valid from `not_before` to `not_after` is past half
|
|
/// its life at `now`.
|
|
pub fn cert_is_due(not_before: i64, not_after: i64, now: i64) -> bool {
|
|
let half = not_after.saturating_sub(not_before) / 2;
|
|
now >= not_before.saturating_add(half)
|
|
}
|
|
|
|
/// A new queue credential for `agent`, minted at `now`. Shared with agent
|
|
/// creation, so every secret this daemon writes carries a mint time.
|
|
///
|
|
/// # Errors
|
|
/// When the kernel will not supply randomness.
|
|
pub fn fresh(agent: &str, now: i64) -> Result<queue::AgentCredential> {
|
|
Ok(queue::AgentCredential {
|
|
value: crate::agent_identity::generate_queue_secret()?,
|
|
agent: agent.to_owned(),
|
|
minted_at: Some(now),
|
|
})
|
|
}
|
|
|
|
/// Every agent declared on at least one hive in a state other than
|
|
/// `Destroyed`.
|
|
pub fn live_agents(declarations: &[HiveWanted]) -> BTreeSet<String> {
|
|
declarations
|
|
.iter()
|
|
.flat_map(|d| &d.agents)
|
|
.filter(|(_, wanted)| wanted.state != AgentState::Destroyed)
|
|
.map(|(agent, _)| agent.clone())
|
|
.collect()
|
|
}
|
|
|
|
/// What one pass found for one credential of one live agent.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Observed {
|
|
/// Stored and due for replacement, with its age in seconds.
|
|
Due(i64),
|
|
/// A queue secret stored with no mint time: due for
|
|
/// [`SecretStep::Backfill`], never for a new value.
|
|
Unstamped,
|
|
/// Stored and not yet due.
|
|
Current,
|
|
/// Nothing stored (a certificate: or stored with no cert-auth role, see
|
|
/// [`cert_needing_role`]). A certificate is minted for a live agent; a queue
|
|
/// secret is left alone (see the module doc).
|
|
Absent,
|
|
/// The read or the decode failed. Nothing is known, so nothing is done.
|
|
Unknown,
|
|
}
|
|
|
|
/// What one pass found for one live agent.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct AgentObserved {
|
|
pub cert: Observed,
|
|
pub secret: Observed,
|
|
}
|
|
|
|
/// One agent's share of a pass: which of its credentials to renew.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct Renewal {
|
|
pub agent: String,
|
|
pub cert: bool,
|
|
pub secret: bool,
|
|
}
|
|
|
|
/// The renewals a pass queues: one per agent with a certificate that is
|
|
/// [`Observed::Due`] or [`Observed::Absent`], or a queue secret that is
|
|
/// [`Observed::Due`] or [`Observed::Unstamped`].
|
|
pub fn plan(observed: &[(String, AgentObserved)]) -> Vec<Renewal> {
|
|
observed
|
|
.iter()
|
|
.map(|(agent, o)| Renewal {
|
|
agent: agent.clone(),
|
|
cert: matches!(o.cert, Observed::Due(_) | Observed::Absent),
|
|
secret: matches!(o.secret, Observed::Due(_) | Observed::Unstamped),
|
|
})
|
|
.filter(|r| r.cert || r.secret)
|
|
.collect()
|
|
}
|
|
|
|
/// An age in seconds, for a log line: whole days.
|
|
struct Age(i64);
|
|
|
|
impl std::fmt::Display for Age {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
write!(f, "{}d", self.0 / 86_400)
|
|
}
|
|
}
|
|
|
|
/// Backfill or re-mint `agent`'s queue secret if it is still stored and still
|
|
/// needs it ([`secret_step`]). The whole job of the `RenewAgentQueueCredential`
|
|
/// node.
|
|
///
|
|
/// Re-decides rather than trusting the pass that queued it, so a node queued
|
|
/// twice acts once. Reads the write back before reporting success.
|
|
///
|
|
/// # Errors
|
|
/// When the store refuses a step, or returns a different object than the one
|
|
/// just written.
|
|
pub async fn renew(agent: &str) -> Result<()> {
|
|
let path = queue::agent_queue_path(agent)?;
|
|
let store = crate::store::connect()
|
|
.await
|
|
.context("logging in to the swarm secret store")?;
|
|
let Some(stored) = store
|
|
.read_optional::<queue::AgentCredential>(&path)
|
|
.await
|
|
.with_context(|| format!("reading {path}"))?
|
|
else {
|
|
tracing::info!(agent, %path, "agent queue credential: nothing stored, nothing to renew");
|
|
return Ok(());
|
|
};
|
|
let now = unix_now();
|
|
let step = secret_step(&stored, now);
|
|
let Some(written) = apply_secret_step(step, &stored, agent, now)? else {
|
|
tracing::debug!(agent, "agent queue credential is current; left as it is");
|
|
return Ok(());
|
|
};
|
|
store
|
|
.write(&path, &written)
|
|
.await
|
|
.with_context(|| format!("writing the agent queue credential at {path}"))?;
|
|
let read_back: queue::AgentCredential = store
|
|
.read(&path)
|
|
.await
|
|
.with_context(|| format!("reading {path} back"))?;
|
|
if read_back != written {
|
|
bail!("the store returned a different object at {path} than the one just written");
|
|
}
|
|
if let Some(at) = stored.minted_at {
|
|
tracing::info!(
|
|
agent,
|
|
%path,
|
|
old_age = %Age(now.saturating_sub(at)),
|
|
"agent queue credential re-minted; the agent presents it from its next restart"
|
|
);
|
|
} else {
|
|
tracing::info!(
|
|
agent,
|
|
%path,
|
|
"agent queue credential had no mint time; stamped now, value unchanged"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// What the store says about `agent`'s certificate.
|
|
async fn observe_cert(store: &SecretStore, agent: &str, now: i64) -> Observed {
|
|
let stored = match mtls::identity_path(agent) {
|
|
Ok(path) => store.read_optional::<mtls::Credential>(&path).await,
|
|
Err(e) => Err(e),
|
|
};
|
|
let credential = match stored {
|
|
Ok(Some(credential)) => credential,
|
|
Ok(None) => return Observed::Absent,
|
|
Err(e) => {
|
|
tracing::warn!(agent, error = %e, "agent certificate: store read failed");
|
|
return Observed::Unknown;
|
|
}
|
|
};
|
|
match cert::validity(&credential.cert) {
|
|
Ok((not_before, not_after)) if cert_is_due(not_before, not_after, now) => {
|
|
Observed::Due(now.saturating_sub(not_before))
|
|
}
|
|
Ok(_) => Observed::Current,
|
|
Err(e) => {
|
|
tracing::warn!(agent, error = %format!("{e:#}"), "agent certificate: unreadable");
|
|
Observed::Unknown
|
|
}
|
|
}
|
|
}
|
|
|
|
/// What the store says about `agent`'s queue secret.
|
|
async fn observe_secret(store: &SecretStore, agent: &str, now: i64) -> Observed {
|
|
let stored = match queue::agent_queue_path(agent) {
|
|
Ok(path) => store.read_optional::<queue::AgentCredential>(&path).await,
|
|
Err(e) => Err(e),
|
|
};
|
|
match stored {
|
|
Ok(Some(stored)) => match (secret_step(&stored, now), stored.minted_at) {
|
|
(SecretStep::Remint, Some(at)) => Observed::Due(now.saturating_sub(at)),
|
|
(SecretStep::Backfill, _) => Observed::Unstamped,
|
|
_ => Observed::Current,
|
|
},
|
|
Ok(None) => Observed::Absent,
|
|
Err(e) => {
|
|
tracing::warn!(agent, error = %e, "agent queue credential: store read failed");
|
|
Observed::Unknown
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Every readable wanted-state declaration of `hives`. Fails as a whole when
|
|
/// one cannot be read, since that hive may be the one declaring an agent
|
|
/// destroyed.
|
|
pub(crate) async fn read_declarations(
|
|
wanted: &WantedWriter,
|
|
hives: &[String],
|
|
) -> Result<Vec<HiveWanted>> {
|
|
let mut declarations = Vec::with_capacity(hives.len());
|
|
for hive in hives {
|
|
if let Some(d) = wanted
|
|
.view(hive)
|
|
.await
|
|
.with_context(|| format!("reading {hive}'s wanted-state declaration"))?
|
|
{
|
|
declarations.push(d);
|
|
}
|
|
}
|
|
Ok(declarations)
|
|
}
|
|
|
|
/// A stored, current certificate whose cert-auth role is not in `roles` is as
|
|
/// good as absent: the agent cannot log in with it, and the mint is the only
|
|
/// thing that writes the role. `None` (the role listing failed) and every
|
|
/// other observation come back unchanged.
|
|
pub fn cert_needing_role(
|
|
cert: Observed,
|
|
roles: Option<&BTreeSet<String>>,
|
|
agent: &str,
|
|
) -> Observed {
|
|
match (cert, roles) {
|
|
(Observed::Current, Some(roles)) if !roles.contains(agent) => Observed::Absent,
|
|
_ => cert,
|
|
}
|
|
}
|
|
|
|
/// One pass: every live agent, observed. Fails as a whole when any hive's
|
|
/// declaration cannot be read.
|
|
async fn observe_all(
|
|
wanted: &WantedWriter,
|
|
hives: &[String],
|
|
) -> Result<Vec<(String, AgentObserved)>> {
|
|
let declarations = read_declarations(wanted, hives).await?;
|
|
let store = crate::store::connect()
|
|
.await
|
|
.context("logging in to the swarm secret store")?;
|
|
let now = unix_now();
|
|
let roles = match store.list_cert_roles(DEFAULT_CERT_MOUNT).await {
|
|
Ok(names) => Some(
|
|
policy::agents_from_role_names(&names)
|
|
.into_iter()
|
|
.collect::<BTreeSet<_>>(),
|
|
),
|
|
Err(e) => {
|
|
tracing::warn!(error = %e, "agent certificate: listing the cert-auth roles failed");
|
|
None
|
|
}
|
|
};
|
|
let mut observed = Vec::new();
|
|
for agent in live_agents(&declarations) {
|
|
let o = AgentObserved {
|
|
cert: cert_needing_role(
|
|
observe_cert(&store, &agent, now).await,
|
|
roles.as_ref(),
|
|
&agent,
|
|
),
|
|
secret: observe_secret(&store, &agent, now).await,
|
|
};
|
|
if let Observed::Due(age) = o.cert {
|
|
tracing::info!(agent, age = %Age(age), "agent certificate past half its life; re-issuing");
|
|
}
|
|
if o.cert == Observed::Absent {
|
|
tracing::info!(
|
|
agent,
|
|
"agent has no usable certificate (none stored, or no cert-auth role); minting its store identity"
|
|
);
|
|
}
|
|
if let Observed::Due(age) = o.secret {
|
|
tracing::info!(agent, age = %Age(age), "agent queue credential due; re-minting");
|
|
}
|
|
if o.secret == Observed::Unstamped {
|
|
tracing::info!(
|
|
agent,
|
|
"agent queue credential has no mint time; stamping it"
|
|
);
|
|
}
|
|
observed.push((agent, o));
|
|
}
|
|
Ok(observed)
|
|
}
|
|
|
|
/// Wait until `connected` holds, checking every [`CONNECT_POLL`] for at most
|
|
/// `bound`. Returns whether it held.
|
|
pub(crate) async fn wait_for_queue(
|
|
connected: impl Fn() -> bool,
|
|
bound: std::time::Duration,
|
|
) -> bool {
|
|
let deadline = tokio::time::Instant::now() + bound;
|
|
loop {
|
|
if connected() {
|
|
return true;
|
|
}
|
|
if tokio::time::Instant::now() >= deadline {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(CONNECT_POLL).await;
|
|
}
|
|
}
|
|
|
|
/// [`spawn`]'s loop, with the connection check and the pass handed in.
|
|
async fn run<P, F>(connected: impl Fn() -> bool, mut pass: P, enqueue: impl Fn(Vec<Renewal>))
|
|
where
|
|
P: FnMut() -> F,
|
|
F: std::future::Future<Output = Result<Vec<(String, AgentObserved)>>>,
|
|
{
|
|
if !wait_for_queue(connected, RECONCILE_INTERVAL).await {
|
|
tracing::warn!(
|
|
waited_s = RECONCILE_INTERVAL.as_secs(),
|
|
"agent credential renewal: queue still not connected; running the first pass anyway"
|
|
);
|
|
}
|
|
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
|
|
loop {
|
|
ticker.tick().await;
|
|
match pass().await {
|
|
Ok(observed) => {
|
|
let renewals = plan(&observed);
|
|
if renewals.is_empty() {
|
|
tracing::debug!(
|
|
checked = observed.len(),
|
|
"agent credential renewal: none due"
|
|
);
|
|
} else {
|
|
tracing::info!(
|
|
checked = observed.len(),
|
|
renewing = renewals.len(),
|
|
"agent credential renewal: queueing"
|
|
);
|
|
enqueue(renewals);
|
|
}
|
|
}
|
|
Err(e) => tracing::warn!(
|
|
error = %format!("{e:#}"),
|
|
retry_in_s = RECONCILE_INTERVAL.as_secs(),
|
|
"agent credential renewal: pass failed; retrying next tick"
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Check every live agent once the queue is connected and every
|
|
/// [`RECONCILE_INTERVAL`] after, and hand the renewals due to `enqueue`, which
|
|
/// inserts a job for each.
|
|
///
|
|
/// Only the first pass waits for the connection, and for at most one
|
|
/// [`RECONCILE_INTERVAL`]. A pass that fails is logged and retried on the next
|
|
/// tick; it never stops the daemon.
|
|
pub fn spawn(
|
|
wanted: Arc<WantedWriter>,
|
|
hives: Vec<String>,
|
|
enqueue: impl Fn(Vec<Renewal>) + Send + 'static,
|
|
) {
|
|
tokio::spawn(async move {
|
|
run(
|
|
|| wanted.is_connected(),
|
|
|| observe_all(&wanted, &hives),
|
|
enqueue,
|
|
)
|
|
.await;
|
|
});
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use swarm_wire::wanted::AgentWanted;
|
|
|
|
const NOW: i64 = 1_790_000_000;
|
|
const DAY: i64 = 86_400;
|
|
|
|
/// A 90-day window, 2026-01-01T00:00:00Z to 2026-04-01T00:00:00Z.
|
|
const CERT_NOT_BEFORE: i64 = 1_767_225_600;
|
|
const CERT_NOT_AFTER: i64 = 1_775_001_600;
|
|
|
|
fn minted(at: Option<i64>) -> queue::AgentCredential {
|
|
queue::AgentCredential {
|
|
value: "Ab9_-zSECRET".to_owned(),
|
|
agent: "atlas".to_owned(),
|
|
minted_at: at,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_secret_without_a_mint_time_is_backfilled_with_its_value_unchanged() {
|
|
let stored = minted(None);
|
|
assert_eq!(secret_step(&stored, NOW), SecretStep::Backfill);
|
|
let written = apply_secret_step(SecretStep::Backfill, &stored, "atlas", NOW)
|
|
.expect("no randomness needed")
|
|
.expect("a backfill writes");
|
|
assert_eq!(written.value, stored.value, "no holder of it is cut off");
|
|
assert_eq!(written.agent, stored.agent);
|
|
assert_eq!(written.minted_at, Some(NOW), "its clock starts now");
|
|
assert_eq!(secret_step(&written, NOW), SecretStep::Keep);
|
|
}
|
|
|
|
#[test]
|
|
fn a_secret_is_re_minted_from_forty_five_days_with_a_new_value_and_mint_time() {
|
|
for age in [45 * DAY, 90 * DAY] {
|
|
let stored = minted(Some(NOW - age));
|
|
assert_eq!(secret_step(&stored, NOW), SecretStep::Remint, "{age}");
|
|
let written = apply_secret_step(SecretStep::Remint, &stored, "atlas", NOW)
|
|
.expect("the kernel supplies randomness")
|
|
.expect("a re-mint writes");
|
|
assert_ne!(written.value, stored.value);
|
|
assert_eq!(written.minted_at, Some(NOW));
|
|
assert_eq!(written.agent, "atlas");
|
|
assert_eq!(secret_step(&written, NOW), SecretStep::Keep);
|
|
}
|
|
let a = fresh("atlas", NOW).expect("randomness");
|
|
let b = fresh("atlas", NOW).expect("randomness");
|
|
assert_ne!(a.value, b.value, "two re-mints must not agree");
|
|
}
|
|
|
|
/// Includes a mint time ahead of the clock, which must not overflow into a
|
|
/// large age.
|
|
#[test]
|
|
fn a_secret_younger_than_forty_five_days_is_left_alone() {
|
|
for at in [NOW, NOW - 45 * DAY + 1, NOW + DAY] {
|
|
let stored = minted(Some(at));
|
|
assert_eq!(secret_step(&stored, NOW), SecretStep::Keep, "{at}");
|
|
assert_eq!(
|
|
apply_secret_step(SecretStep::Keep, &stored, "atlas", NOW).expect("no IO"),
|
|
None
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_ninety_day_certificate_is_due_from_day_forty_five() {
|
|
let (nb, na) = (CERT_NOT_BEFORE, CERT_NOT_AFTER);
|
|
assert!(!cert_is_due(nb, na, nb));
|
|
assert!(!cert_is_due(nb, na, nb + 45 * DAY - 1));
|
|
assert!(cert_is_due(nb, na, nb + 45 * DAY));
|
|
assert!(cert_is_due(nb, na, na + DAY), "an expired one too");
|
|
}
|
|
|
|
fn declared(agents: &[(&str, AgentState)]) -> HiveWanted {
|
|
let mut d = HiveWanted::default();
|
|
for (agent, state) in agents {
|
|
d.agents
|
|
.insert((*agent).to_owned(), AgentWanted { state: *state });
|
|
}
|
|
d
|
|
}
|
|
|
|
#[test]
|
|
fn a_destroyed_agent_is_not_live_and_every_other_state_is() {
|
|
let live = live_agents(&[declared(&[
|
|
("up", AgentState::Up),
|
|
("offline", AgentState::Offline),
|
|
("paused", AgentState::Paused),
|
|
("gone", AgentState::Destroyed),
|
|
])]);
|
|
assert_eq!(
|
|
live.into_iter().collect::<Vec<_>>(),
|
|
["offline", "paused", "up"]
|
|
);
|
|
}
|
|
|
|
/// An agent destroyed on one hive and declared on another is live: it
|
|
/// still runs somewhere.
|
|
#[test]
|
|
fn an_agent_live_on_any_hive_is_live() {
|
|
let live = live_agents(&[
|
|
declared(&[("atlas", AgentState::Destroyed)]),
|
|
declared(&[("atlas", AgentState::Up)]),
|
|
]);
|
|
assert!(live.contains("atlas"), "{live:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn no_declaration_is_no_live_agent() {
|
|
assert!(live_agents(&[]).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn due_and_missing_certificates_and_due_secrets_are_planned() {
|
|
use Observed::{Absent, Current, Due, Unknown, Unstamped};
|
|
let o = |cert, secret| AgentObserved { cert, secret };
|
|
let observed = [
|
|
("both".to_owned(), o(Due(46 * DAY), Due(45 * DAY))),
|
|
("cert".to_owned(), o(Due(46 * DAY), Current)),
|
|
("secret".to_owned(), o(Current, Due(45 * DAY))),
|
|
("unstamped".to_owned(), o(Current, Unstamped)),
|
|
("neither".to_owned(), o(Current, Absent)),
|
|
("unknown".to_owned(), o(Unknown, Unknown)),
|
|
("no identity".to_owned(), o(Absent, Absent)),
|
|
("no cert, old secret".to_owned(), o(Absent, Due(45 * DAY))),
|
|
("unreadable cert".to_owned(), o(Unknown, Absent)),
|
|
];
|
|
let r = |agent: &str, cert, secret| Renewal {
|
|
agent: agent.to_owned(),
|
|
cert,
|
|
secret,
|
|
};
|
|
assert_eq!(
|
|
plan(&observed),
|
|
[
|
|
r("both", true, true),
|
|
r("cert", true, false),
|
|
r("secret", false, true),
|
|
r("unstamped", false, true),
|
|
r("no identity", true, false),
|
|
r("no cert, old secret", true, true),
|
|
]
|
|
);
|
|
}
|
|
|
|
/// A pass over declarations: the agent with no stored identity is minted
|
|
/// for when it is live and not when it is destroyed. The destroyed agent is
|
|
/// never observed, which is what [`live_agents`] does before the store is
|
|
/// read.
|
|
#[test]
|
|
fn a_missing_identity_is_minted_for_a_live_agent_and_not_a_destroyed_one() {
|
|
let declarations = [declared(&[
|
|
("sock", AgentState::Up),
|
|
("paused", AgentState::Paused),
|
|
("gone", AgentState::Destroyed),
|
|
])];
|
|
let nothing_stored = AgentObserved {
|
|
cert: Observed::Absent,
|
|
secret: Observed::Absent,
|
|
};
|
|
let observed: Vec<_> = live_agents(&declarations)
|
|
.into_iter()
|
|
.map(|agent| (agent, nothing_stored))
|
|
.collect();
|
|
let planned: Vec<_> = plan(&observed)
|
|
.into_iter()
|
|
.map(|r| (r.agent, r.cert, r.secret))
|
|
.collect();
|
|
assert_eq!(
|
|
planned,
|
|
[
|
|
("paused".to_owned(), true, false),
|
|
("sock".to_owned(), true, false)
|
|
]
|
|
);
|
|
}
|
|
|
|
/// A current certificate with no cert-auth role listed is minted for; the
|
|
/// controls are the same certificate with its role listed, a failed role
|
|
/// listing, and a certificate that is not `Current` anyway.
|
|
#[test]
|
|
fn a_current_certificate_without_its_role_is_minted_for() {
|
|
let roles: BTreeSet<String> = ["sock".to_owned()].into();
|
|
let planned = |cert: Observed, roles: Option<&BTreeSet<String>>, agent: &str| {
|
|
let observed = [(
|
|
agent.to_owned(),
|
|
AgentObserved {
|
|
cert: cert_needing_role(cert, roles, agent),
|
|
secret: Observed::Current,
|
|
},
|
|
)];
|
|
plan(&observed).iter().map(|r| r.cert).collect::<Vec<_>>()
|
|
};
|
|
assert_eq!(planned(Observed::Current, Some(&roles), "paused"), [true]);
|
|
assert_eq!(
|
|
planned(Observed::Current, Some(&roles), "sock"),
|
|
[] as [bool; 0]
|
|
);
|
|
assert_eq!(planned(Observed::Current, None, "paused"), [] as [bool; 0]);
|
|
assert_eq!(
|
|
planned(Observed::Unknown, Some(&roles), "paused"),
|
|
[] as [bool; 0]
|
|
);
|
|
assert_eq!(planned(Observed::Due(1), Some(&roles), "sock"), [true]);
|
|
}
|
|
|
|
#[test]
|
|
fn the_age_logged_is_whole_days() {
|
|
assert_eq!(Age(46 * DAY + 5).to_string(), "46d");
|
|
}
|
|
|
|
/// [`run`] over a connection the test flips and a pass that only counts
|
|
/// itself.
|
|
fn counting_run() -> (
|
|
Arc<std::sync::atomic::AtomicBool>,
|
|
Arc<std::sync::atomic::AtomicUsize>,
|
|
tokio::task::JoinHandle<()>,
|
|
) {
|
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
|
let connected = Arc::new(AtomicBool::new(false));
|
|
let passes = Arc::new(AtomicUsize::new(0));
|
|
let (c, p) = (Arc::clone(&connected), Arc::clone(&passes));
|
|
let task = tokio::spawn(run(
|
|
move || c.load(Ordering::SeqCst),
|
|
move || {
|
|
p.fetch_add(1, Ordering::SeqCst);
|
|
std::future::ready(Ok(Vec::new()))
|
|
},
|
|
|_| {},
|
|
));
|
|
(connected, passes, task)
|
|
}
|
|
|
|
/// The sleeps end between [`CONNECT_POLL`] boundaries, so a check and an
|
|
/// assertion never fall on the same instant.
|
|
#[tokio::test(start_paused = true)]
|
|
async fn the_first_pass_waits_for_the_queue_and_later_passes_do_not() {
|
|
use std::sync::atomic::Ordering;
|
|
let (connected, passes, task) = counting_run();
|
|
|
|
tokio::time::sleep(std::time::Duration::from_secs(62)).await;
|
|
assert_eq!(passes.load(Ordering::SeqCst), 0, "no pass while Pending");
|
|
|
|
connected.store(true, Ordering::SeqCst);
|
|
tokio::time::sleep(CONNECT_POLL).await;
|
|
assert_eq!(passes.load(Ordering::SeqCst), 1, "a pass once connected");
|
|
|
|
connected.store(false, Ordering::SeqCst);
|
|
tokio::time::sleep(RECONCILE_INTERVAL).await;
|
|
assert_eq!(
|
|
passes.load(Ordering::SeqCst),
|
|
2,
|
|
"a later disconnect does not hold back the next pass"
|
|
);
|
|
task.abort();
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn a_queue_that_never_connects_gets_a_pass_after_one_interval() {
|
|
use std::sync::atomic::Ordering;
|
|
let (_connected, passes, task) = counting_run();
|
|
|
|
tokio::time::sleep(RECONCILE_INTERVAL.saturating_sub(std::time::Duration::from_secs(2)))
|
|
.await;
|
|
assert_eq!(passes.load(Ordering::SeqCst), 0);
|
|
|
|
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
|
|
assert_eq!(passes.load(Ordering::SeqCst), 1);
|
|
task.abort();
|
|
}
|
|
}
|