From 7bc4b25f16c3144ac165969c86736e5f9d0428fb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 28 Sep 2026 22:24:07 +0200 Subject: [PATCH] swarm-controller: backfill a missing queue-secret mint time instead of re-minting it A stored queue secret with no `minted_at` counted as due, and no secret minted before the renewal pass has one, so the first pass after deploy would re-mint every agent's secret. Every reconnect before that agent's next restart would then be refused. Such a secret is now stamped instead: `minted_at = now` is written beside the unchanged `value`, and its 45-day clock starts there. Only a secret whose recorded mint time is at least 45 days old gets a new value. The decision is `secret_step` (Keep / Backfill / Remint), and the pass reports an unstamped secret as `Observed::Unstamped`. Backfill and re-mint log different lines. --- docs/swarm/credentials.md | 28 ++-- swarm-controller/src/agent_renewal.rs | 203 +++++++++++++++++--------- swarm-secret-client/src/queue.rs | 3 +- 3 files changed, 150 insertions(+), 84 deletions(-) diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 28356e39..306f5b57 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -59,20 +59,20 @@ of the cell says how. -| store path | minter | reader — pulls at runtime, holds in memory | automatic re-mint | automatic re-pull | -| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | ✅ the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ `hive-matrix-daemon` exits when the homeserver rejects its token, and a five-minute timer restarts it, which reads the store again | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | must be stated | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | ❌ `swarm-matrix-ctl` mints it once; the container keeps its copy and republishes it when the store's differs | ✅ the controller reads it on every five-minute matrix pass | -| `swarm/controller/swarm-controller/oidc/client` | authelia, at its first boot, where the controller registers its client; `swarm-secret-publish` copies it in | `swarm-controller`, under its own certificate, once at start | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ❌ read once at start; the controller holds the old value until it restarts | -| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | ✅ `swarm-controller`'s five-minute pass re-issues a live agent's leaf once it's past half its validity (45 of 90 days, read from the certificate itself) | ❌ `hive-c0re` reads it when it writes the container config, so the agent presents a new leaf from its next start; the old leaf stays valid until it expires | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | ✅ `swarm-controller`'s five-minute pass re-mints a live agent's secret once it's 45 days old or has no recorded mint time (`minted_at` on the stored object) | ❌ fetched when the container starts. The queue checks the secret only at connect, so an open connection survives a re-mint, but a reconnect before the next restart is denied | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | ✅ the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the agent re-fetches on a 10-minute timer | -| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | must be stated | -| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | must be stated | -| `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | -| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | must be stated | -| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | must be stated | +| store path | minter | reader — pulls at runtime, holds in memory | automatic re-mint | automatic re-pull | +| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | ✅ the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ `hive-matrix-daemon` exits when the homeserver rejects its token, and a five-minute timer restarts it, which reads the store again | +| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | must be stated | +| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | ❌ `swarm-matrix-ctl` mints it once; the container keeps its copy and republishes it when the store's differs | ✅ the controller reads it on every five-minute matrix pass | +| `swarm/controller/swarm-controller/oidc/client` | authelia, at its first boot, where the controller registers its client; `swarm-secret-publish` copies it in | `swarm-controller`, under its own certificate, once at start | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ❌ read once at start; the controller holds the old value until it restarts | +| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | ✅ `swarm-controller`'s five-minute pass re-issues a live agent's leaf once it's past half its validity (45 of 90 days, read from the certificate itself) | ❌ `hive-c0re` reads it when it writes the container config, so the agent presents a new leaf from its next start; the old leaf stays valid until it expires | +| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | ✅ `swarm-controller`'s five-minute pass re-mints a live agent's secret once it's 45 days old by `minted_at` on the stored object; a secret with no `minted_at` gets one stamped, value unchanged | ❌ fetched when the container starts. The queue checks the secret only at connect, so an open connection survives a re-mint, but a reconnect before the next restart is denied | +| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | ✅ the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the agent re-fetches on a 10-minute timer | +| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | must be stated | +| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | must be stated | +| `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | +| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | must be stated | +| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | must be stated | diff --git a/swarm-controller/src/agent_renewal.rs b/swarm-controller/src/agent_renewal.rs index 66ab4164..f315322d 100644 --- a/swarm-controller/src/agent_renewal.rs +++ b/swarm-controller/src/agent_renewal.rs @@ -6,7 +6,7 @@ //! The certificate's age is its own `notBefore`/`notAfter`. The queue secret //! has no expiry and `swarm-nats-auth` checks nothing about time, so its age is //! the controller's own record, [`queue::AgentCredential::minted_at`]; a secret -//! without one is due. +//! without one is stamped, value unchanged ([`SecretStep::Backfill`]). //! //! Either replacement reaches the agent at its next start, when the container //! is handed the certificate and fetches the secret; nothing pulls either into @@ -20,7 +20,7 @@ //! ([`crate::agent_identity::mint_and_verify`], which keeps the queue secret as //! it is) and `RenewAgentQueueCredential` ([`renew`]) for the secret, the //! second after the first when both are due. The decisions are pure -//! ([`live_agents`], [`cert_is_due`], [`secret_is_due`], [`plan`]) so the tests +//! ([`live_agents`], [`cert_is_due`], [`secret_step`], [`plan`]) so the tests //! pin them; the IO on either side only reads or acts. //! //! **Only agents some hive is declared to run are renewed** ([`live_agents`]): @@ -54,13 +54,47 @@ pub fn unix_now() -> i64 { .map_or(0, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX)) } -/// Whether a queue secret is due for a re-mint at `now`: it has no mint time, -/// or it is at least [`SECRET_RENEW_AFTER`] old. -pub fn secret_is_due(stored: &queue::AgentCredential, now: i64) -> bool { +/// What a stored queue secret needs at `now`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretStep { + /// Younger than [`SECRET_RENEW_AFTER`]. + Keep, + /// No mint time recorded: stamp `now` beside the unchanged value, so its + /// clock starts here and no holder of it is cut off. + Backfill, + /// At least [`SECRET_RENEW_AFTER`] old: replace the value. + Remint, +} + +/// Decide [`SecretStep`] for `stored` at `now`. +pub fn secret_step(stored: &queue::AgentCredential, now: i64) -> SecretStep { let renew_after = i64::try_from(SECRET_RENEW_AFTER.as_secs()).unwrap_or(i64::MAX); - stored - .minted_at - .is_none_or(|at| now.saturating_sub(at) >= renew_after) + match stored.minted_at { + None => SecretStep::Backfill, + Some(at) if now.saturating_sub(at) >= renew_after => SecretStep::Remint, + Some(_) => SecretStep::Keep, + } +} + +/// The object `step` writes for `agent` at `now`, or `None` for +/// [`SecretStep::Keep`]. +/// +/// # Errors +/// When a re-mint cannot draw randomness from the kernel. +pub fn apply_secret_step( + step: SecretStep, + stored: &queue::AgentCredential, + agent: &str, + now: i64, +) -> Result> { + Ok(match step { + SecretStep::Keep => None, + SecretStep::Backfill => Some(queue::AgentCredential { + minted_at: Some(now), + ..stored.clone() + }), + SecretStep::Remint => Some(fresh(agent, now)?), + }) } /// A certificate's validity window, `(notBefore, notAfter)` in unix seconds. @@ -111,8 +145,11 @@ pub fn live_agents(declarations: &[HiveWanted]) -> BTreeSet { /// What one pass found for one credential of one live agent. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Observed { - /// Stored and due, with its age in seconds when that is known. - Due(Option), + /// Stored and due for replacement, with its age in seconds. + Due(i64), + /// A queue secret stored with no mint time: due for + /// [`SecretStep::Backfill`], never for a new value. + Unstamped, /// Stored and not yet due. Current, /// Nothing stored. Never renewed: see the module doc. @@ -136,37 +173,36 @@ pub struct Renewal { pub secret: bool, } -/// The renewals a pass queues: one per agent with at least one -/// [`Observed::Due`] credential. +/// The renewals a pass queues: one per agent with a certificate that is +/// [`Observed::Due`], or a queue secret that is [`Observed::Due`] or +/// [`Observed::Unstamped`]. pub fn plan(observed: &[(String, AgentObserved)]) -> Vec { observed .iter() .map(|(agent, o)| Renewal { agent: agent.clone(), cert: matches!(o.cert, Observed::Due(_)), - secret: matches!(o.secret, Observed::Due(_)), + secret: matches!(o.secret, Observed::Due(_) | Observed::Unstamped), }) .filter(|r| r.cert || r.secret) .collect() } -/// An age for a log line: whole days, or `unrecorded`. -struct Age(Option); +/// An age in seconds, for a log line: whole days. +struct Age(i64); impl std::fmt::Display for Age { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self.0 { - Some(secs) => write!(f, "{}d", secs / 86_400), - None => f.write_str("unrecorded"), - } + write!(f, "{}d", self.0 / 86_400) } } -/// Re-mint `agent`'s queue secret if it is still stored and still due. The -/// whole job of the `RenewAgentQueueCredential` node. +/// Backfill or re-mint `agent`'s queue secret if it is still stored and still +/// needs it ([`secret_step`]). The whole job of the `RenewAgentQueueCredential` +/// node. /// /// Re-decides rather than trusting the pass that queued it, so a node queued -/// twice renews once. Reads the write back before reporting success. +/// twice acts once. Reads the write back before reporting success. /// /// # Errors /// When the store refuses a step, or returns a different object than the one @@ -185,28 +221,36 @@ pub async fn renew(agent: &str) -> Result<()> { return Ok(()); }; let now = unix_now(); - if !secret_is_due(&stored, now) { + let step = secret_step(&stored, now); + let Some(written) = apply_secret_step(step, &stored, agent, now)? else { tracing::debug!(agent, "agent queue credential is current; left as it is"); return Ok(()); - } - let renewed = fresh(agent, now)?; + }; store - .write(&path, &renewed) + .write(&path, &written) .await - .with_context(|| format!("writing the re-minted agent queue credential at {path}"))?; + .with_context(|| format!("writing the agent queue credential at {path}"))?; let read_back: queue::AgentCredential = store .read(&path) .await .with_context(|| format!("reading {path} back"))?; - if read_back != renewed { + if read_back != written { bail!("the store returned a different object at {path} than the one just written"); } - tracing::info!( - agent, - %path, - old_age = %Age(stored.minted_at.map(|at| now.saturating_sub(at))), - "agent queue credential re-minted; the agent presents it from its next restart" - ); + if let Some(at) = stored.minted_at { + tracing::info!( + agent, + %path, + old_age = %Age(now.saturating_sub(at)), + "agent queue credential re-minted; the agent presents it from its next restart" + ); + } else { + tracing::info!( + agent, + %path, + "agent queue credential had no mint time; stamped now, value unchanged" + ); + } Ok(()) } @@ -226,7 +270,7 @@ async fn observe_cert(store: &SecretStore, agent: &str, now: i64) -> Observed { }; match cert_validity(&credential.cert) { Ok((not_before, not_after)) if cert_is_due(not_before, not_after, now) => { - Observed::Due(Some(now.saturating_sub(not_before))) + Observed::Due(now.saturating_sub(not_before)) } Ok(_) => Observed::Current, Err(e) => { @@ -243,10 +287,11 @@ async fn observe_secret(store: &SecretStore, agent: &str, now: i64) -> Observed Err(e) => Err(e), }; match stored { - Ok(Some(stored)) if secret_is_due(&stored, now) => { - Observed::Due(stored.minted_at.map(|at| now.saturating_sub(at))) - } - Ok(Some(_)) => Observed::Current, + Ok(Some(stored)) => match (secret_step(&stored, now), stored.minted_at) { + (SecretStep::Remint, Some(at)) => Observed::Due(now.saturating_sub(at)), + (SecretStep::Backfill, _) => Observed::Unstamped, + _ => Observed::Current, + }, Ok(None) => Observed::Absent, Err(e) => { tracing::warn!(agent, error = %e, "agent queue credential: store read failed"); @@ -288,6 +333,12 @@ async fn observe_all( if let Observed::Due(age) = o.secret { tracing::info!(agent, age = %Age(age), "agent queue credential due; re-minting"); } + if o.secret == Observed::Unstamped { + tracing::info!( + agent, + "agent queue credential has no mint time; stamping it" + ); + } observed.push((agent, o)); } Ok(observed) @@ -369,35 +420,48 @@ hWx3sOwmUgjkRQXoxY+p } #[test] - fn a_secret_without_a_mint_time_is_due() { - assert!(secret_is_due(&minted(None), NOW)); + fn a_secret_without_a_mint_time_is_backfilled_with_its_value_unchanged() { + let stored = minted(None); + assert_eq!(secret_step(&stored, NOW), SecretStep::Backfill); + let written = apply_secret_step(SecretStep::Backfill, &stored, "atlas", NOW) + .expect("no randomness needed") + .expect("a backfill writes"); + assert_eq!(written.value, stored.value, "no holder of it is cut off"); + assert_eq!(written.agent, stored.agent); + assert_eq!(written.minted_at, Some(NOW), "its clock starts now"); + assert_eq!(secret_step(&written, NOW), SecretStep::Keep); } #[test] - fn a_secret_is_due_from_forty_five_days_and_not_before() { - assert!(!secret_is_due(&minted(Some(NOW)), NOW)); - assert!(!secret_is_due(&minted(Some(NOW - 45 * DAY + 1)), NOW)); - assert!(secret_is_due(&minted(Some(NOW - 45 * DAY)), NOW)); - assert!(secret_is_due(&minted(Some(NOW - 90 * DAY)), NOW)); + fn a_secret_is_re_minted_from_forty_five_days_with_a_new_value_and_mint_time() { + for age in [45 * DAY, 90 * DAY] { + let stored = minted(Some(NOW - age)); + assert_eq!(secret_step(&stored, NOW), SecretStep::Remint, "{age}"); + let written = apply_secret_step(SecretStep::Remint, &stored, "atlas", NOW) + .expect("the kernel supplies randomness") + .expect("a re-mint writes"); + assert_ne!(written.value, stored.value); + assert_eq!(written.minted_at, Some(NOW)); + assert_eq!(written.agent, "atlas"); + assert_eq!(secret_step(&written, NOW), SecretStep::Keep); + } + let a = fresh("atlas", NOW).expect("randomness"); + let b = fresh("atlas", NOW).expect("randomness"); + assert_ne!(a.value, b.value, "two re-mints must not agree"); } - /// A mint time ahead of the clock is not due, rather than overflowing - /// into a large age. + /// Includes a mint time ahead of the clock, which must not overflow into a + /// large age. #[test] - fn a_mint_time_in_the_future_is_not_due() { - assert!(!secret_is_due(&minted(Some(NOW + DAY)), NOW)); - } - - #[test] - fn a_re_mint_is_a_new_value_with_the_mint_time_for_the_same_agent() { - let old = minted(None); - let renewed = fresh("atlas", NOW).expect("the kernel supplies randomness"); - assert_ne!(renewed.value, old.value); - assert_eq!(renewed.minted_at, Some(NOW)); - assert_eq!(renewed.agent, "atlas"); - assert!(!secret_is_due(&renewed, NOW), "a fresh secret is not due"); - let again = fresh("atlas", NOW).expect("twice"); - assert_ne!(again.value, renewed.value, "two re-mints must not agree"); + fn a_secret_younger_than_forty_five_days_is_left_alone() { + for at in [NOW, NOW - 45 * DAY + 1, NOW + DAY] { + let stored = minted(Some(at)); + assert_eq!(secret_step(&stored, NOW), SecretStep::Keep, "{at}"); + assert_eq!( + apply_secret_step(SecretStep::Keep, &stored, "atlas", NOW).expect("no IO"), + None + ); + } } #[test] @@ -464,12 +528,13 @@ hWx3sOwmUgjkRQXoxY+p #[test] fn only_due_credentials_are_planned() { - use Observed::{Absent, Current, Due, Unknown}; + use Observed::{Absent, Current, Due, Unknown, Unstamped}; let o = |cert, secret| AgentObserved { cert, secret }; let observed = [ - ("both".to_owned(), o(Due(Some(DAY)), Due(None))), - ("cert".to_owned(), o(Due(None), Current)), - ("secret".to_owned(), o(Absent, Due(None))), + ("both".to_owned(), o(Due(46 * DAY), Due(45 * DAY))), + ("cert".to_owned(), o(Due(46 * DAY), Current)), + ("secret".to_owned(), o(Absent, Due(45 * DAY))), + ("unstamped".to_owned(), o(Current, Unstamped)), ("neither".to_owned(), o(Current, Absent)), ("unknown".to_owned(), o(Unknown, Unknown)), ]; @@ -484,13 +549,13 @@ hWx3sOwmUgjkRQXoxY+p r("both", true, true), r("cert", true, false), r("secret", false, true), + r("unstamped", false, true), ] ); } #[test] - fn the_age_logged_is_whole_days_or_unrecorded() { - assert_eq!(Age(Some(46 * DAY + 5)).to_string(), "46d"); - assert_eq!(Age(None).to_string(), "unrecorded"); + fn the_age_logged_is_whole_days() { + assert_eq!(Age(46 * DAY + 5).to_string(), "46d"); } } diff --git a/swarm-secret-client/src/queue.rs b/swarm-secret-client/src/queue.rs index 160d034b..6d72e51f 100644 --- a/swarm-secret-client/src/queue.rs +++ b/swarm-secret-client/src/queue.rs @@ -103,7 +103,8 @@ pub struct AgentCredential { pub agent: String, /// When `value` was minted, in unix seconds. `swarm-controller` re-mints - /// the secret once this is old enough, and treats `None` as due. + /// the secret once this is old enough, and fills in `None` with the time it + /// first sees it, leaving `value` alone. /// /// `Option` because objects written before this field existed lack it and /// must still decode; skipped when `None` so such an object re-serialises