From 12c7d7c01a0ba2f8366f55393821cc3092ad0fb6 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 4 Nov 2026 17:36:57 +0100 Subject: [PATCH] cap the kept bearer's age at one hour --- docs/swarm/credentials-reference.md | 2 +- docs/swarm/services.md | 7 +- swarm-metrics-auth/src/main.rs | 119 ++++++++++++++++++++++++++-- 3 files changed, 116 insertions(+), 12 deletions(-) diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index 37ab0b8c..88b3b047 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -43,7 +43,7 @@ the rest of the cell says how. | `swarm/hives//oidc/client` | authelia, first boot, straight into the store | the hive | ❌ minted once, straight into the store; minted again only when the store holds none | ✅ re-read when authelia refuses it | | `swarm/services//oidc/client` | authelia | that service | must be stated | must be stated | | `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge, at start and every 5 minutes | `swarm-bao-ci-runner` on the runner's host | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change | -| `swarm/services//metrics/bearer` | `swarm-bao-grafana-metrics-bearer` on Grafana's host, when none is stored | `swarm-metrics-auth` and `swarm-logs-grafana-auth` | ❌ delete it and restart `swarm-bao-grafana-metrics-bearer` | ✅ the auth services re-read within a minute, and until they can reach the store they keep accepting the value they last read; ❌ Grafana restarts when its copy changes | +| `swarm/services//metrics/bearer` | `swarm-bao-grafana-metrics-bearer` on Grafana's host, when none is stored | `swarm-metrics-auth` and `swarm-logs-grafana-auth` | ❌ delete it and restart `swarm-bao-grafana-metrics-bearer` | ✅ the auth services re-read within a minute, and while they can't reach the store they keep accepting the value they last read, for an hour at most; ❌ Grafana restarts when its copy changes | | `swarm/services//secret-key` | `swarm-bao-grafana-secret-key` on Grafana's host, from the key an earlier release generated; a new one only when there is none | Grafana | ❌ minted once; a different key leaves Grafana unable to read its stored datasource secrets | ❌ Grafana reads it at start; the unit restarts Grafana when the file changes | | `swarm/services/swarm-authelia/keys/`, for `jwt`, `session`, `storage-encryption`, `oidc-hmac` and `oidc-issuer` | authelia's `authelia-swarm-secrets` unit in its container, under the secret publisher's identity, from the key file an earlier release generated; a new one only when there is none | authelia | ❌ minted once; a different storage-encryption key leaves authelia's database unreadable and a different issuer key invalidates every issued token | ❌ read when the unit runs; the unit restarts authelia when the file changes | | `swarm/services/review-classifier/anthropic-api-key` | you, in the store's UI, field `api_key` | `swarm-review-classifier` | ❌ write a new key | ✅ read on every run | diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 1a1e646d..4eebef0b 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -270,9 +270,10 @@ minute, so a replaced bearer starts working up to a minute later. When the store doesn't answer within five seconds, it checks requests against the last value it read instead of denying, and still refuses a wrong bearer; it tries the store again after three seconds. A revoked or rotated bearer -stays accepted for as long as the store is down. The store's next answer -replaces the value, and a process that has read none yet (it started while -the store was down) denies. `swarm-logs-grafana-auth` +stays accepted while the store is down, for an hour after the last read at +most; after that the responder denies again. The store's next answer +replaces the value, and a process that holds none (it started while the +store was down) denies. `swarm-logs-grafana-auth` is the same binary and does the same. `swarm-metrics-agent-auth` is the log store's `swarm-logs-auth` binary, with diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs index d03863b4..fd4bc4f2 100644 --- a/swarm-metrics-auth/src/main.rs +++ b/swarm-metrics-auth/src/main.rs @@ -19,11 +19,13 @@ //! failure. //! //! When the store does not answer, requests are checked against the last -//! value it did return, however old, and a wrong bearer is still refused. The -//! first such request of each failure streak logs it. A revoked or rotated -//! bearer therefore stays accepted for as long as the store cannot be reached; -//! the first answer replaces the value, or drops it when the store holds none. -//! A process that has not yet read the store has no value and denies. +//! value it did return, up to `MAX_KEPT_AGE` after that read, and a wrong +//! bearer is still refused. The first such request of each failure streak logs +//! it. A revoked or rotated bearer therefore stays accepted for as long as the +//! store cannot be reached, for at most `MAX_KEPT_AGE`; the first answer +//! replaces the value, or drops it when the store holds none. Past that age the +//! value is dropped, and a process that has not yet read the store has none: +//! both deny. use std::future::Future; use std::os::unix::fs::PermissionsExt as _; @@ -51,6 +53,12 @@ const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); /// within seconds. const LOOKUP_FAILURE_WINDOW: Duration = Duration::from_secs(3); +/// How old the last bearer read from the store may be and still be checked +/// against when the store does not answer. It covers the store's slow minutes +/// after a boot (5 to 20 so far), and caps how long a revoked or rotated +/// bearer can survive an outage. Counted from the last successful read. +const MAX_KEPT_AGE: Duration = Duration::from_hours(1); + #[derive(Parser)] #[command(about = "auth_request responder: checks Grafana's metrics bearer against the store")] struct Args { @@ -123,8 +131,9 @@ struct Cached { #[derive(Default)] struct Cache { - /// Kept past `--cache-ttl`: an expired value is replaced by the next - /// answer from the store, and checked against while there is none. + /// Kept past `--cache-ttl`, up to `MAX_KEPT_AGE`: an expired value is + /// replaced by the next answer from the store, and checked against while + /// there is none. stored: Option, /// Until when requests skip the lookup, after one failed or timed out. /// Stamped when that lookup ends: the requests queued behind it carry the @@ -135,6 +144,21 @@ struct Cache { } impl Cache { + /// Drops `stored` once it is `MAX_KEPT_AGE` old at `now`, returning the line + /// to log: `Some` once, when it goes. + fn expire_kept(&mut self, now: Instant) -> Option { + let kept = self.stored.as_ref()?; + if now.saturating_duration_since(kept.read_at) < MAX_KEPT_AGE { + return None; + } + let read_at = kept.read_at_wall; + self.stored = None; + Some(format!( + "the store did not answer for too long; dropping the bearer read at {}", + read_at.to_rfc3339_opts(SecondsFormat::Secs, true) + )) + } + /// The line to log when a request is checked against `stored` because the /// store did not answer: `Some` for the first such request of a failure /// streak, `None` for the rest. @@ -223,8 +247,11 @@ async fn decide( Refresh::Replaced => {} Refresh::Empty => return StatusCode::UNAUTHORIZED, Refresh::Unanswered => { + if let Some(line) = cache.expire_kept(now) { + tracing::warn!("{line}"); + } if cache.stored.is_none() { - tracing::warn!("the store did not answer and no bearer was read yet; denying"); + tracing::warn!("the store did not answer and no bearer is held; denying"); return StatusCode::UNAUTHORIZED; } if let Some(line) = cache.fallback_notice() { @@ -596,4 +623,80 @@ mod tests { assert_eq!(decide(&r, Some(REPLACED), recovered).await, StatusCode::OK); assert!(!r.cached.lock().await.fallback_logged); } + + /// A store that answers the leading `answers` once each, in order, then + /// fails every lookup. + fn then_down(answers: &[Answer]) -> MockBearerSource { + let mut source = store(answers); + source.expect_lookup().returning(fails); + source + } + + /// Within an hour of the read an unanswering store leaves the + /// matching bearer accepted and a wrong one refused; from that age on both + /// are refused. + #[tokio::test] + async fn the_kept_bearer_is_checked_against_only_until_the_max_age() { + let r = responder(then_down(&[secret])); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + let last_moment = t0 + Duration::from_secs(59 * 60 + 59); + assert_eq!(decide(&r, Some(GOOD), last_moment).await, StatusCode::OK); + assert_eq!( + decide(&r, Some("Bearer 0f1e2d3c4b5b"), last_moment).await, + StatusCode::UNAUTHORIZED + ); + let expired = t0 + Duration::from_hours(1); + assert_eq!( + decide(&r, Some(GOOD), expired).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + decide(&r, Some(GOOD), expired + Duration::from_secs(1)).await, + StatusCode::UNAUTHORIZED + ); + } + + /// An answer from the store restarts the age: a value read again after + /// 50 minutes is still checked against 20 minutes into the next outage, + /// 70 minutes after the first read. + #[tokio::test] + async fn an_answer_from_the_store_resets_the_age() { + let r = responder(then_down(&[secret, secret])); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + let second_read = t0 + Duration::from_mins(50); + assert_eq!(decide(&r, Some(GOOD), second_read).await, StatusCode::OK); + let outage = t0 + Duration::from_mins(70); + assert_eq!(decide(&r, Some(GOOD), outage).await, StatusCode::OK); + } + + /// The expiry yields its line once, naming the read time, and leaves + /// nothing to check against; a younger value is left alone. + #[test] + fn the_expiry_of_the_kept_bearer_is_noticed_once() { + let t0 = Instant::now(); + let mut cache = Cache { + stored: Some(Cached { + value: SECRET.to_owned(), + read_at: t0, + read_at_wall: "2026-11-04T07:12:11Z".parse().unwrap(), + }), + ..Cache::default() + }; + assert_eq!( + cache.expire_kept(t0 + MAX_KEPT_AGE.saturating_sub(Duration::from_secs(1))), + None + ); + assert!(cache.stored.is_some()); + assert_eq!( + cache.expire_kept(t0 + MAX_KEPT_AGE).as_deref(), + Some( + "the store did not answer for too long; dropping the bearer read at \ + 2026-11-04T07:12:11Z" + ) + ); + assert!(cache.stored.is_none()); + assert_eq!(cache.expire_kept(t0 + MAX_KEPT_AGE * 2), None); + } }