cap the kept bearer's age at one hour

swarm-metrics-auth/src/main.rs

@@ -19,11 +19,13 @@
 //! failure.
 //!
 //! When the store does not answer, requests are checked against the last
-//! value it did return, however old, and a wrong bearer is still refused. The
-//! first such request of each failure streak logs it. A revoked or rotated
-//! bearer therefore stays accepted for as long as the store cannot be reached;
-//! the first answer replaces the value, or drops it when the store holds none.
-//! A process that has not yet read the store has no value and denies.
+//! value it did return, up to `MAX_KEPT_AGE` after that read, and a wrong
+//! bearer is still refused. The first such request of each failure streak logs
+//! it. A revoked or rotated bearer therefore stays accepted for as long as the
+//! store cannot be reached, for at most `MAX_KEPT_AGE`; the first answer
+//! replaces the value, or drops it when the store holds none. Past that age the
+//! value is dropped, and a process that has not yet read the store has none:
+//! both deny.
 
 use std::future::Future;
 use std::os::unix::fs::PermissionsExt as _;
@@ -51,6 +53,12 @@ const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5);
 /// within seconds.
 const LOOKUP_FAILURE_WINDOW: Duration = Duration::from_secs(3);
 
+/// How old the last bearer read from the store may be and still be checked
+/// against when the store does not answer. It covers the store's slow minutes
+/// after a boot (5 to 20 so far), and caps how long a revoked or rotated
+/// bearer can survive an outage. Counted from the last successful read.
+const MAX_KEPT_AGE: Duration = Duration::from_hours(1);
+
 #[derive(Parser)]
 #[command(about = "auth_request responder: checks Grafana's metrics bearer against the store")]
 struct Args {
@@ -123,8 +131,9 @@ struct Cached {
 
 #[derive(Default)]
 struct Cache {
-    /// Kept past `--cache-ttl`: an expired value is replaced by the next
-    /// answer from the store, and checked against while there is none.
+    /// Kept past `--cache-ttl`, up to `MAX_KEPT_AGE`: an expired value is
+    /// replaced by the next answer from the store, and checked against while
+    /// there is none.
     stored: Option<Cached>,
     /// Until when requests skip the lookup, after one failed or timed out.
     /// Stamped when that lookup ends: the requests queued behind it carry the
@@ -135,6 +144,21 @@ struct Cache {
 }
 
 impl Cache {
+    /// Drops `stored` once it is `MAX_KEPT_AGE` old at `now`, returning the line
+    /// to log: `Some` once, when it goes.
+    fn expire_kept(&mut self, now: Instant) -> Option<String> {
+        let kept = self.stored.as_ref()?;
+        if now.saturating_duration_since(kept.read_at) < MAX_KEPT_AGE {
+            return None;
+        }
+        let read_at = kept.read_at_wall;
+        self.stored = None;
+        Some(format!(
+            "the store did not answer for too long; dropping the bearer read at {}",
+            read_at.to_rfc3339_opts(SecondsFormat::Secs, true)
+        ))
+    }
+
     /// The line to log when a request is checked against `stored` because the
     /// store did not answer: `Some` for the first such request of a failure
     /// streak, `None` for the rest.
@@ -223,8 +247,11 @@ async fn decide(
             Refresh::Replaced => {}
             Refresh::Empty => return StatusCode::UNAUTHORIZED,
             Refresh::Unanswered => {
+                if let Some(line) = cache.expire_kept(now) {
+                    tracing::warn!("{line}");
+                }
                 if cache.stored.is_none() {
-                    tracing::warn!("the store did not answer and no bearer was read yet; denying");
+                    tracing::warn!("the store did not answer and no bearer is held; denying");
                     return StatusCode::UNAUTHORIZED;
                 }
                 if let Some(line) = cache.fallback_notice() {
@@ -596,4 +623,80 @@ mod tests {
         assert_eq!(decide(&r, Some(REPLACED), recovered).await, StatusCode::OK);
         assert!(!r.cached.lock().await.fallback_logged);
     }
+
+    /// A store that answers the leading `answers` once each, in order, then
+    /// fails every lookup.
+    fn then_down(answers: &[Answer]) -> MockBearerSource {
+        let mut source = store(answers);
+        source.expect_lookup().returning(fails);
+        source
+    }
+
+    /// Within an hour of the read an unanswering store leaves the
+    /// matching bearer accepted and a wrong one refused; from that age on both
+    /// are refused.
+    #[tokio::test]
+    async fn the_kept_bearer_is_checked_against_only_until_the_max_age() {
+        let r = responder(then_down(&[secret]));
+        let t0 = Instant::now();
+        assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK);
+        let last_moment = t0 + Duration::from_secs(59 * 60 + 59);
+        assert_eq!(decide(&r, Some(GOOD), last_moment).await, StatusCode::OK);
+        assert_eq!(
+            decide(&r, Some("Bearer 0f1e2d3c4b5b"), last_moment).await,
+            StatusCode::UNAUTHORIZED
+        );
+        let expired = t0 + Duration::from_hours(1);
+        assert_eq!(
+            decide(&r, Some(GOOD), expired).await,
+            StatusCode::UNAUTHORIZED
+        );
+        assert_eq!(
+            decide(&r, Some(GOOD), expired + Duration::from_secs(1)).await,
+            StatusCode::UNAUTHORIZED
+        );
+    }
+
+    /// An answer from the store restarts the age: a value read again after
+    /// 50 minutes is still checked against 20 minutes into the next outage,
+    /// 70 minutes after the first read.
+    #[tokio::test]
+    async fn an_answer_from_the_store_resets_the_age() {
+        let r = responder(then_down(&[secret, secret]));
+        let t0 = Instant::now();
+        assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK);
+        let second_read = t0 + Duration::from_mins(50);
+        assert_eq!(decide(&r, Some(GOOD), second_read).await, StatusCode::OK);
+        let outage = t0 + Duration::from_mins(70);
+        assert_eq!(decide(&r, Some(GOOD), outage).await, StatusCode::OK);
+    }
+
+    /// The expiry yields its line once, naming the read time, and leaves
+    /// nothing to check against; a younger value is left alone.
+    #[test]
+    fn the_expiry_of_the_kept_bearer_is_noticed_once() {
+        let t0 = Instant::now();
+        let mut cache = Cache {
+            stored: Some(Cached {
+                value: SECRET.to_owned(),
+                read_at: t0,
+                read_at_wall: "2026-11-04T07:12:11Z".parse().unwrap(),
+            }),
+            ..Cache::default()
+        };
+        assert_eq!(
+            cache.expire_kept(t0 + MAX_KEPT_AGE.saturating_sub(Duration::from_secs(1))),
+            None
+        );
+        assert!(cache.stored.is_some());
+        assert_eq!(
+            cache.expire_kept(t0 + MAX_KEPT_AGE).as_deref(),
+            Some(
+                "the store did not answer for too long; dropping the bearer read at \
+                 2026-11-04T07:12:11Z"
+            )
+        );
+        assert!(cache.stored.is_none());
+        assert_eq!(cache.expire_kept(t0 + MAX_KEPT_AGE * 2), None);
+    }
 }