vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-matrix-mcp: quieter retries, unique snapshot tmp file, accurate comments

Browse source 
Audit follow-ups on the secondary-account retry.

The accounts snapshot is written by the heartbeat task and, now, by the
retry loop. Both used one fixed tmp path, so concurrent writers could
clobber it or fail the rename. Each write now names its own tmp file
(pid + counter); a test hammers it from four threads.

A permanently bad secondary logged three warnings per attempt, every 16
minutes, for as long as the daemon ran. DownAccounts::failed now reports
whether a failure is news (first, or a different reason from the last);
only that is a warning, repeats are debug. The two per-attempt warnings
in client.rs and credential.rs are covered by that line, which names the
reason, so the store-token one drops to debug.

The comments in client.rs and a credential.rs test, and persistence.md,
still described the old skip-until-restart behaviour or implied that a
store-listed account without a credential is retried here; that case is
handled by the linked-name set changing, not by this loop.
This commit is contained in:
atlas 2026-10-13 21:38:37 +02:00 • committed by mara
commit 53049962b8
5 changed files with 110 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

85
hive-matrix-mcp/src/main.rs
View file

@ -89,6 +89,13 @@ const LINKED_REFRESH: std::time::Duration = std::time::Duration::from_mins(2);
/// takes to come up.
const MAX_SECONDARY_RETRY: std::time::Duration = std::time::Duration::from_mins(16);
/// Why a secondary is down when it had no token to try.
const NO_TOKEN: &str = "no token";
/// Why a secondary is down when start-up gave up on it; the reason was logged
/// there.
const FAILED_AT_START: &str = "failed at start";
/// The exit status the daemon leaves with when the store's linked accounts
/// changed. `nix/agent-modules/matrix.nix` names the same number in the unit's
/// `RestartForceExitStatus` and `SuccessExitStatus`, so systemd restarts it
@ -146,7 +153,7 @@ async fn main() -> Result<ExitCode> {
}
Ok(None) => {
tracing::warn!(account = %cfg.name, "secondary matrix account has no token; will retry");
down.failed(cfg.name.clone(), (cfg, tag), Instant::now());
down.failed(cfg.name.clone(), (cfg, tag), Instant::now(), NO_TOKEN);
}
// The primary failing to restore is fatal (propagate so
// systemd retries on a transient blip — matches legacy
@ -158,7 +165,14 @@ async fn main() -> Result<ExitCode> {
sync_loops.push(named(cfg.name.clone(), sync_loop));
registry.insert(cfg.name, client);
}
None => down.failed(cfg.name.clone(), (cfg, tag), Instant::now()),
None => {
down.failed(
cfg.name.clone(),
(cfg, tag),
Instant::now(),
FAILED_AT_START,
);
}
},
}
}
@ -426,6 +440,8 @@ struct Down<T> {
retry: T,
failures: u32,
not_before: Instant,
/// Why the last try failed.
reason: String,
}
impl<T> Default for DownAccounts<T> {
@ -437,16 +453,22 @@ impl<T> Default for DownAccounts<T> {
}
impl<T> DownAccounts<T> {
/// Record that `name` failed to come up at `at`. `retry` is kept from the
/// first failure on.
fn failed(&mut self, name: String, retry: T, at: Instant) {
/// Record that `name` failed to come up at `at`, for `reason`. `retry` is
/// kept from the first failure on. True when the failure is news — the
/// account's first, or for a different reason than the last — and so
/// worth a warning.
fn failed(&mut self, name: String, retry: T, at: Instant, reason: &str) -> bool {
let down = self.by_name.entry(name).or_insert(Down {
retry,
failures: 0,
not_before: at,
reason: String::new(),
});
let news = down.failures == 0 || down.reason != reason;
reason.clone_into(&mut down.reason);
down.failures += 1;
down.not_before = at + retry_delay(down.failures);
news
}
/// Record that `name` came up; it is never due again.
@ -495,16 +517,19 @@ async fn retry_down(
let _ = up.send(named(name, sync_loop));
}
Ok(None) => {
tracing::warn!(account = %name, "secondary matrix account still has no token; will retry");
down.failed(name, (cfg, tag), now);
if down.failed(name.clone(), (cfg, tag), now, NO_TOKEN) {
tracing::warn!(account = %name, "secondary matrix account has no token; will keep retrying");
} else {
tracing::debug!(account = %name, "secondary matrix account still has no token");
}
}
Err(e) => {
tracing::warn!(
account = %name,
error = %format!("{e:#}"),
"secondary matrix account still failing; will retry"
);
down.failed(name, (cfg, tag), now);
let reason = format!("{e:#}");
if down.failed(name.clone(), (cfg, tag), now, &reason) {
tracing::warn!(account = %name, error = %reason, "secondary matrix account failing; will keep retrying");
} else {
tracing::debug!(account = %name, error = %reason, "secondary matrix account still failing");
}
}
}
}
@ -668,7 +693,7 @@ mod tests {
fn a_failed_account_is_due_once_its_wait_is_over_and_not_before() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
down.failed("ccc".to_owned(), 7, t0);
down.failed("ccc".to_owned(), 7, t0, "r");
assert_eq!(down.due(t0), None);
assert_eq!(down.due(t0 + LINKED_REFRESH - Duration::from_secs(1)), None);
@ -679,9 +704,9 @@ mod tests {
fn each_failure_in_a_row_pushes_the_next_try_further_out() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
down.failed("ccc".to_owned(), (), t0);
down.failed("ccc".to_owned(), (), t0, "r");
let t1 = t0 + LINKED_REFRESH;
down.failed("ccc".to_owned(), (), t1);
down.failed("ccc".to_owned(), (), t1, "r");
assert_eq!(down.due(t1 + LINKED_REFRESH), None);
assert_eq!(down.due(t1 + LINKED_REFRESH * 2), Some(("ccc", &())));
@ -691,7 +716,7 @@ mod tests {
fn an_account_that_came_up_is_never_due_again() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
down.failed("ccc".to_owned(), (), t0);
down.failed("ccc".to_owned(), (), t0, "r");
down.came_up("ccc");
assert_eq!(down.due(t0 + MAX_SECONDARY_RETRY * 10), None);
@ -701,21 +726,37 @@ mod tests {
fn the_longest_overdue_account_goes_first_and_the_others_keep_their_schedule() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
down.failed("late".to_owned(), "late cfg", t0 + Duration::from_secs(30));
down.failed("early".to_owned(), "early cfg", t0);
down.failed(
"late".to_owned(),
"late cfg",
t0 + Duration::from_secs(30),
"r",
);
down.failed("early".to_owned(), "early cfg", t0, "r");
let now = t0 + LINKED_REFRESH * 2;
assert_eq!(down.due(now), Some(("early", &"early cfg")));
down.failed("early".to_owned(), "early cfg", now);
down.failed("early".to_owned(), "early cfg", now, "r");
assert_eq!(down.due(now), Some(("late", &"late cfg")));
}
#[test]
fn a_failure_is_news_the_first_time_and_when_its_reason_changes() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
assert!(down.failed("ccc".to_owned(), (), t0, "rejected"));
assert!(!down.failed("ccc".to_owned(), (), t0, "rejected"));
assert!(down.failed("ccc".to_owned(), (), t0, "no token"));
assert!(!down.failed("ccc".to_owned(), (), t0, "no token"));
assert!(down.failed("other".to_owned(), (), t0, "no token"));
}
#[test]
fn the_first_retry_target_is_kept_across_failures() {
let t0 = Instant::now();
let mut down = DownAccounts::default();
down.failed("ccc".to_owned(), "first", t0);
down.failed("ccc".to_owned(), "second", t0);
down.failed("ccc".to_owned(), "first", t0, "r");
down.failed("ccc".to_owned(), "second", t0, "r");
assert_eq!(down.due(t0 + MAX_SECONDARY_RETRY), Some(("ccc", &"first")));
}