From 02831fb5e966caa0fc61cba973543694c1ebeb02 Mon Sep 17 00:00:00 2001 From: damocles Date: Mon, 12 Oct 2026 22:21:04 +0200 Subject: [PATCH] hive-c0re: replace the agents_crashing banner with a crash-watcher log line + grafana alert --- docs/agent-lifecycle/config-changes.md | 54 ++++++------- docs/scheduler/observability.md | 31 ++++++- docs/turn-loop/mcp.md | 13 +-- docs/web-ui/dashboard.md | 10 ++- hive-agent/src/main.rs | 5 +- hive-c0re/src/coordinator.rs | 80 +----------------- hive-c0re/src/dashboard/state_snapshot.rs | 18 +---- hive-c0re/src/main.rs | 6 +- hive-c0re/src/stats/host_stats.rs | 74 ++++------------- hive-c0re/src/workers/crash_watch.rs | 23 +++--- hive-sh4re/src/manager.rs | 6 -- nix/host-modules/otel.nix | 98 ++++++++++++++++++++--- nix/host-modules/swarm-grafana.nix | 18 +++++ 13 files changed, 213 insertions(+), 223 deletions(-) diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index 39d56f3a..c22725a1 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -359,42 +359,42 @@ the system prompt template. ## Helper events -`Coordinator::notify_agent(agent, &HelperEvent)` pushes the event into -the agent's inbox, and the harness delivers it as a regular `system` -inbox message so it drives a normal claude turn. A "FYI, check when -convenient" event doesn't need a message — hive-c0re publishes those as -swarm lifecycle notices instead (`swarm_notices::notify`). Variants -(`hive_sh4re::manager::HelperEvent`): +`hive_sh4re::manager::HelperEvent` is the wire type for an event urgent +enough to drive an immediate claude turn (delivered as a regular +`system` inbox message, same `recv` path as any other message). It +currently has one variant, `NeedsUpdate { agent }` (sub-agent's +recorded flake rev is stale, operator rebuilds from the dashboard) — +and nothing in the tree constructs it; the push path +(`Coordinator::notify_agent`) was removed once its last caller +(`notify_manager`, used only for the crash event below) went away. +`HelperEvent` is kept as the type for whatever comes along that +genuinely needs to interrupt an agent's turn loop, not because +anything live uses it today. -- `ContainerCrash { agent, note }` — `crash_watch`: a previously- - running container went away with no operator-initiated transient - state (Stopping / Restarting / Destroying / Rebuilding) AND nothing - cleared that transient in the last 30s (`RECENT_TRANSIENT_GRACE` - tombstone, three `POLL_INTERVAL`s — closes the race where a - lifecycle op finishes between two crash-watch polls and the - container shows briefly as "stopped without transient" before - the next start). When `systemctl is-active` fails or runs past its - 10s bound on a poll, the container keeps its previous state for that - poll, so an unreadable unit never counts as a crash. - The root agent escalates to the operator, who starts it again from - the dashboard. -- `NeedsUpdate { agent }` — sub-agent's recorded flake rev is - stale. The operator rebuilds it from the dashboard. +**A previously-running container crashing** used to be exactly that +case (`ContainerCrash`, routed to the root agent via +`Coordinator::notify_manager`) — it no longer drives any agent turn at +all. `crash_watch` now only `tracing::warn!`s `"container crash +detected"`; a swarm-level grafana alert matches that log line directly +and pages the operator, with no agent (and no per-hive dashboard +banner) in the loop. See +[`docs/scheduler/observability.md`](../scheduler/observability.md#container-crash-alerting-hive-c0re). The remaining lower-urgency lifecycle notices — `Rebuilt`, `Killed`, `Destroyed`, `NeedsLogin`, `LoggedIn` — are "FYI, check when convenient" events with no reason to drive an immediate turn, so -they go out via `swarm_notices::notify` (see above) instead +they go out via `swarm_notices::notify` instead of a `HelperEvent`: `subsystem = "core"`, `key = ":"`, and a single free-text `summary` (`rebuilt_todo_summary` renders `Rebuilt`'s `ok`/`note`/`sha`/`tag` fields into that string). -To add a new lifecycle notice: if it needs to drive an immediate turn -(something genuinely urgent, like `ContainerCrash`), add a -`HelperEvent` variant + call sites + update `prompts/system.md`'s -message-event list. If it's "FYI, check when convenient," call -`swarm_notices::notify` instead — no new wire type -needed. +To add a new lifecycle notice: if it's "FYI, check when convenient," +call `swarm_notices::notify` — no new wire type needed. If it needs to +drive an immediate turn, decide first whether it actually needs an +*agent* in the loop at all (the crash case above didn't, once a +swarm-level alert could carry the same signal) — only reach for a new +`HelperEvent` variant + `notify_agent`-equivalent call site + +`prompts/system.md` update when the answer is yes. ## Autoupdate on startup diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index a45bd2e7..bad51fd5 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -5,8 +5,10 @@ swarm's metrics and log stores and, if you name one, exports upstream; each hive runs a collector of its own that forwards its agents' telemetry there. What flows through it: each agent's Claude Code statistics (token usage, cost, tool call counts) via Claude Code's built-in OpenTelemetry -integration, each agent container's journal, and the hyperhive metrics -catalogued below. +integration, each agent container's journal, the hive tier's own +`hive-c0re.service` journal line for a detected container crash (see +[Container-crash alerting](#container-crash-alerting-hive-c0re)), and the +hyperhive metrics catalogued below. ## The two collectors @@ -327,6 +329,31 @@ credential stays on the swarm tier. +## Container-crash alerting (hive-c0re) + +The crash watcher (`crash_watch.rs`, a 10s poll of every managed container) +classifies a container that stopped without an operator-initiated action as +a crash and `tracing::warn!`s `"container crash detected"` — nothing else: +no broker event, no per-hive dashboard banner. Both of those used to exist; +they were removed in favour of this. + +When `otel.enable` is set, the hive tier's own collector runs a `journald` +receiver scoped to `hive-c0re.service` (no bind mount needed — unlike an +agent's journal, this collector already shares a filesystem with the unit it +reads, being a host systemd service rather than a container) and forwards +that one log line alongside the rest of the hive's telemetry. A +swarm-level grafana alert, `hh-container-crashed` in +[`nix/host-modules/swarm-grafana.nix`](../../nix/host-modules/swarm-grafana.nix), +matches it directly and pages the operator — no agent, and no per-hive +dashboard, in the loop. + +Requires `services.journald.storage = "persistent"` (the NixOS default) on +every hive host with `otel.enable` set — asserted at eval time. + +⚠️ **A hive with `otel.enable` off has no operator-visible crash signal at +all** — the per-hive dashboard banner this replaces is gone, and there is no +fallback path. Turning on `otel.enable` is what makes a crash reach anyone. + ## Agent-emitted per-turn metrics (`hive-agent`) diff --git a/docs/turn-loop/mcp.md b/docs/turn-loop/mcp.md index ea7b9bdd..146d7802 100644 --- a/docs/turn-loop/mcp.md +++ b/docs/turn-loop/mcp.md @@ -62,12 +62,13 @@ is opt-in via the P3RM1SS10NS tab. Transient pings (sentinel id 0) have nothing to ack and show no marker. **System messages** (from sender `system`): the broker delivers -`hive_sh4re::manager::HelperEvent` (two variants: `ContainerCrash`, -`NeedsUpdate` — declared, but no call site constructs it) as regular -inbox messages (same `recv` path; body is a JSON object with an `event` -discriminant field). -`ContainerCrash` always goes to `ruth` (`Coordinator::notify_manager`, -hardcoded — `hive-c0re/src/workers/crash_watch.rs`). Lifecycle +`hive_sh4re::manager::HelperEvent` (one variant, `NeedsUpdate` — +declared, but no call site constructs it) as regular inbox messages +(same `recv` path; body is a JSON object with an `event` discriminant +field). A container crash used to be the other variant +(`ContainerCrash`, routed to `ruth`) but that path is gone — a crash +now only produces a log line + swarm-level grafana alert, no agent +turn at all (`docs/scheduler/observability.md`). Lifecycle transitions the job-queue scheduler or crash watcher drive directly — stop/kill, destroy, a flake-rev login or logout state change — reach no individual agent: they publish onto a swarm-wide NATS diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index 0766dd1f..b6c86cec 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -40,9 +40,13 @@ from the dashboard tab strip. that registry. Producers today include the host disk-pressure check (a `statvfs` probe of `/nix`: ≥85% used → `warn`, ≥95% → `crit`, e.g. `⚠ host nix store N% full (G GiB free) — garbage-collect …`), forge - provisioning/CI-runner boot failures, and agent-state warnings - (`pending_logins`, `agents_crashing`) computed alongside the container - snapshot. Hidden when there are no warnings. + provisioning/CI-runner boot failures, and an agent-state warning + (`pending_logins`) computed alongside the container snapshot. A + container crash used to have a sibling `agents_crashing` entry here — + removed in favour of a swarm-level grafana alert on the crash + watcher's own log line (`docs/scheduler/observability.md`), so a + hive with no otel export now has no operator-visible crash signal at + all. Hidden when there are no warnings. - **Browser tab title** — `hyperhive // h1ve-c0re` by default; updated to ` / // h1ve-c0re` once `hive_name` / `swarm_name` arrive in the state snapshot. diff --git a/hive-agent/src/main.rs b/hive-agent/src/main.rs index 2c58d6e1..84577fe2 100644 --- a/hive-agent/src/main.rs +++ b/hive-agent/src/main.rs @@ -118,8 +118,9 @@ async fn main() -> Result<()> { // ---------- shared turn helpers ---------- /// Surface a `SYSTEM_SENDER` message in the live event bus + tracing -/// log. Both agents and the manager receive `ContainerCrash`, -/// notifications, and friends; the parse and log path is +/// log. Both agents and the manager receive `HelperEvent` notifications +/// (just `NeedsUpdate`, now that a container crash is a log line instead +/// of a `ContainerCrash` event); the parse and log path is /// identical. Quiet no-op when `from` isn't /// `SYSTEM_SENDER`. fn log_system_event(bus: &Bus, from: &str, body: &str) { diff --git a/hive-c0re/src/coordinator.rs b/hive-c0re/src/coordinator.rs index c59ce644..927d470e 100644 --- a/hive-c0re/src/coordinator.rs +++ b/hive-c0re/src/coordinator.rs @@ -128,14 +128,6 @@ pub struct Coordinator { /// `StopForUpdate` (`true`) leaves the tombstone reading `false`, and the /// crash watcher then reports an intentional stop as a **crash**. recent_transient: Mutex>, - /// Timestamps of recent unexpected container crashes, keyed by agent. - /// Fed by `crash_watch` each time it classifies a stop as a crash (so - /// a crash-looping container — which `Restart=on-failure` flips back - /// to running between polls — accumulates one entry per down-transition, - /// not just whatever its point-in-time state happens to be). Read by - /// the dashboard's `agents_crashing` banner warning via - /// `recent_crash_counts`, which prunes entries older than its window. - recent_crashes: Mutex>>, /// Agents with a graceful stop in progress. Set by the `GracefulStop` /// orchestration; read by `socket_server::handle_recv`, which returns /// `Response::GracefulStop` (instead of polling the broker) while an @@ -490,7 +482,6 @@ impl Coordinator { model_prices, agents: Mutex::new(HashMap::new()), recent_transient: Mutex::new(HashMap::new()), - recent_crashes: Mutex::new(HashMap::new()), graceful_stop_pending: Mutex::new(HashSet::new()), pause_pending: Mutex::new(HashSet::new()), last_stopped_running: Mutex::new(None), @@ -833,7 +824,7 @@ impl Coordinator { /// watcher reads**, for the job-queue scheduler. /// /// 🚨 The stamp is not bookkeeping. Without it the clear-then-poll race - /// produced a spurious `ContainerCrash` on **every** operator stop/restart: + /// produced a spurious crash warning on **every** operator stop/restart: /// the transient is gone by the time the 10s poll looks, so a deliberate /// stop is indistinguishable from a crash. `recent_transient_within` is what /// closes that window, which is why the clear has to be an *event* — a @@ -955,33 +946,6 @@ impl Coordinator { fold_tombstones_by_agent(map.iter().map(|((a, _), (d, _))| (a.as_str(), *d))) } - /// Record an unexpected crash for `agent`. Called by the crash - /// watcher whenever it classifies a container stop as a crash (not an - /// operator action). Append-only here; pruning happens lazily on read - /// in `recent_crash_counts`. - pub fn record_crash(&self, agent: &str) { - self.recent_crashes - .lock() - .unwrap() - .entry(agent.to_owned()) - .or_default() - .push(std::time::Instant::now()); - } - - /// Per-agent count of crashes within the last `window`. Lazily reaps - /// older timestamps and drops agents with none left, so the map stays - /// bounded and only lists agents actively crashing. Powers the - /// dashboard's `agents_crashing` banner warning. - pub fn recent_crash_counts(&self, window: std::time::Duration) -> HashMap { - let now = std::time::Instant::now(); - let mut map = self.recent_crashes.lock().unwrap(); - map.retain(|_, times| { - times.retain(|ts| now.duration_since(*ts) <= window); - !times.is_empty() - }); - map.iter().map(|(k, v)| (k.clone(), v.len())).collect() - } - /// Every live transient, keyed by agent. /// /// **Derived on read, stored nowhere.** Straight off the running graph, so @@ -1036,48 +1000,6 @@ impl Coordinator { } } - /// Push a `HelperEvent` into the manager's inbox. Encoded as JSON in - /// `Message::body`; sender = `SYSTEM_SENDER`. The manager harness - /// recognises the sender and parses the body. Best-effort: a serde or - /// broker error is logged but does not propagate. - pub fn notify_manager(&self, event: &hive_sh4re::manager::HelperEvent) { - self.notify_agent(hive_sh4re::manager::MANAGER_AGENT, event); - } - - /// Push a `HelperEvent` into an arbitrary agent's inbox. Encoded - /// the same way as `notify_manager` (sender = `SYSTEM_SENDER`, - /// body = JSON-encoded event) — e.g. `ContainerCrash`, `NeedsUpdate`. - pub fn notify_agent(&self, agent: &str, event: &hive_sh4re::manager::HelperEvent) { - self.notify_agent_from(hive_sh4re::manager::SYSTEM_SENDER, agent, event); - } - - /// Same as `notify_agent` but with an explicit sender. Use this - /// when the event originates from a known agent or the operator - /// rather than the system itself, so the recipient's terminal - /// shows the right name instead of `system`. - pub fn notify_agent_from( - &self, - from: &str, - agent: &str, - event: &hive_sh4re::manager::HelperEvent, - ) { - let body = match serde_json::to_string(event) { - Ok(s) => s, - Err(e) => { - tracing::warn!(error = ?e, "failed to encode helper event"); - return; - } - }; - if let Err(e) = self.broker.send(&hive_sh4re::inbox::Message { - from: hive_sh4re::manager::trusted_sender(from), - to: agent.to_owned(), - body, - in_reply_to: None, - }) { - tracing::warn!(error = ?e, target = %agent, "failed to push helper event"); - } - } - /// Deliver `body` to every currently-registered agent except the sender, /// appending the standard broadcast hint. Returns a list of per-agent /// error strings for any that failed (empty = all ok). diff --git a/hive-c0re/src/dashboard/state_snapshot.rs b/hive-c0re/src/dashboard/state_snapshot.rs index 40d756e3..c647fd74 100644 --- a/hive-c0re/src/dashboard/state_snapshot.rs +++ b/hive-c0re/src/dashboard/state_snapshot.rs @@ -120,13 +120,6 @@ struct TransientView { secs: u64, } -/// Window over which container crashes count toward the `agents_crashing` -/// banner warning. Wide enough that a crash-looping container (restarted -/// by `Restart=on-failure` every few seconds) keeps the warning lit -/// between flaps, short enough that a single recovered crash clears within -/// minutes. -const CRASH_WARNING_WINDOW: std::time::Duration = std::time::Duration::from_mins(10); - /// Cold-load snapshot of the whole dashboard. /// /// Includes the roster, tombstones, job queue, meta inputs, and more. Live @@ -195,15 +188,12 @@ pub(super) async fn api_state( let tombstones = build_tombstone_views(&state.coord, &containers); let port_conflicts = build_port_conflicts(&containers); - // Banner warnings: host probes (disk) + agent-state (pending logins, - // crashing agents). Built before the response struct because the - // agent-state producer borrows `containers`, which moves in below. + // Banner warnings: host probes (disk) + agent-state (pending logins). + // Built before the response struct because the agent-state producer + // borrows `containers`, which moves in below. let server_warnings = { let mut w = crate::host_stats::server_warnings(); - w.extend(crate::host_stats::agent_state_warnings( - &containers, - &state.coord.recent_crash_counts(CRASH_WARNING_WINDOW), - )); + w.extend(crate::host_stats::agent_state_warnings(&containers)); w }; diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 0fd7efa6..4ebfec67 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -416,9 +416,9 @@ async fn cmd_serve( // build_logs.sqlite vacuum: c0re-side (single db). Failures kept // 30d, successes 24h — see `build_logs::vacuum` for the rule. crate::build_logs::spawn_vacuum(&coord); - // Container crash watcher: emits HelperEvent::ContainerCrash - // when a previously-running container goes away without an - // operator-initiated transient state. + // Container crash watcher: warn!()s "container crash detected" when a + // previously-running container goes away without an operator-initiated + // transient state — a swarm-level grafana alert matches that line. crash_watch::spawn(coord.clone()); // Agent-sockets marker poll: re-fires `agent_sockets::write` // and `gateway_nginx::write` every 10s so the JSON and nginx diff --git a/hive-c0re/src/stats/host_stats.rs b/hive-c0re/src/stats/host_stats.rs index 76b6e1a1..6a38db23 100644 --- a/hive-c0re/src/stats/host_stats.rs +++ b/hive-c0re/src/stats/host_stats.rs @@ -15,8 +15,6 @@ //! [`refresh_disk_warning`] drives a held guard from a periodic task (see //! `spawn` in `main.rs`) instead of a `statvfs` per `/api/state`. -use std::collections::HashMap; - use serde::Serialize; use utoipa::ToSchema; @@ -99,19 +97,20 @@ pub fn refresh_disk_warning(guard: &mut Option) { } /// Agent-state warnings derived from the live container snapshot the -/// dashboard already holds: agents that need a claude login, and agents -/// that are crashing. Kept separate from [`server_warnings`] (host -/// probes) because the caller owns the container list + crash counts; -/// the dashboard concatenates both into one banner list. +/// dashboard already holds: agents that need a claude login. Kept +/// separate from [`server_warnings`] (host probes) because the caller +/// owns the container list; the dashboard concatenates both into one +/// banner list. /// -/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent → -/// number of crashes inside that window — so a crash-looping agent shows -/// its repeat count rather than a single point-in-time flap. +/// Container crashes used to have a sibling `agents_crashing` entry here, +/// fed by `Coordinator::recent_crash_counts` — removed in favour of a +/// swarm-level grafana alert on the crash-watcher's own log line (see +/// `nix/host-modules/swarm-grafana.nix`'s `hh-container-crashed` rule and +/// `docs/scheduler/observability.md`). That ships the same signal to the +/// operator without needing an agent (or this per-hive banner) in the +/// loop at all. #[must_use] -pub fn agent_state_warnings( - containers: &[ContainerView], - crash_counts: &HashMap, -) -> Vec { +pub fn agent_state_warnings(containers: &[ContainerView]) -> Vec { let mut out = Vec::new(); // `needs_login` is already running-gated in `container_view::build_all`, @@ -137,27 +136,6 @@ pub fn agent_state_warnings( }); } - if !crash_counts.is_empty() { - let mut crashing: Vec<(&String, usize)> = - crash_counts.iter().map(|(a, n)| (a, *n)).collect(); - crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); - let list = crashing - .iter() - .map(|(a, n)| format!("{a} (×{n})")) - .collect::>() - .join(", "); - out.push(ServerWarning { - kind: "agents_crashing", - level: "crit", - message: format!( - "{n} agent{plural} crashing: {list} \ - — check the container journal (`hivectl logs `)", - n = crashing.len(), - plural = if crashing.len() == 1 { "" } else { "s" }, - ), - }); - } - out } @@ -264,13 +242,13 @@ mod tests { #[test] fn no_agent_warnings_when_all_healthy() { let containers = [cv("alice", false), cv("bob", false)]; - assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty()); + assert!(agent_state_warnings(&containers).is_empty()); } #[test] fn pending_logins_lists_sorted_agents() { let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)]; - let w = agent_state_warnings(&containers, &HashMap::new()); + let w = agent_state_warnings(&containers); assert_eq!(w.len(), 1); assert_eq!(w[0].kind, "pending_logins"); assert_eq!(w[0].level, "warn"); @@ -284,29 +262,7 @@ mod tests { #[test] fn singular_grammar_for_one_agent() { let containers = [cv("solo", true)]; - let w = agent_state_warnings(&containers, &HashMap::new()); + let w = agent_state_warnings(&containers); assert!(w[0].message.starts_with("1 agent needs claude login: solo")); } - - #[test] - fn crashing_warning_is_crit_and_count_ordered() { - let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]); - let w = agent_state_warnings(&[], &crashes); - assert_eq!(w.len(), 1); - assert_eq!(w[0].kind, "agents_crashing"); - assert_eq!(w[0].level, "crit"); - // higher crash count first - assert!(w[0].message.contains("flap (×5), blip (×1)")); - } - - #[test] - fn both_warnings_coexist() { - let containers = [cv("a", true)]; - let crashes = HashMap::from([("b".to_owned(), 2)]); - let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes) - .iter() - .map(|w| w.kind) - .collect(); - assert_eq!(kinds, ["pending_logins", "agents_crashing"]); - } } diff --git a/hive-c0re/src/workers/crash_watch.rs b/hive-c0re/src/workers/crash_watch.rs index acf163f8..55802e0d 100644 --- a/hive-c0re/src/workers/crash_watch.rs +++ b/hive-c0re/src/workers/crash_watch.rs @@ -1,7 +1,10 @@ //! Per-container crash and login-state watcher. Polls every managed -//! container on a 10s interval. Fires `ContainerCrash`, `LoggedIn`, -//! and `NeedsLogin` helper events. Event semantics and the -//! `RECENT_TRANSIENT_GRACE` window: `docs/agent-lifecycle/config-changes.md::Helper events`. +//! container on a 10s interval. A classified crash only `tracing::warn!`s +//! (`"container crash detected"`) — a swarm-level grafana alert matches +//! that line directly; see the module doc on [`emit_crash_transitions`]. +//! Login-state transitions (`needs_login`/`logged_in`) publish onto +//! `swarm_notices` instead. The `RECENT_TRANSIENT_GRACE` window: +//! `docs/agent-lifecycle/config-changes.md::Helper events`. use std::collections::HashSet; use std::sync::Arc; @@ -143,12 +146,12 @@ fn emit_crash_transitions(coord: &Coordinator, prev: &HashSet, current: if is_deliberate_stop(active, recently_cleared) { continue; } + // The one place this fires: a swarm-level grafana alert + // (`hh-container-crashed` in `nix/host-modules/swarm-grafana.nix`) + // matches this exact message against the log store and pages the + // operator directly — no agent (and no per-hive dashboard banner) + // in the loop. See `docs/scheduler/observability.md`. tracing::warn!(agent = %stopped, "container crash detected"); - coord.record_crash(stopped); - coord.notify_manager(&hive_sh4re::manager::HelperEvent::ContainerCrash { - agent: stopped.clone(), - note: Some("container stopped without an operator action".into()), - }); } } @@ -230,7 +233,7 @@ mod tests { /// Control: a real drop from running to absent is a *readable*, empty /// list — it must still be processed (and so still reach - /// `emit_crash_transitions`, which fires the crash event), unlike a + /// `emit_crash_transitions`, which warns the crash), unlike a /// read failure. #[test] fn readable_empty_list_still_processes() { @@ -261,7 +264,7 @@ mod tests { #[test] fn not_deliberate_with_no_transient_at_all() { - // The real-crash case — fires the ContainerCrash event. + // The real-crash case — warns "container crash detected". assert!(!is_deliberate_stop(None, None)); } diff --git a/hive-sh4re/src/manager.rs b/hive-sh4re/src/manager.rs index f879c980..50689dbd 100644 --- a/hive-sh4re/src/manager.rs +++ b/hive-sh4re/src/manager.rs @@ -55,12 +55,6 @@ pub fn trusted_sender(s: &str) -> Ident { pub enum HelperEvent { /// A sub-agent's recorded flake rev is stale relative to hyperhive. NeedsUpdate { agent: String }, - /// Container exited without an operator-initiated stop (crash). - ContainerCrash { - agent: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - note: Option, - }, } #[cfg(test)] diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index bb0c2586..bf6e9bc2 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -399,6 +399,21 @@ in # prove this config STARTS, never mind that a sample arrives. validateConfigFile = true; settings = { + # The journald receiver's cursor store, same shape as the agent + # tier's own (`nix/agent-modules/otel.nix`): a restart resumes + # from the last-read position instead of re-reading from the + # journal's tail (silently dropping whatever landed while the + # collector was down) or its head (re-sending everything). + # `create_directory` matters at BUILD time: `validateConfigFile` + # runs `otelcol validate` in the nix sandbox, where systemd's + # `StateDirectory=` has never run and the path does not exist + # yet — without it the validator refuses with "directory must + # exist". + extensions.file_storage = { + directory = "/var/lib/opentelemetry-collector"; + create_directory = true; + }; + receivers = { otlp.protocols.http.endpoint = listen; @@ -452,6 +467,31 @@ in system = { }; }; }; + + # This tier's own host journal, scoped to the one unit whose + # crash-watch warning (`"container crash detected"`, + # `crash_watch.rs`) a swarm-level grafana alert + # (`hh-container-crashed` in `swarm-grafana.nix`) depends on + # reaching VictoriaLogs. Unlike the swarm tier's journald + # receiver (`swarm-otel.nix`) this needs no bind mount: this + # collector is a HOST systemd unit, not a container, so + # `/var/log/journal` is the same filesystem hive-c0re itself + # logs into — no id-mapping gap to cross. + # + # `units`, unlike the swarm tier's unfiltered receiver: this + # collector has no `filter/exclude-user-sessions` processor, + # and a hive host can be someone's workstation too, so an + # allowlist is load-bearing here rather than a tidiness + # preference. + journald = { + directory = "/var/log/journal"; + storage = "file_storage"; + units = [ "hive-c0re.service" ]; + # Severity table shared with the agent and swarm tiers' + # journald receivers — see ../journald-severity.nix for why + # it is its own file. + operators = import ../journald-severity.nix; + }; } # Only when a service has actually declared a target. An enabled # `prometheus` receiver with an empty `scrape_configs` renders, @@ -499,8 +539,10 @@ in // lib.optionalAttrs senderAuth { auth.authenticator = authName; }; # ⚠️ An extension not listed here is INERT: the collector starts - # clean and the exporter naming it sends nothing authenticated. - service.extensions = lib.optional senderAuth authName; + # clean and the exporter naming it sends nothing authenticated — + # and, for `file_storage`, the journald receiver's `storage:` + # above would name a component the collector never starts. + service.extensions = [ "file_storage" ] ++ lib.optional senderAuth authName; # Host identity, so a sample can say WHICH machine it came from. # Without this the host's metrics carry `hive` and no host @@ -584,21 +626,29 @@ in exporters = [ swarmName ]; }; - # The second signal this tier carries, and the reason it exists - # is that an agent's journal is unreadable from here: the - # host-side per-container journal directory is an id-mapped bind - # mount journald never writes into, so each agent container runs - # its own forwarder (nix/agent-modules/otel.nix) and pushes over - # the `otlp` receiver above. Without this pipeline that receiver - # answers 404 on `/v1/logs` — measured, and indistinguishable - # from a route that was never meant to exist. + # The second signal this tier carries. Two distinct sources, for + # two distinct reasons: + # - `otlp`: an agent's journal is unreadable from here (the + # host-side per-container journal directory is an id-mapped + # bind mount journald never writes into), so each agent + # container runs its own forwarder (nix/agent-modules/otel.nix) + # and pushes over the `otlp` receiver above. Without this + # pipeline that receiver answers 404 on `/v1/logs` — measured, + # and indistinguishable from a route that was never meant to + # exist. + # - `journald`: this collector's own host journal, scoped to + # `hive-c0re.service` above — no id-mapping gap to cross here, + # since this collector IS the host, unlike an agent. # - # One receiver, two pipelines: `otlp` accepts both signals and + # One `otlp` receiver, two pipelines: it accepts both signals and # each pipeline takes the one it names. The exporter needs no # change either — its `endpoint` is a base, and otlphttp appends # `/v1/logs` to it exactly as it appends `/v1/metrics`. service.pipelines.logs = { - receivers = [ "otlp" ]; + receivers = [ + "otlp" + "journald" + ]; # `resourcedetection` and NOT `deltatocumulative`: that one is # a metrics-only processor, and naming it here is a startup # failure rather than a no-op. Host identity applies to both @@ -863,6 +913,30 @@ in docs/swarm/secrets.md. ''; } + { + # Same guard as ./swarm-otel.nix's sibling journald receiver, and + # the same reason: the receiver reads /var/log/journal, which only + # exists when journald stores persistently. With "volatile" the + # journal lives in /run/log/journal instead and with "none" there + # is none at all — either way `nixos-container`'s analogue for a + # host unit is simpler but no less fatal: the collector's + # `journald` receiver fails to open the directory and the unit + # never starts, with nothing here saying why. + assertion = + !(lib.elem config.services.journald.storage [ + "volatile" + "none" + ]); + message = '' + services.hyperhive.otel.enable is true, which runs a journald + receiver reading /var/log/journal for hive-c0re's own crash-watch + log line — but services.journald.storage is + "${config.services.journald.storage}" on this host. + + Set services.journald.storage = "persistent" (the NixOS + default). + ''; + } ]; } ); diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 6e70f69a..d8fcc75f 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1176,6 +1176,24 @@ in for = "0s"; summary = "{{ $labels._HOSTNAME }} refused to deploy merged config commit {{ $labels.F_REV }} of agent {{ $labels.F_AGENT }}; the reason is the `F_ERROR` field of the hive-c0re log line"; }) + # Logged once per detected crash, by `emit_crash_transitions` + # in hive-c0re's crash_watch.rs (10s poll interval). Replaces + # the old per-hive dashboard `agents_crashing` banner — see + # that function's module doc for why the banner went away + # in favour of this. Host journal records carry the + # tracing fields as `F_` and no `hive` attribute, + # so the machine is `_HOSTNAME` + # (same shape as `hh-config-rev-refused` above). The unit + # filter keeps out an agent's own log line if one ever + # quotes the same message. + (logCountRule { + uid = "hh-container-crashed"; + title = "container crashed"; + windowSeconds = 900; + expr = ''_time:15m _SYSTEMD_UNIT:="hive-c0re.service" _msg:"container crash detected" | stats by (_HOSTNAME, F_AGENT) count() as crashes''; + for = "0s"; + summary = "{{ $labels._HOSTNAME }} detected agent {{ $labels.F_AGENT }}'s container crash (not an operator-initiated stop)"; + }) ] # One rule per declared hive rather than one grouped by # `hive`, so a hive that sends nothing at all still has