diff --git a/CLAUDE.md b/CLAUDE.md index 4cda3e3e..ec067ede 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ hive-c0re/ host daemon + sibling operator CLI (lib + 2 bins) src/agent_sockets.rs writes `/var/lib/hyperhive/agent-sockets.json` on meta sync; gateway reads name→socket-path map for unix-domain per-agent UI binding. - Entries are filtered by a `.bound` marker file + Entries are filtered by a `hyperhive-socket-bound` marker file the harness drops next to its socket after a successful `bind()` — pre-bind agents stay out of the map so the gateway never races a diff --git a/docs/gateway.md b/docs/gateway.md index 9c1ecd0c..e6e00aa1 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -84,21 +84,22 @@ unix-domain socket as each agent opts in. The mechanism: harness's `unlink + bind(2)` cycle on socket replace. Per-agent subdir keeps each agent's container blind to siblings' sockets. 3. **Marker gate**. After successful `bind_unix`, the harness drops - `/.bound` next to the socket. c0re's `agent_sockets::write` - filters its JSON map by marker presence — only agents whose - harness has actually bound the socket appear there. Without this - filter, the gateway would `proxy_pass` to a non-existent socket - for every sub-agent that hasn't opted in yet. + `/hyperhive-socket-bound` next to the socket. c0re's + `agent_sockets::write` filters its JSON map by marker presence — + only agents whose harness has actually bound the socket appear there. + Without this filter, the gateway would `proxy_pass` to a non-existent + socket for every sub-agent that hasn't opted in yet. (Legacy name + `.bound` also accepted during the transition window.) 4. **Gateway side**. `gateway_nginx::write` generates `/var/lib/hyperhive/agents.conf` — a plain nginx include file with one `location /agent//` block per agent. UDS upstream - (`http://unix:/run/hive-agent//web.sock:/`) when `.bound` - marker present; TCP loopback otherwise. The gateway container - bind-mounts `/var/lib/hyperhive/` at `/run/hive-state/`; nginx - includes `/run/hive-state/agents.conf`. A systemd path unit - (`hive-gateway-agents-conf.path`) inside the container watches the - file and fires `nginx -s reload` on every atomic rename from c0re - — no `nixos-rebuild` needed (#869). + (`http://unix:/run/hive-agent//web.sock:/`) when + `hyperhive-socket-bound` marker present; TCP loopback otherwise. + The gateway container bind-mounts `/var/lib/hyperhive/` at + `/run/hive-state/`; nginx includes `/run/hive-state/agents.conf`. + A systemd path unit (`hive-gateway-agents-conf.path`) inside the + container watches the file and fires `nginx -s reload` on every + atomic rename from c0re — no `nixos-rebuild` needed. c0re regenerates `agents.conf` (and fires the path unit → reload) on two triggers: every topology change (new/removed agents) and every diff --git a/docs/persistence.md b/docs/persistence.md index f2d3404b..f5de1554 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -104,17 +104,28 @@ the turn loop continue. No host-side vacuum yet — tracked separately. Target retention ~90 days, age-only sweep like events_vacuum. -### `/state/hyperhive-rate-limited` (per agent) +### `/state/hyperhive-harness.json` (per agent) -Sentinel file written by `Bus::emit_status("rate_limited")` when the -harness detects a 429 / rate-limit response from the Claude API, and -removed when the retry sleep expires (any subsequent status emit -clears it). The file's presence is checked by hive-c0re's -`container_view::is_rate_limited` on each `build_all` sweep (~10s) to -populate `ContainerView.rate_limited` for the dashboard. Survives a -harness restart (the Bus reads it back at boot and restores the flag), -so the badge remains accurate if hive-c0re restarts while the harness -is mid-sleep. +Consolidated harness state file written atomically (`.tmp` + rename) by +`Bus::emit_status` whenever rate-limited or login-failed flags change. +Shape: + +```json +{ "rate_limited": false, "needs_login": false } +``` + +- `rate_limited` — set when the harness detects a 429 from the Claude + API; cleared by any subsequent status emit. Drives + `ContainerView.rate_limited` on the dashboard. +- `needs_login` — set when a turn hits 401 (expired OAuth credentials); + cleared by `"online"` status (re-auth completed). Drives the + `needs_login` flag alongside the `claude_has_session` check. + +hive-c0re reads this file on each `build_all` sweep (~10s) via +`container_view::read_harness_flags`. Falls back to the legacy individual +sentinel files (`hyperhive-rate-limited`, `hyperhive-needs-login`) if the +JSON is absent, so existing containers keep working through the transition +window before their next rebuild. ### `/state/hyperhive-model` (per agent) diff --git a/hive-ag3nt/src/events.rs b/hive-ag3nt/src/events.rs index 0f93eda6..448cb806 100644 --- a/hive-ag3nt/src/events.rs +++ b/hive-ag3nt/src/events.rs @@ -56,6 +56,59 @@ fn persist_model(name: &str) -> std::io::Result<()> { std::fs::write(path, format!("{name}\n")) } +// --------------------------------------------------------------------------- +// Consolidated harness state file +// --------------------------------------------------------------------------- +// +// `hyperhive-harness.json` replaces the two legacy boolean sentinel files +// (`hyperhive-rate-limited`, `hyperhive-needs-login`) that grew organically +// and had no shared schema. A single JSON file is self-documenting, atomic +// to write, and cheaper for hive-c0re to read on each sweep (one fopen vs +// two stat calls). See `docs/persistence.md::Harness state files`. +// +// Legacy sentinel files written by older harness builds are still honoured +// by `read_harness_state` so in-place upgrades don't lose state (the new +// harness re-normalises on first write). Old files are not deleted — they +// expire naturally when the state dir is purged. `hive-c0re::container_view` +// also checks the legacy paths as a fallback during the transition window. + +const HARNESS_JSON: &str = "hyperhive-harness.json"; + +fn harness_json_path() -> PathBuf { + crate::paths::state_dir().join(HARNESS_JSON) +} + +fn read_harness_state() -> (bool, bool) { + // Try the new consolidated file first. + if let Ok(raw) = std::fs::read_to_string(harness_json_path()) { + if let Ok(v) = serde_json::from_str::(&raw) { + let rate_limited = v.get("rate_limited").and_then(|x| x.as_bool()).unwrap_or(false); + let needs_login = v.get("needs_login").and_then(|x| x.as_bool()).unwrap_or(false); + return (rate_limited, needs_login); + } + } + // Fall back to legacy sentinel files written by older harness builds. + let state_dir = crate::paths::state_dir(); + let rate_limited = state_dir.join("hyperhive-rate-limited").exists(); + let needs_login = state_dir.join("hyperhive-needs-login").exists(); + (rate_limited, needs_login) +} + +/// Write harness state atomically via a `.tmp` + `rename` pair so +/// hive-c0re never reads a partial file. +fn write_harness_state(rate_limited: bool, needs_login: bool) { + let path = harness_json_path(); + let body = serde_json::json!({ + "rate_limited": rate_limited, + "needs_login": needs_login, + }) + .to_string(); + let tmp = path.with_extension("json.tmp"); + if std::fs::write(&tmp, &body).is_ok() { + let _ = std::fs::rename(&tmp, &path); + } +} + fn now_unix() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -456,11 +509,10 @@ impl Bus { || load_model().unwrap_or_else(|| DEFAULT_MODEL.to_owned()), str::to_owned, ); - // Restore rate_limited from the sentinel file — if the harness - // crashed while parked, we should still show the right status on - // cold load until the next turn clears it. - let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited"); - let was_rate_limited = sentinel.exists(); + // Restore rate_limited (and needs_login) from the consolidated + // harness state file so the dashboard shows the correct status + // on cold load if the harness crashed while parked. + let (was_rate_limited, _was_needs_login) = read_harness_state(); Self { tx: Arc::new(tx), event_seq: Arc::new(AtomicU64::new(0)), @@ -688,34 +740,37 @@ impl Bus { /// `Arc>` should also call this so the web UI /// drops its periodic /api/state poll while a turn loop is running. /// - /// Sentinel files survive harness restart so the host-side dashboard - /// can render the status without a live socket call: - /// - `"rate_limited"` writes `{state_dir}/hyperhive-rate-limited` - /// (cleared by any other status). - /// - `"needs_login_idle"` writes `{state_dir}/hyperhive-needs-login` - /// so a 401-triggered re-auth flag persists across harness restart. - /// The web UI's `/login` POST handler clears it via - /// `clear_needs_login_sentinel` once the operator re-auths. - /// - `"online"` clears both sentinels — the agent is healthy again. + /// `hyperhive-harness.json` persists across harness restarts so the + /// host-side dashboard can render the status without a live socket call: + /// - `"rate_limited"` sets `rate_limited: true` in the JSON. + /// - `"needs_login_idle"` sets `needs_login: true` in the JSON. + /// - `"online"` clears both fields — the agent is healthy again. + /// - Other statuses clear `rate_limited` only; `needs_login` is sticky + /// until `"online"` (re-auth completed successfully). + /// + /// Writes are atomic (`.tmp` + `rename`) so hive-c0re never reads a + /// partial file during its ~10s sweep. pub fn emit_status(&self, status: impl Into) { let status = status.into(); - let rate_limited_path = crate::paths::state_dir().join("hyperhive-rate-limited"); - let needs_login_path = crate::paths::state_dir().join("hyperhive-needs-login"); - if status == "rate_limited" { + let new_rate_limited = status == "rate_limited"; + if new_rate_limited { self.rate_limited.store(true, Ordering::Relaxed); - let _ = std::fs::write(&rate_limited_path, b""); } else { self.rate_limited.store(false, Ordering::Relaxed); - let _ = std::fs::remove_file(&rate_limited_path); } - if status == "needs_login_idle" { - let _ = std::fs::write(&needs_login_path, b""); + // Read the current persisted needs_login so we don't flip it on + // statuses that shouldn't touch it (e.g. `needs_login_in_progress` + // is a transient mid-flow status; only `needs_login_idle` and + // `online` should change the persistent flag). + let (_, current_needs_login) = read_harness_state(); + let new_needs_login = if status == "needs_login_idle" { + true } else if status == "online" { - // Re-auth completed (or manual flip back to online) — drop - // the sentinel. `needs_login_in_progress` is a transient - // mid-flow status and shouldn't clear yet. - let _ = std::fs::remove_file(&needs_login_path); - } + false + } else { + current_needs_login + }; + write_harness_state(new_rate_limited, new_needs_login); self.emit(LiveEvent::StatusChanged { status }); } diff --git a/hive-ag3nt/src/web_ui.rs b/hive-ag3nt/src/web_ui.rs index 590555a8..3546abb9 100644 --- a/hive-ag3nt/src/web_ui.rs +++ b/hive-ag3nt/src/web_ui.rs @@ -194,15 +194,15 @@ fn bind_unix(path: &Path) -> Result { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) .with_context(|| format!("set perms on {}", path.display()))?; - // Best-effort .bound marker: failed write isn't fatal (the harness + // Best-effort ready marker: failed write isn't fatal (the harness // still binds + serves), it just means the gateway side keeps the // TCP upstream for one more sync tick. if let Some(parent) = path.parent() { - let marker = parent.join(".bound"); + let marker = parent.join("hyperhive-socket-bound"); if let Err(e) = std::fs::write(&marker, b"") { tracing::warn!( marker = %marker.display(), error = %e, - "failed to write .bound marker — gateway may keep TCP upstream" + "failed to write hyperhive-socket-bound marker — gateway may keep TCP upstream" ); } } diff --git a/hive-c0re/src/agent_sockets.rs b/hive-c0re/src/agent_sockets.rs index 66abe522..30a2aa09 100644 --- a/hive-c0re/src/agent_sockets.rs +++ b/hive-c0re/src/agent_sockets.rs @@ -4,8 +4,8 @@ //! excluded from the map (manager UI routes via the dashboard //! upstream, not per-agent `/agent//`). //! -//! Full mechanism — per-agent subdir bind-mount, `.bound` marker -//! gate, gateway UDS upstream, transition vs `agent-ports.json`, +//! Full mechanism — per-agent subdir bind-mount, `hyperhive-socket-bound` +//! marker gate, gateway UDS upstream, transition vs `agent-ports.json`, //! 10s poll loop: `docs/gateway.md::Per-agent unix-socket upstream`. use std::collections::BTreeMap; @@ -36,7 +36,13 @@ pub const SOCKET_FILENAME: &str = "web.sock"; /// publish the unix upstream for this agent yet". Without this gate /// the gateway would `proxy_pass` to a non-existent socket for every /// sub-agent that hasn't flipped the option yet. -pub const READY_MARKER: &str = ".bound"; +/// +/// Renamed from `.bound` (legacy) to match the `hyperhive-` prefix +/// convention for all harness-written state files (#838). `build_map` +/// checks both names during the transition window so existing containers +/// don't lose gateway routing before their next rebuild. +pub const READY_MARKER: &str = "hyperhive-socket-bound"; +const READY_MARKER_LEGACY: &str = ".bound"; #[must_use] pub fn host_sockets_path() -> PathBuf { @@ -72,18 +78,27 @@ pub fn socket_path_for(name: &str) -> PathBuf { /// `proxy_pass` to a non-existent socket for every sub-agent that /// hasn't yet flipped `hyperhive.web.useUnixSocket = true`. /// +/// Accepts either the new `hyperhive-socket-bound` marker or the legacy +/// `.bound` marker so existing containers keep their gateway routing +/// through the transition window (before their next rebuild writes the +/// new marker name). +/// /// `BTreeMap` keeps the JSON output sorted by key so a re-emit /// without churn produces byte-identical output — same idempotency /// shape `agent_ports::write` relies on. #[must_use] pub fn build_map(names: &[String]) -> BTreeMap { - build_map_with(names, |name| ready_marker_for(name).exists()) + build_map_with(names, |name| { + ready_marker_for(name).exists() + || agent_dir_for(name).join(READY_MARKER_LEGACY).exists() + }) } /// Body of `build_map` with the ready-check parameterised. Tests /// pass a predicate they control (no real filesystem access). /// Production callers go through `build_map` which wires the -/// predicate to the on-disk `.bound` marker check. +/// predicate to the on-disk `hyperhive-socket-bound` (or legacy +/// `.bound`) marker check. fn build_map_with(names: &[String], is_ready: F) -> BTreeMap where F: Fn(&str) -> bool, @@ -96,7 +111,7 @@ where .collect() } -/// Path to the per-agent `.bound` marker file the harness writes +/// Path to the `hyperhive-socket-bound` marker file the harness writes /// after a successful `bind_unix`. Lives next to `web.sock` in the /// per-agent subdir so it's covered by the same bind-mount and same /// per-agent isolation as the socket itself. @@ -271,10 +286,9 @@ mod tests { #[test] fn build_map_filters_by_ready_predicate() { - // The new gate: only ready agents (with `.bound` marker) get - // published. Pin the behaviour so a future refactor that - // drops the filter surfaces here, not as a 502-spew in the - // gateway. + // Only ready agents (with `hyperhive-socket-bound` marker) get + // published. Pin the behaviour so a future refactor that drops + // the filter surfaces here, not as a 502-spew in the gateway. let names: Vec = ["iris", "argus", "atlas"] .iter() .map(|s| (*s).to_owned()) @@ -296,7 +310,7 @@ mod tests { let marker = ready_marker_for("iris"); let socket = socket_path_for("iris"); assert_eq!(marker.parent(), socket.parent()); - assert_eq!(marker, Path::new("/run/hive-agent/iris/.bound")); + assert_eq!(marker, Path::new("/run/hive-agent/iris/hyperhive-socket-bound")); } #[test] diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index bd5571e0..a01b1330 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -211,24 +211,31 @@ fn read_dashboard_links(name: &str) -> Vec { serde_json::from_str::>(&text).unwrap_or_default() } -/// Returns true if the agent's harness is currently parked after an API -/// rate-limit response. Detected via the sentinel file written by -/// `hive_ag3nt::events::Bus::emit_status("rate_limited")`. +/// Read `rate_limited` + `needs_login` from the consolidated +/// `hyperhive-harness.json`. Falls back to the legacy individual +/// sentinel files written by older harness builds so in-place upgrades +/// don't lose state during the transition window. +fn read_harness_flags(name: &str) -> (bool, bool) { + let dir = Coordinator::agent_notes_dir(name); + if let Ok(raw) = std::fs::read_to_string(dir.join("hyperhive-harness.json")) { + if let Ok(v) = serde_json::from_str::(&raw) { + let rl = v.get("rate_limited").and_then(|x| x.as_bool()).unwrap_or(false); + let nl = v.get("needs_login").and_then(|x| x.as_bool()).unwrap_or(false); + return (rl, nl); + } + } + // Legacy fallback: presence of individual sentinel files. + let rate_limited = dir.join("hyperhive-rate-limited").exists(); + let needs_login = dir.join("hyperhive-needs-login").exists(); + (rate_limited, needs_login) +} + fn is_rate_limited(name: &str) -> bool { - Coordinator::agent_notes_dir(name) - .join("hyperhive-rate-limited") - .exists() + read_harness_flags(name).0 } -/// True when the harness wrote `{state_dir}/hyperhive-needs-login` -/// after a 401 mid-turn. Lets the dashboard surface `needs_login` for -/// agents whose `/root/.claude/` dir still exists (so -/// `claude_has_session` returns true) but whose OAuth credentials -/// inside it have actually expired. fn auth_failed_sentinel(name: &str) -> bool { - Coordinator::agent_notes_dir(name) - .join("hyperhive-needs-login") - .exists() + read_harness_flags(name).1 } /// Read the agent's free-text status and the Unix timestamp when it was last set