diff --git a/CLAUDE.md b/CLAUDE.md index ec067ede..4cda3e3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ hive-c0re/ host daemon + sibling operator CLI (lib + 2 bins) src/agent_sockets.rs writes `/var/lib/hyperhive/agent-sockets.json` on meta sync; gateway reads name→socket-path map for unix-domain per-agent UI binding. - Entries are filtered by a `hyperhive-socket-bound` marker file + Entries are filtered by a `.bound` marker file the harness drops next to its socket after a successful `bind()` — pre-bind agents stay out of the map so the gateway never races a diff --git a/docs/gateway.md b/docs/gateway.md index e6e00aa1..9c1ecd0c 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -84,22 +84,21 @@ unix-domain socket as each agent opts in. The mechanism: harness's `unlink + bind(2)` cycle on socket replace. Per-agent subdir keeps each agent's container blind to siblings' sockets. 3. **Marker gate**. After successful `bind_unix`, the harness drops - `/hyperhive-socket-bound` next to the socket. c0re's - `agent_sockets::write` filters its JSON map by marker presence — - only agents whose harness has actually bound the socket appear there. - Without this filter, the gateway would `proxy_pass` to a non-existent - socket for every sub-agent that hasn't opted in yet. (Legacy name - `.bound` also accepted during the transition window.) + `/.bound` next to the socket. c0re's `agent_sockets::write` + filters its JSON map by marker presence — only agents whose + harness has actually bound the socket appear there. Without this + filter, the gateway would `proxy_pass` to a non-existent socket + for every sub-agent that hasn't opted in yet. 4. **Gateway side**. `gateway_nginx::write` generates `/var/lib/hyperhive/agents.conf` — a plain nginx include file with one `location /agent//` block per agent. UDS upstream - (`http://unix:/run/hive-agent//web.sock:/`) when - `hyperhive-socket-bound` marker present; TCP loopback otherwise. - The gateway container bind-mounts `/var/lib/hyperhive/` at - `/run/hive-state/`; nginx includes `/run/hive-state/agents.conf`. - A systemd path unit (`hive-gateway-agents-conf.path`) inside the - container watches the file and fires `nginx -s reload` on every - atomic rename from c0re — no `nixos-rebuild` needed. + (`http://unix:/run/hive-agent//web.sock:/`) when `.bound` + marker present; TCP loopback otherwise. The gateway container + bind-mounts `/var/lib/hyperhive/` at `/run/hive-state/`; nginx + includes `/run/hive-state/agents.conf`. A systemd path unit + (`hive-gateway-agents-conf.path`) inside the container watches the + file and fires `nginx -s reload` on every atomic rename from c0re + — no `nixos-rebuild` needed (#869). c0re regenerates `agents.conf` (and fires the path unit → reload) on two triggers: every topology change (new/removed agents) and every diff --git a/docs/persistence.md b/docs/persistence.md index f5de1554..f2d3404b 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -104,28 +104,17 @@ the turn loop continue. No host-side vacuum yet — tracked separately. Target retention ~90 days, age-only sweep like events_vacuum. -### `/state/hyperhive-harness.json` (per agent) +### `/state/hyperhive-rate-limited` (per agent) -Consolidated harness state file written atomically (`.tmp` + rename) by -`Bus::emit_status` whenever rate-limited or login-failed flags change. -Shape: - -```json -{ "rate_limited": false, "needs_login": false } -``` - -- `rate_limited` — set when the harness detects a 429 from the Claude - API; cleared by any subsequent status emit. Drives - `ContainerView.rate_limited` on the dashboard. -- `needs_login` — set when a turn hits 401 (expired OAuth credentials); - cleared by `"online"` status (re-auth completed). Drives the - `needs_login` flag alongside the `claude_has_session` check. - -hive-c0re reads this file on each `build_all` sweep (~10s) via -`container_view::read_harness_flags`. Falls back to the legacy individual -sentinel files (`hyperhive-rate-limited`, `hyperhive-needs-login`) if the -JSON is absent, so existing containers keep working through the transition -window before their next rebuild. +Sentinel file written by `Bus::emit_status("rate_limited")` when the +harness detects a 429 / rate-limit response from the Claude API, and +removed when the retry sleep expires (any subsequent status emit +clears it). The file's presence is checked by hive-c0re's +`container_view::is_rate_limited` on each `build_all` sweep (~10s) to +populate `ContainerView.rate_limited` for the dashboard. Survives a +harness restart (the Bus reads it back at boot and restores the flag), +so the badge remains accurate if hive-c0re restarts while the harness +is mid-sleep. ### `/state/hyperhive-model` (per agent) diff --git a/hive-ag3nt/src/events.rs b/hive-ag3nt/src/events.rs index 2a45e7f6..0f93eda6 100644 --- a/hive-ag3nt/src/events.rs +++ b/hive-ag3nt/src/events.rs @@ -56,59 +56,6 @@ fn persist_model(name: &str) -> std::io::Result<()> { std::fs::write(path, format!("{name}\n")) } -// --------------------------------------------------------------------------- -// Consolidated harness state file -// --------------------------------------------------------------------------- -// -// `hyperhive-harness.json` replaces the two legacy boolean sentinel files -// (`hyperhive-rate-limited`, `hyperhive-needs-login`) that grew organically -// and had no shared schema. A single JSON file is self-documenting, atomic -// to write, and cheaper for hive-c0re to read on each sweep (one fopen vs -// two stat calls). See `docs/persistence.md::Harness state files`. -// -// Legacy sentinel files written by older harness builds are still honoured -// by `read_harness_state` so in-place upgrades don't lose state (the new -// harness re-normalises on first write). Old files are not deleted — they -// expire naturally when the state dir is purged. `hive-c0re::container_view` -// also checks the legacy paths as a fallback during the transition window. - -const HARNESS_JSON: &str = "hyperhive-harness.json"; - -fn harness_json_path() -> PathBuf { - crate::paths::state_dir().join(HARNESS_JSON) -} - -fn read_harness_state() -> (bool, bool) { - // Try the new consolidated file first. - if let Ok(raw) = std::fs::read_to_string(harness_json_path()) { - if let Ok(v) = serde_json::from_str::(&raw) { - let rate_limited = v.get("rate_limited").and_then(|x| x.as_bool()).unwrap_or(false); - let needs_login = v.get("needs_login").and_then(|x| x.as_bool()).unwrap_or(false); - return (rate_limited, needs_login); - } - } - // Fall back to legacy sentinel files written by older harness builds. - let state_dir = crate::paths::state_dir(); - let rate_limited = state_dir.join("hyperhive-rate-limited").exists(); - let needs_login = state_dir.join("hyperhive-needs-login").exists(); - (rate_limited, needs_login) -} - -/// Write harness state atomically via a `.tmp` + `rename` pair so -/// hive-c0re never reads a partial file. -fn write_harness_state(rate_limited: bool, needs_login: bool) { - let path = harness_json_path(); - let body = serde_json::json!({ - "rate_limited": rate_limited, - "needs_login": needs_login, - }) - .to_string(); - let tmp = path.with_extension("json.tmp"); - if std::fs::write(&tmp, &body).is_ok() { - let _ = std::fs::rename(&tmp, &path); - } -} - fn now_unix() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -509,10 +456,11 @@ impl Bus { || load_model().unwrap_or_else(|| DEFAULT_MODEL.to_owned()), str::to_owned, ); - // Restore rate_limited (and needs_login) from the consolidated - // harness state file so the dashboard shows the correct status - // on cold load if the harness crashed while parked. - let (was_rate_limited, _was_needs_login) = read_harness_state(); + // Restore rate_limited from the sentinel file — if the harness + // crashed while parked, we should still show the right status on + // cold load until the next turn clears it. + let sentinel = crate::paths::state_dir().join("hyperhive-rate-limited"); + let was_rate_limited = sentinel.exists(); Self { tx: Arc::new(tx), event_seq: Arc::new(AtomicU64::new(0)), @@ -740,46 +688,34 @@ impl Bus { /// `Arc>` should also call this so the web UI /// drops its periodic /api/state poll while a turn loop is running. /// - /// `hyperhive-harness.json` persists across harness restarts so the - /// host-side dashboard can render the status without a live socket call: - /// - `"rate_limited"` sets `rate_limited: true` in the JSON. - /// - `"needs_login_idle"` sets `needs_login: true` in the JSON. - /// - `"online"` clears both fields — the agent is healthy again. - /// - Other statuses clear `rate_limited` only; `needs_login` is sticky - /// until `"online"` (re-auth completed successfully). - /// - /// Writes are atomic (`.tmp` + `rename`) so hive-c0re never reads a - /// partial file during its ~10s sweep. + /// Sentinel files survive harness restart so the host-side dashboard + /// can render the status without a live socket call: + /// - `"rate_limited"` writes `{state_dir}/hyperhive-rate-limited` + /// (cleared by any other status). + /// - `"needs_login_idle"` writes `{state_dir}/hyperhive-needs-login` + /// so a 401-triggered re-auth flag persists across harness restart. + /// The web UI's `/login` POST handler clears it via + /// `clear_needs_login_sentinel` once the operator re-auths. + /// - `"online"` clears both sentinels — the agent is healthy again. pub fn emit_status(&self, status: impl Into) { let status = status.into(); - let new_rate_limited = status == "rate_limited"; - if new_rate_limited { + let rate_limited_path = crate::paths::state_dir().join("hyperhive-rate-limited"); + let needs_login_path = crate::paths::state_dir().join("hyperhive-needs-login"); + if status == "rate_limited" { self.rate_limited.store(true, Ordering::Relaxed); + let _ = std::fs::write(&rate_limited_path, b""); } else { self.rate_limited.store(false, Ordering::Relaxed); + let _ = std::fs::remove_file(&rate_limited_path); } - // Read the current persisted needs_login so we don't flip it on - // statuses that shouldn't touch it (e.g. `needs_login_in_progress` - // is a transient mid-flow status; only `needs_login_idle` and - // `online` should change the persistent flag). - // - // Known non-atomicity: this is a read-modify-write. Two concurrent - // `emit_status` calls could clobber each other's `needs_login` - // change if they raced between the read and the write. In practice - // this is safe: the turn loop is sequential and the login flow - // (`needs_login_idle` / `online`) only fires outside of active - // turns, so the two callers never overlap. Documented rather than - // locked because adding a Mutex here would be overkill for the - // actual call pattern. - let (_, current_needs_login) = read_harness_state(); - let new_needs_login = if status == "needs_login_idle" { - true + if status == "needs_login_idle" { + let _ = std::fs::write(&needs_login_path, b""); } else if status == "online" { - false - } else { - current_needs_login - }; - write_harness_state(new_rate_limited, new_needs_login); + // Re-auth completed (or manual flip back to online) — drop + // the sentinel. `needs_login_in_progress` is a transient + // mid-flow status and shouldn't clear yet. + let _ = std::fs::remove_file(&needs_login_path); + } self.emit(LiveEvent::StatusChanged { status }); } diff --git a/hive-ag3nt/src/web_ui.rs b/hive-ag3nt/src/web_ui.rs index 3546abb9..590555a8 100644 --- a/hive-ag3nt/src/web_ui.rs +++ b/hive-ag3nt/src/web_ui.rs @@ -194,15 +194,15 @@ fn bind_unix(path: &Path) -> Result { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) .with_context(|| format!("set perms on {}", path.display()))?; - // Best-effort ready marker: failed write isn't fatal (the harness + // Best-effort .bound marker: failed write isn't fatal (the harness // still binds + serves), it just means the gateway side keeps the // TCP upstream for one more sync tick. if let Some(parent) = path.parent() { - let marker = parent.join("hyperhive-socket-bound"); + let marker = parent.join(".bound"); if let Err(e) = std::fs::write(&marker, b"") { tracing::warn!( marker = %marker.display(), error = %e, - "failed to write hyperhive-socket-bound marker — gateway may keep TCP upstream" + "failed to write .bound marker — gateway may keep TCP upstream" ); } } diff --git a/hive-c0re/src/agent_sockets.rs b/hive-c0re/src/agent_sockets.rs index 30a2aa09..66abe522 100644 --- a/hive-c0re/src/agent_sockets.rs +++ b/hive-c0re/src/agent_sockets.rs @@ -4,8 +4,8 @@ //! excluded from the map (manager UI routes via the dashboard //! upstream, not per-agent `/agent//`). //! -//! Full mechanism — per-agent subdir bind-mount, `hyperhive-socket-bound` -//! marker gate, gateway UDS upstream, transition vs `agent-ports.json`, +//! Full mechanism — per-agent subdir bind-mount, `.bound` marker +//! gate, gateway UDS upstream, transition vs `agent-ports.json`, //! 10s poll loop: `docs/gateway.md::Per-agent unix-socket upstream`. use std::collections::BTreeMap; @@ -36,13 +36,7 @@ pub const SOCKET_FILENAME: &str = "web.sock"; /// publish the unix upstream for this agent yet". Without this gate /// the gateway would `proxy_pass` to a non-existent socket for every /// sub-agent that hasn't flipped the option yet. -/// -/// Renamed from `.bound` (legacy) to match the `hyperhive-` prefix -/// convention for all harness-written state files (#838). `build_map` -/// checks both names during the transition window so existing containers -/// don't lose gateway routing before their next rebuild. -pub const READY_MARKER: &str = "hyperhive-socket-bound"; -const READY_MARKER_LEGACY: &str = ".bound"; +pub const READY_MARKER: &str = ".bound"; #[must_use] pub fn host_sockets_path() -> PathBuf { @@ -78,27 +72,18 @@ pub fn socket_path_for(name: &str) -> PathBuf { /// `proxy_pass` to a non-existent socket for every sub-agent that /// hasn't yet flipped `hyperhive.web.useUnixSocket = true`. /// -/// Accepts either the new `hyperhive-socket-bound` marker or the legacy -/// `.bound` marker so existing containers keep their gateway routing -/// through the transition window (before their next rebuild writes the -/// new marker name). -/// /// `BTreeMap` keeps the JSON output sorted by key so a re-emit /// without churn produces byte-identical output — same idempotency /// shape `agent_ports::write` relies on. #[must_use] pub fn build_map(names: &[String]) -> BTreeMap { - build_map_with(names, |name| { - ready_marker_for(name).exists() - || agent_dir_for(name).join(READY_MARKER_LEGACY).exists() - }) + build_map_with(names, |name| ready_marker_for(name).exists()) } /// Body of `build_map` with the ready-check parameterised. Tests /// pass a predicate they control (no real filesystem access). /// Production callers go through `build_map` which wires the -/// predicate to the on-disk `hyperhive-socket-bound` (or legacy -/// `.bound`) marker check. +/// predicate to the on-disk `.bound` marker check. fn build_map_with(names: &[String], is_ready: F) -> BTreeMap where F: Fn(&str) -> bool, @@ -111,7 +96,7 @@ where .collect() } -/// Path to the `hyperhive-socket-bound` marker file the harness writes +/// Path to the per-agent `.bound` marker file the harness writes /// after a successful `bind_unix`. Lives next to `web.sock` in the /// per-agent subdir so it's covered by the same bind-mount and same /// per-agent isolation as the socket itself. @@ -286,9 +271,10 @@ mod tests { #[test] fn build_map_filters_by_ready_predicate() { - // Only ready agents (with `hyperhive-socket-bound` marker) get - // published. Pin the behaviour so a future refactor that drops - // the filter surfaces here, not as a 502-spew in the gateway. + // The new gate: only ready agents (with `.bound` marker) get + // published. Pin the behaviour so a future refactor that + // drops the filter surfaces here, not as a 502-spew in the + // gateway. let names: Vec = ["iris", "argus", "atlas"] .iter() .map(|s| (*s).to_owned()) @@ -310,7 +296,7 @@ mod tests { let marker = ready_marker_for("iris"); let socket = socket_path_for("iris"); assert_eq!(marker.parent(), socket.parent()); - assert_eq!(marker, Path::new("/run/hive-agent/iris/hyperhive-socket-bound")); + assert_eq!(marker, Path::new("/run/hive-agent/iris/.bound")); } #[test] diff --git a/hive-c0re/src/container_view.rs b/hive-c0re/src/container_view.rs index a01b1330..bd5571e0 100644 --- a/hive-c0re/src/container_view.rs +++ b/hive-c0re/src/container_view.rs @@ -211,31 +211,24 @@ fn read_dashboard_links(name: &str) -> Vec { serde_json::from_str::>(&text).unwrap_or_default() } -/// Read `rate_limited` + `needs_login` from the consolidated -/// `hyperhive-harness.json`. Falls back to the legacy individual -/// sentinel files written by older harness builds so in-place upgrades -/// don't lose state during the transition window. -fn read_harness_flags(name: &str) -> (bool, bool) { - let dir = Coordinator::agent_notes_dir(name); - if let Ok(raw) = std::fs::read_to_string(dir.join("hyperhive-harness.json")) { - if let Ok(v) = serde_json::from_str::(&raw) { - let rl = v.get("rate_limited").and_then(|x| x.as_bool()).unwrap_or(false); - let nl = v.get("needs_login").and_then(|x| x.as_bool()).unwrap_or(false); - return (rl, nl); - } - } - // Legacy fallback: presence of individual sentinel files. - let rate_limited = dir.join("hyperhive-rate-limited").exists(); - let needs_login = dir.join("hyperhive-needs-login").exists(); - (rate_limited, needs_login) -} - +/// Returns true if the agent's harness is currently parked after an API +/// rate-limit response. Detected via the sentinel file written by +/// `hive_ag3nt::events::Bus::emit_status("rate_limited")`. fn is_rate_limited(name: &str) -> bool { - read_harness_flags(name).0 + Coordinator::agent_notes_dir(name) + .join("hyperhive-rate-limited") + .exists() } +/// True when the harness wrote `{state_dir}/hyperhive-needs-login` +/// after a 401 mid-turn. Lets the dashboard surface `needs_login` for +/// agents whose `/root/.claude/` dir still exists (so +/// `claude_has_session` returns true) but whose OAuth credentials +/// inside it have actually expired. fn auth_failed_sentinel(name: &str) -> bool { - read_harness_flags(name).1 + Coordinator::agent_notes_dir(name) + .join("hyperhive-needs-login") + .exists() } /// Read the agent's free-text status and the Unix timestamp when it was last set