refactor(hive-c0re): group src-root files into submodules
stores/ (sqlite-backed host stores + db helper), stats/, agent_config/, workers/ — pure git-mv moves; crate-root re-exports keep every crate::<module> path compiling. flake_check stays at root (synchronous approval-flow validation, not a background worker)
This commit is contained in:
parent
b489454dc2
commit
0e4b5a1120
26 changed files with 68 additions and 24 deletions
95
hive-c0re/src/agent_config/capabilities.rs
Normal file
95
hive-c0re/src/agent_config/capabilities.rs
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
//! Per-agent capability configuration. Stored at
|
||||
//! `/var/lib/hyperhive/meta/capabilities.json` alongside `topology.json`
|
||||
//! and `tool-groups.json`.
|
||||
//!
|
||||
//! Format: a JSON object mapping agent name to an array of
|
||||
//! `hive_sh4re::Capability` `snake_case` strings:
|
||||
//!
|
||||
//! ```json
|
||||
//! {
|
||||
//! "atlas": ["read_host_journal"],
|
||||
//! "ruth": ["query_agent_state"]
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! An absent entry (or an absent file) means "no extra capabilities".
|
||||
//! `render_flake` in `meta.rs` reads this file and injects
|
||||
//! `HIVE_CAPABILITIES` into each agent's systemd service env; absent
|
||||
//! entries emit no env var so agents without capabilities don't trigger
|
||||
//! a spurious rebuild.
|
||||
//!
|
||||
//! Write path: `set_caps` is called from the dashboard action handler
|
||||
//! that the operator uses to grant/revoke capabilities per agent.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const CAPABILITIES_FILE: &str = "capabilities.json";
|
||||
|
||||
#[must_use]
|
||||
pub fn capabilities_path() -> PathBuf {
|
||||
crate::meta::meta_dir().join(CAPABILITIES_FILE)
|
||||
}
|
||||
|
||||
/// Read the per-agent capability map. Returns an empty map when the
|
||||
/// file is absent or unparsable — callers treat a missing entry as
|
||||
/// "no extra capabilities".
|
||||
#[must_use]
|
||||
pub fn read() -> BTreeMap<String, Vec<String>> {
|
||||
let path = capabilities_path();
|
||||
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
serde_json::from_str(&raw).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Look up the configured capabilities for one agent. Returns an empty
|
||||
/// vec when the agent has no entry.
|
||||
#[must_use]
|
||||
pub fn caps_for(name: &str) -> Vec<String> {
|
||||
read().get(name).cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Check whether an agent holds a specific capability.
|
||||
#[must_use]
|
||||
pub fn has_cap(name: &str, cap: hive_sh4re::Capability) -> bool {
|
||||
caps_for(name)
|
||||
.iter()
|
||||
.any(|s| s.eq_ignore_ascii_case(cap.as_str()))
|
||||
}
|
||||
|
||||
/// Persist the full capability map. Sorted JSON output keeps diffs
|
||||
/// minimal. Best-effort — returns `io::Error` so callers decide
|
||||
/// whether to abort or log.
|
||||
pub fn write(map: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||
let path = capabilities_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let text = serde_json::to_string_pretty(map)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, format!("{text}\n"))
|
||||
}
|
||||
|
||||
/// Set the capabilities for one agent and persist the map. An empty
|
||||
/// `caps` vec removes the entry (agent has no capabilities).
|
||||
pub fn set_caps(name: &str, caps: &[String]) -> std::io::Result<()> {
|
||||
let mut current = read();
|
||||
if caps.is_empty() {
|
||||
current.remove(name);
|
||||
} else {
|
||||
current.insert(name.to_owned(), caps.to_vec());
|
||||
}
|
||||
write(¤t)
|
||||
}
|
||||
|
||||
/// Remove an agent from the capability map entirely. Called by
|
||||
/// `meta::sync_agents` when an agent is deprovisioned so stale entries
|
||||
/// don't accumulate. No-op if the agent has no entry.
|
||||
pub fn remove_agent(name: &str) -> std::io::Result<()> {
|
||||
let mut current = read();
|
||||
if current.remove(name).is_some() {
|
||||
write(¤t)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
176
hive-c0re/src/agent_config/limits.rs
Normal file
176
hive-c0re/src/agent_config/limits.rs
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
//! Wire-protocol size limits shared across the agent + manager
|
||||
//! sockets. Caps on inline message bodies stop a single chatty agent
|
||||
//! (or a misbehaving extra-MCP server) from flooding the broker
|
||||
//! sqlite with megabyte-sized rows that then bloat every recipient's
|
||||
//! wake-prompt context. Anything genuinely larger should be written
|
||||
//! to a state file and the path sent as the body.
|
||||
//!
|
||||
//! Reminders get a separate auto-file escape hatch (see
|
||||
//! `socket_server::handle_remind`) so callers don't have to think
|
||||
//! about it — oversized reminder bodies get persisted to disk
|
||||
//! transparently and the inbox sees a pointer.
|
||||
|
||||
/// Per-message body cap. Applies to `send`, `ask` question text,
|
||||
/// `answer` body, and the stored inline form of a reminder. 4 KiB
|
||||
/// catches the bulk of conversational overflow (status reports,
|
||||
/// bullet-list summaries, short proposals) while staying small
|
||||
/// enough that a backed-up inbox of ~10 unread messages only adds
|
||||
/// ~40 KiB to the recipient's wake-prompt context. Genuinely
|
||||
/// long-form artifacts (audit reports, full diffs, transcripts)
|
||||
/// still belong in a state file — the error message on overflow
|
||||
/// points callers at that escape hatch.
|
||||
pub const MESSAGE_MAX_BYTES: usize = 4096;
|
||||
|
||||
/// Validate that `body` fits under [`MESSAGE_MAX_BYTES`]. Returns a
|
||||
/// caller-ready error string (caller wraps in
|
||||
/// `AgentResponse::Err`/`ManagerResponse::Err`) on failure.
|
||||
///
|
||||
/// `label` shows up in the error message verbatim — pass a short
|
||||
/// noun like `"send"`, `"question"`, `"broadcast"` so the model can
|
||||
/// tell which call got rejected.
|
||||
pub fn check_size(label: &str, body: &str) -> Result<(), String> {
|
||||
if body.len() > MESSAGE_MAX_BYTES {
|
||||
Err(format!(
|
||||
"{label} body too long ({} bytes, max {MESSAGE_MAX_BYTES}); write the \
|
||||
payload to a file under your `/agents/<you>/state/` dir and send the \
|
||||
path as the body instead",
|
||||
body.len()
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-status soft cap. `set_status` renders as a short chip on the
|
||||
/// dashboard agent card — the front-end truncates long strings to
|
||||
/// keep the row layout intact, so a multi-paragraph "session report"
|
||||
/// is wasted bytes that just bloat the rescan emit + container view
|
||||
/// payload. Cap at 200 chars to fit the chip plus a little
|
||||
/// descriptive padding without forcing the operator to read a
|
||||
/// scrolling chunk.
|
||||
/// NOTE: `hive-ag3nt/src/mcp.rs::write_status_file` mirrors this constant
|
||||
/// client-side so invalid text is caught before the file is written.
|
||||
/// Keep in sync if this value changes.
|
||||
pub const STATUS_MAX_CHARS: usize = 200;
|
||||
|
||||
/// Validate a `set_status` payload. Single-line + bounded so
|
||||
/// callers can't dump multi-paragraph session reports into the
|
||||
/// dashboard chip. Whitespace trim is done by the caller before the
|
||||
/// store-to-disk step — we run validation on the trimmed form so
|
||||
/// surrounding whitespace doesn't push a borderline-legal status
|
||||
/// past the cap.
|
||||
///
|
||||
/// Empty / all-whitespace input is accepted: the call site treats
|
||||
/// that as "clear the status" and removes the on-disk sentinel. Tests
|
||||
/// + caller cover both directions.
|
||||
///
|
||||
/// Returns a caller-ready error string suitable for surfacing in the
|
||||
/// `*Response::Err` shape.
|
||||
pub fn check_status_text(text: &str) -> Result<(), String> {
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
// Empty = clear-status sentinel; nothing to validate.
|
||||
return Ok(());
|
||||
}
|
||||
// Newline / carriage-return: status is a single-line chip on the
|
||||
// dashboard. Multi-line session reports should go to a state file.
|
||||
if trimmed.contains('\n') || trimmed.contains('\r') {
|
||||
return Err(
|
||||
"set_status text must be a single line — write multi-line context to \
|
||||
a file under your `/agents/<you>/state/` dir and reference that path \
|
||||
from the chip instead"
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
let len = trimmed.chars().count();
|
||||
if len > STATUS_MAX_CHARS {
|
||||
return Err(format!(
|
||||
"set_status text too long ({len} chars, max {STATUS_MAX_CHARS}); the \
|
||||
dashboard chip truncates anything longer, so trim to a short summary \
|
||||
and write the detail to `/agents/<you>/state/<file>` instead"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accepts_short_body() {
|
||||
assert!(check_size("send", "hello").is_ok());
|
||||
assert!(check_size("send", &"x".repeat(MESSAGE_MAX_BYTES)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversize_body() {
|
||||
let err = check_size("send", &"x".repeat(MESSAGE_MAX_BYTES + 1)).unwrap_err();
|
||||
assert!(err.contains("send body too long"));
|
||||
assert!(err.contains(&format!("max {MESSAGE_MAX_BYTES}")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_threads_through() {
|
||||
let err = check_size("question", &"x".repeat(MESSAGE_MAX_BYTES + 1)).unwrap_err();
|
||||
assert!(err.starts_with("question body too long"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_accepts_short_single_line() {
|
||||
assert!(check_status_text("idle").is_ok());
|
||||
assert!(check_status_text("processing matrix messages").is_ok());
|
||||
// Boundary: exactly STATUS_MAX_CHARS chars trimmed is still
|
||||
// accepted; one more rejects.
|
||||
let max = "a".repeat(STATUS_MAX_CHARS);
|
||||
assert!(check_status_text(&max).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_accepts_empty_and_whitespace() {
|
||||
// Empty + whitespace-only are the "clear status" sentinel and
|
||||
// bypass the rest of the checks.
|
||||
assert!(check_status_text("").is_ok());
|
||||
assert!(check_status_text(" ").is_ok());
|
||||
assert!(check_status_text("\n\t ").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_rejects_multi_line() {
|
||||
let err = check_status_text("line one\nline two").unwrap_err();
|
||||
assert!(err.contains("single line"), "err = {err}");
|
||||
// Carriage return alone also rejects (windows linebreak / CR-only).
|
||||
assert!(check_status_text("a\rb").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_rejects_oversize() {
|
||||
let too_long = "a".repeat(STATUS_MAX_CHARS + 1);
|
||||
let err = check_status_text(&too_long).unwrap_err();
|
||||
assert!(err.contains("too long"), "err = {err}");
|
||||
assert!(err.contains(&format!("max {STATUS_MAX_CHARS}")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_counts_chars_not_bytes() {
|
||||
// Multi-byte chars (emoji, accented letters) count once each
|
||||
// per char — chars().count() not byte len. STATUS_MAX_CHARS
|
||||
// worth of 4-byte chars is still legal.
|
||||
let emoji = "💜".repeat(STATUS_MAX_CHARS);
|
||||
assert!(
|
||||
check_status_text(&emoji).is_ok(),
|
||||
"{STATUS_MAX_CHARS} emoji should fit"
|
||||
);
|
||||
let too_many = "💜".repeat(STATUS_MAX_CHARS + 1);
|
||||
assert!(check_status_text(&too_many).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn check_status_validates_post_trim() {
|
||||
// Leading/trailing whitespace is trimmed before the length
|
||||
// check — a borderline-legal payload with spaces around it
|
||||
// still passes.
|
||||
let padded = format!(" {} ", "a".repeat(STATUS_MAX_CHARS));
|
||||
assert!(check_status_text(&padded).is_ok());
|
||||
}
|
||||
}
|
||||
10
hive-c0re/src/agent_config/mod.rs
Normal file
10
hive-c0re/src/agent_config/mod.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
//! Per-agent configuration registries: tool groups, capabilities,
|
||||
//! topology (all JSON files under `/var/lib/hyperhive/meta/`) and the
|
||||
//! shared wire-protocol size limits. Each submodule is re-exported at
|
||||
//! the crate root, so `crate::topology::…` etc. keep working
|
||||
//! unchanged.
|
||||
|
||||
pub mod capabilities;
|
||||
pub mod limits;
|
||||
pub mod tool_groups;
|
||||
pub mod topology;
|
||||
120
hive-c0re/src/agent_config/tool_groups.rs
Normal file
120
hive-c0re/src/agent_config/tool_groups.rs
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
//! Per-agent tool-group configuration. Stored at
|
||||
//! `/var/lib/hyperhive/meta/tool-groups.json` alongside `topology.json`
|
||||
//! and the meta `flake.nix`.
|
||||
//!
|
||||
//! Format: a JSON object mapping agent name to an array of
|
||||
//! `hive_sh4re::ToolGroup` `snake_case` strings:
|
||||
//!
|
||||
//! ```json
|
||||
//! {
|
||||
//! "alice": ["messaging", "meta", "inbox", "lifecycle"],
|
||||
//! "bob": ["messaging", "meta", "inbox"]
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! An absent entry (or an absent file) means "use the harness default"
|
||||
//! (`AGENT_DEFAULT`: `messaging + meta + inbox + execution`). `render_flake`
|
||||
//! in `meta.rs` reads this file and injects `HIVE_TOOL_GROUPS` into each
|
||||
//! agent's systemd service env; agents with no entry get no env var and the
|
||||
//! harness falls back to `AGENT_DEFAULT`.
|
||||
//!
|
||||
//! Write path: `set_groups` is called from the dashboard action handler
|
||||
//! that the operator uses to grant/revoke tool groups per agent.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Context as _;
|
||||
|
||||
const TOOL_GROUPS_FILE: &str = "tool-groups.json";
|
||||
|
||||
#[must_use]
|
||||
pub fn tool_groups_path() -> PathBuf {
|
||||
crate::meta::meta_dir().join(TOOL_GROUPS_FILE)
|
||||
}
|
||||
|
||||
/// Read the per-agent tool-group map. Returns an empty map when the
|
||||
/// file is absent or unparsable — callers treat a missing entry as
|
||||
/// "use role default".
|
||||
#[must_use]
|
||||
pub fn read() -> BTreeMap<String, Vec<String>> {
|
||||
let path = tool_groups_path();
|
||||
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
serde_json::from_str(&raw).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Look up the configured tool groups for one agent. Returns an empty
|
||||
/// vec when the agent has no entry — callers should treat this as
|
||||
/// "use the harness role default."
|
||||
#[must_use]
|
||||
pub fn groups_for(name: &str) -> Vec<String> {
|
||||
read().get(name).cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Persist the full tool-groups map. Sorted JSON output keeps diffs
|
||||
/// minimal. Use `set_groups` (or `remove_agent`) from outside this
|
||||
/// module — they go through the validated write path.
|
||||
fn write(map: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||
let path = tool_groups_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let text = serde_json::to_string_pretty(map)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, format!("{text}\n"))
|
||||
}
|
||||
|
||||
/// Validate a slice of group name strings against `ToolGroup::ALL`.
|
||||
/// Returns `Ok(())` when all names are known, or `Err` listing the
|
||||
/// unrecognised names so callers can surface a useful error message.
|
||||
pub fn validate_groups(groups: &[String]) -> anyhow::Result<()> {
|
||||
let valid: std::collections::BTreeSet<&str> = hive_sh4re::ToolGroup::ALL
|
||||
.iter()
|
||||
.map(|g| g.as_str())
|
||||
.collect();
|
||||
let unknown: Vec<&str> = groups
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.filter(|s| !valid.contains(s))
|
||||
.collect();
|
||||
if unknown.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
anyhow::bail!(
|
||||
"unknown tool group(s): {}; valid names are: {}",
|
||||
unknown.join(", "),
|
||||
hive_sh4re::ToolGroup::ALL
|
||||
.iter()
|
||||
.map(|g| g.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the tool groups for one agent and persist the map. An empty
|
||||
/// `groups` vec removes the entry (agent reverts to role default).
|
||||
/// Returns an error if any name is not in `ToolGroup::ALL`.
|
||||
pub fn set_groups(name: &str, groups: &[String]) -> anyhow::Result<()> {
|
||||
if !groups.is_empty() {
|
||||
validate_groups(groups)?;
|
||||
}
|
||||
let mut current = read();
|
||||
if groups.is_empty() {
|
||||
current.remove(name);
|
||||
} else {
|
||||
current.insert(name.to_owned(), groups.to_vec());
|
||||
}
|
||||
write(¤t).with_context(|| format!("write tool-groups for {name}"))
|
||||
}
|
||||
|
||||
/// Drop the entry for an agent that is being destroyed. Idempotent.
|
||||
pub fn remove_agent(name: &str) -> std::io::Result<()> {
|
||||
let mut current = read();
|
||||
if current.remove(name).is_some() {
|
||||
write(¤t)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
912
hive-c0re/src/agent_config/topology.rs
Normal file
912
hive-c0re/src/agent_config/topology.rs
Normal file
|
|
@ -0,0 +1,912 @@
|
|||
//! Agent topology storage — single source of truth for parent/child
|
||||
//! relations in the hive. Persisted as a flat JSON map of `name →
|
||||
//! parent name | null` at `/var/lib/hyperhive/meta/topology.json`,
|
||||
//! alongside the meta `flake.nix`, so topology changes thread through
|
||||
//! the same git commit log as deploys.
|
||||
//!
|
||||
//! Agent roles are stored alongside in `roles.json` as a flat map of
|
||||
//! `name → [role, ...]`. Roles gate additional bind-mount grants; see
|
||||
//! `lifecycle::set_nspawn_flags` for the consumer.
|
||||
//!
|
||||
//! Format, rationale, read/reconcile/inject/surface flow, and target
|
||||
//! enforcement semantics: `docs/agent-hierarchy.md::Current state`.
|
||||
//! `<parent>` sentinel resolution (delivered by [`resolve_recipient`]):
|
||||
//! `docs/conventions.md::Recipient sentinels`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const TOPOLOGY_FILE: &str = "topology.json";
|
||||
|
||||
#[must_use]
|
||||
pub fn topology_path() -> PathBuf {
|
||||
crate::meta::meta_dir().join(TOPOLOGY_FILE)
|
||||
}
|
||||
|
||||
/// Snapshot of the topology map. Read on every `container_view::build_all`
|
||||
/// and every `render_flake` call. The file is small (one line per agent),
|
||||
/// so we re-read rather than caching — keeps the source of truth on disk.
|
||||
///
|
||||
/// Returns an empty map when the file is absent or unparsable; callers
|
||||
/// treat that as "no recorded parents", which falls back to every agent
|
||||
/// being root-level. Safe degradation for fresh installs that haven't
|
||||
/// run through `meta::sync_agents` yet.
|
||||
#[must_use]
|
||||
pub fn read() -> BTreeMap<String, Option<String>> {
|
||||
let path = topology_path();
|
||||
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
serde_json::from_str(&raw).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Look up one agent's parent. Returns `None` when the agent is root
|
||||
/// or absent from the file. Cheap convenience over `read()` for
|
||||
/// callers that want a single entry.
|
||||
#[must_use]
|
||||
pub fn parent_of(name: &str) -> Option<String> {
|
||||
read().get(name).cloned().flatten()
|
||||
}
|
||||
|
||||
/// Return the direct children of `name` — agents whose `topology.json`
|
||||
/// entry has `name` as their parent. Reads the map once and scans all
|
||||
/// entries; cheap enough for the fan-out path (one disk read per send
|
||||
/// to `<children>`).
|
||||
#[must_use]
|
||||
pub fn children_of(name: &str) -> Vec<String> {
|
||||
children_of_in(&read(), name)
|
||||
}
|
||||
|
||||
/// Pure form of [`children_of`] for unit tests.
|
||||
#[must_use]
|
||||
pub fn children_of_in(topo: &BTreeMap<String, Option<String>>, name: &str) -> Vec<String> {
|
||||
topo.iter()
|
||||
.filter_map(|(agent, parent)| {
|
||||
if parent.as_deref() == Some(name) {
|
||||
Some(agent.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return every agent that has no parent in the topology. These are the
|
||||
/// "top-level" agents a `can_manage_top_level_agents` role holder is
|
||||
/// granted access to. No agent name is hardcoded — the set is derived
|
||||
/// purely from topology structure.
|
||||
///
|
||||
/// In normal operation this is just the manager, but any agent the
|
||||
/// operator explicitly places outside the hierarchy is also included.
|
||||
#[must_use]
|
||||
pub fn top_level_agents() -> Vec<String> {
|
||||
top_level_agents_in(&read())
|
||||
}
|
||||
|
||||
/// Pure form of [`top_level_agents`] for unit tests.
|
||||
#[must_use]
|
||||
pub fn top_level_agents_in(topo: &BTreeMap<String, Option<String>>) -> Vec<String> {
|
||||
topo.iter()
|
||||
.filter_map(|(name, parent)| {
|
||||
if parent.is_none() {
|
||||
Some(name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Resolve a magic recipient sentinel (currently just
|
||||
/// [`hive_sh4re::PARENT_RECIPIENT`]) to a real broker recipient at
|
||||
/// send time. Returns an owned `String` so callers can plug it
|
||||
/// straight into [`crate::broker::Broker::send`] without
|
||||
/// borrow-juggling around the temporary lookup.
|
||||
///
|
||||
/// Rules + rationale: `docs/conventions.md::Recipient sentinels`.
|
||||
/// Fast path: ordinary recipient names short-circuit before any
|
||||
/// disk read — only `<parent>` triggers `read()` on `topology.json`.
|
||||
#[must_use]
|
||||
pub fn resolve_recipient(sender: &str, to: &str) -> String {
|
||||
// Early exit: only sentinel recipients need topology lookup. This
|
||||
// keeps the cost of a normal `send` at one string comparison.
|
||||
if to != hive_sh4re::PARENT_RECIPIENT {
|
||||
return to.to_owned();
|
||||
}
|
||||
resolve_recipient_in(&read(), sender, to)
|
||||
}
|
||||
|
||||
/// Pure form of [`resolve_recipient`] taking the topology map
|
||||
/// explicitly. Split out so unit tests can exercise the sentinel
|
||||
/// rules without writing a `topology.json` to disk.
|
||||
#[must_use]
|
||||
pub fn resolve_recipient_in(
|
||||
topo: &BTreeMap<String, Option<String>>,
|
||||
sender: &str,
|
||||
to: &str,
|
||||
) -> String {
|
||||
if to == hive_sh4re::PARENT_RECIPIENT {
|
||||
topo.get(sender)
|
||||
.cloned()
|
||||
.flatten()
|
||||
.unwrap_or_else(|| hive_sh4re::OPERATOR_RECIPIENT.to_owned())
|
||||
} else {
|
||||
to.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// True when `candidate` is `ancestor` or any descendant of
|
||||
/// `ancestor` per the current topology. Walks parents from
|
||||
/// `candidate` upward; the walk terminates at root or on a cycle
|
||||
/// (cycle defence: bounded to 32 hops, more than any plausible
|
||||
/// hive depth). Used by the cancel-authorization check in
|
||||
/// `socket_server::handle_cancel_schedule` to enforce
|
||||
/// "managers can cancel anything their subtree owns."
|
||||
#[must_use]
|
||||
pub fn is_descendant_of(candidate: &str, ancestor: &str) -> bool {
|
||||
if candidate == ancestor {
|
||||
return true;
|
||||
}
|
||||
let topo = read();
|
||||
let mut cur = candidate.to_owned();
|
||||
for _ in 0..32 {
|
||||
let Some(parent) = topo.get(&cur).cloned().flatten() else {
|
||||
return false;
|
||||
};
|
||||
if parent == ancestor {
|
||||
return true;
|
||||
}
|
||||
cur = parent;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Persist the topology map. Sorted JSON output (`BTreeMap` is sorted by
|
||||
/// key) keeps git diffs minimal across re-writes. Best-effort —
|
||||
/// returns an `io::Error` so callers can decide whether a failure
|
||||
/// should abort their op (`sync_agents`, `RequestSetParent`) or just log.
|
||||
pub fn write(topology: &BTreeMap<String, Option<String>>) -> std::io::Result<()> {
|
||||
let path = topology_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let text = serde_json::to_string_pretty(topology)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, format!("{text}\n"))
|
||||
}
|
||||
|
||||
/// Compute the default topology for a fresh install: every agent is a
|
||||
/// root (parent = null). There is no structural "manager" — agents
|
||||
/// arrange themselves via explicit parent edges (an agent-requested
|
||||
/// sub-agent gets a requester-as-parent edge at `init_config`; the
|
||||
/// operator reparents via the dashboard / `RequestSetParent` API).
|
||||
/// Used by `meta::sync_agents` on first call to seed `topology.json`.
|
||||
///
|
||||
/// As soon as an explicit write lands (dashboard / `RequestSetParent`
|
||||
/// API), this seeding stops touching pre-existing entries —
|
||||
/// `sync_agents` only adds rows for newly-spawned agents against
|
||||
/// whatever the operator has configured.
|
||||
#[must_use]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "kept for the dashboard / RequestSetParent write API; \
|
||||
`sync_agents` does its own seeding today"
|
||||
)]
|
||||
pub fn default_seed(agent_names: &[String]) -> BTreeMap<String, Option<String>> {
|
||||
let mut out = BTreeMap::new();
|
||||
for name in agent_names {
|
||||
out.insert(name.clone(), None);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Pure validation + apply for [`set_parent`]. Splits off so tests
|
||||
/// can exercise the rules (cycle / unknown) on an in-memory
|
||||
/// `BTreeMap` without touching the on-disk `topology.json`. Returns
|
||||
/// either the post-move map (caller writes it back) or a
|
||||
/// user-readable error string.
|
||||
///
|
||||
/// The manager is reparentable like any other agent — its special
|
||||
/// powers come from the privileged MCP socket, not its tree
|
||||
/// position. The cycle walk below covers "moving X under its own
|
||||
/// descendant" for the manager as much as any other agent.
|
||||
/// `docs/agent-hierarchy.md::Current state` has the rationale.
|
||||
pub fn apply_set_parent(
|
||||
topo: &BTreeMap<String, Option<String>>,
|
||||
child: &str,
|
||||
new_parent: Option<&str>,
|
||||
) -> Result<BTreeMap<String, Option<String>>, String> {
|
||||
if !topo.contains_key(child) {
|
||||
return Err(format!("unknown agent: {child}"));
|
||||
}
|
||||
if let Some(p) = new_parent {
|
||||
if !topo.contains_key(p) {
|
||||
return Err(format!("unknown parent: {p}"));
|
||||
}
|
||||
if p == child {
|
||||
return Err("an agent cannot be its own parent".to_owned());
|
||||
}
|
||||
// Cycle check: walk `p`'s ancestors in the EXISTING map. If
|
||||
// we hit `child`, then making `child`'s parent = `p` would
|
||||
// close the loop (child → … → p → child).
|
||||
let mut cur = p.to_owned();
|
||||
for _ in 0..32 {
|
||||
if cur == child {
|
||||
return Err(format!(
|
||||
"cycle: {p} is in {child}'s subtree (would create a loop)"
|
||||
));
|
||||
}
|
||||
let Some(next) = topo.get(&cur).cloned().flatten() else {
|
||||
break;
|
||||
};
|
||||
cur = next;
|
||||
}
|
||||
}
|
||||
let mut next = topo.clone();
|
||||
next.insert(child.to_owned(), new_parent.map(str::to_owned));
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Operator-driven parent move. Set `child`'s parent to `new_parent`
|
||||
/// (or `None` to promote to root). See [`apply_set_parent`] for the
|
||||
/// validation rules. The operator-set parent sticks across
|
||||
/// `reconcile()` calls (which preserves existing entries).
|
||||
///
|
||||
/// No bind-mount / container churn today — the hierarchy is
|
||||
/// currently logical-only. Once sub-manager bind mounts land, the
|
||||
/// caller adds an umount-old / mount-new / restart-cascade step on
|
||||
/// top.
|
||||
pub fn set_parent(child: &str, new_parent: Option<&str>) -> Result<(), String> {
|
||||
let current = read();
|
||||
// Idempotent no-op fast path: skip the disk write when nothing
|
||||
// changes. apply_set_parent still runs to surface validation
|
||||
// errors (e.g. unknown child) so the caller gets a real signal.
|
||||
let next = apply_set_parent(¤t, child, new_parent)?;
|
||||
if next == current {
|
||||
return Ok(());
|
||||
}
|
||||
write(&next).map_err(|e| format!("write topology.json: {e}"))
|
||||
}
|
||||
|
||||
/// Declare a brand-new agent's parent edge before the agent exists in
|
||||
/// the container set. Unlike [`set_parent`] (which reparents an entry
|
||||
/// that must already be present), this inserts a fresh `child -> parent`
|
||||
/// row. Used by the `InitConfig` approval to place a just-scaffolded
|
||||
/// sub-agent under its requesting parent, so the edge is in place
|
||||
/// before the first apply-commit spawns the container (and before
|
||||
/// [`reconcile`] would otherwise default it to the manager).
|
||||
///
|
||||
/// Idempotent when the edge already exists. Refuses to clobber an entry
|
||||
/// whose parent differs (one agent can't steal another's child) and
|
||||
/// validates that `parent` is itself a known agent.
|
||||
pub fn add_child(child: &str, parent: &str) -> Result<(), String> {
|
||||
let current = read();
|
||||
match apply_add_child(¤t, child, parent)? {
|
||||
Some(next) => write(&next).map_err(|e| format!("write topology.json: {e}")),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure form of [`add_child`] for unit tests. Returns the post-insert
|
||||
/// map (caller writes it back), `None` for an idempotent no-op (edge
|
||||
/// already present), or an error string (unknown parent / name owned by
|
||||
/// a different parent).
|
||||
pub fn apply_add_child(
|
||||
topo: &BTreeMap<String, Option<String>>,
|
||||
child: &str,
|
||||
parent: &str,
|
||||
) -> Result<Option<BTreeMap<String, Option<String>>>, String> {
|
||||
if !topo.contains_key(parent) {
|
||||
return Err(format!("unknown parent: {parent}"));
|
||||
}
|
||||
match topo.get(child) {
|
||||
Some(Some(p)) if p == parent => return Ok(None),
|
||||
Some(existing) => {
|
||||
return Err(format!(
|
||||
"agent {child} already exists in topology (parent: {existing:?}); \
|
||||
refusing to reparent via add_child"
|
||||
));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
let mut next = topo.clone();
|
||||
next.insert(child.to_owned(), Some(parent.to_owned()));
|
||||
Ok(Some(next))
|
||||
}
|
||||
|
||||
/// Reconcile `topology.json` against the current agent set. Adds an
|
||||
/// entry (default: parent = null — a new agent with no declared parent
|
||||
/// is its own root) for any agent missing from the file; removes
|
||||
/// entries for agents no longer
|
||||
/// present. Existing entries are preserved as-is — operator/manager
|
||||
/// choices stick across regenerations. Returns true when the file
|
||||
/// changed and should be re-committed by the caller.
|
||||
///
|
||||
/// `pending` lists agents that have an operator-approved proposed config
|
||||
/// repo but no container yet (init'd, not yet spawned). They are KEPT
|
||||
/// (not dropped) so the parent edge written at `InitConfig` approval
|
||||
/// survives until the first apply-commit, but they are NOT seeded with a
|
||||
/// default parent here and NOT added to roles — that happens when the
|
||||
/// container actually spawns and the name moves into `agent_names`.
|
||||
pub fn reconcile(agent_names: &[String], pending: &[String]) -> std::io::Result<bool> {
|
||||
let (next, changed) = apply_reconcile(&read(), agent_names, pending);
|
||||
if changed {
|
||||
write(&next)?;
|
||||
}
|
||||
// Keep roles in sync with the live agent set (pending agents get
|
||||
// roles when they spawn).
|
||||
reconcile_roles(agent_names)?;
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// Pure form of [`reconcile`] for unit tests. Adds missing live agents
|
||||
/// at their default position, drops entries for agents that are neither
|
||||
/// live nor pending-init, and reports whether anything changed.
|
||||
#[must_use]
|
||||
pub fn apply_reconcile(
|
||||
current: &BTreeMap<String, Option<String>>,
|
||||
agent_names: &[String],
|
||||
pending: &[String],
|
||||
) -> (BTreeMap<String, Option<String>>, bool) {
|
||||
let mut next = current.clone();
|
||||
let mut changed = false;
|
||||
for name in agent_names {
|
||||
if !next.contains_key(name) {
|
||||
// A new agent with no declared parent defaults to root
|
||||
// (parent = null). Agent-requested sub-agents always carry an
|
||||
// explicit requester-as-parent edge (written at init_config
|
||||
// approval), so they never hit this default — only
|
||||
// user/operator-initiated spawns do, and those are roots. No
|
||||
// agent is structurally privileged here: "root-ness" is just
|
||||
// a null parent.
|
||||
next.insert(name.clone(), None);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
let known: std::collections::HashSet<&String> =
|
||||
agent_names.iter().chain(pending.iter()).collect();
|
||||
next.retain(|name, _| {
|
||||
let keep = known.contains(name);
|
||||
if !keep {
|
||||
changed = true;
|
||||
}
|
||||
keep
|
||||
});
|
||||
(next, changed)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Roles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Agents with this role have every parentless agent in the topology
|
||||
/// (see `top_level_agents`) added as virtual children for bind-mount
|
||||
/// and config-change purposes. Enables recovery: if a top-level agent
|
||||
/// is down, a role holder can still read its state and update its config.
|
||||
///
|
||||
/// The manager receives this role by default on first `reconcile_roles`
|
||||
/// call; operators can revoke it with `set_role`.
|
||||
pub const ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS: &str = "can_manage_top_level_agents";
|
||||
|
||||
const ROLES_FILE: &str = "roles.json";
|
||||
|
||||
#[must_use]
|
||||
pub fn roles_path() -> std::path::PathBuf {
|
||||
crate::meta::meta_dir().join(ROLES_FILE)
|
||||
}
|
||||
|
||||
/// Read the roles map from disk. Returns an empty map when absent or
|
||||
/// unparsable — same safe-degradation pattern as `topology::read`.
|
||||
#[must_use]
|
||||
pub fn read_roles() -> BTreeMap<String, Vec<String>> {
|
||||
let Ok(raw) = std::fs::read_to_string(roles_path()) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
serde_json::from_str(&raw).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Persist the roles map. Sorted output keeps git diffs minimal.
|
||||
pub fn write_roles(roles: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||
let path = roles_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let text = serde_json::to_string_pretty(roles)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, format!("{text}\n"))
|
||||
}
|
||||
|
||||
/// Return true when `name` holds `role`.
|
||||
#[must_use]
|
||||
pub fn has_role(name: &str, role: &str) -> bool {
|
||||
has_role_in(&read_roles(), name, role)
|
||||
}
|
||||
|
||||
/// Pure form of [`has_role`] for unit tests.
|
||||
#[must_use]
|
||||
pub fn has_role_in(roles: &BTreeMap<String, Vec<String>>, name: &str, role: &str) -> bool {
|
||||
roles
|
||||
.get(name)
|
||||
.is_some_and(|rs| rs.iter().any(|r| r == role))
|
||||
}
|
||||
|
||||
/// Grant or revoke a role for `name`. Idempotent — no disk write when the
|
||||
/// state is already correct.
|
||||
///
|
||||
/// Empty role lists are kept in the map (never removed). An absent key means
|
||||
/// "never seen" (seed on next `reconcile_roles`); an empty list means
|
||||
/// "explicitly revoked" (do not re-seed). Callers that want to remove an
|
||||
/// agent from the map entirely should use `reconcile_roles` (agent departure).
|
||||
pub fn set_role(name: &str, role: &str, enabled: bool) -> Result<(), String> {
|
||||
let mut roles = read_roles();
|
||||
let list = roles.entry(name.to_owned()).or_default();
|
||||
let held = list.iter().any(|r| r == role);
|
||||
match (enabled, held) {
|
||||
(true, false) => list.push(role.to_owned()),
|
||||
(false, true) => list.retain(|r| r != role),
|
||||
_ => return Ok(()),
|
||||
}
|
||||
// Intentionally do NOT remove empty entries — an empty list signals an
|
||||
// explicit revoke and prevents reconcile_roles from re-seeding the role.
|
||||
write_roles(&roles).map_err(|e| format!("write roles.json: {e}"))
|
||||
}
|
||||
|
||||
/// Reconcile `roles.json` against the current agent set:
|
||||
/// - Seeds root's default `can_manage_top_level_agents` role on first
|
||||
/// appearance (operator can revoke with `set_role`).
|
||||
/// - Drops entries for agents that no longer exist.
|
||||
///
|
||||
/// Returns true when the file changed.
|
||||
pub fn reconcile_roles(agent_names: &[String]) -> std::io::Result<bool> {
|
||||
let mut roles = read_roles();
|
||||
let mut changed = false;
|
||||
|
||||
let root = crate::lifecycle::MANAGER_NAME;
|
||||
if agent_names.iter().any(|n| n == root) && !roles.contains_key(root) {
|
||||
roles.insert(
|
||||
root.to_owned(),
|
||||
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||
);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
let known: std::collections::HashSet<_> = agent_names.iter().collect();
|
||||
roles.retain(|name, _| {
|
||||
let keep = known.contains(name);
|
||||
if !keep {
|
||||
changed = true;
|
||||
}
|
||||
keep
|
||||
});
|
||||
|
||||
if changed {
|
||||
write_roles(&roles)?;
|
||||
}
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_seed_makes_every_agent_root() {
|
||||
// No structural manager: every agent defaults to root (null
|
||||
// parent). Explicit edges (init_config / dashboard) are layered
|
||||
// on later.
|
||||
let agents = vec![
|
||||
"alice".to_owned(),
|
||||
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||
"bob".to_owned(),
|
||||
];
|
||||
let seed = default_seed(&agents);
|
||||
assert_eq!(seed.get(crate::lifecycle::MANAGER_NAME), Some(&None));
|
||||
assert_eq!(seed.get("alice"), Some(&None));
|
||||
assert_eq!(seed.get("bob"), Some(&None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_seed_handles_empty_input() {
|
||||
let seed = default_seed(&[]);
|
||||
assert!(seed.is_empty());
|
||||
}
|
||||
|
||||
fn topo_three_level() -> BTreeMap<String, Option<String>> {
|
||||
let mut m = BTreeMap::new();
|
||||
m.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||
m.insert(
|
||||
"alice".to_owned(),
|
||||
Some(crate::lifecycle::MANAGER_NAME.to_owned()),
|
||||
);
|
||||
m.insert("bob".to_owned(), Some("alice".to_owned()));
|
||||
m.insert("carol".to_owned(), Some("alice".to_owned()));
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_promotes_to_root() {
|
||||
let next = apply_set_parent(&topo_three_level(), "alice", None).unwrap();
|
||||
assert_eq!(next.get("alice"), Some(&None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_reparents_under_sibling_subtree() {
|
||||
// bob and carol both under alice; move carol under bob.
|
||||
let next = apply_set_parent(&topo_three_level(), "carol", Some("bob")).unwrap();
|
||||
assert_eq!(next.get("carol"), Some(&Some("bob".to_owned())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_allows_manager_move() {
|
||||
// The manager is reparentable like any other agent (its
|
||||
// privileges live on the MCP socket, not its tree position).
|
||||
// Build a topo with an unrelated root-level agent `peer` so
|
||||
// moving the manager under it doesn't trip the cycle walk
|
||||
// (every non-manager agent in topo_three_level descends from
|
||||
// the manager, so that fixture can't exercise a legal
|
||||
// manager move).
|
||||
let mut topo = BTreeMap::new();
|
||||
topo.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||
topo.insert("peer".to_owned(), None);
|
||||
let next = apply_set_parent(&topo, crate::lifecycle::MANAGER_NAME, Some("peer"))
|
||||
.expect("manager move should succeed");
|
||||
assert_eq!(
|
||||
next.get(crate::lifecycle::MANAGER_NAME),
|
||||
Some(&Some("peer".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_manager_under_own_descendant() {
|
||||
// Moving the manager under `bob` (who already lives under
|
||||
// `alice` who lives under the manager) would close the loop.
|
||||
// The general cycle walk catches this; no separate manager
|
||||
// guard needed.
|
||||
let err = apply_set_parent(
|
||||
&topo_three_level(),
|
||||
crate::lifecycle::MANAGER_NAME,
|
||||
Some("bob"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.contains("cycle"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_unknown_child() {
|
||||
let err = apply_set_parent(&topo_three_level(), "nobody", Some("alice")).unwrap_err();
|
||||
assert!(err.contains("unknown agent"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_unknown_parent() {
|
||||
let err = apply_set_parent(&topo_three_level(), "bob", Some("nobody")).unwrap_err();
|
||||
assert!(err.contains("unknown parent"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_self() {
|
||||
let err = apply_set_parent(&topo_three_level(), "alice", Some("alice")).unwrap_err();
|
||||
assert!(err.contains("own parent"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_cycle() {
|
||||
// bob's parent is alice; trying to make alice's parent =
|
||||
// bob would close the loop alice → bob → alice.
|
||||
let err = apply_set_parent(&topo_three_level(), "alice", Some("bob")).unwrap_err();
|
||||
assert!(err.contains("cycle"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_refuses_deep_cycle() {
|
||||
// Three-deep chain: manager → alice → bob → carol. Moving
|
||||
// alice under carol would create the loop alice → carol → bob → alice.
|
||||
let mut topo = BTreeMap::new();
|
||||
topo.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||
topo.insert(
|
||||
"alice".to_owned(),
|
||||
Some(crate::lifecycle::MANAGER_NAME.to_owned()),
|
||||
);
|
||||
topo.insert("bob".to_owned(), Some("alice".to_owned()));
|
||||
topo.insert("carol".to_owned(), Some("bob".to_owned()));
|
||||
let err = apply_set_parent(&topo, "alice", Some("carol")).unwrap_err();
|
||||
assert!(err.contains("cycle"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_set_parent_is_idempotent_noop() {
|
||||
// bob is already under alice — same value returned.
|
||||
let next = apply_set_parent(&topo_three_level(), "bob", Some("alice")).unwrap();
|
||||
assert_eq!(next, topo_three_level());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_add_child_inserts_new_edge_under_parent() {
|
||||
// alice spawns a brand-new child `dora`: the edge lands
|
||||
// with alice as parent without disturbing the rest of the tree.
|
||||
let next = apply_add_child(&topo_three_level(), "dora", "alice")
|
||||
.unwrap()
|
||||
.expect("brand-new edge should produce a map");
|
||||
assert_eq!(next.get("dora"), Some(&Some("alice".to_owned())));
|
||||
// existing entries untouched.
|
||||
assert_eq!(next.get("bob"), Some(&Some("alice".to_owned())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_add_child_is_idempotent_when_edge_exists() {
|
||||
// bob already under alice — re-init returns a no-op (None).
|
||||
assert!(
|
||||
apply_add_child(&topo_three_level(), "bob", "alice")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_add_child_refuses_unknown_parent() {
|
||||
let err = apply_add_child(&topo_three_level(), "dora", "nobody").unwrap_err();
|
||||
assert!(err.contains("unknown parent"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_add_child_refuses_name_owned_by_other_parent() {
|
||||
// bob lives under alice; the manager can't claim it via add_child.
|
||||
let err = apply_add_child(&topo_three_level(), "bob", crate::lifecycle::MANAGER_NAME)
|
||||
.unwrap_err();
|
||||
assert!(err.contains("already exists"), "err = {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_reconcile_adds_missing_live_agent_as_root() {
|
||||
// A live agent with no prior topology entry defaults to root
|
||||
// (null parent) — no structural manager to hang it under.
|
||||
let live = vec![
|
||||
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||
"newbie".to_owned(),
|
||||
];
|
||||
let (next, changed) = apply_reconcile(&BTreeMap::new(), &live, &[]);
|
||||
assert!(changed);
|
||||
assert_eq!(next.get("newbie"), Some(&None));
|
||||
assert_eq!(next.get(crate::lifecycle::MANAGER_NAME), Some(&None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_reconcile_drops_vanished_agent() {
|
||||
let live = vec![
|
||||
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||
"alice".to_owned(),
|
||||
];
|
||||
// carol + bob are gone from the live set and not pending.
|
||||
let (next, changed) = apply_reconcile(&topo_three_level(), &live, &[]);
|
||||
assert!(changed);
|
||||
assert!(!next.contains_key("bob"));
|
||||
assert!(!next.contains_key("carol"));
|
||||
assert!(next.contains_key("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_reconcile_keeps_pending_init_agent_edge() {
|
||||
// `dora` was init'd under alice (edge present) but has no
|
||||
// container yet, so it's absent from the live set. It must NOT
|
||||
// be dropped, and its alice-parent edge must be preserved (not
|
||||
// re-seeded under the manager).
|
||||
let mut topo = topo_three_level();
|
||||
topo.insert("dora".to_owned(), Some("alice".to_owned()));
|
||||
let live = vec![
|
||||
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||
"alice".to_owned(),
|
||||
"bob".to_owned(),
|
||||
"carol".to_owned(),
|
||||
];
|
||||
let pending = vec!["dora".to_owned()];
|
||||
let (next, changed) = apply_reconcile(&topo, &live, &pending);
|
||||
assert!(!changed, "no change expected: {next:?}");
|
||||
assert_eq!(next.get("dora"), Some(&Some("alice".to_owned())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_recipient_passes_through_ordinary_names() {
|
||||
let topo = topo_three_level();
|
||||
// Real labels, broadcast, and the operator literal all
|
||||
// shortcut through unchanged — no resolution magic.
|
||||
assert_eq!(resolve_recipient_in(&topo, "bob", "alice"), "alice");
|
||||
assert_eq!(resolve_recipient_in(&topo, "bob", "*"), "*");
|
||||
assert_eq!(
|
||||
resolve_recipient_in(&topo, "bob", hive_sh4re::OPERATOR_RECIPIENT),
|
||||
hive_sh4re::OPERATOR_RECIPIENT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_recipient_rewrites_parent_sentinel_to_parent_label() {
|
||||
let topo = topo_three_level();
|
||||
// bob's parent is alice → `<parent>` from bob goes to alice.
|
||||
assert_eq!(
|
||||
resolve_recipient_in(&topo, "bob", hive_sh4re::PARENT_RECIPIENT),
|
||||
"alice"
|
||||
);
|
||||
// alice's parent is the manager — same one-hop rewrite.
|
||||
assert_eq!(
|
||||
resolve_recipient_in(&topo, "alice", hive_sh4re::PARENT_RECIPIENT),
|
||||
crate::lifecycle::MANAGER_NAME
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_recipient_falls_back_to_operator_for_root_agent() {
|
||||
let topo = topo_three_level();
|
||||
// Manager is structurally root (parent = None) → `<parent>`
|
||||
// resolves to the operator (the "no parent → tell mara"
|
||||
// fallback documented in conventions.md).
|
||||
assert_eq!(
|
||||
resolve_recipient_in(
|
||||
&topo,
|
||||
crate::lifecycle::MANAGER_NAME,
|
||||
hive_sh4re::PARENT_RECIPIENT
|
||||
),
|
||||
hive_sh4re::OPERATOR_RECIPIENT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_recipient_falls_back_to_operator_for_unknown_sender() {
|
||||
// Sender absent from topology entirely — defensive fallback
|
||||
// covers the race window where an agent's spawn has registered
|
||||
// its socket but the meta-flake `sync_agents` hasn't yet added
|
||||
// its row.
|
||||
let topo = topo_three_level();
|
||||
assert_eq!(
|
||||
resolve_recipient_in(&topo, "nobody", hive_sh4re::PARENT_RECIPIENT),
|
||||
hive_sh4re::OPERATOR_RECIPIENT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn children_of_in_returns_direct_descendants() {
|
||||
let topo = topo_three_level();
|
||||
// alice's children: bob, carol.
|
||||
let mut children = children_of_in(&topo, "alice");
|
||||
children.sort();
|
||||
assert_eq!(children, vec!["bob", "carol"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn children_of_in_manager_returns_root_level_agents() {
|
||||
let topo = topo_three_level();
|
||||
// Only alice's parent is manager; bob+carol are under alice.
|
||||
let children = children_of_in(&topo, crate::lifecycle::MANAGER_NAME);
|
||||
assert_eq!(children, vec!["alice"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn children_of_in_leaf_returns_empty() {
|
||||
let topo = topo_three_level();
|
||||
// bob and carol have no children.
|
||||
assert!(children_of_in(&topo, "bob").is_empty());
|
||||
assert!(children_of_in(&topo, "carol").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn children_of_in_unknown_sender_returns_empty() {
|
||||
let topo = topo_three_level();
|
||||
assert!(children_of_in(&topo, "nobody").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_agents_in_returns_parentless_agents() {
|
||||
let topo = topo_three_level();
|
||||
// Only the manager has no parent (alice/bob/carol all have parents).
|
||||
let top = top_level_agents_in(&topo);
|
||||
assert_eq!(top, vec![crate::lifecycle::MANAGER_NAME]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_agents_in_multi_root_returns_all_parentless() {
|
||||
let mut topo = topo_three_level();
|
||||
// Simulate a second parentless agent alongside the manager.
|
||||
topo.insert("orphan".to_owned(), None);
|
||||
let mut top = top_level_agents_in(&topo);
|
||||
top.sort();
|
||||
let mut expected = vec![crate::lifecycle::MANAGER_NAME, "orphan"];
|
||||
expected.sort_unstable();
|
||||
assert_eq!(top, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_agents_in_empty_topo_returns_empty() {
|
||||
let topo = BTreeMap::new();
|
||||
assert!(top_level_agents_in(&topo).is_empty());
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Roles tests (no disk I/O — use the pure `has_role_in` / in-memory maps)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn has_role_in_returns_true_when_role_held() {
|
||||
let mut roles = BTreeMap::new();
|
||||
roles.insert(
|
||||
"alice".to_owned(),
|
||||
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||
);
|
||||
assert!(has_role_in(
|
||||
&roles,
|
||||
"alice",
|
||||
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_role_in_returns_false_for_absent_agent() {
|
||||
let roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
assert!(!has_role_in(
|
||||
&roles,
|
||||
"alice",
|
||||
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_role_in_returns_false_for_empty_list() {
|
||||
let mut roles = BTreeMap::new();
|
||||
roles.insert("alice".to_owned(), vec![]);
|
||||
assert!(!has_role_in(
|
||||
&roles,
|
||||
"alice",
|
||||
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||
));
|
||||
}
|
||||
|
||||
/// Revoking a role must leave the key present with an empty list so
|
||||
/// `reconcile_roles` does not re-seed it.
|
||||
#[test]
|
||||
fn set_role_revoke_keeps_empty_entry_as_tombstone() {
|
||||
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||
// Build an in-memory roles map as set_role would see it after granting.
|
||||
let mut roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
roles.insert(
|
||||
mgr.to_owned(),
|
||||
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||
);
|
||||
|
||||
// Simulate the revoke path of set_role (in-memory, no disk).
|
||||
let list = roles.entry(mgr.to_owned()).or_default();
|
||||
let held = list.iter().any(|r| r == ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS);
|
||||
assert!(held);
|
||||
list.retain(|r| r != ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS);
|
||||
|
||||
// Key must still be present (tombstone), just with an empty list.
|
||||
assert!(roles.contains_key(mgr), "empty entry must not be removed");
|
||||
assert!(roles[mgr].is_empty());
|
||||
}
|
||||
|
||||
/// `reconcile_roles` must not re-seed the manager when its entry exists
|
||||
/// but is empty (operator explicitly revoked the role).
|
||||
#[test]
|
||||
fn reconcile_roles_in_does_not_reseed_after_explicit_revoke() {
|
||||
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||
let agent_names = [mgr.to_owned(), "alice".to_owned()];
|
||||
let mut roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
// Tombstone: manager was seen before but all roles were revoked.
|
||||
roles.insert(mgr.to_owned(), vec![]);
|
||||
|
||||
let mgr_present = agent_names.iter().any(|n| n == mgr);
|
||||
let should_seed = mgr_present && !roles.contains_key(mgr);
|
||||
// should_seed must be false because manager key is present (tombstone).
|
||||
assert!(
|
||||
!should_seed,
|
||||
"reconcile_roles must not re-seed an explicit revoke"
|
||||
);
|
||||
}
|
||||
|
||||
/// `reconcile_roles` seeds the manager on first appearance (no prior entry).
|
||||
#[test]
|
||||
fn reconcile_roles_in_seeds_root_when_absent() {
|
||||
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||
let agent_names = [mgr.to_owned(), "alice".to_owned()];
|
||||
let roles: BTreeMap<String, Vec<String>> = BTreeMap::new(); // empty
|
||||
|
||||
let should_seed = agent_names.iter().any(|n| n == mgr) && !roles.contains_key(mgr);
|
||||
assert!(should_seed, "reconcile_roles must seed manager when absent");
|
||||
}
|
||||
}
|
||||
|
|
@ -11,45 +11,45 @@
|
|||
//! Every module is re-exported `pub` so anything in the crate is
|
||||
//! addressable from either binary; the lib doesn't have a curated
|
||||
//! surface beyond "this is where the modules live".
|
||||
//!
|
||||
//! Cohesive clusters live in directory submodules (`stores`, `stats`,
|
||||
//! `agent_config`, `workers`); each of their children is re-exported
|
||||
//! at the crate root so pre-existing `crate::broker::…` /
|
||||
//! `hive_c0re::broker::…` paths keep compiling unchanged.
|
||||
|
||||
pub mod actions;
|
||||
pub mod agent_sockets;
|
||||
pub mod approvals;
|
||||
pub mod audit_log;
|
||||
pub mod auto_update;
|
||||
pub mod broker;
|
||||
pub mod build_logs;
|
||||
pub mod capabilities;
|
||||
pub mod agent_config;
|
||||
pub mod client;
|
||||
pub mod container_stats;
|
||||
pub mod container_view;
|
||||
pub mod coordinator;
|
||||
pub mod crash_watch;
|
||||
pub mod dashboard;
|
||||
pub mod dashboard_events;
|
||||
pub mod db;
|
||||
pub mod flake_check;
|
||||
pub mod forge;
|
||||
pub mod gateway_nginx;
|
||||
pub mod hive_stats;
|
||||
pub mod host_stats;
|
||||
pub mod job_queue;
|
||||
pub mod knowledge;
|
||||
pub mod lifecycle;
|
||||
pub mod limits;
|
||||
pub mod loose_ends;
|
||||
pub mod matrix;
|
||||
pub mod meta;
|
||||
pub mod migrate;
|
||||
pub mod operator_questions;
|
||||
pub mod paths;
|
||||
pub mod power;
|
||||
pub mod priv_client;
|
||||
pub mod questions;
|
||||
pub mod reminder_scheduler;
|
||||
pub mod scheduled_prompts;
|
||||
pub mod scheduled_prompts_worker;
|
||||
pub mod server;
|
||||
pub mod socket_server;
|
||||
pub mod tool_groups;
|
||||
pub mod topology;
|
||||
pub mod stats;
|
||||
pub mod stores;
|
||||
pub mod workers;
|
||||
|
||||
// Root re-exports: keep every pre-grouping `crate::<module>` /
|
||||
// `hive_c0re::<module>` path compiling without touching consumers.
|
||||
pub use agent_config::{capabilities, limits, tool_groups, topology};
|
||||
pub use stats::{container_stats, hive_stats, host_stats};
|
||||
pub use stores::{
|
||||
approvals, audit_log, broker, build_logs, db, operator_questions, power, scheduled_prompts,
|
||||
};
|
||||
pub use workers::{
|
||||
agent_sockets, auto_update, crash_watch, knowledge, reminder_scheduler,
|
||||
scheduled_prompts_worker,
|
||||
};
|
||||
|
|
|
|||
307
hive-c0re/src/stats/container_stats.rs
Normal file
307
hive-c0re/src/stats/container_stats.rs
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
//! Live per-agent-container resource load (CPU + memory) for the
|
||||
//! dashboard. Reads cgroup v2 stat files for each running agent
|
||||
//! machine directly — same privsep-clean posture as the turn-stats
|
||||
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
|
||||
//! `hive-priv` needed). Runs host-side in hive-c0re, where
|
||||
//! `/sys/fs/cgroup/machine.slice/` holds the per-container cgroups. Because
|
||||
//! nixos-container runs `systemd-nspawn --keep-unit` (with
|
||||
//! `Slice = "machine.slice"`), each container's cgroup is its launching
|
||||
//! service unit `container@<machine>.service` — not a machined
|
||||
//! `machine-<name>.scope`. See [`scope_dir`].
|
||||
//!
|
||||
//! No network: agents share the host network namespace
|
||||
//! (`privateNetwork = false`), so there is no per-container net
|
||||
//! counter to read. Per-agent network only becomes meaningful with the
|
||||
//! netns-isolation roadmap (`docs/network.md`).
|
||||
//!
|
||||
//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
|
||||
//! meaningless — we sample every machine's counter, sleep one short
|
||||
//! interval, sample again, and divide the delta by `interval × nproc`
|
||||
//! to get a host-normalised percentage (0..100 across all cores).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
|
||||
/// Sampling interval for the two CPU reads. Short enough to keep the
|
||||
/// endpoint snappy, long enough that the delta isn't dominated by read
|
||||
/// jitter.
|
||||
const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
||||
|
||||
const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ContainerResource {
|
||||
/// Agent name (without the `h-` machine prefix).
|
||||
pub name: String,
|
||||
/// Host-normalised CPU usage over the sample interval, as a
|
||||
/// percentage of total host CPU (0..100 across all cores).
|
||||
pub cpu_pct: f64,
|
||||
/// Current memory usage (`memory.current`), bytes.
|
||||
pub mem_current_bytes: u64,
|
||||
/// High-water memory usage since container start (`memory.peak`),
|
||||
/// bytes. `None` if the kernel doesn't expose `memory.peak`.
|
||||
pub mem_peak_bytes: Option<u64>,
|
||||
/// Memory ceiling (`memory.max`), bytes. `None` when unlimited
|
||||
/// (the file reads `max`).
|
||||
pub mem_max_bytes: Option<u64>,
|
||||
/// Last-sampled total disk usage — the agent's state dir plus the
|
||||
/// container's writable rootfs, excluding the shared read-only nix
|
||||
/// store — in bytes. `None` until the slow background sampler
|
||||
/// ([`disk_sampler_loop`]) has populated the cache at least once.
|
||||
/// Decoupled from this hot per-poll path because a `du` tree-walk is
|
||||
/// far too expensive at the 5s `/api/container-load` cadence.
|
||||
pub disk_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// Cadence for the slow disk sampler. Disk size changes slowly relative
|
||||
/// to CPU/mem, and a `du` walk is expensive, so this runs far less often
|
||||
/// than the per-poll cgroup reads — the row just carries the last value.
|
||||
const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(5);
|
||||
|
||||
/// Root of the per-container writable rootfs trees that nixos-containers
|
||||
/// manages (`/var/lib/nixos-containers/<machine>`).
|
||||
const NIXOS_CONTAINERS_ROOT: &str = "/var/lib/nixos-containers";
|
||||
|
||||
/// Last-sampled per-agent disk usage (bytes), keyed by agent name.
|
||||
/// Written by [`disk_sampler_loop`] (~every 5 min), read by [`gather`]
|
||||
/// so the hot poll never runs `du`.
|
||||
fn disk_cache() -> &'static RwLock<HashMap<String, u64>> {
|
||||
static CACHE: OnceLock<RwLock<HashMap<String, u64>>> = OnceLock::new();
|
||||
CACHE.get_or_init(|| RwLock::new(HashMap::new()))
|
||||
}
|
||||
|
||||
/// `du -sxb <path>` → apparent total bytes. `-s` summary, `-b` apparent
|
||||
/// size, `-x` stay on one filesystem so the walk never descends into the
|
||||
/// bind-mounted read-only shared nix store (or any other bind mount) —
|
||||
/// exactly the per-container-only measure we want. Best-effort: a
|
||||
/// missing/unreadable path (e.g. a stopped container with no rootfs)
|
||||
/// yields `None`, which the caller treats as 0.
|
||||
async fn du_bytes(path: &std::path::Path) -> Option<u64> {
|
||||
let out = tokio::process::Command::new("du")
|
||||
.arg("-sxb")
|
||||
.arg(path)
|
||||
.output()
|
||||
.await
|
||||
.ok()?;
|
||||
if !out.status.success() {
|
||||
return None;
|
||||
}
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
stdout.split_whitespace().next()?.parse::<u64>().ok()
|
||||
}
|
||||
|
||||
/// Total per-agent disk: the agent's host-side state dir
|
||||
/// (`/var/lib/hyperhive/agents/<name>/state`) plus the container's writable
|
||||
/// rootfs (`/var/lib/nixos-containers/h-<name>`). Each measured with `du -sxb`
|
||||
/// so the shared nix store and other bind mounts are excluded. A path that
|
||||
/// doesn't exist contributes 0.
|
||||
///
|
||||
/// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same
|
||||
/// host path the dashboard reads agent state from. Hardcoding the
|
||||
/// container-internal bind-mount path (`/agents/<name>/state`) instead made
|
||||
/// `du` fail host-side (that path only exists inside the container), so every
|
||||
/// agent's state-dir contribution was 0; with the writable rootfs nearly empty
|
||||
/// (almost everything is bind-mounted), that surfaced as all agents reporting
|
||||
/// 0 disk.
|
||||
async fn measure_agent_disk(name: &str) -> u64 {
|
||||
let state_dir = Coordinator::agent_notes_dir(name);
|
||||
let rootfs = PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}"));
|
||||
let mut total = 0u64;
|
||||
for path in [state_dir, rootfs] {
|
||||
if let Some(bytes) = du_bytes(&path).await {
|
||||
total = total.saturating_add(bytes);
|
||||
}
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
/// Background loop: every [`DISK_SAMPLE_INTERVAL`], recompute disk usage
|
||||
/// for every kept-state agent and refresh [`disk_cache`]. The first pass
|
||||
/// runs immediately so the dashboard shows disk shortly after boot.
|
||||
/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
|
||||
pub async fn disk_sampler_loop() {
|
||||
loop {
|
||||
for name in Coordinator::kept_state_names() {
|
||||
let bytes = measure_agent_disk(&name).await;
|
||||
if let Ok(mut cache) = disk_cache().write() {
|
||||
cache.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
sleep(DISK_SAMPLE_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the slow disk sampler as a detached background task. Called once
|
||||
/// from `main` alongside the other host-side loops.
|
||||
pub fn spawn_disk_sampler() {
|
||||
tokio::spawn(disk_sampler_loop());
|
||||
}
|
||||
|
||||
/// Cgroup directory for a container's machine.
|
||||
///
|
||||
/// nixos-container runs `systemd-nspawn --keep-unit` with
|
||||
/// `Slice = "machine.slice"` (see nixpkgs
|
||||
/// `virtualisation/nixos-containers.nix`). `--keep-unit` means nspawn does
|
||||
/// **not** create a separate machined `machine-<name>.scope` — the
|
||||
/// container's cgroup *is* the launching service unit,
|
||||
/// `container@<machine>.service`, placed under `machine.slice`.
|
||||
/// systemd-machined still logs "New machine <name>" (registration), but the
|
||||
/// cgroup stays on the service unit. So the path is
|
||||
/// `machine.slice/container@<machine>.service`, and the service unit name is
|
||||
/// used verbatim — no `\x2d` escaping (that only applies when a string is
|
||||
/// converted *into* a scope/slice unit name, not to an already-formed
|
||||
/// instance unit; the journal shows the literal `container@h-<agent>.service`).
|
||||
fn scope_dir(machine: &str) -> PathBuf {
|
||||
PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service"))
|
||||
}
|
||||
|
||||
/// Read a single unsigned integer from a one-line cgroup file.
|
||||
fn read_u64(path: &std::path::Path) -> Option<u64> {
|
||||
std::fs::read_to_string(path)
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// `memory.max` reads `max` when there's no limit — map that to `None`.
|
||||
fn read_mem_max(path: &std::path::Path) -> Option<u64> {
|
||||
let s = std::fs::read_to_string(path).ok()?;
|
||||
let s = s.trim();
|
||||
if s == "max" {
|
||||
None
|
||||
} else {
|
||||
s.parse::<u64>().ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
|
||||
/// scope's `cpu.stat`.
|
||||
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
|
||||
let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
|
||||
for line in text.lines() {
|
||||
if let Some(rest) = line.strip_prefix("usage_usec ") {
|
||||
return rest.trim().parse::<u64>().ok();
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Total online host CPUs, for normalising CPU% to "% of total host".
|
||||
/// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
|
||||
/// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
|
||||
/// reflects the hive-core process's CPU affinity — if hive-core is ever
|
||||
/// affinity-pinned to a subset, that would under-count the denominator
|
||||
/// and inflate the percentage. Falls back to the process count, then 1.
|
||||
fn host_nproc() -> usize {
|
||||
fn from_sysfs() -> Option<usize> {
|
||||
let s = std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?;
|
||||
let mut count = 0usize;
|
||||
for part in s.trim().split(',') {
|
||||
let part = part.trim();
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some((a, b)) = part.split_once('-') {
|
||||
let a: usize = a.trim().parse().ok()?;
|
||||
let b: usize = b.trim().parse().ok()?;
|
||||
count += b.saturating_sub(a) + 1;
|
||||
} else {
|
||||
part.parse::<usize>().ok()?;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
(count > 0).then_some(count)
|
||||
}
|
||||
from_sysfs()
|
||||
.or_else(|| std::thread::available_parallelism().ok().map(usize::from))
|
||||
.unwrap_or(1)
|
||||
}
|
||||
|
||||
/// Sample CPU + memory for every running agent container. Best-effort:
|
||||
/// agents whose scope dir is absent (not running) or unreadable are
|
||||
/// skipped, never fatal. Sorted by name for a stable display order.
|
||||
pub async fn gather() -> Vec<ContainerResource> {
|
||||
// Machines that currently have a cgroup scope (= running). Agent
|
||||
// machine name is `h-<state-dir name>`.
|
||||
let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
|
||||
.into_iter()
|
||||
.filter_map(|name| {
|
||||
let dir = scope_dir(&format!("h-{name}"));
|
||||
dir.join("cpu.stat").exists().then_some((name, dir))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// First CPU sample for all, then one shared sleep, then the second
|
||||
// — so N agents cost one interval, not N.
|
||||
let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
|
||||
sleep(CPU_SAMPLE).await;
|
||||
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||
)]
|
||||
let nproc = host_nproc() as f64;
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||
)]
|
||||
let interval_usec = CPU_SAMPLE.as_micros() as f64;
|
||||
|
||||
// Snapshot the slow disk cache once (cheap clone of a small map) so
|
||||
// the per-agent loop below is a plain map lookup — no `du` on this
|
||||
// hot path. Agents not yet sampled simply carry `disk_bytes = None`.
|
||||
let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
|
||||
|
||||
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
|
||||
for (i, (name, dir)) in candidates.iter().enumerate() {
|
||||
let cpu_pct = match (t0[i], read_usage_usec(dir)) {
|
||||
(Some(a), Some(b)) => {
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||
)]
|
||||
let delta = b.saturating_sub(a) as f64;
|
||||
(delta / (interval_usec * nproc)) * 100.0
|
||||
}
|
||||
_ => 0.0,
|
||||
};
|
||||
out.push(ContainerResource {
|
||||
name: name.clone(),
|
||||
cpu_pct,
|
||||
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
|
||||
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
|
||||
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
|
||||
disk_bytes: disk.get(name).copied(),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scope_dir_is_the_keep_unit_service_under_machine_slice() {
|
||||
// nixos-container `--keep-unit` keeps the cgroup on the service
|
||||
// unit `container@<machine>.service` (literal name, no `\x2d`),
|
||||
// under machine.slice — NOT a machined `machine-<name>.scope`.
|
||||
assert_eq!(
|
||||
scope_dir("h-atlas"),
|
||||
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service")
|
||||
);
|
||||
// Underscores in agent names are likewise verbatim.
|
||||
assert_eq!(
|
||||
scope_dir("h-foo_bar"),
|
||||
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service")
|
||||
);
|
||||
}
|
||||
}
|
||||
450
hive-c0re/src/stats/hive_stats.rs
Normal file
450
hive-c0re/src/stats/hive_stats.rs
Normal file
|
|
@ -0,0 +1,450 @@
|
|||
//! Hive-wide turn-stats aggregation for the dashboard's swarm stats
|
||||
//! view. Reads every agent's per-agent
|
||||
//! `hyperhive-turn-stats.sqlite` read-only and rolls the rows up into
|
||||
//! swarm totals + a per-agent rollup + model mix + a *labelled*
|
||||
//! cost estimate.
|
||||
//!
|
||||
//! Why re-read the rows here instead of reusing `hive-ag3nt`'s
|
||||
//! `stats.rs`: that module lives in a different crate (the agent
|
||||
//! harness) which hive-c0re can't import. The stable contract is the
|
||||
//! turn-stats *schema*, so we run a focused query against it. If we
|
||||
//! ever want a single source of truth, the row-read + aggregation can
|
||||
//! be lifted into a shared crate — overkill for now.
|
||||
//!
|
||||
//! Privsep: the sqlite files are mode 0644 owned by the agent user;
|
||||
//! `hive-core` can read them fine, but cannot write/delete (which is why
|
||||
//! retention sweeps run agent-side in the harness, not here). We open
|
||||
//! read-only so an in-flight harness writer never blocks us.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use rusqlite::{Connection, OpenFlags};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::coordinator::Coordinator;
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
|
||||
/// Window accepted by `/api/stats-hive?window=`. Maps to a lookback
|
||||
/// span; the hive view is a flat rollup (no per-bucket trend — the
|
||||
/// per-agent `/stats` page owns the trend charts).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Window {
|
||||
Hour,
|
||||
FourHour,
|
||||
Day,
|
||||
ThreeDay,
|
||||
Week,
|
||||
Month,
|
||||
/// All available data: aggregate over every recorded turn since the
|
||||
/// earliest (`from == 0`), with no fixed lookback.
|
||||
All,
|
||||
}
|
||||
|
||||
impl Window {
|
||||
#[must_use]
|
||||
pub fn parse(s: &str) -> Self {
|
||||
match s {
|
||||
"1h" => Self::Hour,
|
||||
"4h" => Self::FourHour,
|
||||
"3d" => Self::ThreeDay,
|
||||
"7d" => Self::Week,
|
||||
"30d" => Self::Month,
|
||||
"all" => Self::All,
|
||||
_ => Self::Day,
|
||||
}
|
||||
}
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Hour => "1h",
|
||||
Self::FourHour => "4h",
|
||||
Self::Day => "24h",
|
||||
Self::ThreeDay => "3d",
|
||||
Self::Week => "7d",
|
||||
Self::Month => "30d",
|
||||
Self::All => "all",
|
||||
}
|
||||
}
|
||||
|
||||
fn span_secs(self) -> i64 {
|
||||
match self {
|
||||
Self::Hour => 3600,
|
||||
Self::FourHour => 4 * 3600,
|
||||
Self::Day => 24 * 3600,
|
||||
Self::ThreeDay => 3 * 24 * 3600,
|
||||
Self::Week => 7 * 24 * 3600,
|
||||
Self::Month => 30 * 24 * 3600,
|
||||
// `All` has no fixed lookback; `hive_snapshot` uses `from == 0`
|
||||
// (every recorded turn). 0 here is a consistent safe fallback.
|
||||
Self::All => 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Approximate USD price per **million** tokens for one model class.
|
||||
/// The four fields map to the four token kinds the turn-stats schema
|
||||
/// records. This is a deliberately rough estimate — Anthropic list
|
||||
/// pricing drifts, so the dashboard labels the figure as an estimate.
|
||||
///
|
||||
/// Operators keep the table current without a code change via the
|
||||
/// `services.hyperhive.modelPrices` nix option (passed to
|
||||
/// `hive-c0re serve --model-prices <json>`, held on the `Coordinator`
|
||||
/// as a [`PriceTable`]). Any model not covered by the operator table
|
||||
/// falls back to [`builtin_prices`].
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct Prices {
|
||||
pub input: f64,
|
||||
pub output: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_write: f64,
|
||||
}
|
||||
|
||||
/// Operator-tunable model→price map. Keys are matched
|
||||
/// case-insensitively as a substring of the model id (e.g. `"sonnet"`
|
||||
/// matches `"claude-sonnet-4-5"`); the longest matching key wins so a
|
||||
/// specific entry beats a generic family name. An empty map means
|
||||
/// every model uses [`builtin_prices`].
|
||||
pub type PriceTable = HashMap<String, Prices>;
|
||||
|
||||
/// Built-in fallback pricing, per **million** tokens. Used when the
|
||||
/// operator's [`PriceTable`] has no key matching the model. Figures are
|
||||
/// current Anthropic list prices for the Claude 4.x family (Opus 4.x,
|
||||
/// Sonnet 4.x, Haiku 4.5); `cache_write` uses the 1-hour cache-TTL price,
|
||||
/// which is the default through the Claude subscription the agents run
|
||||
/// on. Keep in sync with the `services.hyperhive.modelPrices` nix default
|
||||
/// (`nix/modules/hive-c0re.nix`).
|
||||
fn builtin_prices(model: &str) -> Prices {
|
||||
let m = model.to_ascii_lowercase();
|
||||
if m.contains("opus") {
|
||||
Prices {
|
||||
input: 5.0,
|
||||
output: 25.0,
|
||||
cache_read: 0.5,
|
||||
cache_write: 10.0,
|
||||
}
|
||||
} else if m.contains("haiku") {
|
||||
Prices {
|
||||
input: 1.0,
|
||||
output: 5.0,
|
||||
cache_read: 0.1,
|
||||
cache_write: 2.0,
|
||||
}
|
||||
} else {
|
||||
// sonnet + unknown fallback
|
||||
Prices {
|
||||
input: 3.0,
|
||||
output: 15.0,
|
||||
cache_read: 0.3,
|
||||
cache_write: 6.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the price for `model`: prefer the operator-provided
|
||||
/// `table` (longest matching key wins, so a specific
|
||||
/// `claude-3-5-sonnet` entry beats a generic `sonnet`), else fall back
|
||||
/// to [`builtin_prices`].
|
||||
fn resolve_prices(model: &str, table: &PriceTable) -> Prices {
|
||||
let m = model.to_ascii_lowercase();
|
||||
let mut best: Option<(usize, Prices)> = None;
|
||||
for (k, v) in table {
|
||||
let key = k.to_ascii_lowercase();
|
||||
if key.is_empty() || !m.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
let better = match best {
|
||||
Some((blen, _)) => key.len() > blen,
|
||||
None => true,
|
||||
};
|
||||
if better {
|
||||
best = Some((key.len(), *v));
|
||||
}
|
||||
}
|
||||
match best {
|
||||
Some((_, p)) => p,
|
||||
None => builtin_prices(&m),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct KeyCount {
|
||||
pub key: String,
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AgentRollup {
|
||||
pub name: String,
|
||||
pub turns: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cache_read_tokens: u64,
|
||||
pub cache_creation_tokens: u64,
|
||||
/// Labelled estimate — see [`Prices`].
|
||||
pub est_cost_usd: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct HiveStats {
|
||||
pub window: &'static str,
|
||||
pub from: i64,
|
||||
pub now: i64,
|
||||
/// Number of agents that had at least one turn in the window.
|
||||
pub active_agents: u64,
|
||||
pub total_turns: u64,
|
||||
pub total_input_tokens: u64,
|
||||
pub total_output_tokens: u64,
|
||||
pub total_cache_read_tokens: u64,
|
||||
pub total_cache_creation_tokens: u64,
|
||||
/// Labelled estimate — see [`Prices`].
|
||||
pub est_cost_usd: f64,
|
||||
/// Per-agent rollup, busiest (most turns) first.
|
||||
pub agents: Vec<AgentRollup>,
|
||||
/// Turns per model across the whole swarm, busiest first.
|
||||
pub model_mix: Vec<KeyCount>,
|
||||
/// Most-run normalised bash-command heads ("favorite tools") across
|
||||
/// the whole swarm, busiest first, capped to 10. Empty until the
|
||||
/// hive-bash-mcp capture has recorded `bash_commands` rows on at
|
||||
/// least one active agent.
|
||||
pub bash_mix: Vec<KeyCount>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AgentAgg {
|
||||
turns: u64,
|
||||
input: u64,
|
||||
output: u64,
|
||||
cache_read: u64,
|
||||
cache_creation: u64,
|
||||
cost: f64,
|
||||
models: HashMap<String, u64>,
|
||||
/// Normalised bash-command head → invocation count, from the agent's
|
||||
/// `bash_commands` table (written by hive-bash-mcp). Empty when that
|
||||
/// capture hasn't run for this agent (table absent).
|
||||
bash: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::cast_sign_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
reason = "v.max(0) clamps to a non-negative value first, so the i64->u64 cast is exact: no sign loss, and no truncation since u64 covers all non-negative i64"
|
||||
)]
|
||||
fn u64_from_i64(v: i64) -> u64 {
|
||||
v.max(0) as u64
|
||||
}
|
||||
|
||||
/// Aggregate one agent's turn-stats over `[from, now]`. Errors bubble
|
||||
/// up so the caller can skip a single bad/locked db without failing
|
||||
/// the whole endpoint.
|
||||
fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<AgentAgg> {
|
||||
let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||
// turn_stats is rollback-journal (not WAL): a read landing while the
|
||||
// harness is mid-INSERT would get SQLITE_BUSY and drop that active
|
||||
// agent from the rollup. Wait out the brief write instead.
|
||||
conn.busy_timeout(Duration::from_millis(500))?;
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT model, input_tokens, output_tokens,
|
||||
cache_read_input_tokens, cache_creation_input_tokens
|
||||
FROM turn_stats
|
||||
WHERE started_at >= ?1",
|
||||
)?;
|
||||
let mut agg = AgentAgg::default();
|
||||
let rows = stmt.query_map([from], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
u64_from_i64(row.get::<_, i64>(1)?),
|
||||
u64_from_i64(row.get::<_, i64>(2)?),
|
||||
u64_from_i64(row.get::<_, i64>(3)?),
|
||||
u64_from_i64(row.get::<_, i64>(4)?),
|
||||
))
|
||||
})?;
|
||||
for r in rows {
|
||||
let (model, input, output, cache_read, cache_creation) = r?;
|
||||
agg.turns += 1;
|
||||
agg.input = agg.input.saturating_add(input);
|
||||
agg.output = agg.output.saturating_add(output);
|
||||
agg.cache_read = agg.cache_read.saturating_add(cache_read);
|
||||
agg.cache_creation = agg.cache_creation.saturating_add(cache_creation);
|
||||
let p = resolve_prices(&model, prices);
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "token counts stay well under f64's 2^53 exact-integer range, so this cost computation loses no precision"
|
||||
)]
|
||||
{
|
||||
agg.cost += (input as f64 * p.input
|
||||
+ output as f64 * p.output
|
||||
+ cache_read as f64 * p.cache_read
|
||||
+ cache_creation as f64 * p.cache_write)
|
||||
/ 1_000_000.0;
|
||||
}
|
||||
*agg.models.entry(model).or_insert(0) += 1;
|
||||
}
|
||||
agg.bash = read_bash_heads(&conn, from);
|
||||
Ok(agg)
|
||||
}
|
||||
|
||||
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
||||
/// table (`ts INTEGER, head TEXT`, written by hive-bash-mcp) over
|
||||
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
||||
/// a missing table (capture hasn't run for this agent) or any read error
|
||||
/// yields an empty map rather than propagating, so the favorite-tools
|
||||
/// rollup simply omits that agent and never fails the whole endpoint.
|
||||
/// `ts` is unix seconds, matching the `from` cutoff.
|
||||
fn read_bash_heads(conn: &Connection, from: i64) -> HashMap<String, u64> {
|
||||
let mut out: HashMap<String, u64> = HashMap::new();
|
||||
let Ok(mut stmt) =
|
||||
conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts >= ?1 GROUP BY head")
|
||||
else {
|
||||
return out;
|
||||
};
|
||||
let Ok(rows) = stmt.query_map([from], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
u64_from_i64(row.get::<_, i64>(1)?),
|
||||
))
|
||||
}) else {
|
||||
return out;
|
||||
};
|
||||
for (head, count) in rows.flatten() {
|
||||
*out.entry(head).or_insert(0) += count;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Build the swarm-wide rollup. Best-effort: a missing or unreadable
|
||||
/// per-agent db is skipped (logged), never fatal.
|
||||
#[must_use]
|
||||
pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||
let now = now_unix();
|
||||
// Fixed windows look back a constant span; `all` aggregates every
|
||||
// recorded turn (`from == 0`). The hive rollup isn't time-bucketed, so
|
||||
// unlike the per-agent snapshot it needs no adaptive bucket sizing.
|
||||
let from = match window {
|
||||
Window::All => 0,
|
||||
_ => now - window.span_secs(),
|
||||
};
|
||||
|
||||
let mut agents: Vec<AgentRollup> = Vec::new();
|
||||
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
||||
let mut total_turns = 0u64;
|
||||
let mut total_input = 0u64;
|
||||
let mut total_output = 0u64;
|
||||
let mut total_cache_read = 0u64;
|
||||
let mut total_cache_creation = 0u64;
|
||||
let mut total_cost = 0.0f64;
|
||||
let mut active_agents = 0u64;
|
||||
|
||||
for name in Coordinator::kept_state_names() {
|
||||
let path = Coordinator::agent_harness_dir(&name).join("hyperhive-turn-stats.sqlite");
|
||||
if !path.exists() {
|
||||
continue;
|
||||
}
|
||||
let agg = match read_agent(&path, from, prices) {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
tracing::warn!(agent = %name, error = ?e, "hive-stats: read failed; skipping");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if agg.turns == 0 {
|
||||
continue;
|
||||
}
|
||||
active_agents += 1;
|
||||
total_turns += agg.turns;
|
||||
total_input = total_input.saturating_add(agg.input);
|
||||
total_output = total_output.saturating_add(agg.output);
|
||||
total_cache_read = total_cache_read.saturating_add(agg.cache_read);
|
||||
total_cache_creation = total_cache_creation.saturating_add(agg.cache_creation);
|
||||
total_cost += agg.cost;
|
||||
for (m, c) in &agg.models {
|
||||
*model_mix.entry(m.clone()).or_insert(0) += c;
|
||||
}
|
||||
for (h, c) in &agg.bash {
|
||||
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
||||
}
|
||||
agents.push(AgentRollup {
|
||||
name,
|
||||
turns: agg.turns,
|
||||
input_tokens: agg.input,
|
||||
output_tokens: agg.output,
|
||||
cache_read_tokens: agg.cache_read,
|
||||
cache_creation_tokens: agg.cache_creation,
|
||||
est_cost_usd: agg.cost,
|
||||
});
|
||||
}
|
||||
|
||||
// Busiest agents first.
|
||||
agents.sort_by(|a, b| b.turns.cmp(&a.turns).then_with(|| a.name.cmp(&b.name)));
|
||||
|
||||
let mut model_mix: Vec<KeyCount> = model_mix
|
||||
.into_iter()
|
||||
.map(|(key, count)| KeyCount { key, count })
|
||||
.collect();
|
||||
model_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
|
||||
// Busiest commands first, capped to a top-10 "favorite tools" list.
|
||||
let mut bash_mix: Vec<KeyCount> = bash_mix
|
||||
.into_iter()
|
||||
.map(|(key, count)| KeyCount { key, count })
|
||||
.collect();
|
||||
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||
bash_mix.truncate(10);
|
||||
|
||||
HiveStats {
|
||||
window: window.label(),
|
||||
from,
|
||||
now,
|
||||
active_agents,
|
||||
total_turns,
|
||||
total_input_tokens: total_input,
|
||||
total_output_tokens: total_output,
|
||||
total_cache_read_tokens: total_cache_read,
|
||||
total_cache_creation_tokens: total_cache_creation,
|
||||
est_cost_usd: total_cost,
|
||||
agents,
|
||||
model_mix,
|
||||
bash_mix,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `read_bash_heads` tallies per-head counts over the window and is
|
||||
/// isolated from a missing table (returns empty, never errors).
|
||||
#[test]
|
||||
fn bash_heads_tally_and_window() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
||||
.unwrap();
|
||||
let now = now_unix();
|
||||
for (ts, head) in [
|
||||
(now - 100, "cargo"),
|
||||
(now - 200, "cargo"),
|
||||
(now - 300, "git"),
|
||||
(now - 10_000, "rg"), // outside a 1h-ish cutoff below
|
||||
] {
|
||||
conn.execute(
|
||||
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
||||
rusqlite::params![ts, head],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let heads = read_bash_heads(&conn, now - 3600);
|
||||
assert_eq!(heads.get("cargo").copied(), Some(2));
|
||||
assert_eq!(heads.get("git").copied(), Some(1));
|
||||
assert_eq!(heads.get("rg").copied(), None); // window cutoff excludes it
|
||||
}
|
||||
|
||||
/// A missing `bash_commands` table degrades to an empty tally rather
|
||||
/// than erroring — the pre-capture window on a fresh agent.
|
||||
#[test]
|
||||
fn bash_heads_missing_table_is_empty() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
assert!(read_bash_heads(&conn, 0).is_empty());
|
||||
}
|
||||
}
|
||||
260
hive-c0re/src/stats/host_stats.rs
Normal file
260
hive-c0re/src/stats/host_stats.rs
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
//! Host-system probes and the derived **server-warning** list that the
|
||||
//! dashboard renders as a top-of-page banner.
|
||||
//!
|
||||
//! The first (and currently only) producer is a host disk-usage check:
|
||||
//! the nix store filling up is what ENOSPC-failed CI before the GC
|
||||
//! guardrails were documented. hive-c0re runs on the host (not inside a
|
||||
//! container), so it can `statvfs` the store path directly and warn the
|
||||
//! operator *before* an ENOSPC, not after.
|
||||
//!
|
||||
//! [`server_warnings`] is the public surface: it returns a flat list of
|
||||
//! [`ServerWarning`]s for `/api/state`. The dashboard renders whatever it
|
||||
//! returns, coloured by `level`, so adding a new system warning (memory
|
||||
//! pressure, a failed unit, …) is a backend-only change — no frontend
|
||||
//! edit. Keep producers cheap; this runs on every `/api/state` assembly.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::container_view::ContainerView;
|
||||
|
||||
/// One server-level warning for the dashboard's top-of-page banner.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ServerWarning {
|
||||
/// Stable kind id (e.g. `"disk_pressure"`) — lets the frontend dedupe
|
||||
/// or special-case without parsing the message.
|
||||
pub kind: &'static str,
|
||||
/// Severity: `"warn"` (amber) or `"crit"` (red). The banner picks its
|
||||
/// colour from this; everything else is just the message text.
|
||||
pub level: &'static str,
|
||||
/// Human-readable, already-formatted message shown in the banner.
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Percent-used past which the host nix store earns a disk-pressure
|
||||
/// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`.
|
||||
const DISK_WARN_PCT: f64 = 85.0;
|
||||
const DISK_CRIT_PCT: f64 = 95.0;
|
||||
|
||||
/// Collect the current server-level warnings for the dashboard banner.
|
||||
/// Each producer pushes zero or more [`ServerWarning`]s; the frontend
|
||||
/// renders whatever this returns. Cheap to call on every `/api/state`
|
||||
/// assembly (currently a single `statvfs`).
|
||||
#[must_use]
|
||||
pub fn server_warnings() -> Vec<ServerWarning> {
|
||||
let mut out = Vec::new();
|
||||
if let Some(d) = nix_disk_usage()
|
||||
&& d.used_pct >= DISK_WARN_PCT
|
||||
{
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision"
|
||||
)]
|
||||
let free_gib = d.free_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
|
||||
out.push(ServerWarning {
|
||||
kind: "disk_pressure",
|
||||
level: if d.used_pct >= DISK_CRIT_PCT {
|
||||
"crit"
|
||||
} else {
|
||||
"warn"
|
||||
},
|
||||
message: format!(
|
||||
"host nix store {:.0}% full ({free_gib:.1} GiB free) \
|
||||
— garbage-collect the store before it runs out of space",
|
||||
d.used_pct
|
||||
),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Agent-state warnings derived from the live container snapshot the
|
||||
/// dashboard already holds: agents that need a claude login, and agents
|
||||
/// that are crashing. Kept separate from [`server_warnings`] (host
|
||||
/// probes) because the caller owns the container list + crash counts;
|
||||
/// the dashboard concatenates both into one banner list.
|
||||
///
|
||||
/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent →
|
||||
/// number of crashes inside that window — so a crash-looping agent shows
|
||||
/// its repeat count rather than a single point-in-time flap.
|
||||
#[must_use]
|
||||
pub fn agent_state_warnings<S: std::hash::BuildHasher>(
|
||||
containers: &[ContainerView],
|
||||
crash_counts: &HashMap<String, usize, S>,
|
||||
) -> Vec<ServerWarning> {
|
||||
let mut out = Vec::new();
|
||||
|
||||
// `needs_login` is already running-gated in `container_view::build_all`,
|
||||
// so a stopped container never lights this.
|
||||
let mut pending: Vec<&str> = containers
|
||||
.iter()
|
||||
.filter(|c| c.needs_login)
|
||||
.map(|c| c.name.as_str())
|
||||
.collect();
|
||||
if !pending.is_empty() {
|
||||
pending.sort_unstable();
|
||||
out.push(ServerWarning {
|
||||
kind: "pending_logins",
|
||||
level: "warn",
|
||||
message: format!(
|
||||
"{n} agent{plural} {verb} claude login: {list} \
|
||||
— open the agent's page in the dashboard to complete the login",
|
||||
n = pending.len(),
|
||||
plural = if pending.len() == 1 { "" } else { "s" },
|
||||
verb = if pending.len() == 1 { "needs" } else { "need" },
|
||||
list = pending.join(", "),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
if !crash_counts.is_empty() {
|
||||
let mut crashing: Vec<(&String, usize)> =
|
||||
crash_counts.iter().map(|(a, n)| (a, *n)).collect();
|
||||
crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
|
||||
let list = crashing
|
||||
.iter()
|
||||
.map(|(a, n)| format!("{a} (×{n})"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
out.push(ServerWarning {
|
||||
kind: "agents_crashing",
|
||||
level: "crit",
|
||||
message: format!(
|
||||
"{n} agent{plural} crashing: {list} \
|
||||
— check the container journal (`hivectl logs <agent>`)",
|
||||
n = crashing.len(),
|
||||
plural = if crashing.len() == 1 { "" } else { "s" },
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Disk usage for the filesystem backing the host nix store — internal to
|
||||
/// the disk-pressure producer above.
|
||||
struct DiskUsage {
|
||||
/// Space available to unprivileged writers, in bytes.
|
||||
free_bytes: u64,
|
||||
/// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used +
|
||||
/// available)` — so the threshold means what the operator sees in `df`.
|
||||
used_pct: f64,
|
||||
}
|
||||
|
||||
/// Probe disk usage for the filesystem containing the nix store (`/nix`),
|
||||
/// falling back to `/` when `/nix` isn't its own mount. Returns `None` if
|
||||
/// the `statvfs` syscall fails (path missing, permission, etc.).
|
||||
fn nix_disk_usage() -> Option<DiskUsage> {
|
||||
disk_usage("/nix").or_else(|| disk_usage("/"))
|
||||
}
|
||||
|
||||
fn disk_usage(path: &str) -> Option<DiskUsage> {
|
||||
let c_path = std::ffi::CString::new(path).ok()?;
|
||||
// SAFETY: `statvfs` reads only through the valid NUL-terminated
|
||||
// `c_path` pointer and writes into the zeroed `stat` we own. We check
|
||||
// the return code before reading any field.
|
||||
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) };
|
||||
if rc != 0 {
|
||||
return None;
|
||||
}
|
||||
// statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this
|
||||
// runs on); the arithmetic below stays in that native width.
|
||||
let frsize = stat.f_frsize;
|
||||
let total_blocks = stat.f_blocks;
|
||||
let free_blocks = stat.f_bfree;
|
||||
let avail_blocks = stat.f_bavail;
|
||||
if total_blocks == 0 || frsize == 0 {
|
||||
return None;
|
||||
}
|
||||
let free_bytes = avail_blocks.saturating_mul(frsize);
|
||||
// df's use%: used / (used + available). `used` counts root-reserved
|
||||
// blocks (total - bfree); `available` is the unprivileged free
|
||||
// (bavail), so the percentage matches what `df` reports.
|
||||
let used_blocks = total_blocks.saturating_sub(free_blocks);
|
||||
let capacity = used_blocks.saturating_add(avail_blocks);
|
||||
let used_pct = if capacity == 0 {
|
||||
0.0
|
||||
} else {
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision"
|
||||
)]
|
||||
let raw = used_blocks as f64 / capacity as f64 * 100.0;
|
||||
raw
|
||||
};
|
||||
Some(DiskUsage {
|
||||
free_bytes,
|
||||
used_pct,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cv(name: &str, needs_login: bool) -> ContainerView {
|
||||
ContainerView {
|
||||
name: name.to_owned(),
|
||||
container: format!("h-{name}"),
|
||||
port: 0,
|
||||
running: true,
|
||||
needs_update: false,
|
||||
needs_login,
|
||||
deployed_sha: None,
|
||||
pending_reminders: 0,
|
||||
parent: None,
|
||||
active_model: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_agent_warnings_when_all_healthy() {
|
||||
let containers = [cv("alice", false), cv("bob", false)];
|
||||
assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_logins_lists_sorted_agents() {
|
||||
let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
|
||||
let w = agent_state_warnings(&containers, &HashMap::new());
|
||||
assert_eq!(w.len(), 1);
|
||||
assert_eq!(w[0].kind, "pending_logins");
|
||||
assert_eq!(w[0].level, "warn");
|
||||
// sorted, login-needing only, count reflected
|
||||
assert!(
|
||||
w[0].message
|
||||
.starts_with("2 agents need claude login: bob, zoe")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn singular_grammar_for_one_agent() {
|
||||
let containers = [cv("solo", true)];
|
||||
let w = agent_state_warnings(&containers, &HashMap::new());
|
||||
assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crashing_warning_is_crit_and_count_ordered() {
|
||||
let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]);
|
||||
let w = agent_state_warnings(&[], &crashes);
|
||||
assert_eq!(w.len(), 1);
|
||||
assert_eq!(w[0].kind, "agents_crashing");
|
||||
assert_eq!(w[0].level, "crit");
|
||||
// higher crash count first
|
||||
assert!(w[0].message.contains("flap (×5), blip (×1)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_warnings_coexist() {
|
||||
let containers = [cv("a", true)];
|
||||
let crashes = HashMap::from([("b".to_owned(), 2)]);
|
||||
let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes)
|
||||
.iter()
|
||||
.map(|w| w.kind)
|
||||
.collect();
|
||||
assert_eq!(kinds, ["pending_logins", "agents_crashing"]);
|
||||
}
|
||||
}
|
||||
8
hive-c0re/src/stats/mod.rs
Normal file
8
hive-c0re/src/stats/mod.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
//! Metrics aggregation for the dashboard: hive-wide turn-stats
|
||||
//! rollups, host-system probes / server warnings, and live
|
||||
//! per-container cgroup load. Each submodule is re-exported at the
|
||||
//! crate root, so `crate::hive_stats::…` etc. keep working unchanged.
|
||||
|
||||
pub mod container_stats;
|
||||
pub mod hive_stats;
|
||||
pub mod host_stats;
|
||||
515
hive-c0re/src/stores/approvals.rs
Normal file
515
hive-c0re/src/stores/approvals.rs
Normal file
|
|
@ -0,0 +1,515 @@
|
|||
//! Approval queue. Manager submits via `RequestApplyCommit`; the user
|
||||
//! approves/denies via the host admin CLI; on approval the host runs the
|
||||
//! corresponding action (Phase 5a: `lifecycle::rebuild(agent)`).
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use hive_sh4re::{Approval, ApprovalKind, ApprovalStatus};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
const SCHEMA: &str = r"
|
||||
CREATE TABLE IF NOT EXISTS approvals (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
agent TEXT NOT NULL,
|
||||
commit_ref TEXT NOT NULL,
|
||||
requested_at INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
resolved_at INTEGER,
|
||||
note TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_approvals_pending
|
||||
ON approvals (id) WHERE status = 'pending';
|
||||
";
|
||||
|
||||
/// Additive column migrations for pre-existing databases, applied via
|
||||
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||
const MIGRATIONS: &[&str] = &[
|
||||
// `kind` (pre-Phase-8 dbs): legacy rows default to `apply_commit`,
|
||||
// which matches their actual semantics.
|
||||
"ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit'",
|
||||
// `description`: manager-supplied note shown on the dashboard
|
||||
// approval card at submission time (distinct from `note`, set on
|
||||
// denial/failure).
|
||||
"ALTER TABLE approvals ADD COLUMN description TEXT",
|
||||
// `fetched_sha`: the canonical sha hive-c0re vouched for at
|
||||
// `request_apply_commit` time. Distinct from `commit_ref`
|
||||
// (manager-supplied, may not even resolve by approve time).
|
||||
"ALTER TABLE approvals ADD COLUMN fetched_sha TEXT",
|
||||
// `submitter`: the agent that submitted the approval (the
|
||||
// authenticated socket caller); approval-scoped helper events
|
||||
// route to it. Legacy rows are NULL → callers fall back to the
|
||||
// root agent.
|
||||
"ALTER TABLE approvals ADD COLUMN submitter TEXT",
|
||||
];
|
||||
|
||||
pub struct Approvals {
|
||||
conn: Mutex<Connection>,
|
||||
}
|
||||
|
||||
impl Approvals {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
let conn = crate::db::open(path, "approvals")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply approvals schema")?;
|
||||
crate::db::apply_migrations(&conn, "approvals", MIGRATIONS)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn submit_kind(
|
||||
&self,
|
||||
agent: &str,
|
||||
kind: ApprovalKind,
|
||||
commit_ref: &str,
|
||||
description: Option<&str>,
|
||||
submitter: &str,
|
||||
) -> Result<i64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO approvals
|
||||
(agent, kind, commit_ref, requested_at, status, description, submitter)
|
||||
VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6)",
|
||||
params![
|
||||
agent,
|
||||
kind.as_str(),
|
||||
commit_ref,
|
||||
now_unix(),
|
||||
description,
|
||||
submitter
|
||||
],
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
/// The agent that submitted approval `id` (the authenticated socket
|
||||
/// caller at submit time). `None` for legacy rows predating the
|
||||
/// `submitter` column — callers route those to the root agent.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the sqlite prepare/query fails. A missing row
|
||||
/// or a `NULL` submitter is not an error — both yield `Ok(None)`.
|
||||
pub fn submitter_of(&self, id: i64) -> Result<Option<String>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let submitter: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT submitter FROM approvals WHERE id = ?1",
|
||||
params![id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten();
|
||||
Ok(submitter)
|
||||
}
|
||||
|
||||
/// Record the canonical sha hive-c0re fetched from the proposed repo
|
||||
/// into applied at submission time. Idempotent on identical values.
|
||||
pub fn set_fetched_sha(&self, id: i64, sha: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE approvals SET fetched_sha = ?1 WHERE id = ?2",
|
||||
params![sha, id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Last `limit` resolved approvals (approved / denied / failed),
|
||||
/// newest-first. Drives the history tab on the dashboard.
|
||||
pub fn recent_resolved(&self, limit: u64) -> Result<Vec<Approval>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||
FROM approvals
|
||||
WHERE status IN ('approved', 'denied', 'failed', 'cancelled')
|
||||
ORDER BY resolved_at DESC, id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map([limit], row_to_approval)?;
|
||||
Ok(collect_lenient(rows))
|
||||
}
|
||||
|
||||
pub fn pending(&self) -> Result<Vec<Approval>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||
FROM approvals
|
||||
WHERE status = 'pending'
|
||||
ORDER BY id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], row_to_approval)?;
|
||||
Ok(collect_lenient(rows))
|
||||
}
|
||||
|
||||
pub fn get(&self, id: i64) -> Result<Option<Approval>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.query_row(
|
||||
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||
FROM approvals WHERE id = ?1",
|
||||
params![id],
|
||||
row_to_approval,
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Mark pending -> approved (or fail if not pending). Returns the (now-updated)
|
||||
/// approval so the caller can run the action and pass the agent name.
|
||||
pub fn mark_approved(&self, id: i64) -> Result<Approval> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let row: Option<ApprovalLookup> = conn
|
||||
.query_row(
|
||||
ApprovalLookup::SELECT,
|
||||
params![id],
|
||||
ApprovalLookup::from_row,
|
||||
)
|
||||
.optional()?;
|
||||
let Some(row) = row else {
|
||||
bail!("approval {id} not found");
|
||||
};
|
||||
if row.status != "pending" {
|
||||
bail!("approval {id} is {}, not pending", row.status);
|
||||
}
|
||||
let resolved_at = now_unix();
|
||||
conn.execute(
|
||||
"UPDATE approvals SET status = 'approved', resolved_at = ?1 WHERE id = ?2",
|
||||
params![resolved_at, id],
|
||||
)?;
|
||||
Ok(Approval {
|
||||
id,
|
||||
agent: row.agent,
|
||||
kind: kind_from_str(&row.kind)?,
|
||||
commit_ref: row.commit_ref,
|
||||
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
|
||||
status: ApprovalStatus::Approved,
|
||||
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
|
||||
note: None,
|
||||
fetched_sha: row.fetched_sha,
|
||||
description: row.description,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn mark_denied(&self, id: i64, note: Option<&str>) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let affected = conn.execute(
|
||||
"UPDATE approvals SET status = 'denied', resolved_at = ?1, note = ?2
|
||||
WHERE id = ?3 AND status = 'pending'",
|
||||
params![now_unix(), note, id],
|
||||
)?;
|
||||
if affected == 0 {
|
||||
bail!("approval {id} not pending");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn mark_failed(&self, id: i64, note: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2 WHERE id = ?3",
|
||||
params![now_unix(), note, id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Withdraw a pending approval. Returns the now-updated
|
||||
/// row so the caller can emit `ApprovalResolved` with the right
|
||||
/// kind / agent / sha. Errors if the approval isn't pending — once
|
||||
/// it's approved/denied/failed/cancelled, the resolution is final.
|
||||
pub fn mark_cancelled(&self, id: i64, canceller: &str) -> Result<Approval> {
|
||||
let mut conn = self.conn.lock().unwrap();
|
||||
let tx = conn.transaction()?;
|
||||
let row: Option<ApprovalLookup> = tx
|
||||
.query_row(
|
||||
ApprovalLookup::SELECT,
|
||||
params![id],
|
||||
ApprovalLookup::from_row,
|
||||
)
|
||||
.optional()?;
|
||||
let Some(row) = row else {
|
||||
bail!("approval {id} not found");
|
||||
};
|
||||
if row.status != "pending" {
|
||||
bail!("approval {id} is {}, not pending", row.status);
|
||||
}
|
||||
let resolved_at = now_unix();
|
||||
let note = format!("cancelled by {canceller}");
|
||||
tx.execute(
|
||||
"UPDATE approvals SET status = 'cancelled', resolved_at = ?1, note = ?2 WHERE id = ?3",
|
||||
params![resolved_at, note, id],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(Approval {
|
||||
id,
|
||||
agent: row.agent,
|
||||
kind: kind_from_str(&row.kind)?,
|
||||
commit_ref: row.commit_ref,
|
||||
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
|
||||
status: ApprovalStatus::Cancelled,
|
||||
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
|
||||
note: Some(note),
|
||||
fetched_sha: row.fetched_sha,
|
||||
description: row.description,
|
||||
})
|
||||
}
|
||||
|
||||
/// Mark every pending approval for `agent` as failed (returns rows affected).
|
||||
/// Used by `destroy` to clear the queue of an agent that no longer exists.
|
||||
pub fn fail_pending_for_agent(&self, agent: &str, note: &str) -> Result<usize> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let n = conn.execute(
|
||||
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2
|
||||
WHERE agent = ?3 AND status = 'pending'",
|
||||
params![now_unix(), note, agent],
|
||||
)?;
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Columns needed to rebuild an [`Approval`] after a status transition,
|
||||
/// shared by `mark_approved` / `mark_cancelled`. Replaces a 7-field
|
||||
/// tuple that tripped `clippy::type_complexity` and was duplicated
|
||||
/// across both callers (one suppressed the lint, the other aliased the
|
||||
/// tuple) — one named projection + mapper now backs both.
|
||||
struct ApprovalLookup {
|
||||
agent: String,
|
||||
kind: String,
|
||||
commit_ref: String,
|
||||
requested_at: i64,
|
||||
status: String,
|
||||
fetched_sha: Option<String>,
|
||||
description: Option<String>,
|
||||
}
|
||||
|
||||
impl ApprovalLookup {
|
||||
/// The single-row lookup by id (`?1`). Column order matches
|
||||
/// [`ApprovalLookup::from_row`].
|
||||
const SELECT: &str = "SELECT agent, kind, commit_ref, requested_at, status, fetched_sha, \
|
||||
description FROM approvals WHERE id = ?1";
|
||||
|
||||
fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
|
||||
Ok(Self {
|
||||
agent: row.get(0)?,
|
||||
kind: row.get(1)?,
|
||||
commit_ref: row.get(2)?,
|
||||
requested_at: row.get(3)?,
|
||||
status: row.get(4)?,
|
||||
fetched_sha: row.get(5)?,
|
||||
description: row.get(6)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect approval rows, dropping (and logging) any that fail to
|
||||
/// deserialize. A single malformed / unknown-kind row must never blank
|
||||
/// the whole list: `collect::<Result<Vec>>()` is all-or-nothing, so one
|
||||
/// bad row used to make `pending()` / `recent_resolved()` error out
|
||||
/// wholesale — the dashboard then rendered an empty approvals queue.
|
||||
fn collect_lenient(rows: impl Iterator<Item = rusqlite::Result<Approval>>) -> Vec<Approval> {
|
||||
rows.filter_map(|r| match r {
|
||||
Ok(a) => Some(a),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "skipping unparseable approval row");
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn row_to_approval(row: &rusqlite::Row<'_>) -> rusqlite::Result<Approval> {
|
||||
// Column order: id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description.
|
||||
let kind: String = row.get(2)?;
|
||||
let kind = match kind.as_str() {
|
||||
"apply_commit" => ApprovalKind::ApplyCommit,
|
||||
"spawn" => ApprovalKind::Spawn,
|
||||
"init_config" => ApprovalKind::InitConfig,
|
||||
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
|
||||
"schedule_prompt" => ApprovalKind::SchedulePrompt,
|
||||
"merge_config_pr" => ApprovalKind::MergeConfigPr,
|
||||
other => {
|
||||
return Err(rusqlite::Error::FromSqlConversionFailure(
|
||||
2,
|
||||
rusqlite::types::Type::Text,
|
||||
format!("unknown approval kind '{other}'").into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let status: String = row.get(5)?;
|
||||
let status = match status.as_str() {
|
||||
"pending" => ApprovalStatus::Pending,
|
||||
"approved" => ApprovalStatus::Approved,
|
||||
"denied" => ApprovalStatus::Denied,
|
||||
"failed" => ApprovalStatus::Failed,
|
||||
"cancelled" => ApprovalStatus::Cancelled,
|
||||
other => {
|
||||
return Err(rusqlite::Error::FromSqlConversionFailure(
|
||||
5,
|
||||
rusqlite::types::Type::Text,
|
||||
format!("unknown approval status '{other}'").into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(Approval {
|
||||
id: row.get(0)?,
|
||||
agent: row.get(1)?,
|
||||
kind,
|
||||
commit_ref: row.get(3)?,
|
||||
requested_at: hive_sh4re::wire_time::from_secs(row.get(4)?),
|
||||
status,
|
||||
resolved_at: row
|
||||
.get::<_, Option<i64>>(6)?
|
||||
.map(hive_sh4re::wire_time::from_secs),
|
||||
note: row.get(7)?,
|
||||
fetched_sha: row.get(8)?,
|
||||
description: row.get(9)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn kind_from_str(s: &str) -> Result<ApprovalKind> {
|
||||
Ok(match s {
|
||||
"apply_commit" => ApprovalKind::ApplyCommit,
|
||||
"spawn" => ApprovalKind::Spawn,
|
||||
"init_config" => ApprovalKind::InitConfig,
|
||||
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
|
||||
"schedule_prompt" => ApprovalKind::SchedulePrompt,
|
||||
"merge_config_pr" => ApprovalKind::MergeConfigPr,
|
||||
other => bail!("unknown approval kind '{other}'"),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hive_sh4re::ApprovalKind;
|
||||
|
||||
fn open_temp() -> (tempfile::TempDir, std::path::PathBuf, Approvals) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("approvals.sqlite");
|
||||
let db = Approvals::open(&path).expect("open approvals db");
|
||||
(dir, path, db)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_config_approval_round_trips() {
|
||||
// Regression test: an `init_config` row used to fail
|
||||
// deserialization (row_to_approval matched only apply_commit +
|
||||
// spawn), erroring out the whole `pending()` query — every
|
||||
// approval then vanished from the dashboard.
|
||||
let (_dir, _path, db) = open_temp();
|
||||
let id = db
|
||||
.submit_kind(
|
||||
"bitburner",
|
||||
ApprovalKind::InitConfig,
|
||||
"",
|
||||
Some("scaffold"),
|
||||
"bitburner",
|
||||
)
|
||||
.expect("submit init_config");
|
||||
let pending = db
|
||||
.pending()
|
||||
.expect("pending() must not error on an init_config row");
|
||||
assert_eq!(pending.len(), 1);
|
||||
assert_eq!(pending[0].id, id);
|
||||
assert!(matches!(pending[0].kind, ApprovalKind::InitConfig));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_kinds_all_listed() {
|
||||
let (_dir, _path, db) = open_temp();
|
||||
db.submit_kind("a", ApprovalKind::ApplyCommit, "deadbeef", None, "a")
|
||||
.unwrap();
|
||||
db.submit_kind("b", ApprovalKind::Spawn, "", None, "b")
|
||||
.unwrap();
|
||||
db.submit_kind("c", ApprovalKind::InitConfig, "", None, "c")
|
||||
.unwrap();
|
||||
let pending = db.pending().expect("pending");
|
||||
assert_eq!(pending.len(), 3, "all three kinds must be visible");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mark_cancelled_transitions_pending_row() {
|
||||
// Manager withdraws a pending approval. Row leaves pending(),
|
||||
// shows up in recent_resolved() with the cancelled status + a
|
||||
// "cancelled by <who>" note.
|
||||
let (_dir, _path, db) = open_temp();
|
||||
let id = db
|
||||
.submit_kind(
|
||||
"bitburner",
|
||||
ApprovalKind::ApplyCommit,
|
||||
"cafef00d",
|
||||
Some("test"),
|
||||
"bitburner",
|
||||
)
|
||||
.unwrap();
|
||||
let row = db.mark_cancelled(id, "manager").expect("cancel");
|
||||
assert_eq!(row.id, id);
|
||||
assert!(matches!(row.status, ApprovalStatus::Cancelled));
|
||||
assert_eq!(row.note.as_deref(), Some("cancelled by manager"));
|
||||
assert!(row.resolved_at.is_some());
|
||||
assert!(db.pending().unwrap().is_empty(), "row leaves pending");
|
||||
let resolved = db.recent_resolved(10).unwrap();
|
||||
assert_eq!(resolved.len(), 1);
|
||||
assert!(matches!(resolved[0].status, ApprovalStatus::Cancelled));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mark_cancelled_refuses_already_resolved_row() {
|
||||
// Once approved/denied/failed/cancelled the resolution is
|
||||
// final — re-cancelling errors instead of silently overwriting.
|
||||
let (_dir, _path, db) = open_temp();
|
||||
let id = db
|
||||
.submit_kind("a", ApprovalKind::Spawn, "deadbeef", None, "a")
|
||||
.unwrap();
|
||||
db.mark_cancelled(id, "manager").expect("first cancel");
|
||||
let err = db
|
||||
.mark_cancelled(id, "manager")
|
||||
.expect_err("second cancel must fail");
|
||||
assert!(err.to_string().contains("not pending"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_kind_row_is_skipped_not_fatal() {
|
||||
// A single malformed / future-kind row must not blank the
|
||||
// whole list — collect_lenient skips it instead of failing.
|
||||
let (_dir, path, db) = open_temp();
|
||||
let good = db
|
||||
.submit_kind("good", ApprovalKind::ApplyCommit, "cafe", None, "good")
|
||||
.unwrap();
|
||||
let raw = Connection::open(&path).unwrap();
|
||||
raw.execute(
|
||||
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
|
||||
VALUES ('weird', 'from_the_future', '', 0, 'pending')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let pending = db
|
||||
.pending()
|
||||
.expect("pending() must survive an unparseable row");
|
||||
assert_eq!(pending.len(), 1);
|
||||
assert_eq!(pending[0].id, good);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn submitter_round_trips_and_legacy_reads_none() {
|
||||
// A submitted approval records its submitter; a legacy row
|
||||
// (inserted without the column) reads back as None so callers
|
||||
// fall back to the root agent.
|
||||
let (_dir, path, db) = open_temp();
|
||||
let id = db
|
||||
.submit_kind("child", ApprovalKind::ApplyCommit, "cafe", None, "parent")
|
||||
.unwrap();
|
||||
assert_eq!(db.submitter_of(id).unwrap().as_deref(), Some("parent"));
|
||||
|
||||
let raw = Connection::open(&path).unwrap();
|
||||
raw.execute(
|
||||
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
|
||||
VALUES ('old', 'apply_commit', '', 0, 'pending')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let legacy_id = raw.last_insert_rowid();
|
||||
assert_eq!(db.submitter_of(legacy_id).unwrap(), None);
|
||||
}
|
||||
}
|
||||
332
hive-c0re/src/stores/audit_log.rs
Normal file
332
hive-c0re/src/stores/audit_log.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
//! Sqlite-backed audit trail of agent-initiated privileged actions.
|
||||
//!
|
||||
//! Surfaces, durably and operator-visibly, the privileged operations
|
||||
//! hive-c0re performs *on behalf of an agent* — the ones that cross the
|
||||
//! agent/operator trust boundary and so warrant a who/what/when record
|
||||
//! beyond hive-priv's low-level journal trace. First entry: infra
|
||||
//! container restarts via the `infra_admin`-gated `restart` tool (the
|
||||
//! follow-up audit trail for that capability).
|
||||
//!
|
||||
//! Deliberately scoped to *agent-initiated* privileged actions. The bulk
|
||||
//! of `PrivRequest` traffic (token writes, nspawn-flag edits) fires
|
||||
//! constantly during normal lifecycle and is hive-c0re's own bookkeeping,
|
||||
//! not an agent crossing the boundary — logging all of it would drown the
|
||||
//! signal the operator actually wants.
|
||||
//!
|
||||
//! Same process-singleton handle pattern as `build_logs`: installed once
|
||||
//! at `Coordinator::open`, fetched via [`global`] so the recording sites
|
||||
//! (e.g. `socket_server::handle_restart_infra`) don't have to thread an
|
||||
//! `Arc<AuditLog>` through every call path. Recording is best-effort: a
|
||||
//! sqlite blip must never fail the underlying privileged action.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::Serialize;
|
||||
|
||||
/// Process-singleton handle, set once at coordinator startup. Mirrors
|
||||
/// `build_logs::GLOBAL` — lets recording sites write without threading an
|
||||
/// `Arc<AuditLog>` through every entry point.
|
||||
static GLOBAL: OnceLock<Arc<AuditLog>> = OnceLock::new();
|
||||
|
||||
/// Install the process-wide `AuditLog` handle. Idempotent: a second call
|
||||
/// silently keeps the first handle.
|
||||
pub fn install(handle: Arc<AuditLog>) {
|
||||
let _ = GLOBAL.set(handle);
|
||||
}
|
||||
|
||||
/// Fetch the process-wide handle, or `None` if `install` hasn't run yet
|
||||
/// (early startup, or unit tests). Callers must gracefully no-op on `None`.
|
||||
#[must_use]
|
||||
pub fn global() -> Option<Arc<AuditLog>> {
|
||||
GLOBAL.get().cloned()
|
||||
}
|
||||
|
||||
/// Retain audit rows for 90 days. Longer than build-log retention — this
|
||||
/// is a security/accountability record, not debug noise; the operator may
|
||||
/// want to review "who restarted what" well after the fact.
|
||||
const KEEP_SECS: i64 = 90 * 24 * 3600;
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts_unix INTEGER NOT NULL,
|
||||
agent TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
outcome TEXT NOT NULL,
|
||||
detail TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_log_ts ON audit_log (ts_unix DESC);
|
||||
";
|
||||
|
||||
/// Outcome of a recorded privileged action. Stored as the literal string
|
||||
/// in the `outcome` column.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AuditOutcome {
|
||||
/// The privileged action succeeded.
|
||||
Ok,
|
||||
/// The privileged action was attempted but failed (e.g. the
|
||||
/// underlying systemctl call errored). Denied-by-capability attempts
|
||||
/// are recorded too — see the recording site.
|
||||
Err,
|
||||
}
|
||||
|
||||
impl AuditOutcome {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ok => "ok",
|
||||
Self::Err => "err",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One audit row as returned to the dashboard.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct AuditEntry {
|
||||
pub id: i64,
|
||||
pub ts_unix: DateTime<Utc>,
|
||||
/// Agent on whose behalf the action was taken.
|
||||
pub agent: String,
|
||||
/// What was done (e.g. `restart_infra`).
|
||||
pub action: String,
|
||||
/// What it acted on (e.g. `hive-ci`).
|
||||
pub target: String,
|
||||
/// `"ok"` | `"err"`.
|
||||
pub outcome: String,
|
||||
/// Optional free-text detail (e.g. the error message on failure).
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
/// Sqlite-backed audit-log store. `Arc<AuditLog>`-friendly: all methods
|
||||
/// take `&self`, an internal `Mutex<Connection>` serializes access.
|
||||
pub struct AuditLog {
|
||||
conn: Mutex<Connection>,
|
||||
}
|
||||
|
||||
impl AuditLog {
|
||||
/// Open (creating if absent) the `audit_log.sqlite` store under
|
||||
/// `db_dir` and apply the schema. `db_dir` is shared with
|
||||
/// `build_logs` (the broker db's parent directory).
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the directory can't be created, the sqlite
|
||||
/// file can't be opened, or applying the schema fails.
|
||||
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||
let path = db_dir.join("audit_log.sqlite");
|
||||
let conn = crate::db::open(&path, "audit_log")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply audit_log schema")?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record one privileged action. Best-effort: a sqlite error is logged
|
||||
/// but never returned, so a transient blip never fails the underlying
|
||||
/// privileged action (the action already happened — losing its audit
|
||||
/// row is strictly less bad than failing the action retroactively).
|
||||
///
|
||||
/// Returns the inserted [`AuditEntry`] (with its assigned id +
|
||||
/// timestamp) on success, or `None` if the insert failed. The
|
||||
/// returned row is the canonical record — callers that also push a
|
||||
/// live event (e.g. the dashboard stream) emit *this* rather than
|
||||
/// re-deriving the fields, so the stored row and the streamed event
|
||||
/// can't drift.
|
||||
#[must_use]
|
||||
pub fn record(
|
||||
&self,
|
||||
agent: &str,
|
||||
action: &str,
|
||||
target: &str,
|
||||
outcome: AuditOutcome,
|
||||
detail: Option<&str>,
|
||||
) -> Option<AuditEntry> {
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
match conn.execute(
|
||||
"INSERT INTO audit_log (ts_unix, agent, action, target, outcome, detail)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![now, agent, action, target, outcome.as_str(), detail],
|
||||
) {
|
||||
Ok(_) => Some(AuditEntry {
|
||||
id: conn.last_insert_rowid(),
|
||||
ts_unix: hive_sh4re::wire_time::from_secs(now),
|
||||
agent: agent.to_owned(),
|
||||
action: action.to_owned(),
|
||||
target: target.to_owned(),
|
||||
outcome: outcome.as_str().to_owned(),
|
||||
detail: detail.map(str::to_owned),
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
%agent, %action, %target,
|
||||
error = ?e,
|
||||
"audit_log: record failed (dropping entry)"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the most recent `limit` rows, newest first. Limit is
|
||||
/// hard-clamped to 500 to bound the worst-case payload.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the query fails to prepare or a row fails to
|
||||
/// deserialize.
|
||||
pub fn list_recent(&self, limit: usize) -> Result<Vec<AuditEntry>> {
|
||||
let limit = limit.min(500);
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, ts_unix, agent, action, target, outcome, detail
|
||||
FROM audit_log
|
||||
ORDER BY ts_unix DESC, id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(500)], row_to_entry)?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Total row count, regardless of the `list_recent` clamp. Lets the
|
||||
/// dashboard show "latest N of TOTAL" instead of silently capping.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the `COUNT(*)` query fails.
|
||||
pub fn count_total(&self) -> Result<i64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let n: i64 = conn.query_row("SELECT COUNT(*) FROM audit_log", [], |r| r.get(0))?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Drop rows older than the retention window. Returns the number of
|
||||
/// rows deleted. Called from the hourly vacuum loop.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the `DELETE` query fails.
|
||||
pub fn vacuum(&self) -> Result<u64> {
|
||||
let cutoff = now_unix() - KEEP_SECS;
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let removed = conn.execute("DELETE FROM audit_log WHERE ts_unix < ?1", params![cutoff])?;
|
||||
Ok(u64::try_from(removed).unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the hourly retention sweep. Mirrors `build_logs::spawn_vacuum`
|
||||
/// in cadence + shutdown handling.
|
||||
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
|
||||
use std::time::Duration;
|
||||
let audit = coord.audit_log.clone();
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
let interval = Duration::from_hours(1);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match audit.vacuum() {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!(removed = n, "audit_log vacuum"),
|
||||
Err(e) => tracing::warn!(error = ?e, "audit_log vacuum failed"),
|
||||
}
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(interval) => {}
|
||||
_ = shutdown.changed() => {
|
||||
tracing::info!("audit_log vacuum: shutdown signal received");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn row_to_entry(r: &rusqlite::Row) -> rusqlite::Result<AuditEntry> {
|
||||
Ok(AuditEntry {
|
||||
id: r.get(0)?,
|
||||
ts_unix: hive_sh4re::wire_time::from_secs(r.get(1)?),
|
||||
agent: r.get(2)?,
|
||||
action: r.get(3)?,
|
||||
target: r.get(4)?,
|
||||
outcome: r.get(5)?,
|
||||
detail: r.get(6)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tmpdb() -> (tempfile::TempDir, AuditLog) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = AuditLog::open(dir.path()).expect("open");
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_list_newest_first() {
|
||||
let (_d, db) = tmpdb();
|
||||
// record() returns the canonical inserted row (id + ts assigned).
|
||||
let entry = db
|
||||
.record("atlas", "restart_infra", "hive-ci", AuditOutcome::Ok, None)
|
||||
.expect("record returns the inserted entry");
|
||||
assert!(entry.id > 0);
|
||||
assert_eq!(entry.target, "hive-ci");
|
||||
assert_eq!(entry.outcome, "ok");
|
||||
assert!(entry.detail.is_none());
|
||||
let _ = db.record(
|
||||
"atlas",
|
||||
"restart_infra",
|
||||
"hive-gateway",
|
||||
AuditOutcome::Err,
|
||||
Some("systemctl failed"),
|
||||
);
|
||||
let rows = db.list_recent(10).expect("list");
|
||||
assert_eq!(rows.len(), 2);
|
||||
// Newest first: the gateway/err row was inserted last.
|
||||
assert_eq!(rows[0].target, "hive-gateway");
|
||||
assert_eq!(rows[0].outcome, "err");
|
||||
assert_eq!(rows[0].detail.as_deref(), Some("systemctl failed"));
|
||||
assert_eq!(rows[1].target, "hive-ci");
|
||||
assert_eq!(rows[1].outcome, "ok");
|
||||
assert!(rows[1].detail.is_none());
|
||||
assert_eq!(rows[0].agent, "atlas");
|
||||
assert_eq!(rows[0].action, "restart_infra");
|
||||
assert_eq!(db.count_total().expect("count"), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_clamps_to_500() {
|
||||
let (_d, db) = tmpdb();
|
||||
let _ = db.record("a", "x", "t", AuditOutcome::Ok, None);
|
||||
let rows = db.list_recent(999_999).expect("list");
|
||||
assert!(rows.len() <= 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vacuum_drops_only_old_rows() {
|
||||
let (_d, db) = tmpdb();
|
||||
let _ = db.record("a", "restart_infra", "hive-ci", AuditOutcome::Ok, None);
|
||||
// Backdate it past the retention window.
|
||||
{
|
||||
let conn = db.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE audit_log SET ts_unix = ?1",
|
||||
params![now_unix() - KEEP_SECS - 60],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let _ = db.record("a", "restart_infra", "hive-forge", AuditOutcome::Ok, None);
|
||||
let removed = db.vacuum().expect("vacuum");
|
||||
assert_eq!(removed, 1, "only the backdated row should be reaped");
|
||||
let rows = db.list_recent(10).expect("list");
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(rows[0].target, "hive-forge");
|
||||
}
|
||||
}
|
||||
1578
hive-c0re/src/stores/broker.rs
Normal file
1578
hive-c0re/src/stores/broker.rs
Normal file
File diff suppressed because it is too large
Load diff
565
hive-c0re/src/stores/build_logs.rs
Normal file
565
hive-c0re/src/stores/build_logs.rs
Normal file
|
|
@ -0,0 +1,565 @@
|
|||
//! Sqlite-backed full build-log capture — stdout + stderr per
|
||||
//! `nixos-container` / `nix build` invocation, accumulated live.
|
||||
//! Schema, indices, retention, and the rationale for replacing
|
||||
//! the old ring buffer: `docs/persistence.md::/var/lib/hyperhive/db/build_logs.sqlite`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
/// Process-singleton handle, set once at coordinator startup. Lets
|
||||
/// the `lifecycle` module's `run` / `prebuild_toplevel` access the
|
||||
/// writer without threading an `Arc<BuildLogs>` through every
|
||||
/// `pub async fn` entry point in the lifecycle surface — there are
|
||||
/// 10+ callsites and the handle is the same `Arc` everywhere
|
||||
/// anyway. Set by `Coordinator::open`.
|
||||
static GLOBAL: OnceLock<Arc<BuildLogs>> = OnceLock::new();
|
||||
|
||||
/// Install the process-wide `BuildLogs` handle. Idempotent: a second
|
||||
/// call (e.g. test harness setup) silently keeps the first handle.
|
||||
pub fn install(handle: Arc<BuildLogs>) {
|
||||
let _ = GLOBAL.set(handle);
|
||||
}
|
||||
|
||||
/// Fetch the process-wide handle, or `None` if `install` hasn't run
|
||||
/// yet (e.g. early in startup before `Coordinator::open`, or in unit
|
||||
/// tests that don't bother with the global). Callers must gracefully
|
||||
/// no-op when this returns `None`.
|
||||
#[must_use]
|
||||
pub fn global() -> Option<Arc<BuildLogs>> {
|
||||
GLOBAL.get().cloned()
|
||||
}
|
||||
|
||||
/// Retain failed-build rows for 30 days — they're what the operator
|
||||
/// needs to investigate when diagnosing a regression.
|
||||
const KEEP_FAIL_SECS: i64 = 30 * 24 * 3600;
|
||||
|
||||
/// Retain successful-build rows for 24 hours — useful for diffing
|
||||
/// what changed across a recent rebuild, but past a day the log is
|
||||
/// noise. In-progress rows are never reaped (they have
|
||||
/// `finished_at IS NULL`).
|
||||
const KEEP_OK_SECS: i64 = 24 * 3600;
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS build_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
agent TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
cmdline TEXT NOT NULL,
|
||||
started_at INTEGER NOT NULL,
|
||||
finished_at INTEGER,
|
||||
status TEXT,
|
||||
stdout TEXT NOT NULL DEFAULT '',
|
||||
stderr TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_build_logs_agent_started
|
||||
ON build_logs (agent, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_build_logs_status_finished
|
||||
ON build_logs (status, finished_at)
|
||||
WHERE finished_at IS NOT NULL;
|
||||
";
|
||||
|
||||
/// Status of a finished build attempt. Stored as the literal string in
|
||||
/// the `status` column; `NULL` while the attempt is still in progress.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BuildStatus {
|
||||
/// Child exited with success.
|
||||
Ok,
|
||||
/// Child exited non-zero (build / eval failure).
|
||||
Fail,
|
||||
}
|
||||
|
||||
impl BuildStatus {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ok => "ok",
|
||||
Self::Fail => "fail",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Header-only row returned by `list_recent_for_agent`. Carries the
|
||||
/// metadata the dashboard's agent-card chip needs (status + age +
|
||||
/// id-to-open) without the multi-MB stdout/stderr payload.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct BuildLogHeader {
|
||||
pub id: i64,
|
||||
pub agent: String,
|
||||
pub kind: String,
|
||||
pub cmdline: String,
|
||||
pub started_at: i64,
|
||||
pub finished_at: Option<i64>,
|
||||
pub status: Option<String>,
|
||||
/// Elapsed seconds from `started_at` to `finished_at`. `None` while
|
||||
/// the build is still in progress.
|
||||
pub runtime_secs: Option<i64>,
|
||||
}
|
||||
|
||||
/// Full row with stdout/stderr text inlined. Returned by `get_full`,
|
||||
/// backs the side-panel viewer's payload.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct BuildLogFull {
|
||||
#[serde(flatten)]
|
||||
pub header: BuildLogHeader,
|
||||
pub stdout: String,
|
||||
pub stderr: String,
|
||||
}
|
||||
|
||||
/// Incremental text returned by `get_progress`. Carries only the new
|
||||
/// bytes since the caller's last cursor positions so the SSE stream
|
||||
/// handler can send deltas without re-transmitting the full log.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BuildLogProgress {
|
||||
/// New stdout bytes beyond `stdout_cursor`.
|
||||
pub stdout_append: String,
|
||||
/// New stderr bytes beyond `stderr_cursor`.
|
||||
pub stderr_append: String,
|
||||
/// `Some(unix_ts)` once the build is finished.
|
||||
pub finished_at: Option<i64>,
|
||||
/// Terminal status string (`"ok"` / `"fail"`) once finished.
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
/// Channel capacity for per-build append notifications. 64 slots is
|
||||
/// plenty — the consumer reads fast relative to line-append rate and
|
||||
/// any lag means "read now, you have new content" rather than a lost
|
||||
/// data line.
|
||||
const NOTIFY_CAP: usize = 64;
|
||||
|
||||
/// Sqlite-backed build-log store. `Arc<BuildLogs>`-friendly: all
|
||||
/// methods take `&self`, internal `Mutex<Connection>` serializes
|
||||
/// access.
|
||||
pub struct BuildLogs {
|
||||
conn: Mutex<Connection>,
|
||||
/// Broadcast channel that fires with the `id` of the row that just
|
||||
/// had a line appended or was finished. The SSE stream handler
|
||||
/// subscribes once per open panel and drives delta reads from this.
|
||||
/// `send()` is non-async and silently drops frames when there are
|
||||
/// no subscribers — safe to call from sync append/finish paths.
|
||||
notify_tx: broadcast::Sender<i64>,
|
||||
}
|
||||
|
||||
impl BuildLogs {
|
||||
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||
let path = db_dir.join("build_logs.sqlite");
|
||||
let conn = crate::db::open(&path, "build_logs")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply build_logs schema")?;
|
||||
let (notify_tx, _) = broadcast::channel(NOTIFY_CAP);
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
notify_tx,
|
||||
})
|
||||
}
|
||||
|
||||
/// Subscribe to per-build append/finish notifications. Each emitted
|
||||
/// value is the `id` of the row that changed. The SSE stream handler
|
||||
/// calls this once and filters for its target id.
|
||||
pub fn subscribe_notifications(&self) -> broadcast::Receiver<i64> {
|
||||
self.notify_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Open a row for a new build attempt. Returns the assigned id
|
||||
/// — the caller threads it through `append_stdout` / `append_stderr`
|
||||
/// while the child runs and into `finish` once it exits.
|
||||
pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result<i64> {
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)",
|
||||
params![agent, kind, cmdline, now],
|
||||
)
|
||||
.context("insert build_logs row")?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
/// Append a single stdout line. Best-effort: errors are logged
|
||||
/// but never returned to the caller, so a transient sqlite blip
|
||||
/// never tears down a rebuild's stdout pump.
|
||||
pub fn append_stdout(&self, id: i64, line: &str) {
|
||||
self.append(id, "stdout", line);
|
||||
}
|
||||
|
||||
/// Append a single stderr line. Same best-effort contract as
|
||||
/// `append_stdout`.
|
||||
pub fn append_stderr(&self, id: i64, line: &str) {
|
||||
self.append(id, "stderr", line);
|
||||
}
|
||||
|
||||
fn append(&self, id: i64, column: &'static str, line: &str) {
|
||||
// `column` is hard-coded by the caller (`stdout` / `stderr`)
|
||||
// — never user-supplied — so the string-format here is safe
|
||||
// and lets us reuse one helper for both streams.
|
||||
let sql = format!("UPDATE build_logs SET {column} = {column} || ?1 || x'0a' WHERE id = ?2");
|
||||
let conn = self.conn.lock().unwrap();
|
||||
if let Err(e) = conn.execute(&sql, params![line, id]) {
|
||||
tracing::warn!(
|
||||
build_log_id = id,
|
||||
column = column,
|
||||
error = ?e,
|
||||
"build_logs: append failed (dropping line)"
|
||||
);
|
||||
}
|
||||
drop(conn);
|
||||
// Notify SSE stream subscribers — non-blocking, no-op when no
|
||||
// subscribers are watching (e.g. no panel is open). Lagged
|
||||
// receivers (channel full) automatically drop frames; the SSE
|
||||
// handler re-reads the full delta on the next notification it
|
||||
// does receive, so no content is lost, only an intermediate
|
||||
// wake-up is coalesced.
|
||||
let _ = self.notify_tx.send(id);
|
||||
}
|
||||
|
||||
/// Finalize a build attempt. Sets `finished_at` to now and
|
||||
/// `status` to the terminal state. Best-effort.
|
||||
pub fn finish(&self, id: i64, status: BuildStatus) {
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
if let Err(e) = conn.execute(
|
||||
"UPDATE build_logs SET finished_at = ?1, status = ?2 WHERE id = ?3",
|
||||
params![now, status.as_str(), id],
|
||||
) {
|
||||
tracing::warn!(
|
||||
build_log_id = id,
|
||||
error = ?e,
|
||||
"build_logs: finish failed"
|
||||
);
|
||||
}
|
||||
drop(conn);
|
||||
// Final notification so the SSE stream handler sees the
|
||||
// finished_at and status, closes the connection cleanly.
|
||||
let _ = self.notify_tx.send(id);
|
||||
}
|
||||
|
||||
/// Return incremental log content beyond the given byte cursors.
|
||||
/// Used by the SSE stream handler to compute deltas between polls.
|
||||
///
|
||||
/// `stdout_cursor` / `stderr_cursor` are byte offsets into the
|
||||
/// stored `stdout` / `stderr` columns from the previous read.
|
||||
/// Slicing is safe because cursors are always derived from prior
|
||||
/// `String::len()` values (valid UTF-8 boundaries).
|
||||
///
|
||||
/// Returns `None` when the row no longer exists (vacuum reap during
|
||||
/// a long-open panel).
|
||||
pub fn get_progress(
|
||||
&self,
|
||||
id: i64,
|
||||
stdout_cursor: usize,
|
||||
stderr_cursor: usize,
|
||||
) -> Result<Option<BuildLogProgress>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT stdout, stderr, finished_at, status \
|
||||
FROM build_logs WHERE id = ?1",
|
||||
)?;
|
||||
let row = stmt
|
||||
.query_row(params![id], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, Option<i64>>(2)?,
|
||||
r.get::<_, Option<String>>(3)?,
|
||||
))
|
||||
})
|
||||
.optional()?;
|
||||
match row {
|
||||
None => Ok(None),
|
||||
Some((stdout, stderr, finished_at, status)) => {
|
||||
let stdout_append = stdout.get(stdout_cursor..).unwrap_or("").to_string();
|
||||
let stderr_append = stderr.get(stderr_cursor..).unwrap_or("").to_string();
|
||||
Ok(Some(BuildLogProgress {
|
||||
stdout_append,
|
||||
stderr_append,
|
||||
finished_at,
|
||||
status,
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the most recent `limit` rows for `agent`, newest first.
|
||||
/// Headers only (no stdout/stderr blobs) — keeps `/api/state`
|
||||
/// payloads light. Limit is hard-clamped to 50 to bound worst-case
|
||||
/// payload regardless of caller input.
|
||||
pub fn list_recent_for_agent(&self, agent: &str, limit: usize) -> Result<Vec<BuildLogHeader>> {
|
||||
let limit = limit.min(50);
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
|
||||
FROM build_logs
|
||||
WHERE agent = ?1
|
||||
ORDER BY started_at DESC
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
let rows = stmt.query_map(
|
||||
params![agent, i64::try_from(limit).unwrap_or(50)],
|
||||
row_to_header,
|
||||
)?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Return the most recent `limit` rows across all agents, newest first.
|
||||
/// Same header-only shape as `list_recent_for_agent`. Limit clamped to 100.
|
||||
pub fn list_recent_all(&self, limit: usize) -> Result<Vec<BuildLogHeader>> {
|
||||
let limit = limit.min(100);
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
|
||||
FROM build_logs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(100)], row_to_header)?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Fetch a single full row (with stdout/stderr text) by id.
|
||||
/// Returns `None` when the id doesn't exist (vacuum sweep already
|
||||
/// reaped it, or the operator passed a stale id from a refresh
|
||||
/// race).
|
||||
pub fn get_full(&self, id: i64) -> Result<Option<BuildLogFull>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, agent, kind, cmdline, started_at, finished_at, status, stdout, stderr
|
||||
FROM build_logs
|
||||
WHERE id = ?1",
|
||||
)?;
|
||||
let row = stmt
|
||||
.query_row(params![id], |r| {
|
||||
let started_at: i64 = r.get(4)?;
|
||||
let finished_at: Option<i64> = r.get(5)?;
|
||||
let runtime_secs = compute_runtime_secs(started_at, finished_at);
|
||||
Ok(BuildLogFull {
|
||||
header: BuildLogHeader {
|
||||
id: r.get(0)?,
|
||||
agent: r.get(1)?,
|
||||
kind: r.get(2)?,
|
||||
cmdline: r.get(3)?,
|
||||
started_at,
|
||||
finished_at,
|
||||
status: r.get(6)?,
|
||||
runtime_secs,
|
||||
},
|
||||
stdout: r.get(7)?,
|
||||
stderr: r.get(8)?,
|
||||
})
|
||||
})
|
||||
.optional()?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Drop rows past their retention window. Returns the number of
|
||||
/// rows deleted. Called from this module's hourly `spawn` loop.
|
||||
///
|
||||
/// Rule:
|
||||
/// - `status = 'fail'` rows kept for `KEEP_FAIL_SECS` past their
|
||||
/// `finished_at` (failures are what operators dig into).
|
||||
/// - `status = 'ok'` rows kept for `KEEP_OK_SECS` past their
|
||||
/// `finished_at` (successes are mostly noise after a day).
|
||||
/// - In-flight rows (`finished_at IS NULL`) are never touched —
|
||||
/// a long-running build shouldn't disappear from its own log
|
||||
/// viewer mid-stream.
|
||||
pub fn vacuum(&self) -> Result<u64> {
|
||||
let now = now_unix();
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let fail_cutoff = now - KEEP_FAIL_SECS;
|
||||
let ok_cutoff = now - KEEP_OK_SECS;
|
||||
let removed = conn.execute(
|
||||
"DELETE FROM build_logs
|
||||
WHERE finished_at IS NOT NULL
|
||||
AND (
|
||||
(status = 'fail' AND finished_at < ?1)
|
||||
OR (status = 'ok' AND finished_at < ?2)
|
||||
)",
|
||||
params![fail_cutoff, ok_cutoff],
|
||||
)?;
|
||||
Ok(u64::try_from(removed).unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the hourly retention sweep. A host-side sweep (`build_logs.sqlite`
|
||||
/// is hive-c0re-owned, so no privsep ownership issue). Runs once at startup
|
||||
/// before its first sleep so a long-uptime instance doesn't accumulate a
|
||||
/// backlog the first hour after restart.
|
||||
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
|
||||
use std::time::Duration;
|
||||
let logs = coord.build_logs.clone();
|
||||
let mut shutdown = coord.shutdown_rx();
|
||||
let interval = Duration::from_hours(1);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match logs.vacuum() {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!(removed = n, "build_logs vacuum"),
|
||||
Err(e) => tracing::warn!(error = ?e, "build_logs vacuum failed"),
|
||||
}
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(interval) => {}
|
||||
_ = shutdown.changed() => {
|
||||
tracing::info!("build_logs vacuum: shutdown signal received");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn compute_runtime_secs(started_at: i64, finished_at: Option<i64>) -> Option<i64> {
|
||||
finished_at.map(|f| f - started_at)
|
||||
}
|
||||
|
||||
fn row_to_header(r: &rusqlite::Row) -> rusqlite::Result<BuildLogHeader> {
|
||||
let started_at: i64 = r.get(4)?;
|
||||
let finished_at: Option<i64> = r.get(5)?;
|
||||
let runtime_secs = compute_runtime_secs(started_at, finished_at);
|
||||
Ok(BuildLogHeader {
|
||||
id: r.get(0)?,
|
||||
agent: r.get(1)?,
|
||||
kind: r.get(2)?,
|
||||
cmdline: r.get(3)?,
|
||||
started_at,
|
||||
finished_at,
|
||||
status: r.get(6)?,
|
||||
runtime_secs,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tmpdb() -> (tempfile::TempDir, BuildLogs) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = BuildLogs::open(dir.path()).expect("open");
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_appends_finish_flow() {
|
||||
let (_d, db) = tmpdb();
|
||||
let id = db
|
||||
.start("alice", "prebuild", "nix build foo")
|
||||
.expect("start");
|
||||
db.append_stdout(id, "building '/nix/store/abc.drv'");
|
||||
db.append_stderr(id, "error: line 12");
|
||||
db.append_stderr(id, " at /nix/store/.../module.nix:5");
|
||||
db.finish(id, BuildStatus::Fail);
|
||||
|
||||
let full = db.get_full(id).expect("get").expect("Some");
|
||||
assert_eq!(full.header.agent, "alice");
|
||||
assert_eq!(full.header.kind, "prebuild");
|
||||
assert_eq!(full.header.status.as_deref(), Some("fail"));
|
||||
assert!(full.header.finished_at.is_some());
|
||||
assert!(full.stdout.contains("/nix/store/abc.drv"));
|
||||
assert!(full.stderr.contains("error: line 12"));
|
||||
assert!(full.stderr.contains("module.nix:5"));
|
||||
// Lines are terminator-delimited so each contributes a trailing
|
||||
// newline — the viewer joins on the existing newlines rather
|
||||
// than re-inserting them.
|
||||
assert!(full.stderr.ends_with('\n'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_recent_orders_newest_first_and_clamps() {
|
||||
let (_d, db) = tmpdb();
|
||||
// Three attempts, two for alice + one for bob. Without sleeping
|
||||
// sqlite's `INTEGER` started_at ties at 1-sec resolution, so we
|
||||
// assert id-ordering (autoincrement) is the tiebreaker — list
|
||||
// sorts by started_at DESC but the ORDER BY still produces the
|
||||
// last-inserted row first when timestamps match.
|
||||
let id_a1 = db.start("alice", "run", "cmd one").expect("start");
|
||||
let _id_b = db.start("bob", "run", "cmd two").expect("start");
|
||||
let id_a2 = db.start("alice", "run", "cmd three").expect("start");
|
||||
db.finish(id_a1, BuildStatus::Ok);
|
||||
|
||||
let alice_rows = db.list_recent_for_agent("alice", 10).expect("list");
|
||||
assert_eq!(alice_rows.len(), 2);
|
||||
// Without distinct started_at values both rows share `now`,
|
||||
// but list_recent already orders by `started_at DESC` then
|
||||
// sqlite's natural insertion-order tiebreak. We rely only on
|
||||
// both IDs being present + correct count + agent isolation.
|
||||
let ids: std::collections::HashSet<i64> = alice_rows.iter().map(|h| h.id).collect();
|
||||
assert!(ids.contains(&id_a1));
|
||||
assert!(ids.contains(&id_a2));
|
||||
|
||||
let bob_rows = db.list_recent_for_agent("bob", 10).expect("list");
|
||||
assert_eq!(bob_rows.len(), 1);
|
||||
|
||||
// Limit clamp at 50.
|
||||
let huge = db.list_recent_for_agent("alice", 999_999).expect("list");
|
||||
assert!(huge.len() <= 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_full_returns_none_for_missing_id() {
|
||||
let (_d, db) = tmpdb();
|
||||
let missing = db.get_full(999_999).expect("get");
|
||||
assert!(missing.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vacuum_drops_old_finished_only_per_status() {
|
||||
let (_d, db) = tmpdb();
|
||||
let id_fresh_fail = db.start("alice", "run", "fresh fail").expect("start");
|
||||
let id_old_fail = db.start("alice", "run", "old fail").expect("start");
|
||||
let id_old_ok = db.start("alice", "run", "old ok").expect("start");
|
||||
let id_running = db.start("alice", "run", "still running").expect("start");
|
||||
db.finish(id_fresh_fail, BuildStatus::Fail);
|
||||
db.finish(id_old_fail, BuildStatus::Fail);
|
||||
db.finish(id_old_ok, BuildStatus::Ok);
|
||||
// Backdate two rows past their retention windows. fresh_fail
|
||||
// stays within KEEP_FAIL_SECS so it survives; old_fail goes
|
||||
// beyond; old_ok goes past KEEP_OK_SECS but inside
|
||||
// KEEP_FAIL_SECS — proves the per-status rule.
|
||||
let now = now_unix();
|
||||
{
|
||||
let conn = db.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
|
||||
params![now - KEEP_FAIL_SECS - 60, id_old_fail],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
|
||||
params![now - KEEP_OK_SECS - 60, id_old_ok],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let removed = db.vacuum().expect("vacuum");
|
||||
assert_eq!(removed, 2, "old_fail + old_ok should be vacuumed");
|
||||
assert!(db.get_full(id_fresh_fail).unwrap().is_some());
|
||||
assert!(db.get_full(id_old_fail).unwrap().is_none());
|
||||
assert!(db.get_full(id_old_ok).unwrap().is_none());
|
||||
// Running row must survive vacuum regardless of retention
|
||||
// windows — finished_at IS NULL gates it out.
|
||||
assert!(db.get_full(id_running).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_after_finish_still_appends() {
|
||||
// Defensive: if a child's stdout pump fires one last line
|
||||
// between the wait-syscall returning and `finish` running, the
|
||||
// append should land on the row (status already set, but the
|
||||
// log stays consistent with what happened).
|
||||
let (_d, db) = tmpdb();
|
||||
let id = db.start("alice", "run", "cmd").expect("start");
|
||||
db.finish(id, BuildStatus::Ok);
|
||||
db.append_stdout(id, "post-finish trailing line");
|
||||
let full = db.get_full(id).expect("get").expect("Some");
|
||||
assert!(full.stdout.contains("post-finish trailing line"));
|
||||
}
|
||||
}
|
||||
58
hive-c0re/src/stores/db.rs
Normal file
58
hive-c0re/src/stores/db.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
//! Shared sqlite connection setup for hive-c0re's host-side stores.
|
||||
//!
|
||||
//! Several modules keep their own tables — and their own
|
||||
//! `Mutex<Connection>` — in the coordinator DB
|
||||
//! (`db/broker.sqlite`: broker, approvals, operator questions,
|
||||
//! scheduled prompts, agent power) or in a sibling file under the same
|
||||
//! `db/` dir (`build_logs.sqlite`, `audit_log.sqlite`). The open dance
|
||||
//! is identical everywhere: ensure the parent dir exists, open the
|
||||
//! connection, set a busy timeout so concurrent same-process writers
|
||||
//! wait each other out instead of surfacing `SQLITE_BUSY`. This helper
|
||||
//! owns that dance; schema creation + column migrations stay with each
|
||||
//! store (they're per-table concerns).
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::Connection;
|
||||
|
||||
/// How long a write waits on another connection's lock before erroring.
|
||||
/// Generous relative to the stores' tiny transactions — a timeout here
|
||||
/// means something is genuinely wedged, not ordinary contention.
|
||||
const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Apply additive, idempotent migrations: run each statement and
|
||||
/// ignore `duplicate column name` errors (sqlite has no
|
||||
/// `ADD COLUMN IF NOT EXISTS`; try-and-ignore is the portable path —
|
||||
/// same pattern as hive-ag3nt's `turn_stats`). Any other error
|
||||
/// propagates. Fits plain `ALTER TABLE ADD COLUMN` (new columns must
|
||||
/// carry a default or tolerate NULL) and `IF NOT EXISTS` index DDL;
|
||||
/// migrations with creation-conditional backfills (broker's
|
||||
/// `acked_at`) stay bespoke in their store.
|
||||
pub fn apply_migrations(conn: &Connection, subsystem: &str, statements: &[&str]) -> Result<()> {
|
||||
for stmt in statements {
|
||||
if let Err(e) = conn.execute_batch(stmt) {
|
||||
if e.to_string().contains("duplicate column name") {
|
||||
continue;
|
||||
}
|
||||
return Err(e).with_context(|| format!("{subsystem} migration failed: {stmt}"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open a connection to the sqlite file at `path`, creating the parent
|
||||
/// directory if needed. `subsystem` labels error contexts (`"broker"`,
|
||||
/// `"approvals"`, …).
|
||||
pub fn open(path: &Path, subsystem: &str) -> Result<Connection> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create {subsystem} db parent {}", parent.display()))?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("open {subsystem} db {}", path.display()))?;
|
||||
conn.busy_timeout(BUSY_TIMEOUT)
|
||||
.with_context(|| format!("set {subsystem} busy_timeout"))?;
|
||||
Ok(conn)
|
||||
}
|
||||
14
hive-c0re/src/stores/mod.rs
Normal file
14
hive-c0re/src/stores/mod.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
//! Sqlite-backed host-side stores (broker, approval / question /
|
||||
//! schedule queues, build logs, audit trail, power intent) plus the
|
||||
//! shared connection open/migration helper (`db`). Each submodule is
|
||||
//! re-exported at the crate root, so `crate::broker::…` etc. keep
|
||||
//! working unchanged.
|
||||
|
||||
pub mod approvals;
|
||||
pub mod audit_log;
|
||||
pub mod broker;
|
||||
pub mod build_logs;
|
||||
pub mod db;
|
||||
pub mod operator_questions;
|
||||
pub mod power;
|
||||
pub mod scheduled_prompts;
|
||||
273
hive-c0re/src/stores/operator_questions.rs
Normal file
273
hive-c0re/src/stores/operator_questions.rs
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
//! Question queue. Agents submit via `Ask`; the answer comes from
|
||||
//! either the operator (via the dashboard, for `target IS NULL`) or
|
||||
//! a peer agent (via `Answer`, for agent-to-agent questions).
|
||||
//!
|
||||
//! Despite the file name (kept for git history sanity), this table
|
||||
//! now stores *all* asynchronous questions in the hive — both the
|
||||
//! operator-targeted ones and the peer-to-peer ones. `target IS
|
||||
//! NULL` is the operator path (back-compat with rows written before
|
||||
//! the column existed); `target = '<agent-name>'` is the
|
||||
//! agent-to-agent path.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
|
||||
const SCHEMA: &str = r"
|
||||
CREATE TABLE IF NOT EXISTS operator_questions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asker TEXT NOT NULL,
|
||||
question TEXT NOT NULL,
|
||||
options_json TEXT NOT NULL,
|
||||
asked_at INTEGER NOT NULL,
|
||||
answered_at INTEGER,
|
||||
answer TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_operator_questions_pending
|
||||
ON operator_questions (id) WHERE answered_at IS NULL;
|
||||
";
|
||||
|
||||
/// Additive column migrations for pre-existing databases, applied via
|
||||
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||
const MIGRATIONS: &[&str] = &[
|
||||
"ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0",
|
||||
"ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER",
|
||||
// `target` = recipient of the question. NULL = operator
|
||||
// (back-compat default for rows written before agent-to-agent
|
||||
// questions existed); a non-null agent name = peer-to-peer
|
||||
// question. Dashboard's `pending()` filters on `target IS NULL`
|
||||
// so peer questions never leak into the operator's queue.
|
||||
"ALTER TABLE operator_questions ADD COLUMN target TEXT",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OpQuestion {
|
||||
pub id: i64,
|
||||
pub asker: String,
|
||||
pub question: String,
|
||||
pub options: Vec<String>,
|
||||
pub multi: bool,
|
||||
pub asked_at: DateTime<Utc>,
|
||||
/// Deadline after which a watchdog auto-resolves the question with
|
||||
/// answer `[expired]`. `None` = no expiry. Surfaced on the
|
||||
/// dashboard as a remaining-time chip.
|
||||
pub deadline_at: Option<DateTime<Utc>>,
|
||||
pub answered_at: Option<DateTime<Utc>>,
|
||||
pub answer: Option<String>,
|
||||
/// Recipient of the question. `None` = the operator (dashboard
|
||||
/// path); `Some(<agent>)` = a peer agent asked via
|
||||
/// `Ask { to: Some(<agent>), ... }`. Agent-to-agent questions
|
||||
/// never appear in `pending()` so the operator's queue stays clean.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target: Option<String>,
|
||||
}
|
||||
|
||||
pub struct OperatorQuestions {
|
||||
conn: Mutex<Connection>,
|
||||
}
|
||||
|
||||
impl OperatorQuestions {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
let conn = crate::db::open(path, "operator_questions")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply operator_questions schema")?;
|
||||
crate::db::apply_migrations(&conn, "operator_questions", MIGRATIONS)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn submit(
|
||||
&self,
|
||||
asker: &str,
|
||||
question: &str,
|
||||
options: &[String],
|
||||
multi: bool,
|
||||
deadline_at: Option<i64>,
|
||||
target: Option<&str>,
|
||||
) -> Result<i64> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let options_json = serde_json::to_string(options).unwrap_or_else(|_| "[]".into());
|
||||
conn.execute(
|
||||
"INSERT INTO operator_questions
|
||||
(asker, question, options_json, multi, deadline_at, target, asked_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||
params![
|
||||
asker,
|
||||
question,
|
||||
options_json,
|
||||
i64::from(multi),
|
||||
deadline_at,
|
||||
target,
|
||||
now_unix(),
|
||||
],
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
/// Mark a pending question answered. `answerer` is who's actually
|
||||
/// answering: `"operator"` for the dashboard path, or an agent's
|
||||
/// own name when responding via `Answer`. Authorisation:
|
||||
///
|
||||
/// - Operator-targeted questions (`target IS NULL`) can only be
|
||||
/// answered by `"operator"`. (Agents must not be able to spoof
|
||||
/// answers to operator questions — the dashboard is the
|
||||
/// privileged path.)
|
||||
/// - Agent-targeted questions can only be answered by the
|
||||
/// declared target agent, OR by `"operator"` (operator override
|
||||
/// for stuck threads — useful when an agent is offline/down
|
||||
/// and someone has to close the loop).
|
||||
///
|
||||
/// Returns `(question, asker, target)` so the caller can fire the
|
||||
/// `QuestionAnswered` event with the right answerer label and route
|
||||
/// it back to the original asker.
|
||||
pub fn answer(
|
||||
&self,
|
||||
id: i64,
|
||||
answer: &str,
|
||||
answerer: &str,
|
||||
) -> Result<(String, String, Option<String>)> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
|
||||
.query_row(
|
||||
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
|
||||
params![id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let Some((question, asker, target, answered_at)) = row else {
|
||||
bail!("question {id} not found");
|
||||
};
|
||||
if answered_at.is_some() {
|
||||
bail!("question {id} already answered");
|
||||
}
|
||||
// Authorisation check: must match the target, or be the operator
|
||||
// (operator-targeted questions are operator-only; the operator
|
||||
// can additionally override agent-to-agent questions to close
|
||||
// stuck threads).
|
||||
let authorised = match target.as_deref() {
|
||||
None => answerer == hive_sh4re::OPERATOR_RECIPIENT,
|
||||
Some(t) => answerer == t || answerer == hive_sh4re::OPERATOR_RECIPIENT,
|
||||
};
|
||||
if !authorised {
|
||||
bail!(
|
||||
"question {id} not addressed to '{answerer}' (target = {:?})",
|
||||
target.as_deref().unwrap_or(hive_sh4re::OPERATOR_RECIPIENT)
|
||||
);
|
||||
}
|
||||
conn.execute(
|
||||
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
|
||||
params![answer, now_unix(), id],
|
||||
)?;
|
||||
Ok((question, asker, target))
|
||||
}
|
||||
|
||||
/// Cancel a pending question on behalf of `canceller`. Returns
|
||||
/// `(question, asker, target)` so the caller can fire the usual
|
||||
/// `QuestionAnswered` event to the asker with a `[cancelled by
|
||||
/// <canceller>]` sentinel.
|
||||
///
|
||||
/// Auth: the canceller must be one of:
|
||||
/// - the original asker (an agent withdrawing their own ask),
|
||||
/// - the operator (already covered by the existing `answer` path
|
||||
/// but allowed here too for symmetry / dashboard cancel),
|
||||
/// - a `privileged` caller (one that arrived on the manager socket —
|
||||
/// privileged hive-wide cleanup; derived from the socket, not a
|
||||
/// name match).
|
||||
///
|
||||
/// Not the target — that's covered by `answer` (responding with
|
||||
/// an actual reply, sentinel or otherwise).
|
||||
pub fn cancel(
|
||||
&self,
|
||||
id: i64,
|
||||
canceller: &str,
|
||||
privileged: bool,
|
||||
) -> Result<(String, String, Option<String>)> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
|
||||
.query_row(
|
||||
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
|
||||
params![id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let Some((question, asker, target, answered_at)) = row else {
|
||||
bail!("question {id} not found");
|
||||
};
|
||||
if answered_at.is_some() {
|
||||
bail!("question {id} already answered/cancelled");
|
||||
}
|
||||
let authorised =
|
||||
privileged || canceller == asker || canceller == hive_sh4re::OPERATOR_RECIPIENT;
|
||||
if !authorised {
|
||||
bail!("question {id}: '{canceller}' not allowed to cancel (asker = '{asker}')");
|
||||
}
|
||||
let sentinel = format!("[cancelled by {canceller}]");
|
||||
conn.execute(
|
||||
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
|
||||
params![sentinel, now_unix(), id],
|
||||
)?;
|
||||
Ok((question, asker, target))
|
||||
}
|
||||
|
||||
/// Every pending question, operator-targeted or peer-to-peer.
|
||||
/// Drives the dashboard's questions pane now that peer threads
|
||||
/// are surfaced for visibility + operator override-answer.
|
||||
pub fn pending_all(&self) -> Result<Vec<OpQuestion>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
|
||||
FROM operator_questions
|
||||
WHERE answered_at IS NULL
|
||||
ORDER BY id ASC",
|
||||
)?;
|
||||
let rows = stmt.query_map([], row_to_question)?;
|
||||
rows.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Last `limit` answered questions across both target kinds,
|
||||
/// newest-first. Companion to `pending_all`.
|
||||
pub fn recent_answered_all(&self, limit: u64) -> Result<Vec<OpQuestion>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
|
||||
FROM operator_questions
|
||||
WHERE answered_at IS NOT NULL
|
||||
ORDER BY answered_at DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
let limit_i = i64::try_from(limit).unwrap_or(i64::MAX);
|
||||
let rows = stmt.query_map(params![limit_i], row_to_question)?;
|
||||
rows.collect::<rusqlite::Result<Vec<_>>>()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
fn row_to_question(row: &rusqlite::Row<'_>) -> rusqlite::Result<OpQuestion> {
|
||||
let options_json: String = row.get(3)?;
|
||||
let options: Vec<String> = serde_json::from_str(&options_json).unwrap_or_default();
|
||||
let multi: i64 = row.get(4)?;
|
||||
Ok(OpQuestion {
|
||||
id: row.get(0)?,
|
||||
asker: row.get(1)?,
|
||||
question: row.get(2)?,
|
||||
options,
|
||||
multi: multi != 0,
|
||||
asked_at: hive_sh4re::wire_time::from_secs(row.get(5)?),
|
||||
answered_at: row
|
||||
.get::<_, Option<i64>>(6)?
|
||||
.map(hive_sh4re::wire_time::from_secs),
|
||||
answer: row.get(7)?,
|
||||
deadline_at: row
|
||||
.get::<_, Option<i64>>(8)?
|
||||
.map(hive_sh4re::wire_time::from_secs),
|
||||
target: row.get(9)?,
|
||||
})
|
||||
}
|
||||
198
hive-c0re/src/stores/power.rs
Normal file
198
hive-c0re/src/stores/power.rs
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the
|
||||
//! spec half of spec-vs-status desired-state reconciliation.
|
||||
//! `container_view` remains the observed *status*; the job queue's
|
||||
//! `Reconcile` nodes are the mechanism that converges the two.
|
||||
//!
|
||||
//! Stored as the `agent_power` table in the coordinator DB
|
||||
//! (`/var/lib/hyperhive/db/broker.sqlite`, one tiny row per agent) —
|
||||
//! same one-file-many-modules pattern as `approvals` /
|
||||
//! `operator_questions` / `scheduled_prompts`, each with its own
|
||||
//! connection. Intent persists across hive-c0re restarts; in-flight
|
||||
//! queue work deliberately does not. Setting `wanted` is never a
|
||||
//! queued node: operator/intent actions update the row synchronously
|
||||
//! at request time, then submit the DAG whose terminal `Reconcile`
|
||||
//! reads the fresh value — rapid toggles are last-writer-wins and the
|
||||
//! reconciles converge. Power toggles never commit to the meta repo.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use hive_sh4re::wire_time::now_unix;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS agent_power (
|
||||
agent TEXT PRIMARY KEY,
|
||||
wanted TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// Per-agent power intent.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Wanted {
|
||||
Up,
|
||||
Offline,
|
||||
}
|
||||
|
||||
impl Wanted {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Wanted::Up => "up",
|
||||
Wanted::Offline => "offline",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"up" => Some(Wanted::Up),
|
||||
"offline" => Some(Wanted::Offline),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed value from an observed running state (first boot after
|
||||
/// this store lands, or an agent spawned outside the normal path).
|
||||
pub fn from_running(running: bool) -> Self {
|
||||
if running { Wanted::Up } else { Wanted::Offline }
|
||||
}
|
||||
}
|
||||
|
||||
/// What a `Reconcile` should do given intent + observation. Pure so
|
||||
/// the `{Up,Offline} × {up,down}` matrix is unit-testable.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReconcileAction {
|
||||
Start,
|
||||
Stop,
|
||||
Noop,
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction {
|
||||
match (wanted, running) {
|
||||
(Wanted::Up, false) => ReconcileAction::Start,
|
||||
(Wanted::Offline, true) => ReconcileAction::Stop,
|
||||
(Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the
|
||||
/// internal `Mutex<Connection>` serializes access.
|
||||
pub struct PowerStore {
|
||||
conn: Mutex<Connection>,
|
||||
}
|
||||
|
||||
impl PowerStore {
|
||||
/// Open (a connection to) the shared coordinator DB and ensure the
|
||||
/// `agent_power` table exists. `db_path` is the same sqlite file
|
||||
/// the broker / approvals / questions stores open.
|
||||
pub fn open(db_path: &Path) -> Result<Self> {
|
||||
let conn = crate::db::open(db_path, "agent_power")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply agent_power schema")?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
/// In-memory store for tests.
|
||||
#[cfg(test)]
|
||||
pub fn open_in_memory() -> Result<Self> {
|
||||
let conn = Connection::open_in_memory().context("open in-memory agent_power db")?;
|
||||
conn.execute_batch(SCHEMA)
|
||||
.context("apply agent_power schema")?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
})
|
||||
}
|
||||
|
||||
/// Read an agent's intent. `None` when the agent has no row yet
|
||||
/// (callers seed from observed state via [`Self::get_or_seed`]).
|
||||
pub fn get(&self, agent: &str) -> Result<Option<Wanted>> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
let row: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT wanted FROM agent_power WHERE agent = ?1",
|
||||
params![agent],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()
|
||||
.context("select agent_power")?;
|
||||
Ok(row.and_then(|s| Wanted::parse(&s)))
|
||||
}
|
||||
|
||||
/// Write an agent's intent (last-writer-wins, synchronous at
|
||||
/// request time).
|
||||
pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
conn.execute(
|
||||
"INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3)
|
||||
ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3",
|
||||
params![agent, wanted.as_str(), now_unix()],
|
||||
)
|
||||
.context("upsert agent_power")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read an agent's intent, seeding the row from the observed
|
||||
/// running state when absent — the migration rule for agents that
|
||||
/// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after
|
||||
/// which the DB is authoritative.
|
||||
pub fn get_or_seed(&self, agent: &str, running: bool) -> Result<Wanted> {
|
||||
if let Some(w) = self.get(agent)? {
|
||||
return Ok(w);
|
||||
}
|
||||
let seeded = Wanted::from_running(running);
|
||||
self.set(agent, seeded)?;
|
||||
tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state");
|
||||
Ok(seeded)
|
||||
}
|
||||
|
||||
/// Drop an agent's row (container destroyed).
|
||||
pub fn remove(&self, agent: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().expect("agent_power mutex poisoned");
|
||||
conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent])
|
||||
.context("delete agent_power")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The full `{Up,Offline} × {up,down}` reconcile matrix:
|
||||
/// start / stop / noop / noop.
|
||||
#[test]
|
||||
fn reconcile_matrix() {
|
||||
assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start);
|
||||
assert_eq!(
|
||||
reconcile_action(Wanted::Offline, true),
|
||||
ReconcileAction::Stop
|
||||
);
|
||||
assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop);
|
||||
assert_eq!(
|
||||
reconcile_action(Wanted::Offline, false),
|
||||
ReconcileAction::Noop
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_set_roundtrip_and_seed() {
|
||||
let store = PowerStore::open_in_memory().expect("open");
|
||||
assert_eq!(store.get("alice").expect("get"), None);
|
||||
// Seed from observed running state, once.
|
||||
assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up);
|
||||
// Thereafter the DB is authoritative — observed state no longer
|
||||
// overrides.
|
||||
assert_eq!(
|
||||
store.get_or_seed("alice", false).expect("seeded"),
|
||||
Wanted::Up
|
||||
);
|
||||
store.set("alice", Wanted::Offline).expect("set");
|
||||
assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline));
|
||||
store.remove("alice").expect("remove");
|
||||
assert_eq!(store.get("alice").expect("get"), None);
|
||||
}
|
||||
}
|
||||
1155
hive-c0re/src/stores/scheduled_prompts.rs
Normal file
1155
hive-c0re/src/stores/scheduled_prompts.rs
Normal file
File diff suppressed because it is too large
Load diff
341
hive-c0re/src/workers/agent_sockets.rs
Normal file
341
hive-c0re/src/workers/agent_sockets.rs
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
//! `/var/lib/hyperhive/run/agent-sockets.json` writer. Atomic
|
||||
//! `<path>.tmp` + `rename()` write so the gateway's nginx worker never
|
||||
//! reads a partial file. Includes manager and sub-agents so the gateway
|
||||
//! can route `/agent/<name>/` for all containers with a bound unix
|
||||
//! socket.
|
||||
//!
|
||||
//! Full mechanism — per-agent subdir bind-mount, `hyperhive-socket-bound`
|
||||
//! marker gate, gateway UDS upstream, 10s poll loop:
|
||||
//! `docs/gateway.md::Per-agent unix-socket upstream`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
/// Host-side parent directory holding per-agent socket subdirs. The
|
||||
/// gateway container bind-mounts this whole tree (read-only) so it
|
||||
/// can `proxy_pass` to any agent. Each agent's container bind-mounts
|
||||
/// only its own `<name>/` subdir — agents can only access their own
|
||||
/// sockets.
|
||||
pub const AGENT_SOCKET_DIR: &str = "/run/hive-agent";
|
||||
|
||||
/// Socket filename inside each per-agent subdir. Fixed so the path
|
||||
/// derives entirely from `(AGENT_SOCKET_DIR, name)` — no second
|
||||
/// degree of freedom for callers to get wrong.
|
||||
pub const SOCKET_FILENAME: &str = "web.sock";
|
||||
|
||||
/// Marker file the harness drops next to the socket after a
|
||||
/// successful `bind_unix`. Presence = "harness has bound the socket,
|
||||
/// unix upstream is live"; absence = "harness hasn't started yet or
|
||||
/// hasn't been rebuilt under the new config — keep TCP fallback".
|
||||
/// Without this gate the gateway would `proxy_pass` to a non-existent
|
||||
/// socket for an agent that's still starting up after a rebuild.
|
||||
///
|
||||
/// Renamed from `.bound` (legacy) to match the `hyperhive-` prefix
|
||||
/// convention for all harness-written state files. `build_map`
|
||||
/// checks both names during the transition window so existing containers
|
||||
/// don't lose gateway routing before their next rebuild.
|
||||
pub const READY_MARKER: &str = "hyperhive-socket-bound";
|
||||
const READY_MARKER_LEGACY: &str = ".bound";
|
||||
|
||||
#[must_use]
|
||||
pub fn host_sockets_path() -> PathBuf {
|
||||
crate::paths::agent_sockets_file()
|
||||
}
|
||||
|
||||
/// Per-agent socket subdir on the host. Lifecycle pre-creates this
|
||||
/// before container start so the bind-mount source exists; the
|
||||
/// harness binds the socket inside it as `web.sock`.
|
||||
#[must_use]
|
||||
pub fn agent_dir_for(name: &str) -> PathBuf {
|
||||
Path::new(AGENT_SOCKET_DIR).join(name)
|
||||
}
|
||||
|
||||
/// Compute the deterministic socket path for an agent. Pure function
|
||||
/// of the agent name so the value matches whatever
|
||||
/// [`agent_sockets::write`] writes for that agent, and whatever the
|
||||
/// harness binds via `HIVE_WEB_SOCKET`.
|
||||
#[must_use]
|
||||
pub fn socket_path_for(name: &str) -> PathBuf {
|
||||
agent_dir_for(name).join(SOCKET_FILENAME)
|
||||
}
|
||||
|
||||
/// Compute the agent-socket map for the given logical agent names.
|
||||
/// Includes manager and sub-agents. Filters by `READY_MARKER`
|
||||
/// presence: only agents whose harness has actually bound the unix
|
||||
/// socket appear in the map. Without this, the gateway would
|
||||
/// `proxy_pass` to a non-existent socket for agents that haven't
|
||||
/// been rebuilt yet or are mid-restart.
|
||||
///
|
||||
/// Accepts either the new `hyperhive-socket-bound` marker or the legacy
|
||||
/// `.bound` marker so existing containers keep their gateway routing
|
||||
/// through the transition window (before their next rebuild writes the
|
||||
/// new marker name).
|
||||
///
|
||||
/// `BTreeMap` keeps the JSON output sorted by key so a re-emit
|
||||
/// without churn produces byte-identical output for idempotent writes.
|
||||
#[must_use]
|
||||
pub fn build_map(names: &[String]) -> BTreeMap<String, PathBuf> {
|
||||
build_map_with(names, |name| {
|
||||
ready_marker_for(name).exists() || agent_dir_for(name).join(READY_MARKER_LEGACY).exists()
|
||||
})
|
||||
}
|
||||
|
||||
/// Body of `build_map` with the ready-check parameterised. Tests
|
||||
/// pass a predicate they control (no real filesystem access).
|
||||
/// Production callers go through `build_map` which wires the
|
||||
/// predicate to the on-disk `hyperhive-socket-bound` (or legacy
|
||||
/// `.bound`) marker check.
|
||||
fn build_map_with<F>(names: &[String], is_ready: F) -> BTreeMap<String, PathBuf>
|
||||
where
|
||||
F: Fn(&str) -> bool,
|
||||
{
|
||||
names
|
||||
.iter()
|
||||
.filter(|n| is_ready(n))
|
||||
.map(|n| (n.clone(), socket_path_for(n)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Path to the `hyperhive-socket-bound` marker file the harness writes
|
||||
/// after a successful `bind_unix`. Lives next to `web.sock` in the
|
||||
/// per-agent subdir so it's covered by the same bind-mount and same
|
||||
/// per-agent isolation as the socket itself.
|
||||
#[must_use]
|
||||
pub fn ready_marker_for(name: &str) -> PathBuf {
|
||||
agent_dir_for(name).join(READY_MARKER)
|
||||
}
|
||||
|
||||
/// Render the map as pretty-printed JSON. Pretty so a human peek at
|
||||
/// `cat /var/lib/hyperhive/run/agent-sockets.json` shows one row per agent
|
||||
/// — keeps the file readable without a separate jq step.
|
||||
fn render(map: &BTreeMap<String, PathBuf>) -> String {
|
||||
// Serialize as strings (PathBuf → JSON string via the Display
|
||||
// impl). BTreeMap → serde_json::to_string_pretty preserves key
|
||||
// order, so the output is deterministic across calls with the
|
||||
// same agent set.
|
||||
let stringly: BTreeMap<&String, String> = map
|
||||
.iter()
|
||||
.map(|(k, v)| (k, v.display().to_string()))
|
||||
.collect();
|
||||
serde_json::to_string_pretty(&stringly)
|
||||
.expect("BTreeMap<&String, String> is always serialisable")
|
||||
}
|
||||
|
||||
/// Atomically write the JSON for `names` to
|
||||
/// `/var/lib/hyperhive/run/agent-sockets.json`. Writes via a sibling
|
||||
/// `<path>.tmp` + rename so a crashing process never leaves a
|
||||
/// partial file behind that the gateway worker would fail to parse.
|
||||
///
|
||||
/// Idempotent — if the rendered content matches what's already on
|
||||
/// disk, the write + rename are skipped so the file's mtime stays
|
||||
/// stable and inotify watchers in the gateway (or any future
|
||||
/// watchers) don't fire spurious reload events.
|
||||
pub fn write(names: &[String]) -> Result<()> {
|
||||
let map = build_map(names);
|
||||
let body = render(&map);
|
||||
let path = host_sockets_path();
|
||||
if std::fs::read_to_string(&path).ok().as_deref() == Some(&body) {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
|
||||
}
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, &path).with_context(|| {
|
||||
format!(
|
||||
"rename {} -> {} (atomic publish)",
|
||||
tmp.display(),
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spawn the marker poll task. Periodically re-runs `write` so the
|
||||
/// JSON map picks up newly-bound sockets after a rebuild (harness
|
||||
/// drops a fresh `.bound` marker on start) without needing an explicit
|
||||
/// hook on container
|
||||
/// start. `write` is idempotent (skips the rename when content
|
||||
/// unchanged) so the steady-state cost is one directory stat per
|
||||
/// agent per poll interval.
|
||||
///
|
||||
/// Also calls `gateway_nginx::reload_if_pending` on every tick to
|
||||
/// retry a gateway nginx reload that may have failed on the previous
|
||||
/// tick (e.g. gateway container temporarily down). This recovers
|
||||
/// gateway routing without needing a manual gateway restart.
|
||||
///
|
||||
/// Mirrors the spawn-loop shape used by `crash_watch`,
|
||||
/// `reminder_scheduler`, etc. — the existing background-task
|
||||
/// convention in `main.rs`.
|
||||
pub fn spawn_poll() {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(10));
|
||||
// First tick fires immediately; that's fine — meta::sync_agents
|
||||
// also writes on boot, this just catches up the window before
|
||||
// the next agent restart.
|
||||
loop {
|
||||
interval.tick().await;
|
||||
match crate::lifecycle::agents_for_meta_listing().await {
|
||||
Ok(agents) => {
|
||||
let names: Vec<String> = agents.into_iter().map(|a| a.name).collect();
|
||||
if let Err(e) = write(&names) {
|
||||
tracing::debug!(error = ?e, "agent_sockets poll write failed");
|
||||
}
|
||||
// Regenerate the gateway nginx include whenever
|
||||
// socket readiness changes — the upstream
|
||||
// selection (UDS vs TCP) depends on .bound markers
|
||||
// which change independently of topology. Write is
|
||||
// idempotent; skips rename when nothing changed.
|
||||
if let Err(e) = crate::gateway_nginx::write(&names).await {
|
||||
tracing::debug!(error = ?e, "gateway_nginx poll write failed");
|
||||
}
|
||||
// Retry a pending nginx reload that failed on a
|
||||
// previous tick (no-op if no reload is pending).
|
||||
crate::gateway_nginx::reload_if_pending().await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(error = ?e, "agent_sockets poll: failed to list agents");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::lifecycle::MANAGER_NAME;
|
||||
|
||||
#[test]
|
||||
fn socket_path_for_uses_subdir_layout() {
|
||||
// Per-agent subdir + fixed socket filename — see module-level
|
||||
// "Per-agent subdir layout" for why this isn't a flat
|
||||
// `<name>.sock`. Pin both ends so a future move (e.g. to
|
||||
// `/run/hyperhive/sockets/`) requires updating both the
|
||||
// constant and the consumers.
|
||||
let p = socket_path_for("iris");
|
||||
assert_eq!(p, Path::new("/run/hive-agent/iris/web.sock"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_dir_for_is_socket_parent() {
|
||||
// `agent_dir_for` is what lifecycle bind-mounts per agent;
|
||||
// `socket_path_for` lives inside it. Keep them in lockstep so
|
||||
// a divergence (e.g. typo in one constant) surfaces here
|
||||
// rather than as a confusing nspawn bind-source-not-found at
|
||||
// container start.
|
||||
let dir = agent_dir_for("iris");
|
||||
let sock = socket_path_for("iris");
|
||||
assert_eq!(sock.parent(), Some(dir.as_path()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_map_includes_manager() {
|
||||
let names: Vec<String> = ["iris", MANAGER_NAME, "argus"]
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect();
|
||||
let map = build_map_with(&names, |_| true);
|
||||
assert!(map.contains_key(MANAGER_NAME));
|
||||
assert!(map.contains_key("iris"));
|
||||
assert!(map.contains_key("argus"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_map_uses_socket_path_for() {
|
||||
// Map values agree with the helper so callers can use either
|
||||
// (build_map for the bulk write, socket_path_for for one-off
|
||||
// lookups) without divergence.
|
||||
let names = vec!["iris".to_owned()];
|
||||
let map = build_map_with(&names, |_| true);
|
||||
assert_eq!(map.get("iris"), Some(&socket_path_for("iris")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_map_handles_empty_input() {
|
||||
let map = build_map_with::<fn(&str) -> bool>(&[], |_| true);
|
||||
assert!(map.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_map_dedupes_via_btreemap_key_collision() {
|
||||
// Duplicate inputs collapse via the map; no callsite passes
|
||||
// dups today, but guarding the invariant here means a future
|
||||
// bug doesn't surface as a corrupt JSON doc (two `"iris":`
|
||||
// keys).
|
||||
let names = vec!["iris".to_owned(), "iris".to_owned()];
|
||||
let map = build_map_with(&names, |_| true);
|
||||
assert_eq!(map.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_map_filters_by_ready_predicate() {
|
||||
// Only ready agents (with `hyperhive-socket-bound` marker) get
|
||||
// published. Pin the behaviour so a future refactor that drops
|
||||
// the filter surfaces here, not as a 502-spew in the gateway.
|
||||
let names: Vec<String> = ["iris", "argus", "atlas"]
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect();
|
||||
// Pretend only `atlas` has flipped + bound — the gate makes
|
||||
// sure only opted-in agents get a UDS upstream.
|
||||
let map = build_map_with(&names, |name| name == "atlas");
|
||||
assert!(map.contains_key("atlas"));
|
||||
assert!(!map.contains_key("iris"));
|
||||
assert!(!map.contains_key("argus"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_marker_path_is_sibling_of_socket() {
|
||||
// Marker lives in the same per-agent subdir as the socket so
|
||||
// the same bind-mount covers both; harness writes both inside
|
||||
// the container, host (and gateway via shared bind-mount)
|
||||
// sees both at the deterministic path.
|
||||
let marker = ready_marker_for("iris");
|
||||
let socket = socket_path_for("iris");
|
||||
assert_eq!(marker.parent(), socket.parent());
|
||||
assert_eq!(
|
||||
marker,
|
||||
Path::new("/run/hive-agent/iris/hyperhive-socket-bound")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_is_pretty_and_sorted() {
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"zeta".to_owned(),
|
||||
PathBuf::from("/run/hive-agent/zeta/web.sock"),
|
||||
);
|
||||
map.insert(
|
||||
"alpha".to_owned(),
|
||||
PathBuf::from("/run/hive-agent/alpha/web.sock"),
|
||||
);
|
||||
let body = render(&map);
|
||||
// Pretty-print = newlines between keys + indentation.
|
||||
assert!(body.contains('\n'));
|
||||
// BTreeMap sorts → alpha before zeta in output.
|
||||
let alpha_pos = body.find("alpha").expect("alpha in output");
|
||||
let zeta_pos = body.find("zeta").expect("zeta in output");
|
||||
assert!(alpha_pos < zeta_pos, "sorted order broken:\n{body}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_emits_paths_as_strings() {
|
||||
// PathBuf-valued map serialises as plain JSON strings (not
|
||||
// some {"inner": "..."} wrapper). Pin the shape so the
|
||||
// gateway-side reader can deserialise into String values
|
||||
// without nested struct logic.
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"iris".to_owned(),
|
||||
PathBuf::from("/run/hive-agent/iris/web.sock"),
|
||||
);
|
||||
let body = render(&map);
|
||||
assert!(body.contains("\"iris\""));
|
||||
assert!(body.contains("\"/run/hive-agent/iris/web.sock\""));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue